fix: detecting copyright cell

* fix: filename rename

* fix: lint issues

.cloud-build/execute_notebook_helper.py

@@ -26,6 +26,9 @@ from utils import util
 
 # This script is used to execute a notebook and write out the output notebook.
 
+# This is used to force papermill to use this kernel to run the notebook instead of any defined inside the notebook itself
+DEFAULT_KERNEL_NAME = "python3"
+
 
 def execute_notebook(
     notebook_source: str,
@@ -50,14 +53,11 @@ def execute_notebook(
 
     execution_exception = None
 
-
     print("\n=== DOWNLOAD EXECUTED NOTEBOOK ===\n")
-    print(
-                f"Please debug the executed notebook by downloading the executed notebook:"
-            )
+    print(f"Please debug the executed notebook by downloading the executed notebook:")
 
     print("Option 1. Using gsutil. Run the following command in your terminal.")
-    print(f"\tgsutil cp \"{output_file_or_uri}\" .")
+    print(f'\tgsutil cp "{output_file_or_uri}" .')
 
     print("Option 2. Using this link.")
     print(f"\thttps://storage.googleapis.com/{output_file_or_uri[5:]}")
@@ -72,6 +72,7 @@ def execute_notebook(
             output_path=notebook_source,
             progress_bar=should_log_output,
             request_save_on_cell_execute=should_log_output,
+            kernel_name=DEFAULT_KERNEL_NAME,
             log_output=should_log_output,
             stdout_file=sys.stdout if should_log_output else None,
             stderr_file=sys.stderr if should_log_output else None,

community-content/CODEOWNERS

@@ -1,5 +1,6 @@
 *  @vertex-ai-samples-contributors  @GoogleCloudPlatform/cloudml-samples-owners
 /tf_agents_bandits_movie_recommendation_with_kfp_and_vertex_sdk	@yinghsienwu
+/pytorch_pre_built_images_deployment @googleapis/vertex-prediction-team
 /pytorch_text_classification_using_vertex_sdk_and_gcloud  @RajeshThallam
 /pytorch_text_classification_using_vertex_sdk_and_gcloud		@RajeshThallam @ultrons
 /sklearn_text_classification_from_script_using_vertex_sdk       @maxhardt

community-content/cpr-examples/timm_serving/.gitignore

@@ -2,4 +2,5 @@ cpr_model_server.py
 entrypoint.py
 state_dict.pth
 config.json
-**/__pycache__
+**/__pycache__
+!testdata/**

community-content/cpr-examples/timm_serving/README.md

@@ -2,7 +2,7 @@
 
 ## About CPR
 
-CPR ([custom prediction routines](https://github.com/googleapis/python-aiplatform/blob/custom-prediction-routine/google/cloud/aiplatform/prediction/README.md)) is a framework designed by Google Cloud developers to make it easier to combine machine learning models with custom preprocessing and postprocessing logic in a real-time serving application. 
+CPR ([custom prediction routines](https://github.com/googleapis/python-aiplatform/blob/main/google/cloud/aiplatform/prediction/README.md)) is a framework designed by Google Cloud developers to make it easier to combine machine learning models with custom preprocessing and postprocessing logic in a real-time serving application. 
 
 ## Using this example
 
@@ -34,6 +34,23 @@ Finally, install the Python modules required to build and run the model server:
 pip install -r requirements.txt
 ```
 
+### Auth
+
+This example uses Google Cloud Storage for hosting model artifacts and Artifact Registry to store the container image. 
+You'll need to authorize yourself before you can interact with these.
+
+First, log in to GCP with application default credentials:
+```sh
+gcloud auth application-default login
+```
+
+Next, if you haven't done so already, set up the [gcloud credential helper](https://cloud.google.com/artifact-registry/docs/docker/authentication)
+for the Artifact Registry region where you intend to host the image.  
+```
+gcloud auth configure-docker <region>-docker.pkg.dev
+```
+
+
 ### Predictor
 
 The `TimmPredictor` class in `timm_serving/predictor.py` implements most of the important logic for the server.

community-content/cpr-examples/timm_serving/config.py

@@ -60,9 +60,9 @@ class CPRConfig(object):
     image: str = "timm_predictor:latest"
     artifact_local_dir: str = ""
     region: str = "us-central1"
-    project_id: str = "samthrasher-experimental"
+    project_id: str = "<your project ID here>"
     repository: str = "cpr-images"
-    artifact_gcs_dir: str = "gs://samthrasher-cpr-example/timm-vit224/"
+    artifact_gcs_dir: str = "gs://<your bucket ID here>/timm-vit224/"
     model_name: str = ""
     endpoint_name: str = ""
     machine_type: str = "n1-standard-2"

community-content/cpr-examples/timm_serving/requirements.txt

@@ -5,4 +5,4 @@ timm==0.5.4
 smart_open==6.0.0
 
 google-cloud-storage>=1.26.0,<2.0.0dev
-google-cloud-aiplatform[prediction] @ git+https://github.com/googleapis/python-aiplatform.git@custom-prediction-routine
+google-cloud-aiplatform[prediction]>=1.16.0

community-content/cpr-examples/timm_serving/test.py

@@ -70,7 +70,10 @@ class PredictorUnitTests(absltest.TestCase):
     def setUp(self):
         super().setUp()
         self.config = CPRConfig()
-        self.config.load()
+        try:
+            self.config.load()
+        except FileNotFoundError:
+            logging.info("No saved config file found, using default values.")
         self.predictor = predictor.TimmPredictor()
 
     def test_load_from_saved_state_dict_ok(self):
@@ -170,7 +173,10 @@ class ServerEndToEndTests(absltest.TestCase):
     def setUp(self):
         super().setUp()
         self.config = CPRConfig()
-        self.config.load()
+        try:
+            self.config.load()
+        except FileNotFoundError:
+            logging.info("No saved config file found, using default values.")
         self.local_model = cpr.LocalModel(
             serving_container_spec=aiplatform.gapic.ModelContainerSpec(
                 image_uri=self.config.image

community-content/cpr-examples/timm_serving/testdata/bad_model_1/state_dict.pth

@@ -0,0 +1 @@
+blah

community-content/pytorch_pre_built_images_deployment/README.md

@@ -0,0 +1,30 @@
+# PyTorch Deployment on Google Cloud: Text Classification
+
+**This is an Experimental release**, covered by the Pre-GA Offerings Terms of your Google Cloud Platform [Terms of Service](https://cloud.google.com/terms).
+
+Experiments are focused on validating a prototype and are not guaranteed to be released. They are not intended for production use or covered by any SLA, support obligation, or deprecation policy and might be subject to backward-incompatible changes.
+
+**Kindly drop us a note before you run any scale tests.**
+
+**Do not hesitate to contact vertexai-prediction-preview-feedback@google.com if you have any questions or run into any issues.**
+
+The projects need to be added to the allowlist in order to deploy PyTorch models using Vertex AI Prediction pre-built PyTorch images. If you are interested in the feature, please send an email to vertexai-prediction-preview-feedback@google.com to provide your project numbers OR project ids.
+
+## Overview
+
+In the PyTorch on Google Cloud series of blog posts, we aim to share how to deploy PyTorch models at scale on [Vertex AI](https://cloud.google.com/vertex-ai).
+
+This tutorial on text classification shows how to deploy a PyTorch based text classification model on [Vertex AI](https://cloud.google.com/vertex-ai/docs/start/client-libraries#python) using Vertex SDK and [`gcloud ai`](https://cloud.google.com/sdk/gcloud/reference/beta/ai).
+
+## Notebooks
+
+| <h4>Notebook</h4> | <h4>Description</h4>                       |
+| :-------- | :------- |
+| [pytorch-text-classification-vertex-ai-deploy.ipynb](./pytorch-text-classification-vertex-ai-deploy.ipynb) | Notebook to show deploying a PyTorch model on Vertex AI |
+
+## Folders
+
+
+| <h4>Folder Name</h4> | <h4>Description</h4>                       |
+| :-------- | :------- |
+| [`predictor`](./predictor) | Folder with custom prediction handler to deploy a PyTorch model to Vertex Prediction. In the [notebook](./pytorch-text-classification-vertex-ai-deploy.ipynb), this folder is used for deploying a PyTorch model on Vertex AI using Vertex Prediction pre-built PyTorch images |

community-content/pytorch_pre_built_images_deployment/predictor/custom_handler.py

@@ -0,0 +1,91 @@
+
+import os
+import json
+import logging
+
+import torch
+from transformers import AutoModelForSequenceClassification, AutoTokenizer
+from ts.torch_handler.base_handler import BaseHandler
+
+logger = logging.getLogger(__name__)
+
+
+class TransformersClassifierHandler(BaseHandler):
+    """
+    The handler takes an input string and returns the classification text 
+    based on the serialized transformers checkpoint.
+    """
+    def __init__(self):
+        super(TransformersClassifierHandler, self).__init__()
+        self.initialized = False
+
+    def initialize(self, ctx):
+        """ Loads the model.pt file and initialized the model object.
+        Instantiates Tokenizer for preprocessor to use
+        Loads labels to name mapping file for post-processing inference response
+        """
+        self.manifest = ctx.manifest
+
+        properties = ctx.system_properties
+        model_dir = properties.get("model_dir")
+        self.device = torch.device("cuda:" + str(properties.get("gpu_id")) if torch.cuda.is_available() else "cpu")
+
+        # Read model serialize/pt file
+        serialized_file = self.manifest["model"]["serializedFile"]
+        model_pt_path = os.path.join(model_dir, serialized_file)
+        if not os.path.isfile(model_pt_path):
+            raise RuntimeError("Missing the model.pt or pytorch_model.bin file")
+        
+        # Load model
+        self.model = AutoModelForSequenceClassification.from_pretrained(model_dir)
+        self.model.to(self.device)
+        self.model.eval()
+        logger.debug('Transformer model from path {0} loaded successfully'.format(model_dir))
+        
+        # Ensure to use the same tokenizer used during training
+        self.tokenizer = AutoTokenizer.from_pretrained('bert-base-cased')
+
+        # Read the mapping file, index to object name
+        mapping_file_path = os.path.join(model_dir, "index_to_name.json")
+
+        if os.path.isfile(mapping_file_path):
+            with open(mapping_file_path) as f:
+                self.mapping = json.load(f)
+        else:
+            logger.warning('Missing the index_to_name.json file. Inference output will default.')
+            self.mapping = {"0": "Negative",  "1": "Positive"}
+
+        self.initialized = True
+
+    def preprocess(self, data):
+        """ Preprocessing input request by tokenizing
+            Extend with your own preprocessing steps as needed
+        """
+        text = data[0].get("data")
+        if text is None:
+            text = data[0].get("body")
+        sentences = text.decode('utf-8')
+        logger.info("Received text: '%s'", sentences)
+
+        # Tokenize the texts
+        tokenizer_args = ((sentences,))
+        inputs = self.tokenizer(*tokenizer_args,
+                                padding='max_length',
+                                max_length=128,
+                                truncation=True,
+                                return_tensors = "pt")
+        return inputs
+
+    def inference(self, inputs):
+        """ Predict the class of a text using a trained transformer model.
+        """
+        prediction = self.model(inputs['input_ids'].to(self.device))[0].argmax().item()
+
+        if self.mapping:
+            prediction = self.mapping[str(prediction)]
+
+        logger.info("Model predicted: '%s'", prediction)
+        return [prediction]
+
+    def postprocess(self, inference_output):
+        return inference_output

community-content/pytorch_pre_built_images_deployment/predictor/index_to_name.json

@@ -0,0 +1,5 @@
+
+{
+    "0": "Negative", 
+    "1": "Positive"
+}

community-content/pytorch_text_classification_using_vertex_sdk_and_gcloud/pytorch-text-classification-vertex-ai-train-tune-deploy.ipynb

@@ -658,8 +658,8 @@
       },
       "outputs": [],
       "source": [
-        "datasets = load_dataset(\"imdb\")\n",
-        "datasets"
+        "dataset = load_dataset(\"imdb\")\n",
+        "dataset"
       ]
     },
     {
@@ -668,7 +668,7 @@
         "id": "RzfPtOMoIrIu"
       },
       "source": [
-        "The `datasets` object itself is [`DatasetDict`](https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasetdict), which contains one key for the training, validation and test set."
+        "The `dataset` object itself is [`DatasetDict`](https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasetdict), which contains one key for the training, validation and test set."
       ]
     },
     {
@@ -681,12 +681,12 @@
       "source": [
         "print(\n",
         "    \"Total # of rows in training dataset {} and size {:5.2f} MB\".format(\n",
-        "        datasets[\"train\"].shape[0], datasets[\"train\"].size_in_bytes / (1024 * 1024)\n",
+        "        dataset[\"train\"].shape[0], dataset[\"train\"].size_in_bytes / (1024 * 1024)\n",
         "    )\n",
         ")\n",
         "print(\n",
         "    \"Total # of rows in test dataset {} and size {:5.2f} MB\".format(\n",
-        "        datasets[\"test\"].shape[0], datasets[\"test\"].size_in_bytes / (1024 * 1024)\n",
+        "        dataset[\"test\"].shape[0], dataset[\"test\"].size_in_bytes / (1024 * 1024)\n",
         "    )\n",
         ")"
       ]
@@ -708,7 +708,7 @@
       },
       "outputs": [],
       "source": [
-        "datasets[\"train\"][0]"
+        "dataset[\"train\"][0]"
       ]
     },
     {
@@ -728,7 +728,7 @@
       },
       "outputs": [],
       "source": [
-        "label_list = datasets[\"train\"].unique(\"label\")\n",
+        "label_list = dataset[\"train\"].unique(\"label\")\n",
         "label_list"
       ]
     },
@@ -779,7 +779,7 @@
       },
       "outputs": [],
       "source": [
-        "show_random_elements(datasets[\"train\"])"
+        "show_random_elements(dataset[\"train\"])"
       ]
     },
     {
@@ -883,7 +883,7 @@
       },
       "outputs": [],
       "source": [
-        "example = datasets[\"train\"][4]\n",
+        "example = dataset[\"train\"][4]\n",
         "print(example)"
       ]
     },
@@ -920,7 +920,7 @@
       "source": [
         "# Dataset loading repeated here to make this cell idempotent\n",
         "# Since we are over-writing datasets variable\n",
-        "datasets = load_dataset(\"imdb\")\n",
+        "dataset = load_dataset(\"imdb\")\n",
         "\n",
         "# Mapping labels to ids\n",
         "# NOTE: We can extract this automatically but the `Unique` method of the datasets\n",
@@ -948,7 +948,7 @@
         "\n",
         "\n",
         "# apply preprocessing function to input examples\n",
-        "datasets = datasets.map(preprocess_function, batched=True, load_from_cache_file=True)"
+        "dataset = dataset.map(preprocess_function, batched=True, load_from_cache_file=True)"
       ]
     },
     {
@@ -1091,8 +1091,8 @@
         "trainer = Trainer(\n",
         "    model,\n",
         "    args,\n",
-        "    train_dataset=datasets[\"train\"],\n",
-        "    eval_dataset=datasets[\"test\"],\n",
+        "    train_dataset=dataset[\"train\"],\n",
+        "    eval_dataset=dataset[\"test\"],\n",
         "    data_collator=default_data_collator,\n",
         "    tokenizer=tokenizer,\n",
         "    compute_metrics=compute_metrics,\n",

notebooks/community/CODEOWNERS

@@ -17,6 +17,7 @@
 /explainable_ai/SDK_Custom_Container_XAI.ipynb @brianchunkang
 /matching_engine/sdk_matching_engine_for_indexing.ipynb @ivanmkc
 /matching_engine/matching_engine_for_indexing.ipynb @yinghsienwu
+/matching_engine/stream_update_for_matching_engine.ipynb @peterping666
 /sdk/pytorch_lightning_custom_container_training.ipynb @brianchunkang
 /tensorboard @yfang1
 /feature_store @nayaknishant @morgandu

notebooks/community/feature_store/gapic-feature-store.ipynb

@@ -292,6 +292,37 @@
         "    PROJECT_ID = \"python-docs-samples-tests\"  # @param {type:\"string\"}"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d9f118b92c74"
+      },
+      "source": [
+        "#### UUID\n",
+        "\n",
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3ee72715c0fd"
+      },
+      "outputs": [],
+      "source": [
+        "import random\n",
+        "import string\n",
+        "\n",
+        "\n",
+        "# Generate a uuid of a specifed length(default=8)\n",
+        "def generate_uuid(length: int = 8) -> str:\n",
+        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
+        "\n",
+        "\n",
+        "UUID = generate_uuid()"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -478,7 +509,6 @@
         "from google.cloud.aiplatform_v1.types import \\\n",
         "    featurestore_service as featurestore_service_pb2\n",
         "from google.cloud.aiplatform_v1.types import io as io_pb2\n",
-        "from google.protobuf.duration_pb2 import Duration\n",
         "\n",
         "# Create admin_client for CRUD and data_client for reading feature values.\n",
         "admin_client = FeaturestoreServiceClient(client_options={\"api_endpoint\": API_ENDPOINT})\n",
@@ -542,7 +572,7 @@
       },
       "outputs": [],
       "source": [
-        "FEATURESTORE_ID = \"movie_prediction\"\n",
+        "FEATURESTORE_ID = f\"movie_prediction_{UUID}\"\n",
         "try:\n",
         "    create_lro = admin_client.create_featurestore(\n",
         "        featurestore_service_pb2.CreateFeaturestoreRequest(\n",
@@ -567,7 +597,7 @@
         "id": "ag8pCQ7rNjVf"
       },
       "source": [
-        "You can use [GetFeaturestore](https://cloud.google.com/vertex-ai/docs/reference/rpc/google.cloud.aiplatform.v1beta1#google.cloud.aiplatform.v1beta1.FeaturestoreService.GetFeaturestore) or [ListFeaturestores](https://cloud.google.com/vertex-ai/docs/reference/rpc/google.cloud.aiplatform.v1beta1#google.cloud.aiplatform.v1beta1.FeaturestoreService.ListFeaturestores) to check if the Featurestore was successfully created. The following example gets the details of the Featurestore.\n"
+        "You can use [GetFeaturestore](https://cloud.google.com/vertex-ai/docs/reference/rpc/google.cloud.aiplatform.v1#google.cloud.aiplatform.v1.FeaturestoreService.GetFeaturestore) or [ListFeaturestores](https://cloud.google.com/vertex-ai/docs/reference/rpc/google.cloud.aiplatform.v1#google.cloud.aiplatform.v1.FeaturestoreService.ListFeaturestores) to check if the Featurestore was successfully created. The following example gets the details of the Featurestore.\n"
       ]
     },
     {
@@ -589,7 +619,7 @@
         "id": "018ab19d934f"
       },
       "source": [
-        "Auto scaling is available in v1beta1 since v1.11. Below is the example for the `CreateFeaturestoreRequest` with auto-scaling, use it with `aiplatform_v1beta1.FeaturestoreServiceClient` to create Featurestore:"
+        "Auto scaling is available in v1 since v1.11. Below is the example for the `CreateFeaturestoreRequest` with auto-scaling, use it with `aiplatform_v1.FeaturestoreServiceClient` to create Featurestore:"
       ]
     },
     {
@@ -600,17 +630,17 @@
       },
       "outputs": [],
       "source": [
-        "from google.cloud.aiplatform_v1beta1.types import \\\n",
-        "    featurestore as v1beta1_featurestore_pb2\n",
-        "from google.cloud.aiplatform_v1beta1.types import \\\n",
-        "    featurestore_service as v1beta1_featurestore_service_pb2\n",
+        "from google.cloud.aiplatform_v1.types import \\\n",
+        "    featurestore as v1_featurestore_pb2\n",
+        "from google.cloud.aiplatform_v1.types import \\\n",
+        "    featurestore_service as v1_featurestore_service_pb2\n",
         "\n",
-        "create_featurestore_request = v1beta1_featurestore_service_pb2.CreateFeaturestoreRequest(\n",
+        "create_featurestore_request = v1_featurestore_service_pb2.CreateFeaturestoreRequest(\n",
         "    parent=BASE_RESOURCE_PATH,\n",
         "    featurestore_id=FEATURESTORE_ID,\n",
-        "    featurestore=v1beta1_featurestore_pb2.Featurestore(\n",
-        "        online_serving_config=v1beta1_featurestore_pb2.Featurestore.OnlineServingConfig(\n",
-        "            scaling=v1beta1_featurestore_pb2.Featurestore.OnlineServingConfig.Scaling(\n",
+        "    featurestore=v1_featurestore_pb2.Featurestore(\n",
+        "        online_serving_config=v1_featurestore_pb2.Featurestore.OnlineServingConfig(\n",
+        "            scaling=v1_featurestore_pb2.Featurestore.OnlineServingConfig.Scaling(\n",
         "                min_node_count=1, max_node_count=5\n",
         "            )\n",
         "        ),\n",
@@ -681,7 +711,7 @@
         "id": "dPkT7KDuEvWv"
       },
       "source": [
-        "Feature [monitoring](https://cloud.google.com/vertex-ai/docs/featurestore/monitoring) is in preview, so you need to use v1beta1 Python. Import feature analysis is only available through SDK for now."
+        "Feature [monitoring](https://cloud.google.com/vertex-ai/docs/featurestore/monitoring) is in preview, so you need to use v1 Python. Import feature analysis is only available through SDK for now."
       ]
     },
     {
@@ -692,36 +722,35 @@
       },
       "outputs": [],
       "source": [
-        "from google.cloud.aiplatform_v1beta1 import \\\n",
-        "    FeaturestoreServiceClient as v1beta1_FeaturestoreServiceClient\n",
-        "from google.cloud.aiplatform_v1beta1.types import \\\n",
-        "    entity_type as v1beta1_entity_type_pb2\n",
-        "from google.cloud.aiplatform_v1beta1.types import \\\n",
-        "    featurestore_monitoring as v1beta1_featurestore_monitoring_pb2\n",
-        "from google.cloud.aiplatform_v1beta1.types import \\\n",
-        "    featurestore_service as v1beta1_featurestore_service_pb2\n",
+        "from google.cloud.aiplatform_v1 import \\\n",
+        "    FeaturestoreServiceClient as v1_FeaturestoreServiceClient\n",
+        "from google.cloud.aiplatform_v1.types import entity_type as v1_entity_type_pb2\n",
+        "from google.cloud.aiplatform_v1.types import \\\n",
+        "    featurestore_monitoring as v1_featurestore_monitoring_pb2\n",
+        "from google.cloud.aiplatform_v1.types import \\\n",
+        "    featurestore_service as v1_featurestore_service_pb2\n",
         "\n",
-        "v1beta1_admin_client = v1beta1_FeaturestoreServiceClient(\n",
+        "v1_admin_client = v1_FeaturestoreServiceClient(\n",
         "    client_options={\"api_endpoint\": API_ENDPOINT}\n",
         ")\n",
         "\n",
         "# Enable import feature analysis for users entity type.\n",
         "# All Features belonging to this EntityType will by default inherit the monitoring config.\n",
-        "v1beta1_admin_client.update_entity_type(\n",
-        "    v1beta1_featurestore_service_pb2.UpdateEntityTypeRequest(\n",
-        "        entity_type=v1beta1_entity_type_pb2.EntityType(\n",
+        "v1_admin_client.update_entity_type(\n",
+        "    v1_featurestore_service_pb2.UpdateEntityTypeRequest(\n",
+        "        entity_type=v1_entity_type_pb2.EntityType(\n",
         "            name=admin_client.entity_type_path(\n",
         "                PROJECT_ID, REGION, FEATURESTORE_ID, \"users\"\n",
         "            ),\n",
-        "            monitoring_config=v1beta1_featurestore_monitoring_pb2.FeaturestoreMonitoringConfig(\n",
-        "                import_features_analysis=v1beta1_featurestore_monitoring_pb2.FeaturestoreMonitoringConfig.ImportFeaturesAnalysis(\n",
-        "                    anomaly_detection_baseline=v1beta1_featurestore_monitoring_pb2.FeaturestoreMonitoringConfig.ImportFeaturesAnalysis.Baseline.LATEST_STATS,\n",
-        "                    state=v1beta1_featurestore_monitoring_pb2.FeaturestoreMonitoringConfig.ImportFeaturesAnalysis.State.ENABLED,\n",
+        "            monitoring_config=v1_featurestore_monitoring_pb2.FeaturestoreMonitoringConfig(\n",
+        "                import_features_analysis=v1_featurestore_monitoring_pb2.FeaturestoreMonitoringConfig.ImportFeaturesAnalysis(\n",
+        "                    anomaly_detection_baseline=v1_featurestore_monitoring_pb2.FeaturestoreMonitoringConfig.ImportFeaturesAnalysis.Baseline.LATEST_STATS,\n",
+        "                    state=v1_featurestore_monitoring_pb2.FeaturestoreMonitoringConfig.ImportFeaturesAnalysis.State.ENABLED,\n",
         "                ),\n",
-        "                numerical_threshold_config=v1beta1_featurestore_monitoring_pb2.FeaturestoreMonitoringConfig.ThresholdConfig(\n",
+        "                numerical_threshold_config=v1_featurestore_monitoring_pb2.FeaturestoreMonitoringConfig.ThresholdConfig(\n",
         "                    value=0.001,\n",
         "                ),\n",
-        "                categorical_threshold_config=v1beta1_featurestore_monitoring_pb2.FeaturestoreMonitoringConfig.ThresholdConfig(\n",
+        "                categorical_threshold_config=v1_featurestore_monitoring_pb2.FeaturestoreMonitoringConfig.ThresholdConfig(\n",
         "                    value=0.001,\n",
         "                ),\n",
         "            ),\n",
@@ -736,7 +765,7 @@
         "id": "85b1f59fbf6d"
       },
       "source": [
-        "The easiest way to set up snapshot analysis for now is using [console UI](https://console.cloud.google.com/vertex-ai/features). For completeness, below is example to do this using v1beta1 SDK.\n",
+        "The easiest way to set up snapshot analysis for now is using [console UI](https://console.cloud.google.com/vertex-ai/features). For completeness, below is example to do this using v1 SDK.\n",
         "\n",
         "You can view monitoring statistics on [console UI](https://console.cloud.google.com/vertex-ai/features)."
       ]
@@ -749,36 +778,35 @@
       },
       "outputs": [],
       "source": [
-        "from google.cloud.aiplatform_v1beta1 import \\\n",
-        "    FeaturestoreServiceClient as v1beta1_FeaturestoreServiceClient\n",
-        "from google.cloud.aiplatform_v1beta1.types import \\\n",
-        "    entity_type as v1beta1_entity_type_pb2\n",
-        "from google.cloud.aiplatform_v1beta1.types import \\\n",
-        "    featurestore_monitoring as v1beta1_featurestore_monitoring_pb2\n",
-        "from google.cloud.aiplatform_v1beta1.types import \\\n",
-        "    featurestore_service as v1beta1_featurestore_service_pb2\n",
+        "from google.cloud.aiplatform_v1 import \\\n",
+        "    FeaturestoreServiceClient as v1_FeaturestoreServiceClient\n",
+        "from google.cloud.aiplatform_v1.types import entity_type as v1_entity_type_pb2\n",
+        "from google.cloud.aiplatform_v1.types import \\\n",
+        "    featurestore_monitoring as v1_featurestore_monitoring_pb2\n",
+        "from google.cloud.aiplatform_v1.types import \\\n",
+        "    featurestore_service as v1_featurestore_service_pb2\n",
         "\n",
-        "v1beta1_admin_client = v1beta1_FeaturestoreServiceClient(\n",
+        "v1_admin_client = v1_FeaturestoreServiceClient(\n",
         "    client_options={\"api_endpoint\": API_ENDPOINT}\n",
         ")\n",
         "\n",
         "# Enable snapshot analysis for users entity type.\n",
         "# All Features belonging to this EntityType will by default inherit the monitoring config.\n",
-        "v1beta1_admin_client.update_entity_type(\n",
-        "    v1beta1_featurestore_service_pb2.UpdateEntityTypeRequest(\n",
-        "        entity_type=v1beta1_entity_type_pb2.EntityType(\n",
+        "v1_admin_client.update_entity_type(\n",
+        "    v1_featurestore_service_pb2.UpdateEntityTypeRequest(\n",
+        "        entity_type=v1_entity_type_pb2.EntityType(\n",
         "            name=admin_client.entity_type_path(\n",
         "                PROJECT_ID, REGION, FEATURESTORE_ID, \"users\"\n",
         "            ),\n",
-        "            monitoring_config=v1beta1_featurestore_monitoring_pb2.FeaturestoreMonitoringConfig(\n",
-        "                snapshot_analysis=v1beta1_featurestore_monitoring_pb2.FeaturestoreMonitoringConfig.SnapshotAnalysis(\n",
-        "                    monitoring_interval=Duration(seconds=86400),  # 1 day\n",
+        "            monitoring_config=v1_featurestore_monitoring_pb2.FeaturestoreMonitoringConfig(\n",
+        "                snapshot_analysis=v1_featurestore_monitoring_pb2.FeaturestoreMonitoringConfig.SnapshotAnalysis(\n",
+        "                    monitoring_interval_days=1,  # 1 day\n",
         "                    staleness_days=30,\n",
         "                ),\n",
-        "                numerical_threshold_config=v1beta1_featurestore_monitoring_pb2.FeaturestoreMonitoringConfig.ThresholdConfig(\n",
+        "                numerical_threshold_config=v1_featurestore_monitoring_pb2.FeaturestoreMonitoringConfig.ThresholdConfig(\n",
         "                    value=0.001,\n",
         "                ),\n",
-        "                categorical_threshold_config=v1beta1_featurestore_monitoring_pb2.FeaturestoreMonitoringConfig.ThresholdConfig(\n",
+        "                categorical_threshold_config=v1_featurestore_monitoring_pb2.FeaturestoreMonitoringConfig.ThresholdConfig(\n",
         "                    value=0.001,\n",
         "                ),\n",
         "            ),\n",
@@ -891,8 +919,8 @@
       "source": [
         "## Search created features\n",
         "\n",
-        "While the [ListFeatures](https://cloud.google.com/vertex-ai/docs/reference/rpc/google.cloud.aiplatform.v1beta1#google.cloud.aiplatform.v1beta1.FeaturestoreService.ListFeatures) method allows you to easily view all features of a single\n",
-        "entity type, the [SearchFeatures](https://cloud.google.com/vertex-ai/docs/reference/rpc/google.cloud.aiplatform.v1beta1#google.cloud.aiplatform.v1beta1.FeaturestoreService.SearchFeatures) method searches across all featurestores\n",
+        "While the [ListFeatures](https://cloud.google.com/vertex-ai/docs/reference/rpc/google.cloud.aiplatform.v1#google.cloud.aiplatform.v1.FeaturestoreService.ListFeatures) method allows you to easily view all features of a single\n",
+        "entity type, the [SearchFeatures](https://cloud.google.com/vertex-ai/docs/reference/rpc/google.cloud.aiplatform.v1#google.cloud.aiplatform.v1.FeaturestoreService.SearchFeatures) method searches across all featurestores\n",
         "and entity types in a given location (such as `us-central1`). This can help you discover features that were created by someone else.\n",
         "\n",
         "You can query based on feature properties including feature ID, entity type ID,\n",
@@ -1206,7 +1234,7 @@
       },
       "source": [
         "The\n",
-        "[Online Serving APIs](https://cloud.google.com/vertex-ai/docs/reference/rpc/google.cloud.aiplatform.v1beta1#featurestoreonlineservingservice)\n",
+        "[Online Serving APIs](https://cloud.google.com/vertex-ai/docs/reference/rpc/google.cloud.aiplatform.v1#featurestoreonlineservingservice)\n",
         "lets you serve feature values for small batches of entities. It's designed for latency-sensitive service, such as online model prediction. For example, for a movie service, you might want to quickly shows movies that the current user would most likely watch by using online predictions."
       ]
     },

notebooks/community/ml_ops/stage1/get_started_with_data_labeling.ipynb

@@ -212,7 +212,7 @@
         "\n",
         "3. [Enable the Vertex AI APIs and Compute Engine APIs.](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component)\n",
         "\n",
-        "4. [Google Cloud SDK](https://cloud.google.com/sdk) is already installed in Google Cloud Notebooks.\n",
+        "4. [Google Cloud SDK](https://cloud.google.com/sdk) is already installed in Vertex AI Workbench Notebooks.\n",
         "\n",
         "5. Enter your project ID in the cell below. Then run the  cell to make sure the\n",
         "Cloud SDK uses the right project for all the commands in this notebook.\n",
@@ -374,15 +374,8 @@
         "### Authenticate your Google Cloud account\n",
         "\n",
         "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
-        "authenticated. Skip this step."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "32e1cd21a5d5"
-      },
-      "source": [
+        "authenticated. \n",
+        "\n",
         "**If you are using Colab**, run the cell below and follow the instructions\n",
         "when prompted to authenticate your account via oAuth.\n",
         "\n",

notebooks/community/ml_ops/stage5/get_started_with_vertex_endpoints.ipynb

@@ -340,7 +340,7 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. \n",
         "\n",
         "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
         "\n",
@@ -376,12 +376,11 @@
         "import sys\n",
         "\n",
         "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = False\n",
+        "IS_COLAB = \"google.colab\" in sys.modules\n",
         "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
         "    \"DL_ANACONDA_HOME\"\n",
         "):\n",
         "    if \"google.colab\" in sys.modules:\n",
-        "        IS_COLAB = True\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
         "        google_auth.authenticate_user()\n",
@@ -428,8 +427,9 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
+        "    BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
     {
@@ -785,7 +785,7 @@
       },
       "outputs": [],
       "source": [
-        "endpoint.gca_resource"
+        "print(endpoint.gca_resource)"
       ]
     },
     {
@@ -908,7 +908,7 @@
       },
       "outputs": [],
       "source": [
-        "endpoint.gca_resource.deployed_models[0]"
+        "print(endpoint.gca_resource.deployed_models[0])"
       ]
     },
     {
@@ -1203,12 +1203,10 @@
         "\n",
         "In this pipeline, you create an `Endpoint` resource, and then you deploy a `Model` resource to the `Endpoint` resource. The `Model` resource  to deploy is your existing TFHub model which you previously imported as a `Model` resource. The steps are:\n",
         "\n",
-        "- For pipeline parameters, pass the resource name and resource URI for the existing `Model` resource.\n",
-        "- Use the `importer_node()` component to create a `VertexModel` pipeline artifact for the model.\n",
+        "- For pipeline parameters, pass the resource name for the existing `Model` resource.\n",
+        "- Use the `GetVertexModelOp()` component to create a `VertexModel` pipeline artifact for the model.\n",
         "- Create an `Endpoint` resource.\n",
-        "- Using the `VertexModel` pipeline artifact, deploy the `Model` resource to the `Endpoint` resource.\n",
-        "\n",
-        "*Note:* This example currently blocked by internal issue: b/219835305"
+        "- Using the `VertexModel` pipeline artifact, deploy the `Model` resource to the `Endpoint` resource."
       ]
     },
     {
@@ -1225,20 +1223,6 @@
         "\n",
         "PIPELINE_ROOT = \"{}/pipeline_root/endpoint_example\".format(BUCKET_URI)\n",
         "\n",
-        "# (WORKAROUND  b/219835305)\n",
-        "@component(\n",
-        "    base_image=\"python:3.9\",\n",
-        "    packages_to_install=[\"google-cloud-aiplatform\"],\n",
-        ")\n",
-        "def return_unmanaged_model(\n",
-        "    serving_image: str, artifact_uri: str, resource_name: str, model: Output[Artifact]\n",
-        "):\n",
-        "    model.metadata[\"containerSpec\"] = {\"imageUri\": serving_image}\n",
-        "\n",
-        "    model.metadata[\"resourceName\"] = resource_name\n",
-        "\n",
-        "    model.uri = artifact_uri\n",
-        "\n",
         "\n",
         "@dsl.pipeline(\n",
         "    name=\"create-endpoint-deploy-model\",\n",
@@ -1246,34 +1230,16 @@
         ")\n",
         "def pipeline(\n",
         "    display_name: str,\n",
-        "    resource_uri: str,\n",
         "    resource_name: str,\n",
-        "    # Model properties (WORKAROUND  b/219835305)\n",
-        "    serving_image: str,\n",
-        "    artifact_uri: str,\n",
         "    project: str = PROJECT_ID,\n",
         "    region: str = REGION,\n",
         "):\n",
-        "    from google_cloud_pipeline_components.types import artifact_types\n",
+        "    from google_cloud_pipeline_components.experimental.evaluation import \\\n",
+        "        GetVertexModelOp\n",
         "    from google_cloud_pipeline_components.v1.endpoint import (EndpointCreateOp,\n",
         "                                                              ModelDeployOp)\n",
-        "    from kfp.v2.components import importer_node\n",
         "\n",
-        "    # Desired sequence: blocked by b/219835305\n",
-        "    \"\"\"\n",
-        "    model = importer_node.importer(\n",
-        "        artifact_uri=resource_uri,\n",
-        "        artifact_class=artifact_types.VertexModel,\n",
-        "        metadata={\"resourceName\": resource_name},\n",
-        "    )\n",
-        "    \"\"\"\n",
-        "\n",
-        "    # (WORKAROUND  b/219835305)\n",
-        "    model = return_unmanaged_model(\n",
-        "        serving_image=serving_image,\n",
-        "        artifact_uri=artifact_uri,\n",
-        "        resource_name=resource_name,\n",
-        "    )\n",
+        "    model = GetVertexModelOp(model_resource_name=resource_name)\n",
         "\n",
         "    endpoint_op = EndpointCreateOp(\n",
         "        project=project,\n",
@@ -1281,7 +1247,7 @@
         "        display_name=display_name,\n",
         "    )\n",
         "\n",
-        "    deploy_op = ModelDeployOp(\n",
+        "    _ = ModelDeployOp(\n",
         "        model=model.outputs[\"model\"],\n",
         "        endpoint=endpoint_op.outputs[\"endpoint\"],\n",
         "        dedicated_resources_min_replica_count=1,\n",
@@ -1310,7 +1276,6 @@
         "\n",
         "- `display_name`: The display name for the generated Vertex AI resources.\n",
         "- `resource_name`: The resource name of the existing `Model` resource.\n",
-        "- `resource_uri`: The resource uri of the existing `Model` resource.\n",
         "- `project`: The project ID.\n",
         "- `region`: The region."
       ]
@@ -1323,10 +1288,6 @@
       },
       "outputs": [],
       "source": [
-        "# Model properties (WORKAROUND  b/219835305)\n",
-        "SERVING_CONTAINER_URI = model.gca_resource.container_spec.image_uri\n",
-        "ARTIFACT_URI = model.gca_resource.artifact_uri\n",
-        "\n",
         "try:\n",
         "    pipeline = aip.PipelineJob(\n",
         "        display_name=\"create-endpoint-deploy-pipeline\",\n",
@@ -1335,11 +1296,6 @@
         "        parameter_values={\n",
         "            \"display_name\": \"create_endpoint_and_deploy_model_\" + TIMESTAMP,\n",
         "            \"resource_name\": model.resource_name,\n",
-        "            \"resource_uri\": \"https://us-central1-aiplatform.googleapis.com/v1/\"\n",
-        "            + model.resource_name,\n",
-        "            # Model properties (WORKAROUND  b/219835305)\n",
-        "            \"serving_image\": SERVING_CONTAINER_URI,\n",
-        "            \"artifact_uri\": ARTIFACT_URI,\n",
         "            \"project\": PROJECT_ID,\n",
         "            \"region\": REGION,\n",
         "        },\n",
@@ -1488,7 +1444,7 @@
         "\n",
         "- For pipeline parameters, pass the resource names and resource URIs for the existing `Model` and `Endpoint` resource.\n",
         "- Use the `importer_node()` component to create a `VertexModel` pipeline artifact for the model.\n",
-        "- Use the `importer_node()` component to create a `VertexEndpoint` pipeline artifact for the endpoint.\n",
+        "- Use the `GetVertexModelOp()` component to create a `VertexModel` pipeline artifact for the model.\n",
         "- Using the `VertexModel` and `VertexEndpoint` pipeline artifacts, deploy the `Model` resource to the `Endpoint` resource.\n",
         "\n",
         "*Note:* This example currently blocked by internal issue: b/219835305"
@@ -1504,6 +1460,7 @@
       "source": [
         "PIPELINE_ROOT = \"{}/pipeline_root/endpoint_example_2\".format(BUCKET_URI)\n",
         "\n",
+        "\n",
         "# (WORKAROUND  b/219835305)\n",
         "@component(\n",
         "    base_image=\"python:3.9\",\n",
@@ -1520,35 +1477,23 @@
         ")\n",
         "def pipeline(\n",
         "    display_name: str,\n",
-        "    model_resource_uri: str,\n",
         "    model_resource_name: str,\n",
         "    endpoint_resource_uri: str,\n",
         "    endpoint_resource_name: str,\n",
-        "    # Model properties (WORKAROUND  b/219835305)\n",
-        "    serving_image: str,\n",
-        "    artifact_uri: str,\n",
         "    project: str = PROJECT_ID,\n",
         "    region: str = REGION,\n",
         "):\n",
-        "    from google_cloud_pipeline_components.types import artifact_types\n",
+        "    from google_cloud_pipeline_components.experimental.evaluation import \\\n",
+        "        GetVertexModelOp\n",
         "    from google_cloud_pipeline_components.v1.endpoint import ModelDeployOp\n",
-        "    from kfp.v2.components import importer_node\n",
         "\n",
         "    # Desired sequence: blocked by b/219835305\n",
         "    \"\"\"\n",
-        "    model = importer_node.importer(\n",
-        "        artifact_uri=resource_uri,\n",
-        "        artifact_class=artifact_types.VertexModel,\n",
-        "        metadata={\"resourceName\": resource_name},\n",
-        "    )\n",
+        "    from kfp.v2.components import importer_node\n",
+        "    from google_cloud_pipeline_components.types import artifact_types\n",
         "    \"\"\"\n",
         "\n",
-        "    # (WORKAROUND  b/219835305)\n",
-        "    model = return_unmanaged_model(\n",
-        "        serving_image=serving_image,\n",
-        "        artifact_uri=artifact_uri,\n",
-        "        resource_name=model_resource_name,\n",
-        "    )\n",
+        "    model = GetVertexModelOp(model_resource_name=model_resource_name)\n",
         "\n",
         "    # Desired sequence: blocked by b/219835305\n",
         "    \"\"\"\n",
@@ -1562,7 +1507,7 @@
         "    # (WORKAROUND  b/219835305)\n",
         "    endpoint = return_unmanaged_endpoint(resource_name=endpoint_resource_name)\n",
         "\n",
-        "    deploy_op = ModelDeployOp(\n",
+        "    _ = ModelDeployOp(\n",
         "        model=model.outputs[\"model\"],\n",
         "        endpoint=endpoint.outputs[\"endpoint\"],\n",
         "        dedicated_resources_min_replica_count=1,\n",
@@ -1591,7 +1536,6 @@
         "\n",
         "- `display_name`: The display name for the generated Vertex AI resources.\n",
         "- `model_resource_name`: The resource name of the existing `Model` resource.\n",
-        "- `model_resource_uri`: The resource uri of the existing `Model` resource.\n",
         "- `endpoint_resource_name`: The resource name of the existing `Endpoint` resource.\n",
         "- `endpoint_resource_uri`: The resource uri of the existing `Endpoint` resource.\n",
         "- `project`: The project ID.\n",
@@ -1614,14 +1558,9 @@
         "        parameter_values={\n",
         "            \"display_name\": \"deploy_model_existing_endpoint_\" + TIMESTAMP,\n",
         "            \"model_resource_name\": model.resource_name,\n",
-        "            \"model_resource_uri\": \"https://us-central1-aiplatform.googleapis.com/v1/\"\n",
-        "            + model.resource_name,\n",
         "            \"endpoint_resource_name\": endpoint.resource_name,\n",
         "            \"endpoint_resource_uri\": \"https://us-central1-aiplatform.googleapis.com/v1/\"\n",
         "            + endpoint.resource_name,\n",
-        "            # Model properties (WORKAROUND  b/219835305)\n",
-        "            \"serving_image\": SERVING_CONTAINER_URI,\n",
-        "            \"artifact_uri\": ARTIFACT_URI,\n",
         "            \"project\": PROJECT_ID,\n",
         "            \"region\": REGION,\n",
         "        },\n",

notebooks/community/ml_ops/stage5/get_started_with_vertex_private_endpoints.ipynb

@@ -385,12 +385,11 @@
         "import sys\n",
         "\n",
         "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = False\n",
+        "IS_COLAB = \"google.colab\" in sys.modules\n",
         "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
         "    \"DL_ANACONDA_HOME\"\n",
         "):\n",
         "    if \"google.colab\" in sys.modules:\n",
-        "        IS_COLAB = True\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
         "        google_auth.authenticate_user()\n",
@@ -1068,7 +1067,6 @@
         "- `model`: The `Model` resource.\n",
         "- `deployed_model_displayed_name`: The human readable name for the deployed model instance.\n",
         "- `machine_type`: The machine type for each VM instance.\n",
-        "- `traffic_split`: Set to `{}` to indicate no traffic split.\n",
         "\n",
         "Do to the requirements to provision the resource, this may take upto a few minutes."
       ]
@@ -1085,7 +1083,6 @@
         "    model=model,\n",
         "    deployed_model_display_name=\"example_\" + TIMESTAMP,\n",
         "    machine_type=DEPLOY_COMPUTE,\n",
-        "    traffic_split={},  # no traffic split\n",
         ")\n",
         "\n",
         "print(endpoint)"
@@ -1187,62 +1184,6 @@
         "    f.write(json.dumps({\"instances\": [{serving_input: {\"b64\": b64str}}]}))"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "23e995c35fd6"
-      },
-      "source": [
-        "#### Construct the `Private Endpoint` URI\n",
-        "\n",
-        "Next, you construct the URI for the `Private Endpoint`."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "97b248b2efb5"
-      },
-      "outputs": [],
-      "source": [
-        "endpoint_id = endpoint.resource_name\n",
-        "\n",
-        "ENDPOINT_URL = ! gcloud beta ai endpoints describe {endpoint_id} \\\n",
-        "  --region={REGION} \\\n",
-        "  --format=\"value(deployedModels.privateEndpoints.predictHttpUri)\"\n",
-        "\n",
-        "private_url = ENDPOINT_URL[1]\n",
-        "print(private_url)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "27605b5f0c3a"
-      },
-      "source": [
-        "### Make the prediction request using curl\n",
-        "\n",
-        "Use `curl` to make the prediction request to the private URI."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "6cb568e6bb49"
-      },
-      "outputs": [],
-      "source": [
-        "output = ! curl -X POST -d@instances.json $private_url\n",
-        "\n",
-        "predictions = output[5]\n",
-        "print(predictions)\n",
-        "\n",
-        "! rm test.jpg instances.json"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -1251,7 +1192,7 @@
       "source": [
         "### Make the prediction request using SDK\n",
         "\n",
-        "Finally, use the `Vertex AI SDK` to make a prediction request."
+        "Next, use the `Vertex AI SDK` to make a prediction request."
       ]
     },
     {

notebooks/community/ml_ops/stage6/get_started_with_matching_engine_swivel.ipynb

@@ -1068,7 +1068,7 @@
         "model = aiplatform.Model.upload(\n",
         "    display_name=\"movies_\" + TIMESTAMP,\n",
         "    artifact_uri=SAVEDMODEL_DIR,\n",
-        "    serving_container_image_uri=DELOY_IMAGE,\n",
+        "    serving_container_image_uri=DEPLOY_IMAGE,\n",
         ")"
       ]
     },

notebooks/community/ml_ops/stage6/get_started_with_tf_serving.ipynb

@@ -702,7 +702,7 @@
         "        + f\"/{PRIVATE_REPO}\"\n",
         "        + \"/tf_serving:gpu\"\n",
         "    )\n",
-        "    TF_IMAGE = \"tensorflow/serving:latest-gpu\"\n",
+        "    TF_IMAGE = \"tensorflow/serving:2.5.4-gpu\"\n",
         "else:\n",
         "    DEPLOY_IMAGE = (\n",
         "        f\"{REGION}-docker.pkg.dev/\"\n",
@@ -710,15 +710,15 @@
         "        + f\"/{PRIVATE_REPO}\"\n",
         "        + \"/tf_serving:cpu\"\n",
         "    )\n",
-        "    TF_IMAGE = \"tensorflow/serving:latest\"\n",
+        "    TF_IMAGE = \"tensorflow/serving:2.5.4\"\n",
         "\n",
         "if not IS_COLAB:\n",
         "    if DEPLOY_GPU:\n",
-        "        ! sudo docker pull tensorflow/serving:latest-gpu\n",
+        "        ! sudo docker pull tensorflow/serving:2.5.4-gpu\n",
         "    else:\n",
-        "        ! sudo docker pull tensorflow/serving:latest\n",
+        "        ! sudo docker pull tensorflow/serving:2.5.4\n",
         "\n",
-        "    ! docker tag tensorflow/serving $DEPLOY_IMAGE\n",
+        "    ! docker tag $TF_IMAGE $DEPLOY_IMAGE\n",
         "    ! docker push $DEPLOY_IMAGE\n",
         "else:\n",
         "    # install docker daemon\n",
@@ -1214,6 +1214,20 @@
         "     [1.0,3.0,\"cat1\"],\n",
         "     [2.0,4.0,\"cat2\"]\n",
         "    ]}\n",
+        "    \n",
+        "**BigQuery**\n",
+        "\n",
+        "Each row is converted to a JSON array. For example:\n",
+        "\n",
+        "    [1.0,3.0,\"cat1\"]\n",
+        "    [2.0,4.0,\"cat2\"]\n",
+        "    \n",
+        "The batch server generates the pivot data with the same format. The generated pivot data is then wrapped into a payload request:\n",
+        "\n",
+        "    {\"instances\": [\n",
+        "     [1.0,3.0,\"cat1\"],\n",
+        "     [2.0,4.0,\"cat2\"]\n",
+        "    ]}\n",
         "\n",
         "**TFRecords**\n",
         "\n",
@@ -1420,7 +1434,7 @@
       },
       "outputs": [],
       "source": [
-        "delete_bucket = False\n",
+        "delete_bucket = True\n",
         "delete_model = True\n",
         "delete_endpoint = True\n",
         "delete_batch_job = True\n",

notebooks/community/prediction/custom_prediction_routines/SDK_Custom_Preprocess.ipynb

@@ -673,7 +673,7 @@
         "Before you can deploy your model for serving, Vertex AI needs access to the following files in Cloud Storage:\n",
         "\n",
         "* `model.joblib` (model artifact)\n",
-        "* `preprocessor.pkl` (model artifact)\n",
+        "* `preprocessor.pkl` (preprocessor code)\n",
         "\n",
         "Run the following commands to upload your files:"
       ]

notebooks/notebook_template_review.py

@@ -56,7 +56,7 @@ def parse_notebook(path):
         # cell 1 is copyright
         nth = 0
         cell, nth = get_cell(path, cells, nth)
-        if not cell['source'][0].startswith('# Copyright'):
+        if not 'Copyright' in cell['source'][0]:
             report_error(path, 0, "missing copyright cell")
             
         # check for notices

notebooks/official/CODEOWNERS

@@ -10,12 +10,13 @@
 /tabnet/tabnet_vertex_tutorial.ipynb @longtle
 
 /migration @andrewferlitsch
-/explainabl_ai   
+/explainabl_ai
 /pipelines @andrewferlitsch
 /ml_metadata @andrewferlitsch
 /model_monitoring @andrewferlitsch
 /tensorboard @zbl94
 
+/bigquery_ml/bqml-online-prediction.ipynb @polong-lin
 /model_monitoring/model_monitoring.ipynb @mco-gh
 /ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb @jialuzh
 /ml_metadata/sdk-metric-parameter-tracking-for-locally-trained-models.ipynb @jialuzh
@@ -28,4 +29,9 @@
 /automl/automl_forecasting_bqml_arima_plus_comparison.ipynb @TheMichaelHu
 /automl/automl_tabular_on_vertex_pipelines.ipynb @helinwang
 /custom/custom_training_tensorboard_profiler.ipynb @itseric
-/workbench/spark/spark_sample_notebook.ipynb @bmiro
+/workbench/spark/spark_sample_notebook.ipynb @bradmiro
+/workbench/spark/spark_ml.ipynb @bradmiro
+/model-registry/bqml-vertexai-model-registry.ipynb @soheilazangeneh
+/workbench/exploratory_data_analysis/explore_data_in_bigquery_with_workbench.ipynb @alokpattani
+/model_evaluation/automl_tabular_classification_model_evaluation.ipynb @soheilazangeneh
+/model_evaluation/automl_tabular_regression_model_evaluation.ipynb @soheilazangeneh

notebooks/official/automl/automl-text-classification.ipynb

@@ -2,7 +2,7 @@
   "cells": [
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 54,
       "metadata": {
         "id": "ur8xi4C7S06n"
       },
@@ -17,12 +17,21 @@
         "#     https://www.apache.org/licenses/LICENSE-2.0\n",
         "#\n",
         "# Unless required by applicable law or agreed to in writing, software\n",
-        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# distributed under the Lice`nse is distributed on an \"AS IS\" BASIS,\n",
         "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
         "# See the License for the specific language governing permissions and\n",
         "# limitations under the License."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0d2298941703"
+      },
+      "source": [
+        "# Vertex AI: Create, train, and deploy an AutoML text classification model"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -43,8 +52,8 @@
         "    </a>\n",
         "  </td>\n",
         " <td>\n",
-        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/automl/automl-text-classification.ipynb\">\n",
-        "        <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/automl-text-classification.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
         "  </td>\n",
@@ -54,19 +63,20 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "0259a7ce8120"
+        "id": "1adb10a59bc3"
       },
       "source": [
-        "# Vertex AI: Create, train, and deploy an AutoML text classification model\n",
-        "\n",
         "## Overview\n",
         "\n",
-        "This notebook walks you through the major phases of building and using an AutoML text classification model on [Vertex AI](https://cloud.google.com/vertex-ai/docs/). \n",
-        "\n",
-        "### Dataset\n",
-        "\n",
-        "In this notebook, you use the \"Happy Moments\" sample dataset to train a model. The resulting model classifies happy moments into categores that reflect the causes of happiness. \n",
-        "\n",
+        "This notebook walks you through the major phases of building and using an AutoML text classification model on [Vertex AI](https://cloud.google.com/vertex-ai/docs/). \n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9b9824ae2c91"
+      },
+      "source": [
         "### Objective\n",
         "\n",
         "In this tutorial, you learn how to use `AutoML` to train a text classification model.\n",
@@ -84,8 +94,26 @@
         "* Create an `Endpoint` resource.\n",
         "* Deploy the `Model` resource to the `Endpoint` resource.\n",
         "* Make an online prediction\n",
-        "* Make a batch prediction\n",
+        "* Make a batch prediction"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "f67c62885df4"
+      },
+      "source": [
+        "### Dataset\n",
         "\n",
+        "In this notebook, you use the \"Happy Moments\" sample dataset to train a model. The resulting model classifies happy moments into categores that reflect the causes of happiness. "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0259a7ce8120"
+      },
+      "source": [
         "### Costs\n",
         "\n",
         "This tutorial uses billable components of Google Cloud:\n",
@@ -173,7 +201,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 55,
       "metadata": {
         "id": "b75757581291"
       },
@@ -188,6 +216,7 @@
         ")\n",
         "\n",
         "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "\n",
         "USER_FLAG = \"\"\n",
         "if IS_WORKBENCH_NOTEBOOK:\n",
         "    USER_FLAG = \"--user\"\n",
@@ -208,7 +237,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 56,
       "metadata": {
         "id": "0c0b2427998a"
       },
@@ -254,7 +283,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 1,
       "metadata": {
         "id": "be175254a715"
       },
@@ -311,7 +340,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 4,
       "metadata": {
         "id": "ae43d96c4b1b"
       },
@@ -336,7 +365,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 5,
       "metadata": {
         "id": "953fa6e5ddda"
       },
@@ -424,7 +453,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 6,
       "metadata": {
         "id": "d2de92accb67"
       },
@@ -436,7 +465,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 7,
       "metadata": {
         "id": "5ba09496accc"
       },
@@ -478,7 +507,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 9,
       "metadata": {
         "id": "96ad3d416327"
       },
@@ -498,7 +527,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 10,
       "metadata": {
         "id": "152013538e59"
       },
@@ -521,7 +550,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 11,
       "metadata": {
         "id": "740cd5c67c79"
       },
@@ -551,24 +580,15 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "6caf82e5e84e"
+        "id": "d35b8b6b94ae"
       },
       "outputs": [],
       "source": [
         "# Use a timestamp to ensure unique resources\n",
         "src_uris = \"gs://cloud-ml-data/NL-classification/happiness.csv\"\n",
-        "display_name = f\"e2e-text-dataset-{TIMESTAMP}\""
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "d35b8b6b94ae"
-      },
-      "outputs": [],
-      "source": [
-        "ds = aiplatform.TextDataset.create(\n",
+        "display_name = f\"e2e-text-dataset-{TIMESTAMP}\"\n",
+        "\n",
+        "text_dataset = aiplatform.TextDataset.create(\n",
         "    display_name=display_name,\n",
         "    gcs_source=src_uris,\n",
         "    import_schema_uri=aiplatform.schema.dataset.ioformat.text.single_label_classification,\n",
@@ -582,53 +602,7 @@
         "id": "5b3cc427353a"
       },
       "source": [
-        "## Train your text classification model\n",
-        "\n",
-        "Once your dataset has finished importing data, you are ready to train your model. To do this, you first need the full resource name of your dataset, where the full name has the format `projects/[YOUR_PROJECT]/locations/[YOUR_REGIO)N]/datasets/[YOUR_DATASET_ID]`. If you don't have the resource name handy, you can list all of the datasets in your project using `TextDataset.list()`. \n",
-        "\n",
-        "As shown in the following code block, you can pass in the display name of your dataset in the call to `list()` to filter the results.\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "52cf56f1c8a9"
-      },
-      "outputs": [],
-      "source": [
-        "datasets = aiplatform.TextDataset.list(filter=f'display_name=\"{display_name}\"')\n",
-        "print(datasets)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "58df3e02df82"
-      },
-      "source": [
-        "When you create a new model, you need a reference to the `TextDataset` object that corresponds to your dataset. You can use the `ds` variable you created previously when you created the dataset or you can also list all of your datasets to get a reference to your dataset. Each item returned from `TextDataset.list()` is an instance of `TextDataset`.\n",
-        "\n",
-        "The following code block shows how to instantiate a `TextDataset` object using a dataset ID. Note that this code is intentionally verbose for demonstration purposes."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "aa667203da03"
-      },
-      "outputs": [],
-      "source": [
-        "# Get the dataset ID if it's not available\n",
-        "dataset_id = \"[your-dataset-id]\"\n",
-        "\n",
-        "if dataset_id == \"[your-dataset-id]\":\n",
-        "    # Use the reference to the new dataset captured when we created it\n",
-        "    dataset_id = ds.resource_name.split(\"/\")[-1]\n",
-        "    print(f\"Dataset ID: {dataset_id}\")\n",
-        "\n",
-        "text_dataset = aiplatform.TextDataset(dataset_id)"
+        "## Train your text classification model\n"
       ]
     },
     {
@@ -653,7 +627,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 16,
       "metadata": {
         "id": "0aa0f01805ea"
       },
@@ -682,8 +656,8 @@
         "model = job.run(\n",
         "    dataset=text_dataset,\n",
         "    model_display_name=model_display_name,\n",
-        "    training_fraction_split=0.7,\n",
-        "    validation_fraction_split=0.2,\n",
+        "    training_fraction_split=0.1,\n",
+        "    validation_fraction_split=0.1,\n",
         "    test_fraction_split=0.1,\n",
         "    sync=True,\n",
         ")"
@@ -740,39 +714,11 @@
         "deployed_model_display_name = f\"e2e-deployed-text-classification-model-{TIMESTAMP}\"\n",
         "\n",
         "endpoint = model.deploy(\n",
-        "    deployed_model_display_name=deployed_model_display_name, sync=True\n",
+        "    deployed_model_display_name=deployed_model_display_name, \n",
+        "    sync=True\n",
         ")"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "531da446035b"
-      },
-      "source": [
-        "In case you didn't record the name of the new endpoint, you can get a list of all your endpoints as you did before with datasets and models. For each endpoint, you can list the models deployed to that endpoint. To get a reference to the model that you just deployed, you can check the `display_name` of each model deployed to the endpoint against the model you're looking for."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "f61fb44181b4"
-      },
-      "outputs": [],
-      "source": [
-        "endpoints = aiplatform.Endpoint.list()\n",
-        "\n",
-        "endpoint_with_deployed_model = []\n",
-        "\n",
-        "for endpoint_ in endpoints:\n",
-        "    for model in endpoint_.list_models():\n",
-        "        if model.display_name.find(deployed_model_display_name) == 0:\n",
-        "            endpoint_with_deployed_model.append(endpoint_)\n",
-        "\n",
-        "print(endpoint_with_deployed_model)"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -781,7 +727,7 @@
       "source": [
         "## Get online predictions from your model\n",
         "\n",
-        "Now that you have your endpoint's resource name, you can get online predictions from the text classification model. To get the online prediction, you send a prediction request to your endpoint."
+        "Now that you have your endpoint, you can get online predictions from the text classification model. To get the online prediction, you send a prediction request to your endpoint."
       ]
     },
     {
@@ -792,13 +738,6 @@
       },
       "outputs": [],
       "source": [
-        "endpoint_name = \"[your-endpoint-name]\"\n",
-        "if endpoint_name == \"[your-endpoint-name]\":\n",
-        "    endpoint_name = endpoint.resource_name\n",
-        "\n",
-        "print(f\"Endpoint name: {endpoint_name}\")\n",
-        "\n",
-        "endpoint = aiplatform.Endpoint(endpoint_name)\n",
         "content = \"I got a high score on my math final!\"\n",
         "\n",
         "response = endpoint.predict(instances=[{\"content\": content}])\n",
@@ -835,7 +774,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 23,
       "metadata": {
         "id": "e4b838cbcd99"
       },
@@ -858,7 +797,7 @@
         "\n",
         "+ All of your prediction instances as individual files on Google Cloud Storage, as TXT files for your instances\n",
         "+ A JSONL file that lists the URIs of all your prediction instances\n",
-        "+ A Google Cloud Storage bucket to hold the output from batch prediction\n",
+        "+ A Cloud Storage bucket to hold the output from batch prediction\n",
         "\n",
         "For this tutorial, the following cells create a new Storage bucket, upload individual prediction instances as text files to the bucket, and then create the JSONL file with the URIs of your prediction instances."
       ]
@@ -872,16 +811,15 @@
       "outputs": [],
       "source": [
         "# Instantiate the Storage client and create the new bucket\n",
-        "storage = storage.Client()\n",
-        "bucket = storage.bucket(BUCKET_URI)\n",
-        "\n",
+        "# from google.cloud import  storage\n",
+        "storage_client = storage.Client()\n",
+        "bucket = storage_client.bucket(BUCKET_NAME)\n",
         "# Iterate over the prediction instances, creating a new TXT file\n",
         "# for each.\n",
         "input_file_data = []\n",
         "for count, instance in enumerate(instances):\n",
         "    instance_name = f\"input_{count}.txt\"\n",
         "    instance_file_uri = f\"{BUCKET_URI}/{instance_name}\"\n",
-        "\n",
         "    # Add the data to store in the JSONL input file.\n",
         "    tmp_data = {\"content\": instance_file_uri, \"mimeType\": \"text/plain\"}\n",
         "    input_file_data.append(tmp_data)\n",
@@ -901,7 +839,7 @@
         "id": "31c262320610"
       },
       "source": [
-        "Now that you have the bucket with the prediction instances ready, you can send a batch prediction request to Vertex AI. When you send a request to the service, you must provide the URI of your JSONL file and your output bucket, including the `gs://` protocols.\n",
+        "Now that you have the bucket with the prediction instances ready, you can send a batch prediction rhttps://storage.googleapis.com/upload/storage/v1/b/gs://vertex-ai-devaip-20220728004429/o?uploadType=multipartequest to Vertex AI. When you send a request to the service, you must provide the URI of your JSONL file and your output bucket, including the `gs://` protocols.\n",
         "\n",
         "With the Python SDK, you can create a batch prediction job by calling `Model.batch_predict()`."
       ]
@@ -915,15 +853,13 @@
       "outputs": [],
       "source": [
         "job_display_name = \"e2e-text-classification-batch-prediction-job\"\n",
-        "model = aiplatform.Model(model_name=model_name)\n",
-        "\n",
+        "# model = aiplatform.Model(model_name=model.name)\n",
         "batch_prediction_job = model.batch_predict(\n",
         "    job_display_name=job_display_name,\n",
         "    gcs_source=f\"{BUCKET_URI}/{input_file_name}\",\n",
         "    gcs_destination_prefix=f\"{BUCKET_URI}/output\",\n",
         "    sync=True,\n",
         ")\n",
-        "\n",
         "batch_prediction_job_name = batch_prediction_job.resource_name"
       ]
     },
@@ -938,6 +874,15 @@
         "The following code snippet demonstrates how to create an instance of the `BatchPredictionJob` class to review its status. Note that you need the full resource name printed out from the Python SDK for this snippet.\n"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "cd014de40e2f"
+      },
+      "source": [
+        "## BatchPredictionJob"
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -999,6 +944,8 @@
       },
       "outputs": [],
       "source": [
+        "import os\n",
+        "\n",
         "RESULTS_DIRECTORY = \"prediction_results\"\n",
         "RESULTS_DIRECTORY_FULL = f\"{RESULTS_DIRECTORY}/output\"\n",
         "\n",
@@ -1020,6 +967,15 @@
         "print(f\"Local results folder: {latest_directory}\")"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e375109b7e40"
+      },
+      "source": [
+        "## JsonLines"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -1094,9 +1050,9 @@
         "    ! gsutil rm -r $BUCKET_URI\n",
         "\n",
         "batch_job.delete()\n",
-        "\n",
+        "endpoint.undeploy_all()\n",
         "# `force` parameter ensures that models are undeployed before deletion\n",
-        "endpoint.delete(force=True)\n",
+        "endpoint.delete()\n",
         "\n",
         "model.delete()\n",
         "\n",

notebooks/official/automl/automl_forecasting_bqml_arima_plus_comparison.ipynb

@@ -176,7 +176,10 @@
         "else:\n",
         "    USER_FLAG = \"\"\n",
         "\n",
-        "! pip3 install --upgrade google-cloud-bigquery[pandas] google-cloud-aiplatform google-cloud-pipeline-components $USER_FLAG"
+        "! (pip3 install --upgrade $USER_FLAG \\\n",
+        "    google-cloud-bigquery[pandas]==2.34.4 \\\n",
+        "    google-cloud-aiplatform==1.16.1 \\\n",
+        "    google-cloud-pipeline-components==1.0.18)"
       ]
     },
     {

notebooks/official/automl/sdk_automl_image_object_detection_batch.ipynb

@@ -8,7 +8,7 @@
       },
       "outputs": [],
       "source": [
-        "# Copyright 2022 Google LLC\n",
+        "# Copyright 2021 Google LLC\n",
         "#\n",
         "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
         "# you may not use this file except in compliance with the License.\n",
@@ -66,17 +66,6 @@
         "This tutorial demonstrates how to use the Vertex AI SDK to create image object detection models and do batch prediction using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "dataset:salads,iod"
-      },
-      "source": [
-        "### Dataset\n",
-        "\n",
-        "The dataset used for this tutorial is the Salads category of the [OpenImages dataset](https://www.tensorflow.org/datasets/catalog/open_images_v4) from [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/overview). This dataset does not require any feature engineering. The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket. The trained model predicts the bounding box locations and the corresponding type of salad items in an image from a class of five items: salad, seafood, tomato, baked goods, or cheese."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -101,6 +90,17 @@
         "* Batch Prediction Service: Does a queued (batch) prediction for the entire set of instances in the background and stores the results in a Cloud Storage bucket when ready."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dataset:salads,iod"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the Salads category of the [OpenImages dataset](https://www.tensorflow.org/datasets/catalog/open_images_v4) from [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/overview). This dataset does not require any feature engineering. The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket. The trained model predicts the bounding box locations and the corresponding type of salad items in an image from a class of five items: salad, seafood, tomato, baked goods, or cheese."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -201,7 +201,7 @@
       },
       "outputs": [],
       "source": [
-        "! pip3 install -U google-cloud-storage $USER_FLAG"
+        "! pip3 install -U --upgrade tensorflow google-cloud-storage $USER_FLAG"
       ]
     },
     {
@@ -213,17 +213,6 @@
         "Install the latest version of *tensorflow* library."
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "install_tensorflow"
-      },
-      "outputs": [],
-      "source": [
-        "! pip3 install --upgrade tensorflow $USER_FLAG"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -383,9 +372,9 @@
         "id": "timestamp"
       },
       "source": [
-        "#### Timestamp\n",
+        "#### UUID\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
       ]
     },
     {
@@ -396,9 +385,16 @@
       },
       "outputs": [],
       "source": [
-        "from datetime import datetime\n",
+        "import random\n",
+        "import string\n",
         "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+        "\n",
+        "# Generate a uuid of a specifed length(default=8)\n",
+        "def generate_uuid(length: int = 8) -> str:\n",
+        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
+        "\n",
+        "\n",
+        "UUID = generate_uuid()"
       ]
     },
     {
@@ -409,7 +405,7 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Google Cloud Notebooks**, your environment is already authenticated. Skip this step.\n",
+        "**If you are using Google Cloud Notebooks**, your environment is already authenticated.\n",
         "\n",
         "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
         "\n",
@@ -494,7 +490,7 @@
       "outputs": [],
       "source": [
         "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + UUID"
       ]
     },
     {
@@ -669,7 +665,7 @@
       "outputs": [],
       "source": [
         "dataset = aiplatform.ImageDataset.create(\n",
-        "    display_name=\"Salads\" + \"_\" + TIMESTAMP,\n",
+        "    display_name=\"Salads\" + \"_\" + UUID,\n",
         "    gcs_source=[IMPORT_FILE],\n",
         "    import_schema_uri=aiplatform.schema.dataset.ioformat.image.bounding_box,\n",
         ")\n",
@@ -717,7 +713,7 @@
       "outputs": [],
       "source": [
         "job = aiplatform.AutoMLImageTrainingJob(\n",
-        "    display_name=\"salads_\" + TIMESTAMP,\n",
+        "    display_name=\"salads_\" + UUID,\n",
         "    prediction_type=\"object_detection\",\n",
         "    multi_label=False,\n",
         "    model_type=\"CLOUD\",\n",
@@ -760,7 +756,7 @@
       "source": [
         "model = job.run(\n",
         "    dataset=dataset,\n",
-        "    model_display_name=\"salads_\" + TIMESTAMP,\n",
+        "    model_display_name=\"salads_\" + UUID,\n",
         "    training_fraction_split=0.8,\n",
         "    validation_fraction_split=0.1,\n",
         "    test_fraction_split=0.1,\n",
@@ -790,7 +786,7 @@
       "outputs": [],
       "source": [
         "# Get model resource ID\n",
-        "models = aiplatform.Model.list(filter=\"display_name=salads_\" + TIMESTAMP)\n",
+        "models = aiplatform.Model.list(filter=\"display_name=salads_\" + UUID)\n",
         "\n",
         "# Get a reference to the Model Service client\n",
         "client_options = {\"api_endpoint\": f\"{REGION}-aiplatform.googleapis.com\"}\n",
@@ -961,7 +957,7 @@
       "outputs": [],
       "source": [
         "batch_predict_job = model.batch_predict(\n",
-        "    job_display_name=\"salads_\" + TIMESTAMP,\n",
+        "    job_display_name=\"salads_\" + UUID,\n",
         "    gcs_source=gcs_input_uri,\n",
         "    gcs_destination_prefix=BUCKET_URI,\n",
         "    machine_type=\"n1-standard-4\",\n",

notebooks/official/bigquery_ml/bqml-online-prediction.ipynb

@@ -32,18 +32,18 @@
         "<table align=\"left\">\n",
         "\n",
         "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/bigquery_ml/bqml-online-prediction.ipynb\">\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/official/bigquery_ml/bqml-online-prediction.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://github/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/bigquery_ml/bqml-online-prediction.ipynb\">\n",
+        "    <a href=\"https://github/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/official/bigquery_ml/bqml-online-prediction.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/bigquery_ml/bqml-online-prediction.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/official/bigquery_ml/bqml-online-prediction.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
@@ -63,7 +63,7 @@
         "\n",
         "### Dataset\n",
         "\n",
-        "The dataset, [available publicly on BigQuery](https://console.cloud.google.com/bigquery?project=bigquery-public-data&d=ga4_obfuscated_sample_ecommerce&p=bigquery-public-data&page=dataset), comes from obfuscated [Google Analytics 4 data](https://support.google.com/analytics/answer/10937659) from the [Google Merchandise Store](https://shop.googlemerchandisestore.com/).\n",
+        "The dataset, <a href=\"https://console.cloud.google.com/bigquery?project=bigquery-public-data&d=ga4_obfuscated_sample_ecommerce&p=bigquery-public-data&page=dataset\" target=\"_blank\">available publicly on BigQuery</a>, comes from obfuscated <a href=\"https://support.google.com/analytics/answer/10937659\" target=\"_blank\">Google Analytics 4 data</a> from the <a href=\"https://shop.googlemerchandisestore.com/\" target=\"_blank\">Google Merchandise Store</a>).\n",
         "\n",
         "### Objective\n",
         "\n",
@@ -95,9 +95,9 @@
         "* Vertex AI\n",
         "\n",
         "\n",
-        "Learn about [BigQuery Pricing](https://cloud.google.com/bigquery/pricing), [BigQuery ML pricing](https://cloud.google.com/bigquery-ml/pricing), [Vertex AI\n",
-        "pricing](https://cloud.google.com/vertex-ai/pricing), and use the [Pricing\n",
-        "Calculator](https://cloud.google.com/products/calculator/)\n",
+        "Learn about <a href=\"https://cloud.google.com/bigquery/pricing\" target=\"_blank\">BigQuery Pricing</a>, <a href=\"https://cloud.google.com/bigquery-ml/pricing\" target=\"_blank\">BigQuery ML pricing</a>, <a href=\"https://cloud.google.com/vertex-ai/pricing\" target=\"_blank\">Vertex AI\n",
+        "pricing</a>, and use the <a href=\"https://cloud.google.com/products/calculator/\" target=\"_blank\">Pricing\n",
+        "Calculator</a>\n",
         "to generate a cost estimate based on your projected usage."
       ]
     },
@@ -128,18 +128,18 @@
         "* virtualenv\n",
         "* Jupyter notebook running in a virtual environment with Python 3\n",
         "\n",
-        "The Google Cloud guide to [Setting up a Python development\n",
-        "environment](https://cloud.google.com/python/setup) and the [Jupyter\n",
-        "installation guide](https://jupyter.org/install) provide detailed instructions\n",
+        "The Google Cloud guide to <a href=\"https://cloud.google.com/python/setup\" target=\"_blank\">Setting up a Python development\n",
+        "environment</a> and the <a href=\"https://jupyter.org/install\" target=\"_blank\">Jupyter\n",
+        "installation guide</a> provide detailed instructions\n",
         "for meeting these requirements. The following steps provide a condensed set of\n",
         "instructions:\n",
         "\n",
-        "1. [Install and initialize the Cloud SDK.](https://cloud.google.com/sdk/docs/)\n",
+        "1. <a href=\"https://cloud.google.com/sdk/docs/\" target=\"_blank\">Install and initialize the Cloud SDK.</a>\n",
         "\n",
-        "1. [Install Python 3.](https://cloud.google.com/python/setup#installing_python)\n",
+        "1. <a href=\"https://cloud.google.com/python/setup#installing_python\" target=\"_blank\">Install Python 3.</a>\n",
         "\n",
-        "1. [Install\n",
-        "   virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv)\n",
+        "1. <a href=\"https://cloud.google.com/python/setup#installing_and_using_virtualenv\" target=\"_blank\">Install\n",
+        "   virtualenv</a>\n",
         "   and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
         "\n",
         "1. To install Jupyter, run `pip3 install jupyter` on the\n",
@@ -234,13 +234,13 @@
         "\n",
         "**The following steps are required, regardless of your notebook environment.**\n",
         "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "1. <a href=\"https://console.cloud.google.com/cloud-resource-manager\" target=\"_blank\">Select or create a Google Cloud project</a>. When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
         "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "1. <a href=\"https://cloud.google.com/billing/docs/how-to/modify-project\" target=\"_blank\">Make sure that billing is enabled for your project</a>.\n",
         "\n",
-        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com).\n",
+        "1. <a href=\"https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com\" target=\"_blank\">Enable the Vertex AI API</a>.\n",
         "\n",
-        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "1. If you are running this notebook locally, you will need to install the <a href=\"https://cloud.google.com/sdk\" target=\"_blank\">Cloud SDK</a>.\n",
         "\n",
         "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
         "Cloud SDK uses the right project for all the commands in this notebook.\n",
@@ -267,7 +267,7 @@
       },
       "outputs": [],
       "source": [
-        "PROJECT_ID = \"YOUR-PROJECT-ID\"\n",
+        "PROJECT_ID = \"[YOUR-PROJECT-ID]\"\n",
         "\n",
         "# Get your Google Cloud project ID from gcloud\n",
         "import os\n",
@@ -314,9 +314,9 @@
         "- Europe: `europe-west4`\n",
         "- Asia Pacific: `asia-east1`\n",
         "\n",
-        "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
+        "You might not be able to use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
         "\n",
-        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
+        "Learn more about <a href=\"https://cloud.google.com/vertex-ai/docs/general/locations\" target=\"_blank\">Vertex AI regions</a>."
       ]
     },
     {
@@ -339,9 +339,9 @@
         "id": "06571eb4063b"
       },
       "source": [
-        "#### Timestamp\n",
+        "#### UUID\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append it onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
       ]
     },
     {
@@ -352,9 +352,16 @@
       },
       "outputs": [],
       "source": [
-        "from datetime import datetime\n",
+        "import random\n",
+        "import string\n",
         "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+        "\n",
+        "# Generate a uuid of a specifed length(default=8)\n",
+        "def generate_uuid(length: int = 8) -> str:\n",
+        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
+        "\n",
+        "\n",
+        "UUID = generate_uuid()"
       ]
     },
     {
@@ -380,8 +387,7 @@
         "\n",
         "**Otherwise**, follow these steps:\n",
         "\n",
-        "1. In the Cloud Console, go to the [**Create service account key**\n",
-        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
+        "1. In the Cloud Console, go to the <a href=\"https://console.cloud.google.com/apis/credentials/serviceaccountkey\" target=\"_blank\">**Create service account key** page</a>.\n",
         "\n",
         "2. Click **Create service account**.\n",
         "\n",
@@ -486,7 +492,10 @@
       },
       "outputs": [],
       "source": [
+        "from typing import Union\n",
+        "\n",
         "import google.cloud.aiplatform as vertex_ai\n",
+        "import pandas as pd\n",
         "from google.cloud import bigquery"
       ]
     },
@@ -550,24 +559,17 @@
       "outputs": [],
       "source": [
         "# Wrapper to use BigQuery client to run query/job, return job ID or result as DF\n",
-        "def bq_query(sql):\n",
+        "def run_bq_query(sql: str) -> Union[str, pd.DataFrame]:\n",
         "    \"\"\"\n",
         "    Input: SQL query, as a string, to execute in BigQuery\n",
         "    Returns the query results as a pandas DataFrame, or error, if any\n",
         "    \"\"\"\n",
-        "    # Import Exceptions library to help with dataset error catching\n",
-        "    from google.cloud.exceptions import BadRequest\n",
         "\n",
         "    # Try dry run before executing query to catch any errors\n",
-        "    try:\n",
-        "        job_config = bigquery.QueryJobConfig(dry_run=True, use_query_cache=False)\n",
-        "\n",
-        "        bq_client.query(sql, job_config=job_config)\n",
-        "\n",
-        "    except BadRequest as err:\n",
-        "        print(err)\n",
-        "        return\n",
+        "    job_config = bigquery.QueryJobConfig(dry_run=True, use_query_cache=False)\n",
+        "    bq_client.query(sql, job_config=job_config)\n",
         "\n",
+        "    # If dry run succeeds without errors, proceed to run query\n",
         "    job_config = bigquery.QueryJobConfig()\n",
         "    client_result = bq_client.query(sql, job_config=job_config)\n",
         "\n",
@@ -589,7 +591,7 @@
         "\n",
         "BigQuery ML (BQML) provides the capability to train ML tabular models, such as classification, regression, forecasting, and matrix factorization, in BigQuery using SQL syntax directly. BigQuery ML uses the scalable infrastructure of BigQuery ML so you don't need to set up additional infrastructure for training or batch serving.\n",
         "\n",
-        "Learn more about [BigQuery ML documentation](https://cloud.google.com/bigquery-ml/docs)."
+        "Learn more about <a href=\"https://cloud.google.com/bigquery-ml/docs\" target=\"_blank\">BigQuery ML documentation</a>."
       ]
     },
     {
@@ -600,9 +602,13 @@
       },
       "outputs": [],
       "source": [
-        "BQ_DATASET_NAME = \"ga4_churnprediction\"\n",
+        "BQ_DATASET_NAME = f\"ga4_churnprediction_{UUID}\"\n",
         "\n",
-        "bq_query(f\"\"\"CREATE SCHEMA IF NOT EXISTS {BQ_DATASET_NAME}\"\"\")"
+        "sql_create_dataset = f\"\"\"CREATE SCHEMA IF NOT EXISTS {BQ_DATASET_NAME}\"\"\"\n",
+        "\n",
+        "print(sql_create_dataset)\n",
+        "\n",
+        "run_bq_query(sql_create_dataset)"
       ]
     },
     {
@@ -620,7 +626,7 @@
         "id": "49dd00d5fbe5"
       },
       "source": [
-        "Inpect data that has been pre-processed from [Google Analytics 4 data from the Google Merchandise Store](https://support.google.com/analytics/answer/10937659) so that it can be used for classification. For more information on how this data was prepared, read [this blog post](https://cloud.google.com/blog/topics/developers-practitioners/churn-prediction-game-developers-using-google-analytics-4-ga4-and-bigquery-ml).\n",
+        "Inpect data that has been pre-processed from <a href=\"https://support.google.com/analytics/answer/10937659\" target=\"_blank\">Google Analytics 4 data from the Google Merchandise Store</a> so that it can be used for classification. For more information on how this data was prepared, read <a href=\"https://cloud.google.com/blog/topics/developers-practitioners/churn-prediction-game-developers-using-google-analytics-4-ga4-and-bigquery-ml\" target=\"_blank\">this blog post</a>.\n",
         "\n",
         "As seen below, each row represents a single user, and the columns represent their demographic features, their aggregated behavioral features in the first 24 hours of visiting the Google Merchandise Store, and the label (whether the user churned or returned any time after the first 24 hours)."
       ]
@@ -641,7 +647,7 @@
         "LIMIT\n",
         "    100\n",
         "\"\"\"\n",
-        "bq_query(sql_inspect)"
+        "run_bq_query(sql_inspect)"
       ]
     },
     {
@@ -662,9 +668,9 @@
         "The query below trains a logistic regression model using BigQuery ML. BigQuery resources are used to train the model.\n",
         "\n",
         "In the `OPTIONS` parameter:\n",
-        "* with `model_registry=\"vertex_ai\"`, the BigQuery ML model will automatically be [registered to Vertex AI Model Registry](https://cloud.google.com/vertex-ai/docs/model-registry/model-registry-bqml), which enables you to view all of your registered models and its versions on Google Cloud in one place.\n",
+        "* with `model_registry=\"vertex_ai\"`, the BigQuery ML model will automatically be <a href=\"https://cloud.google.com/vertex-ai/docs/model-registry/model-registry-bqml\" target=\"_blank\">registered to Vertex AI Model Registry</a>, which enables you to view all of your registered models and its versions on Google Cloud in one place.\n",
         "\n",
-        "* `vertex_ai_model_version_aliases allows you to set aliases to help you keep track of your model version ([documentation](https://cloud.google.com/vertex-ai/docs/model-registry/model-alias))."
+        "* `vertex_ai_model_version_aliases allows you to set aliases to help you keep track of your model version (<a href=\"https://cloud.google.com/vertex-ai/docs/model-registry/model-alias\" target=\"_blank\">documentation</a>)."
       ]
     },
     {
@@ -677,7 +683,7 @@
       "source": [
         "# this cell may take ~1 min to run\n",
         "\n",
-        "BQML_MODEL_NAME = \"bqmlmodelchurn\"\n",
+        "BQML_MODEL_NAME = f\"bqml_model_churn_{UUID}\"\n",
         "\n",
         "sql_train_model_bqml = f\"\"\"\n",
         "CREATE OR REPLACE MODEL {BQ_DATASET_NAME}.{BQML_MODEL_NAME}    \n",
@@ -696,7 +702,7 @@
         "\n",
         "print(sql_train_model_bqml)\n",
         "\n",
-        "bq_query(sql_train_model_bqml)"
+        "run_bq_query(sql_train_model_bqml)"
       ]
     },
     {
@@ -714,7 +720,7 @@
         "id": "2aaaae772f67"
       },
       "source": [
-        "With the model created, you can now evaluate the logistic regression model. Behind the scenes, BigQuery ML automatically [split the data](https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-create#data_split_method), which makes it easier to quickly train and evaluate models."
+        "With the model created, you can now evaluate the logistic regression model. Behind the scenes, BigQuery ML automatically <a href=\"https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-create#data_split_method\" target=\"_blank\">split the data</a>, which makes it easier to quickly train and evaluate models."
       ]
     },
     {
@@ -734,7 +740,7 @@
         "\n",
         "print(sql_evaluate_model)\n",
         "\n",
-        "bq_query(sql_evaluate_model)"
+        "run_bq_query(sql_evaluate_model)"
       ]
     },
     {
@@ -745,7 +751,7 @@
       "source": [
         "These metrics help you understand the performance of the model. \n",
         "\n",
-        "There are various metrics for logistic regression and other model types (full list of metrics can be found in the [documentation](https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-evaluate#mlevaluate_output))."
+        "There are various metrics for logistic regression and other model types (full list of metrics can be found in the <a href=\"https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-evaluate#mlevaluate_output\" target=\"_blank\">documentation</a>)."
       ]
     },
     {
@@ -765,7 +771,7 @@
       "source": [
         "Make a batch prediction in BigQuery ML on the original training data to check the probability of churn for each of the users, as seen in the `probability` column, with the predicted label under the `predicted_churn` column.\n",
         "\n",
-        "[ML.EXPLAIN_PREDICT](https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-explain-predict) has built-in [Explainable AI](https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-xai-overview). This allows you to see the top contributing features to each prediction and interpret how it was computed."
+        "<a href=\"https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-explain-predict\" target=\"_blank\">ML.EXPLAIN_PREDICT</a> has built-in <a href=\"https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-xai-overview\" target=\"_blank\">Explainable AI</a>. This allows you to see the top contributing features to each prediction and interpret how it was computed."
       ]
     },
     {
@@ -787,7 +793,7 @@
         "\n",
         "print(sql_explain_predict)\n",
         "\n",
-        "bq_query(sql_explain_predict)"
+        "run_bq_query(sql_explain_predict)"
       ]
     },
     {
@@ -796,7 +802,7 @@
         "id": "fa1f96c0f452"
       },
       "source": [
-        "Since the `top_feature_attributions` is a nested column, you can unnest the array ([documentation](https://cloud.google.com/bigquery/docs/reference/standard-sql/arrays)) into separate rows for each of the features. In other words, since ML.EXPLAIN_PREDICT provides the top 5 most important features, using `UNNEST` results in 5 rows per prediction:"
+        "Since the `top_feature_attributions` is a nested column, you can unnest the array (<a href=\"https://cloud.google.com/bigquery/docs/reference/standard-sql/arrays\" target=\"_blank\">documentation</a>) into separate rows for each of the features. In other words, since ML.EXPLAIN_PREDICT provides the top 5 most important features, using `UNNEST` results in 5 rows per prediction:"
       ]
     },
     {
@@ -827,7 +833,7 @@
         "\n",
         "print(sql_explain_predict)\n",
         "\n",
-        "bq_query(sql_explain_predict)"
+        "run_bq_query(sql_explain_predict)"
       ]
     },
     {
@@ -847,7 +853,7 @@
       "source": [
         "When the model was trained in BigQuery ML, the line `model_registry=\"vertex_ai\"` registered the model to Vertex AI Model Registry automatically upon completion.\n",
         "\n",
-        "You can view the model on the [Vertex AI Model Registry page](https://console.cloud.google.com/vertex-ai/models), or use the code below to check that it was successfully registered:"
+        "You can view the model on the <a href=\"https://console.cloud.google.com/vertex-ai/models\" target=\"_blank\">Vertex AI Model Registry page</a>, or use the code below to check that it was successfully registered:"
       ]
     },
     {
@@ -858,12 +864,7 @@
       },
       "outputs": [],
       "source": [
-        "print(f\"BQML_MODEL_NAME = {BQML_MODEL_NAME}\")\n",
-        "\n",
-        "models = vertex_ai.Model.list(\n",
-        "    filter=f\"display_name={BQML_MODEL_NAME}\", order_by=\"update_time\"\n",
-        ")\n",
-        "model = models[0]\n",
+        "model = vertex_ai.Model(model_name=BQML_MODEL_NAME)\n",
         "\n",
         "print(model.gca_resource)"
       ]
@@ -883,7 +884,7 @@
         "id": "b6120dcc1ff6"
       },
       "source": [
-        "While BigQuery ML supports batch prediction with [ML.PREDICT](https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-predict) and [ML.EXPLAIN_PREDICT](https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-explain-predict), BigQuery ML is not suitable for real-time predictions where you need low latency predictions with potentially high frequency of requests.\n",
+        "While BigQuery ML supports batch prediction with <a href=\"https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-predict\" target=\"_blank\">ML.PREDICT</a> and <a href=\"https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-explain-predict\" target=\"_blank\">ML.EXPLAIN_PREDICT</a>, BigQuery ML is not suitable for real-time predictions where you need low latency predictions with potentially high frequency of requests.\n",
         "\n",
         "In other words, deploying the BigQuery ML model to an endpoint enables you to do online predictions."
       ]
@@ -906,30 +907,6 @@
         "To deploy your model to an endpoint, you will first need to create an endpoint before you deploy the model to it."
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "3ce73125dff6"
-      },
-      "outputs": [],
-      "source": [
-        "def create_endpoint(\n",
-        "    project: str,\n",
-        "    display_name: str,\n",
-        "    location: str,\n",
-        "):\n",
-        "    endpoint = vertex_ai.Endpoint.create(\n",
-        "        display_name=display_name,\n",
-        "        project=project,\n",
-        "        location=location,\n",
-        "    )\n",
-        "\n",
-        "    print(endpoint.display_name)\n",
-        "    print(endpoint.resource_name)\n",
-        "    return endpoint"
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -938,17 +915,16 @@
       },
       "outputs": [],
       "source": [
-        "endpoint_name = f\"{BQML_MODEL_NAME}-{TIMESTAMP}\"\n",
+        "ENDPOINT_NAME = f\"{BQML_MODEL_NAME}-endpoint\"\n",
         "\n",
-        "print(\n",
-        "    f\"\"\"\n",
-        "PROJECT_ID: {PROJECT_ID},\n",
-        "endpoint_name: {endpoint_name}\n",
-        "REGION: {REGION}\n",
-        "\"\"\"\n",
+        "endpoint = vertex_ai.Endpoint.create(\n",
+        "    display_name=ENDPOINT_NAME,\n",
+        "    project=PROJECT_ID,\n",
+        "    location=REGION,\n",
         ")\n",
         "\n",
-        "create_endpoint(PROJECT_ID, endpoint_name, REGION)"
+        "print(endpoint.display_name)\n",
+        "print(endpoint.resource_name)"
       ]
     },
     {
@@ -966,31 +942,7 @@
         "id": "951ed1693f6b"
       },
       "source": [
-        "List the endpoints to make sure it has successfully been created. You can also view your endpoints on the [Vertex AI Endpoints page](https://console.cloud.google.com/vertex-ai/endpoints?project=polong-contentdev)."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "0a9bad8d9ad4"
-      },
-      "outputs": [],
-      "source": [
-        "endpoint = vertex_ai.Endpoint.list(\n",
-        "    # filter=f'display_name={endpoint_name}', # optional: filter by specific endpoint name\n",
-        "    order_by=\"update_time\"\n",
-        ")\n",
-        "endpoint[-1]"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "2431a4d28d97"
-      },
-      "source": [
-        "Retrieve the endpoint id so you can use it in the next step."
+        "List the endpoints to make sure it has successfully been created. (You can also view your endpoints on the <a href=\"https://console.cloud.google.com/vertex-ai/endpoints\" target=\"_blank\">Vertex AI Endpoints page</a>)."
       ]
     },
     {
@@ -1001,7 +953,7 @@
       },
       "outputs": [],
       "source": [
-        "endpoint[-1].to_dict()"
+        "endpoint.list()"
       ]
     },
     {
@@ -1019,74 +971,19 @@
         "id": "6a90be5b77a2"
       },
       "source": [
-        "With the model, you can now deploy it to an endpoint. "
+        "With the new endpoint, you can now deploy your model."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "af323ea42c5b"
-      },
-      "outputs": [],
-      "source": [
-        "from typing import Dict, Optional, Sequence, Tuple\n",
-        "\n",
-        "\n",
-        "def deploy_model_with_automatic_resources_sample(\n",
-        "    project,\n",
-        "    location,\n",
-        "    model_name: str,\n",
-        "    endpoint: Optional[vertex_ai.Endpoint] = None,\n",
-        "    deployed_model_display_name: Optional[str] = None,\n",
-        "    traffic_percentage: Optional[int] = 0,\n",
-        "    traffic_split: Optional[Dict[str, int]] = None,\n",
-        "    min_replica_count: int = 1,\n",
-        "    max_replica_count: int = 1,\n",
-        "    metadata: Optional[Sequence[Tuple[str, str]]] = (),\n",
-        "    sync: bool = True,\n",
-        "):\n",
-        "    \"\"\"\n",
-        "    model_name: A fully-qualified model resource name or model ID.\n",
-        "          Example: \"projects/123/locations/us-central1/models/456\" or\n",
-        "          \"456\" when project and location are initialized or passed.\n",
-        "    \"\"\"\n",
-        "\n",
-        "    model = vertex_ai.Model(model_name=model_name)\n",
-        "\n",
-        "    model.deploy(\n",
-        "        endpoint=endpoint,\n",
-        "        deployed_model_display_name=deployed_model_display_name,\n",
-        "        traffic_percentage=traffic_percentage,\n",
-        "        traffic_split=traffic_split,\n",
-        "        min_replica_count=min_replica_count,\n",
-        "        max_replica_count=max_replica_count,\n",
-        "        metadata=metadata,\n",
-        "        sync=sync,\n",
-        "    )\n",
-        "\n",
-        "    model.wait()\n",
-        "\n",
-        "    print(model.display_name)\n",
-        "    print(model.resource_name)\n",
-        "    return"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "9e6763369af4"
+        "id": "c70ecc568ee5"
       },
       "outputs": [],
       "source": [
         "# deploying the model to the endpoint may take 10-15 minutes\n",
-        "deploy_model_with_automatic_resources_sample(\n",
-        "    project=PROJECT_ID,\n",
-        "    location=REGION,\n",
-        "    model_name=BQML_MODEL_NAME,\n",
-        "    endpoint=endpoint[-1],\n",
-        ")"
+        "model.deploy(endpoint=endpoint)"
       ]
     },
     {
@@ -1095,7 +992,7 @@
         "id": "c303d779477b"
       },
       "source": [
-        "You can also check on the status of your model by visiting the [Vertex AI Endpoints page](https://console.cloud.google.com/vertex-ai/endpoints)."
+        "You can also check on the status of your model by visiting the <a href=\"https://console.cloud.google.com/vertex-ai/endpoints\" target=\"_blank\">Vertex AI Endpoints page</a>."
       ]
     },
     {
@@ -1168,35 +1065,12 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "2c6093ce9f8a"
+        "id": "b4839f31d2f8"
       },
       "outputs": [],
       "source": [
-        "def endpoint_predict_sample(\n",
-        "    project: str, location: str, instances: list, endpoint: str\n",
-        "):\n",
-        "    endpoint = vertex_ai.Endpoint(endpoint)\n",
-        "\n",
-        "    prediction = endpoint.predict(instances=instances)\n",
-        "    return prediction"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "0c41fd6eeb6f"
-      },
-      "outputs": [],
-      "source": [
-        "prediction_response = endpoint_predict_sample(\n",
-        "    project=PROJECT_ID,\n",
-        "    location=REGION,\n",
-        "    instances=df_sample_requests_list,\n",
-        "    endpoint=endpoint[-1].name,\n",
-        ")\n",
-        "\n",
-        "prediction_response"
+        "prediction = endpoint.predict(df_sample_requests_list)\n",
+        "print(prediction)"
       ]
     },
     {
@@ -1216,7 +1090,7 @@
       },
       "outputs": [],
       "source": [
-        "prediction_response.predictions"
+        "prediction.predictions"
       ]
     },
     {
@@ -1227,8 +1101,8 @@
       "source": [
         "## Cleaning up\n",
         "\n",
-        "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
-        "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
+        "To clean up all Google Cloud resources used in this project, you can <a href=\"https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects\" target=\"_blank\">delete the Google Cloud\n",
+        "project</a> you used for the tutorial.\n",
         "\n",
         "Otherwise, you can delete the individual resources you created in this tutorial:"
       ]
@@ -1241,18 +1115,12 @@
       },
       "outputs": [],
       "source": [
-        "# MODEL_ID = model.name\n",
+        "# Undeploy model from endpoint and delete endpoint\n",
+        "endpoint.undeploy_all()\n",
+        "endpoint.delete()\n",
         "\n",
-        "ENDPOINT_ID = int(endpoint[-1].name)\n",
-        "\n",
-        "# Undeploy model from endpoint\n",
-        "endpoint[-1].undeploy_all()\n",
-        "\n",
-        "# Delete endpoint resource\n",
-        "! gcloud ai endpoints delete $ENDPOINT_ID --quiet --region $REGION\n",
-        "\n",
-        "# Delete BigQuery ML model\n",
-        "! bq rm -f --model $PROJECT_ID\\:$BQ_DATASET_NAME\\.$BQML_MODEL_NAME"
+        "# Delete BigQuery dataset, including the BigQuery ML model\n",
+        "! bq rm -r -f $PROJECT_ID:$BQ_DATASET_NAME"
       ]
     }
   ],

notebooks/official/custom/custom-tabular-bq-managed-dataset.ipynb

@@ -44,7 +44,7 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/custom/custom-tabular-bq-managed-dataset.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/custom/custom-tabular-bq-managed-dataset.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
@@ -263,7 +263,7 @@
         "\n",
         "2. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
         "\n",
-        "3. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com).\n",
+        "3. [Enable the following APIs: Vertex AI API, Cloud Resource Manager API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,cloudresourcemanager.googleapis.com).\n",
         "\n",
         "4. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
         "\n",

notebooks/official/experiments/build_model_experimentation_lineage_with_prebuild_code.ipynb

@@ -8,7 +8,7 @@
       },
       "outputs": [],
       "source": [
-        "# Copyright 2022 Google LLC\n",
+        "# Copyright 2021 Google LLC\n",
         "#\n",
         "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
         "# you may not use this file except in compliance with the License.\n",
@@ -54,13 +54,31 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "tvgnzT1CKxrO"
+        "id": "d975c5729f18"
       },
       "source": [
         "## Overview\n",
         "\n",
-        "As a Data Scientist, you want to be able to reuse code path (data preprocessing, feature engineering etc...) that others within your team have written to simplify and standardize all the complex data wrangling. \n",
+        "As a Data Scientist, you want to be able to reuse code path (data preprocessing, feature engineering etc...) that others within your team have written to simplify and standardize all the complex data wrangling. "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3a0f8061b9c1"
+      },
+      "source": [
+        "### Objective\n",
         "\n",
+        "In this notebook, you learn how to integrate preprocessing code in a Vertex AI experiments. Also you build the experiment lineage lets you record, analyze, debug, and audit metadata and artifacts produced along your ML journey."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tvgnzT1CKxrO"
+      },
+      "source": [
         "### Dataset\n",
         "\n",
         "This dataset is the UCI News Aggregator Data Set which contains 422,937 news collected between March 10th, 2014 and August 10th, 2014. Below are example records from the dataset:\n",
@@ -72,13 +90,15 @@
         "|2  |Fed's Charles Plosser sees high bar for change in pace of tapering   |http://www.livemint.com/Politics/H2EvwJSK2VE6OF7iK1g3PP/Feds-Charles-Plosser-sees-high-bar-for-change-in-pace-of-ta.html   |Livemint         |b       |ddUyU0VZz0BRneMioxUPQVP6sIxvM|www.livemint.com   |1394470371207|\n",
         "|3  |US open: Stocks fall after Fed official hints at accelerated tapering|http://www.ifamagazine.com/news/us-open-stocks-fall-after-fed-official-hints-at-accelerated-tapering-294436                |IFA Magazine     |b       |ddUyU0VZz0BRneMioxUPQVP6sIxvM|www.ifamagazine.com|1394470371550|\n",
         "|4  |Fed risks falling 'behind the curve', Charles Plosser says           |http://www.ifamagazine.com/news/fed-risks-falling-behind-the-curve-charles-plosser-says-294430                             |IFA Magazine     |b       |ddUyU0VZz0BRneMioxUPQVP6sIxvM|www.ifamagazine.com|1394470371793|\n",
-        "|5  |Fed's Plosser: Nasty Weather Has Curbed Job Growth                   |http://www.moneynews.com/Economy/federal-reserve-charles-plosser-weather-job-growth/2014/03/10/id/557011                   |Moneynews        |b       |ddUyU0VZz0BRneMioxUPQVP6sIxvM|www.moneynews.com  |1394470372027|\n",
-        "\n",
-        "\n",
-        "### Objective\n",
-        "\n",
-        "In this notebook, you learn how to integrate preprocessing code in a Vertex AI experiments. Also you will build the experiment lineage lets you record, analyze, debug, and audit metadata and artifacts produced along your ML journey.\n",
-        "\n",
+        "|5  |Fed's Plosser: Nasty Weather Has Curbed Job Growth                   |http://www.moneynews.com/Economy/federal-reserve-charles-plosser-weather-job-growth/2014/03/10/id/557011                   |Moneynews        |b       |ddUyU0VZz0BRneMioxUPQVP6sIxvM|www.moneynews.com  |1394470372027|"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5e2eba58ad71"
+      },
+      "source": [
         "### Costs \n",
         "\n",
         "This tutorial uses billable components of Google Cloud:\n",
@@ -150,7 +170,7 @@
       "source": [
         "### Install additional packages\n",
         "\n",
-        "Install additional package dependencies not installed in your notebook environment, such as TensorFlow or Vertex AI SDK. Use the latest major GA version of each package."
+        "Install additional package dependencies not installed in your notebook environment,TensorFlow or Vertex AI SDK. Use the latest major GA version of each package."
       ]
     },
     {
@@ -174,9 +194,8 @@
         "if IS_WORKBENCH_NOTEBOOK:\n",
         "    USER_FLAG = \"--user\"\n",
         "\n",
-        "! pip3 install {USER_FLAG} --upgrade fsspec gcsfs joblib -q\n",
-        "! pip3 install {USER_FLAG} --force-reinstall 'google-cloud-aiplatform>=1.15' -q\n",
-        "! pip3 install {USER_FLAG} --upgrade sklearn"
+        "! pip3 install {USER_FLAG} --upgrade joblib fsspec gcsfs scikit-learn -q\n",
+        "! pip install {USER_FLAG} --upgrade google-cloud-aiplatform -q"
       ]
     },
     {
@@ -209,21 +228,14 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "lWEdiXsJg0XY"
-      },
-      "source": [
-        "## Before you begin"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "BF1j6f9HApxa"
       },
       "source": [
+        "## Before you begin\n",
+        "\n",
         "### Set up your Google Cloud project\n",
         "\n",
         "**The following steps are required, regardless of your notebook environment.**\n",
@@ -232,9 +244,9 @@
         "\n",
         "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
         "\n",
-        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com).\n",
+        "1. [Enable APIs](https://console.cloud.google.com/flows/enableapi?apiid=cloudresourcemanager.googleapis.com,aiplatform.googleapis.com).\n",
         "\n",
-        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
         "\n",
         "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
         "Cloud SDK uses the right project for all the commands in this notebook.\n",
@@ -344,9 +356,9 @@
         "id": "06571eb4063b"
       },
       "source": [
-        "#### Timestamp\n",
+        "#### UUID\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append it onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
       ]
     },
     {
@@ -357,9 +369,16 @@
       },
       "outputs": [],
       "source": [
-        "from datetime import datetime\n",
+        "import random\n",
+        "import string\n",
         "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+        "\n",
+        "# Generate a uuid of a specifed length(default=8)\n",
+        "def generate_uuid(length: int = 8) -> str:\n",
+        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
+        "\n",
+        "\n",
+        "UUID = generate_uuid()"
       ]
     },
     {
@@ -371,7 +390,7 @@
         "### Authenticate your Google Cloud account\n",
         "\n",
         "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
-        "authenticated. Skip this step."
+        "authenticated."
       ]
     },
     {
@@ -393,9 +412,14 @@
         "3. In the **Service account name** field, enter a name, and\n",
         "   click **Create**.\n",
         "\n",
-        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
-        "into the filter box, and select\n",
-        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type and select\n",
+        "the following role into the filter box:\n",
+        "\n",
+        "    *   Storage Admin\n",
+        "    *   Storage Object Admin\n",
+        "    *   Service Account User\n",
+        "    *   Vertex AI Administrator\n",
+        "\n",
         "\n",
         "5. Click *Create*. A JSON file that contains your key downloads to your\n",
         "local environment.\n",
@@ -480,7 +504,7 @@
       "outputs": [],
       "source": [
         "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
         "    BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
@@ -533,17 +557,6 @@
         "### Set project folder"
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "AARD6Fsr-DSi"
-      },
-      "outputs": [],
-      "source": [
-        "DATA_PATH = \"data\""
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -552,6 +565,7 @@
       },
       "outputs": [],
       "source": [
+        "DATA_PATH = \"data\"\n",
         "!mkdir -m 777 -p {DATA_PATH}"
       ]
     },
@@ -564,17 +578,6 @@
         "### Get the data"
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "3V6W2nIo9FtL"
-      },
-      "outputs": [],
-      "source": [
-        "DATASET_URL = \"https://archive.ics.uci.edu/ml/machine-learning-databases/00359/NewsAggregatorDataset.zip\""
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -583,6 +586,7 @@
       },
       "outputs": [],
       "source": [
+        "DATASET_URL = \"https://archive.ics.uci.edu/ml/machine-learning-databases/00359/NewsAggregatorDataset.zip\"\n",
         "!wget --no-parent {DATASET_URL} --directory-prefix={DATA_PATH}\n",
         "!mkdir -m 777 -p {DATA_PATH}/temp {DATA_PATH}/raw\n",
         "!unzip {DATA_PATH}/*.zip -d {DATA_PATH}/temp\n",
@@ -623,10 +627,13 @@
         "logger = logging.getLogger(\"logger\")\n",
         "logging.basicConfig(level=logging.INFO)\n",
         "\n",
+        "import collections\n",
         "import tempfile\n",
         "import time\n",
         "from json import dumps\n",
         "\n",
+        "collections.Iterable = collections.abc.Iterable\n",
+        "\n",
         "# Vertex AI\n",
         "from google.cloud import aiplatform as vertex_ai"
       ]
@@ -655,7 +662,7 @@
         "# Experiments\n",
         "TASK = \"classification\"\n",
         "MODEL_TYPE = \"naivebayes\"\n",
-        "EXPERIMENT_NAME = f\"{TASK}-{MODEL_TYPE}-{TIMESTAMP}\"\n",
+        "EXPERIMENT_NAME = f\"{TASK}-{MODEL_TYPE}-{UUID}\"\n",
         "EXPERIMENT_RUN_NAME = \"run-1\"\n",
         "\n",
         "# Preprocessing\n",
@@ -683,7 +690,7 @@
         "FEATURES = \"title\"\n",
         "TEST_SIZE = 0.2\n",
         "SEED = 8\n",
-        "TRAINED_MODEL_URI = f\"{BUCKET_URI}/deliverables/{TIMESTAMP}\"\n",
+        "TRAINED_MODEL_URI = f\"{BUCKET_URI}/deliverables/{UUID}\"\n",
         "MODEL_NAME = f\"{EXPERIMENT_NAME}-model\""
       ]
     },
@@ -793,7 +800,7 @@
       "source": [
         "#### Create a Dataset Metadata Artifact\n",
         "\n",
-        "First you create the Dataset artifact to track the dataset resource in the Vertex AI ML Metadata and create the experiment lineage."
+        "First you create the Dataset artifact to track the dataset resource in the Vertex ML Metadata and create the experiment lineage."
       ]
     },
     {
@@ -832,7 +839,6 @@
         "Preprocess module\n",
         "\"\"\"\n",
         "\n",
-        "import string\n",
         "\n",
         "import pandas as pd\n",
         "\n",
@@ -862,7 +868,10 @@
       "source": [
         "#### Add the `preprocessing` Execution\n",
         "\n",
-        "You add the preprocessing step to track its execution in the lineage associated to Vertex AI Experiment. "
+        "Vertex AI Experiments supports tracking both executions and artifacts. Executions are steps in an ML workflow that can include but are not limited to data preprocessing, training, and model evaluation. Executions can consume artifacts such as datasets and produce artifacts such as models.\n",
+        "\n",
+        "You add the preprocessing step to track its execution in the lineage associated to Vertex AI Experiment. \n",
+        "For Vertex AI, the parameters are passed inside the message field which we see in the logs. These structures of the logs are predefined."
       ]
     },
     {
@@ -936,7 +945,16 @@
       "source": [
         "#### Create model training module\n",
         "\n",
-        "Below the training module."
+        "Below the training module.\n",
+        "\n",
+        "**get_training_split :** It takes parameters like  x(The data to be split), y(The labels to be split), test_size(The proportion of the data to be reserved for testing) and random_state(The seed used by the random number generator).\n",
+        "This function return training data, testing data , The training labels and The testing labels.\n",
+        "\n",
+        "**get_pipeline :** It return's the model.\n",
+        "\n",
+        "**train_pipeline :** It train the model by using model, training data, training lables and return's the trained model.\n",
+        "\n",
+        "**evaluate_model :** It evaluate the model and return the accuracy of the model.\n"
       ]
     },
     {
@@ -1144,6 +1162,15 @@
         "    exc.assign_output_artifacts([model])"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e595c893de8d"
+      },
+      "source": [
+        "### Stop Experiment run"
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -1163,7 +1190,7 @@
       "source": [
         "### Visualize Experiment Lineage\n",
         "\n",
-        "Below you will get the link to Vertex AI Metadata UI in the console that will show the experiment lineage."
+        "Below you get the link to Vertex AI Metadata UI in the console that show the experiment lineage."
       ]
     },
     {
@@ -1201,17 +1228,8 @@
       "source": [
         "# Delete experiment\n",
         "exp = vertex_ai.Experiment(EXPERIMENT_NAME)\n",
-        "exp.delete()"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "gW8Ddbr8xaKp"
-      },
-      "outputs": [],
-      "source": [
+        "exp.delete()\n",
+        "\n",
         "# Delete model\n",
         "model_list = vertex_ai.Model.list(filter=f'display_name=\"{MODEL_NAME}\"')\n",
         "for model in model_list:\n",
@@ -1223,22 +1241,26 @@
         "        filter=f'display_name=\"{dataset_name}\"'\n",
         "    )\n",
         "    for dataset in dataset_list:\n",
-        "        dataset.delete()"
+        "        dataset.delete()\n",
+        "\n",
+        "# Delete Cloud Storage objects that were created\n",
+        "delete_bucket = True\n",
+        "\n",
+        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "    ! gsutil -m rm -r $BUCKET_URI\n",
+        "\n",
+        "!rm -Rf {DATA_PATH}"
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "sx_vKniMq9ZX"
+        "id": "3f00c455b930"
       },
       "outputs": [],
       "source": [
-        "# Delete Cloud Storage objects that were created\n",
-        "delete_bucket = True\n",
-        "\n",
-        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
-        "    ! gsutil -m rm -r $BUCKET_URI"
+        "!rm -Rf {DATA_PATH}"
       ]
     }
   ],

notebooks/official/experiments/comparing_local_trained_models.ipynb

@@ -56,26 +56,63 @@
       "metadata": {
         "id": "tvgnzT1CKxrO"
       },
+      "source": [
+        "# Compare trained and evaluated model experiments using Vertex AI Experiments"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3a0651225470"
+      },
       "source": [
         "## Overview\n",
         "\n",
-        "As a Data Scientist, you probably start running model experiments locally on your notebook. Depending on the framework you use, you would need to track parameters, training time series and evaluation metrics. In this way, you would be able to explain the modelling approach you would choose. \n",
-        "\n",
-        "### Dataset\n",
-        "\n",
-        "The dataset is the Tensorflow Dataset's Large Yelp Review Dataset. The Yelp reviews dataset consists of reviews from Yelp. For more information, please refer to this [link](http://www.yelp.com/dataset).\n",
-        "\n",
+        "As a Data Scientist, you probably start running model experiments locally on your notebook. Depending on the framework you use, you would need to track parameters, training time series and evaluation metrics. In this way, you would be able to explain the modelling approach you would choose. "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "f3021b2963a6"
+      },
+      "source": [
         "### Objective\n",
         "\n",
-        "In this notebook, you will learn how to \n",
+        "In this tutorial, you learn how to use Vertex AI Experiments to compare and evaluate model experiments.\n",
+        "\n",
+        "This tutorial uses the following Google Cloud ML services and resources:\n",
+        "\n",
+        "- Vertex AI Workbench\n",
+        "- Vertex AI Experiments\n",
+        "\n",
+        "The steps performed include:\n",
         "\n",
         "- log the model parameters\n",
         "- log the loss and metrics on every epoch to TensorBoard\n",
         "- log the evaluation metrics\n",
         "- compare two experiments\n",
         "\n",
-        "in Vertex AI Experiment of a recurrent neural network (RNN) for sentiment analysis.\n",
+        "in Vertex AI Experiment of a recurrent neural network (RNN) for sentiment analysis."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "20a5168cf05e"
+      },
+      "source": [
+        "### Dataset\n",
         "\n",
+        "The dataset is the Tensorflow Dataset's Large Yelp Review Dataset. The Yelp reviews dataset consists of reviews from Yelp. For more information, please refer to this [link](http://www.yelp.com/dataset).\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "de76bb18c85b"
+      },
+      "source": [
         "### Costs \n",
         "\n",
         "This tutorial uses billable components of Google Cloud:\n",
@@ -99,15 +136,8 @@
         "### Set up your local development environment\n",
         "\n",
         "**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets\n",
-        "all the requirements to run this notebook. You can skip this step."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "gCuSR8GkAgzl"
-      },
-      "source": [
+        "all the requirements to run this notebook. You can skip this step.\n",
+        "\n",
         "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
         "You need the following:\n",
         "\n",
@@ -169,17 +199,8 @@
         "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
         "USER_FLAG = \"\"\n",
         "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\""
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "wyy5Lbnzg5fi"
-      },
-      "outputs": [],
-      "source": [
+        "    USER_FLAG = \"--user\"\n",
+        "\n",
         "! pip3 install {USER_FLAG} --upgrade tensorflow==2.8.0 tensorflow_datasets==4.5.2 -q\n",
         "! pip3 install --user --force-reinstall 'google-cloud-aiplatform>=1.15' -q"
       ]
@@ -376,15 +397,8 @@
         "### Authenticate your Google Cloud account\n",
         "\n",
         "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
-        "authenticated. Skip this step."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "sBCra4QMA2wR"
-      },
-      "source": [
+        "authenticated. Skip this step.\n",
+        "\n",
         "**If you are using Colab**, run the cell below and follow the instructions\n",
         "when prompted to authenticate your account via oAuth.\n",
         "\n",
@@ -398,9 +412,14 @@
         "3. In the **Service account name** field, enter a name, and\n",
         "   click **Create**.\n",
         "\n",
-        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
-        "into the filter box, and select\n",
-        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type and select\n",
+        "the following role into the filter box:\n",
+        "\n",
+        "    *   Storage Admin\n",
+        "    *   Storage Object Admin\n",
+        "    *   Service Account User\n",
+        "    *   Vertex AI Administrator\n",
+        "\n",
         "\n",
         "5. Click *Create*. A JSON file that contains your key downloads to your\n",
         "local environment.\n",
@@ -426,16 +445,11 @@
         "import sys\n",
         "\n",
         "# If on Vertex AI Workbench, then don't execute this code\n",
-        "\n",
-        "IS_COLAB = False\n",
-        "\n",
+        "IS_COLAB = \"google.colab\" in sys.modules\n",
         "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
         "    \"DL_ANACONDA_HOME\"\n",
         "):\n",
         "    if \"google.colab\" in sys.modules:\n",
-        "\n",
-        "        IS_COLAB = True\n",
-        "\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
         "        google_auth.authenticate_user()\n",
@@ -443,9 +457,7 @@
         "    # If you are running this notebook locally, replace the string below with the\n",
         "    # path to your service account key and run this cell to authenticate your GCP\n",
         "    # account.\n",
-        "\n",
         "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "\n",
         "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
       ]
     },
@@ -650,7 +662,15 @@
         "\n",
         "Vertex AI enables users to track the steps (for example, preprocessing, training) of an experiment run, and track inputs (for example, algorithm, parameters, datasets) and outputs (for example, models, checkpoints, metrics) of those steps. \n",
         "\n",
-        "Below you have some example of how track experiments to train recurrent neural network for sentiment analysis."
+        "Below you have some example of how track experiments to train recurrent neural network for sentiment analysis. \n",
+        "\n",
+        "To simplify the code, here you have helper function to cover the following steps:\n",
+        "\n",
+        "- Collect training data\n",
+        "- Create text encoder\n",
+        "- Build a RNN as baseline model\n",
+        "- Build a LSTM as challenger model\n",
+        "- Train the model"
       ]
     },
     {
@@ -721,16 +741,16 @@
         "    return encoder\n",
         "\n",
         "\n",
-        "def get_model(encoder, model_params, role):\n",
+        "def get_baseline_model(encoder, model_params):\n",
         "    \"\"\"\n",
-        "    Returns a tf.keras.Model object for the model\n",
+        "    Returns a tf.keras.Model object for the model as baseline\n",
         "    Args:\n",
         "        encoder: A TextVectorization object for the encoder\n",
         "        model_params: A dictionary with model parameters\n",
-        "        role: A variable to set the role of model\n",
         "    Returns:\n",
         "        tf.keras.Model: A tf.keras.Model object for the model\n",
         "    \"\"\"\n",
+        "\n",
         "    model = tf.keras.Sequential()\n",
         "    model.add(encoder)\n",
         "    model.add(\n",
@@ -738,20 +758,46 @@
         "            input_dim=len(encoder.get_vocabulary()), output_dim=64, mask_zero=True\n",
         "        )\n",
         "    )\n",
-        "    if role == \"baseline\":\n",
-        "        model.add(tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(64)))\n",
-        "        model.add(tf.keras.layers.Dense(64, activation=\"relu\"))\n",
-        "        model.add(tf.keras.layers.Dense(1))\n",
-        "    else:\n",
-        "        model.add(\n",
-        "            tf.keras.layers.Bidirectional(\n",
-        "                tf.keras.layers.LSTM(64, return_sequences=True)\n",
-        "            )\n",
-        "        )\n",
-        "        model.add(tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(32)))\n",
-        "        model.add(tf.keras.layers.Dropout(0.5))\n",
-        "        model.add(tf.keras.layers.Dense(1))\n",
+        "    model.add(tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(64)))\n",
+        "    model.add(tf.keras.layers.Dense(64, activation=\"relu\"))\n",
+        "    model.add(tf.keras.layers.Dense(1))\n",
+        "    model.compile(\n",
+        "        loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),\n",
+        "        optimizer=tf.keras.optimizers.Adam(\n",
+        "            learning_rate=model_params[\"learning_rate\"],\n",
+        "            beta_1=model_params[\"beta_1\"],\n",
+        "            beta_2=model_params[\"beta_2\"],\n",
+        "            epsilon=model_params[\"epsilon\"],\n",
+        "        ),\n",
+        "        metrics=[\"accuracy\"],\n",
+        "    )\n",
+        "    return model\n",
         "\n",
+        "\n",
+        "def get_stacked_model(encoder, model_params):\n",
+        "    \"\"\"\n",
+        "    Returns a tf.keras.Model object for the model as challenger\n",
+        "        Args:\n",
+        "            encoder: A TextVectorization object for the encoder\n",
+        "            model_params: A dictionary with model parameters\n",
+        "        Returns:\n",
+        "            tf.keras.Model: A tf.keras.Model object for the model\n",
+        "    \"\"\"\n",
+        "\n",
+        "    model = tf.keras.Sequential()\n",
+        "    model.add(encoder)\n",
+        "    model.add(\n",
+        "        tf.keras.layers.Embedding(\n",
+        "            input_dim=len(encoder.get_vocabulary()), output_dim=64, mask_zero=True\n",
+        "        )\n",
+        "    )\n",
+        "    model.add(\n",
+        "        tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(64, return_sequences=True))\n",
+        "    )\n",
+        "    model.add(tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(32)))\n",
+        "    model.add(tf.keras.layers.Dense(64, activation=\"relu\"))\n",
+        "    model.add(tf.keras.layers.Dropout(0.5))\n",
+        "    model.add(tf.keras.layers.Dense(1))\n",
         "    model.compile(\n",
         "        loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),\n",
         "        optimizer=tf.keras.optimizers.Adam(\n",
@@ -806,7 +852,15 @@
       "source": [
         "#### Run experiment and evaluate experiment runs using `with` statement\n",
         "\n",
-        "This step would takes **10 min** approx. to finish.\n"
+        "This step would takes **10 min** approx. to finish. And it covers the following steps:\n",
+        "\n",
+        "- Initialize an experiment run\n",
+        "- Log the parameters associated to training data\n",
+        "- Log the parameters of the encoder\n",
+        "- Log the parameters of the model\n",
+        "- Train the model\n",
+        "- Log the metrics for each epochs\n",
+        "- Log the overall training metrics"
       ]
     },
     {
@@ -818,7 +872,7 @@
       "outputs": [],
       "source": [
         "# Experiment Settings ----------------------------------------------------------\n",
-        "ID_1 = \"run-1\"\n",
+        "RUN_ID_1 = \"run-1\"\n",
         "BUFFER_SIZE = 10000\n",
         "BATCH_SIZE = 64\n",
         "VOCAB_SIZE = 1000\n",
@@ -832,7 +886,7 @@
         "\n",
         "# Initialize the experiment\n",
         "logging.info(\"Initialize the experiment.\")\n",
-        "with vertex_ai.start_run(ID_1) as run:\n",
+        "with vertex_ai.start_run(RUN_ID_1) as run:\n",
         "\n",
         "    # Get the training and testing datasets\n",
         "    logging.info(\"Get the training and testing datasets.\")\n",
@@ -856,7 +910,7 @@
         "    logging.info(\"Get the model.\")\n",
         "    run.log_params({\"role\": ROLE})\n",
         "    model_params = {\"learning_rate\": LR, \"beta_1\": B_1, \"beta_2\": B_2, \"epsilon\": EPS}\n",
-        "    model = get_model(encoder=encoder, model_params=model_params, role=ROLE)\n",
+        "    model = get_baseline_model(encoder=encoder, model_params=model_params)\n",
         "    run.log_params(model_params)\n",
         "\n",
         "    # Train the model\n",
@@ -907,7 +961,7 @@
         "# Get experiment\n",
         "logging.info(\"Get experiment status.\")\n",
         "experiment_df = vertex_ai.get_experiment_df()\n",
-        "experiment_df"
+        "experiment_df.T"
       ]
     },
     {
@@ -953,17 +1007,17 @@
       "outputs": [],
       "source": [
         "# Experiment Settings ----------------------------------------------------------\n",
-        "ID_2 = \"run-2\"\n",
+        "RUN_ID_2 = \"run-2\"\n",
         "ROLE = \"stacked\"\n",
         "\n",
         "# Initialize the experiment\n",
         "logger.info(\"Initialize the experiment.\")\n",
-        "vertex_ai.start_run(ID_2)\n",
+        "vertex_ai.start_run(RUN_ID_2)\n",
         "\n",
         "# Get the model\n",
         "logging.info(\"Get the model.\")\n",
         "run.log_params({\"role\": ROLE})\n",
-        "model = get_model(encoder=encoder, model_params=model_params, role=ROLE)\n",
+        "model = get_stacked_model(encoder=encoder, model_params=model_params)\n",
         "vertex_ai.log_params(model_params)\n",
         "\n",
         "# Train the model\n",
@@ -1014,7 +1068,7 @@
         "# Get experiment\n",
         "logging.info(\"Get experiment status.\")\n",
         "experiment_df = vertex_ai.get_experiment_df()\n",
-        "experiment_df"
+        "experiment_df.T"
       ]
     },
     {
@@ -1026,10 +1080,10 @@
       "outputs": [],
       "source": [
         "# Get time series metrics\n",
-        "exp_run = vertex_ai.ExperimentRun(ID_2, experiment=EXPERIMENT_NAME)\n",
+        "exp_run = vertex_ai.ExperimentRun(RUN_ID_2, experiment=EXPERIMENT_NAME)\n",
         "logging.info(\"Get time series metrics.\")\n",
         "ts_runs_df = exp_run.get_time_series_data_frame()\n",
-        "print(ts_runs_df)"
+        "ts_runs_df"
       ]
     },
     {
@@ -1056,34 +1110,29 @@
       "source": [
         "# Delete experiment\n",
         "exp = vertex_ai.Experiment(EXPERIMENT_NAME)\n",
-        "exp.delete(delete_backing_tensorboard_runs=True)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "dde8937123d4"
-      },
-      "outputs": [],
-      "source": [
+        "exp.delete(delete_backing_tensorboard_runs=True)\n",
+        "\n",
         "# Delete Tensorboard\n",
-        "vertex_ai_tb.delete()"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "sx_vKniMq9ZX"
-      },
-      "outputs": [],
-      "source": [
+        "vertex_ai_tb.delete()\n",
+        "\n",
         "# Delete Cloud Storage objects that were created\n",
-        "delete_bucket = False\n",
+        "delete_bucket = True\n",
         "\n",
         "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
-        "    ! gsutil rm -rf {BUCKET_URI}"
+        "    ! gsutil rm -rf {BUCKET_URI}\n",
+        "\n",
+        "!rm -Rf $DATA_DIR $LOG_DIR"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "15fbfe47e022"
+      },
+      "outputs": [],
+      "source": [
+        "!rm -Rf $DATA_DIR $LOG_DIR"
       ]
     }
   ],

notebooks/official/experiments/comparing_pipeline_runs.ipynb

@@ -55,7 +55,7 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/experiments/comparing_pipeline_runs.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/experiments/comparing_pipeline_runs.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
@@ -70,6 +70,8 @@
         "id": "tvgnzT1CKxrO"
       },
       "source": [
+        "# Compare pipeline runs with Vertex AI Experiments\n",
+        "\n",
         "## Overview\n",
         "\n",
         "Depending on the model life cycle of your data science team, you would like to experiment and track training Pipeline runs and its associated parameters. Then, you would to compare runs of these Pipelines to each others in order to figure out which is the best configuration generates the model you will register in the Vertex AI Model Registry."
@@ -207,10 +209,8 @@
         "if IS_WORKBENCH_NOTEBOOK:\n",
         "    USER_FLAG = \"--user\"\n",
         "\n",
-        "! pip install {USER_FLAG} --upgrade google-cloud-aiplatform \\\n",
-        "                                    google-cloud-storage \\\n",
-        "                                    google-auth \\\n",
-        "                                    kfp -q"
+        "!pip3 install {USER_FLAG} --force-reinstall 'google-cloud-aiplatform>=1.15' -q --no-warn-conflicts\n",
+        "!pip3 install {USER_FLAG} kfp -q --no-warn-conflicts"
       ]
     },
     {
@@ -270,7 +270,7 @@
         "\n",
         "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
         "\n",
-        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com).\n",
+        "1. [Enable APIs](https://console.cloud.google.com/flows/enableapi?apiid=cloudresourcemanager.googleapis.com,aiplatform.googleapis.com).\n",
         "\n",
         "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
         "\n",
@@ -451,9 +451,14 @@
         "3. In the **Service account name** field, enter a name, and\n",
         "   click **Create**.\n",
         "\n",
-        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
-        "into the filter box, and select\n",
-        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type and select\n",
+        "the following role into the filter box:\n",
+        "\n",
+        "    *   Storage Admin\n",
+        "    *   Storage Object Admin\n",
+        "    *   Service Account User\n",
+        "    *   Vertex AI Administrator\n",
+        "\n",
         "\n",
         "5. Click *Create*. A JSON file that contains your key downloads to your\n",
         "local environment.\n",
@@ -720,6 +725,7 @@
         "import kfp.v2.dsl as dsl\n",
         "# Vertex AI\n",
         "from google.cloud import aiplatform as vertex_ai\n",
+        "from google.cloud.aiplatform_v1.types.pipeline_state import PipelineState\n",
         "from kfp.v2.dsl import Metrics, Model, Output, component"
       ]
     },
@@ -738,6 +744,7 @@
         "EXPERIMENT_NAME = f\"{PROJECT_ID}-{TASK}-{MODEL_TYPE}-{UUID}\"\n",
         "\n",
         "# Pipeline\n",
+        "PIPELINE_TEMPLATE_FILE = \"pipeline.json\"\n",
         "PIPELINE_URI = f\"{BUCKET_URI}/pipelines\"\n",
         "TRAIN_URI = f\"{BUCKET_URI}/iris/iris_data.csv\"\n",
         "LABEL_URI = f\"{BUCKET_URI}/iris/iris_target.csv\"\n",
@@ -819,9 +826,7 @@
       "source": [
         "Before you start running your pipeline experiments, you have to formalize your training as pipeline component.\n",
         "\n",
-        "To do that, you will use the `kfp.v2.dsl.component` decorator to convert your training task into a pipeline component.\n",
-        "\n",
-        "Training code will import required libraries to train,evaluate and save a model with mentioned features. "
+        "To do that, you build the pipeline by using the `kfp.v2.dsl.component` decorator to convert your training task into a pipeline component. "
       ]
     },
     {
@@ -1075,7 +1080,7 @@
         "\n",
         "    job = vertex_ai.PipelineJob(\n",
         "        display_name=f\"{EXPERIMENT_NAME}-pipeline-run-{i}\",\n",
-        "        template_path=\"pipeline.json\",\n",
+        "        template_path=PIPELINE_TEMPLATE_FILE,\n",
         "        pipeline_root=PIPELINE_URI,\n",
         "        parameter_values={\n",
         "            \"train_uri\": TRAIN_URI,\n",
@@ -1160,10 +1165,9 @@
       "source": [
         "# Get the PipelineJob resource using the experiment run name\n",
         "pipeline_experiments_df = vertex_ai.get_experiment_df(EXPERIMENT_NAME)\n",
-        "for i in range(5):\n",
-        "    job = vertex_ai.PipelineJob.get(pipeline_experiments_df.run_name[i])\n",
-        "    print(job.resource_name)\n",
-        "    print(job._dashboard_uri())"
+        "job = vertex_ai.PipelineJob.get(pipeline_experiments_df.run_name[0])\n",
+        "print(\"Pipeline job name: \", job.resource_name)\n",
+        "print(\"Pipeline Run UI link: \", job._dashboard_uri())"
       ]
     },
     {
@@ -1191,13 +1195,16 @@
       "outputs": [],
       "source": [
         "# Delete the pipeline\n",
-        "# Get the PipelineJob resource using the experiment run name\n",
-        "pipeline_experiments_df = vertex_ai.get_experiment_df(EXPERIMENT_NAME)\n",
-        "for i in range(5):\n",
-        "    job = vertex_ai.PipelineJob.get(pipeline_experiments_df.run_name[i])\n",
-        "    print(job.resource_name)\n",
-        "    print(job._dashboard_uri())\n",
-        "    job.delete()\n",
+        "while True:\n",
+        "    for i in range(0, len(runs)):\n",
+        "        pipeline_job = vertex_ai.PipelineJob.get(pipeline_experiments_df.run_name[i])\n",
+        "        if pipeline_job.state != PipelineState.PIPELINE_STATE_SUCCEEDED:\n",
+        "            print(\"Pipeline job is still running...\")\n",
+        "            time.sleep(60)\n",
+        "        else:\n",
+        "            print(\"Pipeline job is complete.\")\n",
+        "            pipeline_job.delete()\n",
+        "    break\n",
         "\n",
         "# Delete experiment\n",
         "exp = vertex_ai.Experiment(EXPERIMENT_NAME)\n",
@@ -1206,7 +1213,11 @@
         "# Delete bucket\n",
         "delete_bucket = False\n",
         "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
-        "    ! gsutil rm -rf {BUCKET_URI}"
+        "    ! gsutil rm -rf {BUCKET_URI}\n",
+        "\n",
+        "# Remove local files\n",
+        "\n",
+        "!rm {PIPELINE_TEMPLATE_FILE}"
       ]
     }
   ],

notebooks/official/explainable_ai/sdk_automl_tabular_binary_classification_batch_explain.ipynb

@@ -29,7 +29,7 @@
         "id": "title"
       },
       "source": [
-        "# Vertex SDK: AutoML training tabular binary classification model for batch explanation\n",
+        "# Vertex AI SDK: AutoML training tabular binary classification model for batch explanation\n",
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
@@ -65,17 +65,6 @@
         "This tutorial demonstrates how to use the Vertex AI SDK to create tabular binary classification models and do batch prediction with explanation using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "dataset:bank,lbn"
-      },
-      "source": [
-        "### Dataset\n",
-        "\n",
-        "The dataset used for this tutorial is the [Bank Marketing](https://pantheon.corp.google.com/storage/browser/_details/cloud-ml-tables-data/bank-marketing.csv) . This dataset does not require any feature engineering. The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -108,6 +97,17 @@
         "* Batch Prediction Service: Does a queued (batch) prediction for the entire set of instances in the background and stores the results in a Cloud Storage bucket when ready."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "7a4881cf39a4"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the Bank Marketing. This dataset does not require any feature engineering. The version of the dataset you use in this tutorial is stored in a public Cloud Storage bucket."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -136,7 +136,7 @@
       "source": [
         "### Set up your local development environment\n",
         "\n",
-        "If you are using Colab or Vertex Workbench Notebook, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
+        "If you are using Colab or Vertex AI Workbench Notebook, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
         "\n",
         "Otherwise, make sure your environment meets this notebook's requirements. You need the following:\n",
         "\n",
@@ -322,7 +322,10 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -331,9 +334,8 @@
         "id": "timestamp"
       },
       "source": [
-        "#### Timestamp\n",
-        "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
+        "#### UUID\n",
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append the uuid onto the name of resources you create in this tutorial."
       ]
     },
     {
@@ -344,9 +346,16 @@
       },
       "outputs": [],
       "source": [
-        "from datetime import datetime\n",
+        "import random\n",
+        "import string\n",
         "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+        "\n",
+        "# Generate a uuid of length 8\n",
+        "def generate_uuid():\n",
+        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=8))\n",
+        "\n",
+        "\n",
+        "UUID = generate_uuid()"
       ]
     },
     {
@@ -357,7 +366,7 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Workbench AI Notebooks**, your environment is already\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
         "authenticated. Skip this step.\n",
         "\n",
         "**If you are using Colab**, run the cell below and follow the instructions\n",
@@ -440,7 +449,7 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
         "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
@@ -453,7 +462,7 @@
       "outputs": [],
       "source": [
         "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
         "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
       ]
     },
@@ -583,7 +592,7 @@
       "source": [
         "#### Quick peek at your data\n",
         "\n",
-        "You will use a version of the Bank Marketing dataset that is stored in a public Cloud Storage bucket, using a CSV index file.\n",
+        "You use a version of the Bank Marketing dataset that is stored in a public Cloud Storage bucket, using a CSV index file.\n",
         "\n",
         "Start by doing a quick peek at the data. You count the number of examples by counting the number of rows in the CSV index file  (`wc -l`) and then peek at the first few rows.\n",
         "\n",
@@ -637,7 +646,7 @@
       "outputs": [],
       "source": [
         "dataset = aip.TabularDataset.create(\n",
-        "    display_name=\"Bank Marketing\" + \"_\" + TIMESTAMP, gcs_source=[IMPORT_FILE]\n",
+        "    display_name=\"Bank Marketing\" + \"_\" + UUID, gcs_source=[IMPORT_FILE]\n",
         ")\n",
         "\n",
         "print(dataset.resource_name)"
@@ -688,7 +697,7 @@
       "outputs": [],
       "source": [
         "dag = aip.AutoMLTabularTrainingJob(\n",
-        "    display_name=\"bank_\" + TIMESTAMP,\n",
+        "    display_name=\"bank_\" + UUID,\n",
         "    optimization_prediction_type=\"classification\",\n",
         "    optimization_objective=\"minimize-log-loss\",\n",
         ")\n",
@@ -730,7 +739,7 @@
       "source": [
         "model = dag.run(\n",
         "    dataset=dataset,\n",
-        "    model_display_name=\"bank_\" + TIMESTAMP,\n",
+        "    model_display_name=\"bank_\" + UUID,\n",
         "    training_fraction_split=0.6,\n",
         "    validation_fraction_split=0.2,\n",
         "    test_fraction_split=0.2,\n",
@@ -784,7 +793,7 @@
       "source": [
         "### Make test items\n",
         "\n",
-        "You will use synthetic data as a test data items. Don't be concerned that we are using synthetic data -- we just want to demonstrate how to make a prediction."
+        "You use synthetic data as a test data items. Don't be concerned that we are using synthetic data."
       ]
     },
     {
@@ -852,11 +861,11 @@
       "outputs": [],
       "source": [
         "batch_predict_job = model.batch_predict(\n",
-        "    job_display_name=\"bank_\" + TIMESTAMP,\n",
+        "    job_display_name=\"bank_\" + UUID,\n",
         "    gcs_source=gcs_input_uri,\n",
         "    gcs_destination_prefix=BUCKET_URI,\n",
         "    instances_format=\"csv\",\n",
-        "    predictions_format=\"csv\",\n",
+        "    predictions_format=\"jsonl\",\n",
         "    generate_explanation=True,\n",
         "    sync=False,\n",
         ")\n",
@@ -952,6 +961,7 @@
         "# Set this to true only if you'd like to delete your bucket\n",
         "delete_bucket = False\n",
         "\n",
+        "dataset.delete()\n",
         "model.delete()\n",
         "batch_predict_job.delete()\n",
         "\n",

notebooks/official/feature_store/sdk-feature-store.ipynb

@@ -29,6 +29,8 @@
         "id": "JAPoU8Sm5E6e"
       },
       "source": [
+        "# Online and Batch predictions using Vertex AI Feature Store\n",
+        "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
         "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/feature_store/sdk-feature-store.ipynb\">\n",
@@ -51,19 +53,22 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "tvgnzT1CKxrO"
+        "id": "c4aaea3bab5e"
       },
       "source": [
         "## Overview\n",
         "\n",
         "This notebook introduces Vertex AI Feature Store, a managed cloud service for machine learning engineers and data scientists to store, serve, manage and share machine learning features at a large scale.\n",
         "\n",
-        "This notebook assumes that you understand basic Google Cloud concepts such as [Project](https://cloud.google.com/storage/docs/projects), [Storage](https://cloud.google.com/storage) and [Vertex AI](https://cloud.google.com/vertex-ai/docs). Some machine learning knowledge is also helpful but not required.\n",
-        "\n",
-        "### Dataset\n",
-        "\n",
-        "This notebook uses a movie recommendation dataset as an example throughout all the sessions. The task is to train a model to predict if a user is going to watch a movie and serve this model online. \n",
-        "\n",
+        "This notebook assumes that you understand basic Google Cloud concepts such as [Project](https://cloud.google.com/storage/docs/projects), [Storage](https://cloud.google.com/storage) and [Vertex AI](https://cloud.google.com/vertex-ai/docs). Some machine learning knowledge is also helpful but not required.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "71779c8088bf"
+      },
+      "source": [
         "### Objective\n",
         "\n",
         "In this notebook, you will learn how to use `Vertex AI Feature Store` to import feature data, and to access the feature data for both online serving and offline tasks, such as training.\n",
@@ -79,8 +84,26 @@
         "- Create featurestore, entity type, and feature resources.\n",
         "- Import feature data into `Vertex AI Feature Store` resource.\n",
         "- Serve online prediction requests using the imported features.\n",
-        "- Access imported features in offline jobs, such as training jobs.\n",
+        "- Access imported features in offline jobs, such as training jobs."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "55e01a856f57"
+      },
+      "source": [
+        "### Dataset\n",
         "\n",
+        "This notebook uses a movie recommendation dataset as an example throughout all the sessions. The task is to train a model to predict if a user is going to watch a movie and serve this model online."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tvgnzT1CKxrO"
+      },
+      "source": [
         "### Costs \n",
         "\n",
         "This tutorial uses billable components of Google Cloud:\n",
@@ -262,7 +285,15 @@
       },
       "outputs": [],
       "source": [
-        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
+        "import os\n",
+        "\n",
+        "PROJECT_ID = \"\"\n",
+        "\n",
+        "# Get your Google Cloud project ID from gcloud\n",
+        "if not os.getenv(\"IS_TESTING\"):\n",
+        "    shell_output = !gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "    PROJECT_ID = shell_output[0]\n",
+        "    print(\"Project ID: \", PROJECT_ID)"
       ]
     },
     {
@@ -275,10 +306,7 @@
       "source": [
         "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
         "    PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}\n",
-        "    # Get your GCP project id from gcloud\n",
-        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
-        "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID:\", PROJECT_ID)"
+        "print(\"Project ID: \", PROJECT_ID)"
       ]
     },
     {
@@ -320,7 +348,9 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+        "REGION = \"[your-region]\"  # @param {type:\"string\"}\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -329,9 +359,9 @@
         "id": "timestamp"
       },
       "source": [
-        "#### Timestamp\n",
+        "#### UUID\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
       ]
     },
     {
@@ -342,9 +372,16 @@
       },
       "outputs": [],
       "source": [
-        "from datetime import datetime\n",
+        "import random\n",
+        "import string\n",
         "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+        "\n",
+        "# Generate a uuid of a specifed length(default=8)\n",
+        "def generate_uuid(length: int = 8) -> str:\n",
+        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
+        "\n",
+        "\n",
+        "UUID = generate_uuid()"
       ]
     },
     {
@@ -441,7 +478,7 @@
       "source": [
         "from google.cloud.aiplatform import Feature, Featurestore\n",
         "\n",
-        "FEATURESTORE_ID = \"movie_prediction\"\n",
+        "FEATURESTORE_ID = \"movie_prediction\" + UUID\n",
         "INPUT_CSV_FILE = \"gs://cloud-samples-data-us-central1/vertex-ai/feature-store/datasets/movie_prediction.csv\"\n",
         "ONLINE_STORE_FIXED_NODE_COUNT = 1"
       ]

notebooks/official/matching_engine/intro-swivel.ipynb

@@ -214,7 +214,7 @@
         "if not os.getenv(\"IS_TESTING\"):\n",
         "    # Automatically restart kernel after installs\n",
         "    import IPython\n",
-        "    \n",
+        "\n",
         "    app = IPython.Application.instance()\n",
         "    app.kernel.do_shutdown(True)"
       ]
@@ -675,6 +675,18 @@
         "!./swivel_template_configuration.sh -pipeline_suffix {YOUR_PIPELINE_SUFFIX} -project_number {PROJECT_NUMBER} -project_id {PROJECT_ID} -machine_type {MACHINE_TYPE} -accelerator_count {ACCELERATOR_COUNT} -accelerator_type {ACCELERATOR_TYPE} -pipeline_root {BUCKET}"
       ]
     },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "1cacea95d68c"
+      },
+      "outputs": [],
+      "source": [
+        "! sed \"s:\\t:    :g\" swivel_pipeline_basic.json >tmp.json\n",
+        "! mv tmp.json swivel_pipeline_basic.json"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -765,19 +777,15 @@
         "# Instantiate PipelineJob object\n",
         "pl = aiplatform.PipelineJob(\n",
         "    display_name=YOUR_PIPELINE_SUFFIX,\n",
-        "\n",
         "    # Whether or not to enable caching\n",
         "    # True = always cache pipeline step result\n",
         "    # False = never cache pipeline step result\n",
         "    # None = defer to cache option for each pipeline component in the pipeline definition\n",
         "    enable_caching=False,\n",
-        "\n",
         "    # Local or GCS path to a compiled pipeline definition\n",
         "    template_path=\"swivel_pipeline_basic.json\",\n",
-        "\n",
         "    # Dictionary containing input parameters for your pipeline\n",
         "    parameter_values=PARAMETER_VALUES,\n",
-        "\n",
         "    # GCS path to act as the pipeline root\n",
         "    pipeline_root=PIPELINE_ROOT,\n",
         ")\n",

notebooks/official/migration/UJ14 Vertex SDK AutoML Video Classification.ipynb

@@ -32,17 +32,24 @@
         "# Vertex AI: Vertex AI Migration: AutoML Video Classificaton\n",
         "\n",
         "<table align=\"left\">\n",
+        "\n",
         "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/ai-platform-samples/blob/master/vertex-ai-samples/tree/master/notebooks/official/migration/UJ14%20Vertex%20SDK%20AutoML%20Video%20Classification.ipynb\">\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ14 Vertex SDK AutoML Video Classification.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/ai-platform-samples/blob/master/vertex-ai-samples/tree/master/notebooks/official/migration/UJ14%20Vertex%20SDK%20AutoML%20Video%20Classification.ipynb\">\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ14 Vertex SDK AutoML Video Classification.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ14 Vertex SDK AutoML Video Classification.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "  </td>                                                                                               \n",
         "</table>\n",
         "<br/><br/><br/>"
       ]
@@ -150,17 +157,6 @@
         "Install the latest GA version of *google-cloud-storage* library as well."
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "install_storage"
-      },
-      "outputs": [],
-      "source": [
-        "! pip3 install -U google-cloud-storage $USER_FLAG"
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -170,7 +166,7 @@
       "outputs": [],
       "source": [
         "if os.getenv(\"IS_TESTING\"):\n",
-        "    ! pip3 install --upgrade tensorflow $USER_FLAG"
+        "    ! pip3 install -U google-cloud-storage --upgrade tensorflow $USER_FLAG"
       ]
     },
     {
@@ -297,7 +293,10 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -306,9 +305,9 @@
         "id": "timestamp"
       },
       "source": [
-        "#### Timestamp\n",
+        "#### UUID\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
       ]
     },
     {
@@ -319,9 +318,16 @@
       },
       "outputs": [],
       "source": [
-        "from datetime import datetime\n",
+        "import random\n",
+        "import string\n",
         "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+        "\n",
+        "# Generate a uuid of a specifed length(default=8)\n",
+        "def generate_uuid(length: int = 8) -> str:\n",
+        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
+        "\n",
+        "\n",
+        "UUID = generate_uuid()"
       ]
     },
     {
@@ -332,7 +338,7 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Google Cloud Notebooks**, your environment is already authenticated. Skip this step.\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
         "\n",
         "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
         "\n",
@@ -404,7 +410,8 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
+        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
     {
@@ -415,8 +422,9 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
+        "    BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
     {
@@ -436,7 +444,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_NAME"
+        "! gsutil mb -l $REGION $BUCKET_URI"
       ]
     },
     {
@@ -456,7 +464,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_NAME"
+        "! gsutil ls -al $BUCKET_URI"
       ]
     },
     {
@@ -501,7 +509,7 @@
       },
       "outputs": [],
       "source": [
-        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_NAME)"
+        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
       ]
     },
     {
@@ -602,7 +610,7 @@
       "outputs": [],
       "source": [
         "dataset = aip.VideoDataset.create(\n",
-        "    display_name=\"MIT Human Motion\" + \"_\" + TIMESTAMP,\n",
+        "    display_name=\"MIT Human Motion\" + \"_\" + UUID,\n",
         "    gcs_source=[IMPORT_FILE],\n",
         "    import_schema_uri=aip.schema.dataset.ioformat.video.classification,\n",
         ")\n",
@@ -677,7 +685,7 @@
       "outputs": [],
       "source": [
         "dag = aip.AutoMLVideoTrainingJob(\n",
-        "    display_name=\"hmdb_\" + TIMESTAMP,\n",
+        "    display_name=\"hmdb_\" + UUID,\n",
         "    prediction_type=\"classification\",\n",
         ")\n",
         "\n",
@@ -725,7 +733,7 @@
       "source": [
         "model = dag.run(\n",
         "    dataset=dataset,\n",
-        "    model_display_name=\"hmdb_\" + TIMESTAMP,\n",
+        "    model_display_name=\"hmdb_\" + UUID,\n",
         "    training_fraction_split=0.8,\n",
         "    test_fraction_split=0.2,\n",
         ")"
@@ -800,7 +808,7 @@
       "outputs": [],
       "source": [
         "# Get model resource ID\n",
-        "models = aip.Model.list(filter=\"display_name=hmdb_\" + TIMESTAMP)\n",
+        "models = aip.Model.list(filter=\"display_name=hmdb_\" + UUID)\n",
         "\n",
         "# Get a reference to the Model Service client\n",
         "client_options = {\"api_endpoint\": f\"{REGION}-aiplatform.googleapis.com\"}\n",
@@ -932,7 +940,7 @@
         "\n",
         "import tensorflow as tf\n",
         "\n",
-        "gcs_input_uri = BUCKET_NAME + \"/test.jsonl\"\n",
+        "gcs_input_uri = BUCKET_URI + \"/test.jsonl\"\n",
         "with tf.io.gfile.GFile(gcs_input_uri, \"w\") as f:\n",
         "    data = {\n",
         "        \"content\": test_item_1,\n",
@@ -978,9 +986,9 @@
       "outputs": [],
       "source": [
         "batch_predict_job = model.batch_predict(\n",
-        "    job_display_name=\"hmdb_\" + TIMESTAMP,\n",
+        "    job_display_name=\"hmdb_\" + UUID,\n",
         "    gcs_source=gcs_input_uri,\n",
-        "    gcs_destination_prefix=BUCKET_NAME,\n",
+        "    gcs_destination_prefix=BUCKET_URI,\n",
         "    sync=False,\n",
         ")\n",
         "\n",
@@ -1208,8 +1216,8 @@
         "    except Exception as e:\n",
         "        print(e)\n",
         "\n",
-        "    if \"BUCKET_NAME\" in globals():\n",
-        "        ! gsutil rm -r $BUCKET_NAME"
+        "    if \"BUCKET_URI\" in globals():\n",
+        "        ! gsutil rm -r $BUCKET_URI"
       ]
     }
   ],

notebooks/official/migration/UJ4 Vertex SDK AutoML Tabular Binary Classification.ipynb

@@ -55,7 +55,7 @@
       "source": [
         "### Dataset\n",
         "\n",
-        "The dataset used for this tutorial is the [Bank Marketing](https://pantheon.corp.google.com/storage/browser/_details/cloud-ml-tables-data/bank-marketing.csv) . This dataset does not require any feature engineering. The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket."
+        "The dataset used for this tutorial is the Bank Marketing. This dataset does not require any feature engineering. The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket."
       ]
     },
     {

notebooks/official/migration/UJ5 Vertex SDK AutoML Image Object Detection.ipynb

@@ -32,18 +32,26 @@
         "# Vertex AI: Vertex AI Migration: AutoML Image Object Detection\n",
         "\n",
         "<table align=\"left\">\n",
+        "\n",
         "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/ai-platform-samples/blob/master/vertex-ai-samples/tree/master/notebooks/official/migration/UJ5%20Vertex%20SDK%20AutoML%20Image%20Object%20Detection.ipynb\">\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ5 Vertex SDK AutoML Image Object Detection.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/ai-platform-samples/blob/master/vertex-ai-samples/tree/master/notebooks/official/migration/UJ5%20Vertex%20SDK%20AutoML%20Image%20Object%20Detection.ipynb\">\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ5 Vertex SDK AutoML Image Object Detection.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ5 Vertex SDK AutoML Image Object Detection.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "  </td>                                                                                               \n",
         "</table>\n",
+        "\n",
         "<br/><br/><br/>"
       ]
     },
@@ -119,7 +127,7 @@
       "source": [
         "## Installation\n",
         "\n",
-        "Install the latest version of Vertex SDK for Python."
+        "Install the latest version of Vertex AI SDK for Python."
       ]
     },
     {
@@ -138,7 +146,7 @@
         "else:\n",
         "    USER_FLAG = \"\"\n",
         "\n",
-        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG"
+        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG -q"
       ]
     },
     {
@@ -150,17 +158,6 @@
         "Install the latest GA version of *google-cloud-storage* library as well."
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "install_storage"
-      },
-      "outputs": [],
-      "source": [
-        "! pip3 install -U google-cloud-storage $USER_FLAG"
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -169,8 +166,9 @@
       },
       "outputs": [],
       "source": [
-        "if os.getenv(\"IS_TESTING\"):\n",
-        "    ! pip3 install --upgrade tensorflow $USER_FLAG"
+        "! pip3 install -U google-cloud-storage $USER_FLAG -q\n",
+        "\n",
+        "! pip3 install --upgrade tensorflow $USER_FLAG -q"
       ]
     },
     {
@@ -297,7 +295,10 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -306,9 +307,9 @@
         "id": "timestamp"
       },
       "source": [
-        "#### Timestamp\n",
+        "#### UUID\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
       ]
     },
     {
@@ -319,9 +320,16 @@
       },
       "outputs": [],
       "source": [
-        "from datetime import datetime\n",
+        "import random\n",
+        "import string\n",
         "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+        "\n",
+        "# Generate a uuid of a specifed length(default=8)\n",
+        "def generate_uuid(length: int = 8) -> str:\n",
+        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
+        "\n",
+        "\n",
+        "UUID = generate_uuid()"
       ]
     },
     {
@@ -332,7 +340,7 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Google Cloud Notebooks**, your environment is already authenticated. Skip this step.\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
         "\n",
         "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
         "\n",
@@ -367,8 +375,11 @@
         "import os\n",
         "import sys\n",
         "\n",
-        "# If on Google Cloud Notebook, then don't execute this code\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = \"google.colab\" in sys.modules\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
         "    if \"google.colab\" in sys.modules:\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
@@ -404,7 +415,8 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
+        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
     {
@@ -415,8 +427,9 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
+        "    BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
     {
@@ -436,7 +449,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_NAME"
+        "! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
       ]
     },
     {
@@ -456,7 +469,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_NAME"
+        "! gsutil ls -al $BUCKET_URI"
       ]
     },
     {
@@ -501,7 +514,7 @@
       },
       "outputs": [],
       "source": [
-        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_NAME)"
+        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
       ]
     },
     {
@@ -603,7 +616,7 @@
       "outputs": [],
       "source": [
         "dataset = aip.ImageDataset.create(\n",
-        "    display_name=\"Salads\" + \"_\" + TIMESTAMP,\n",
+        "    display_name=\"Salads\" + \"_\" + UUID,\n",
         "    gcs_source=[IMPORT_FILE],\n",
         "    import_schema_uri=aip.schema.dataset.ioformat.image.bounding_box,\n",
         ")\n",
@@ -688,7 +701,7 @@
       "outputs": [],
       "source": [
         "dag = aip.AutoMLImageTrainingJob(\n",
-        "    display_name=\"salads_\" + TIMESTAMP,\n",
+        "    display_name=\"salads_\" + UUID,\n",
         "    prediction_type=\"object_detection\",\n",
         "    multi_label=False,\n",
         "    model_type=\"CLOUD\",\n",
@@ -742,7 +755,7 @@
       "source": [
         "model = dag.run(\n",
         "    dataset=dataset,\n",
-        "    model_display_name=\"salads_\" + TIMESTAMP,\n",
+        "    model_display_name=\"salads_\" + UUID,\n",
         "    training_fraction_split=0.8,\n",
         "    validation_fraction_split=0.1,\n",
         "    test_fraction_split=0.1,\n",
@@ -815,7 +828,7 @@
       "outputs": [],
       "source": [
         "# Get model resource ID\n",
-        "models = aip.Model.list(filter=\"display_name=salads_\" + TIMESTAMP)\n",
+        "models = aip.Model.list(filter=\"display_name=salads_\" + UUID)\n",
         "\n",
         "# Get a reference to the Model Service client\n",
         "client_options = {\"api_endpoint\": f\"{REGION}-aiplatform.googleapis.com\"}\n",
@@ -945,11 +958,11 @@
         "file_1 = test_item_1.split(\"/\")[-1]\n",
         "file_2 = test_item_2.split(\"/\")[-1]\n",
         "\n",
-        "! gsutil cp $test_item_1 $BUCKET_NAME/$file_1\n",
-        "! gsutil cp $test_item_2 $BUCKET_NAME/$file_2\n",
+        "! gsutil cp $test_item_1 $BUCKET_URI/$file_1\n",
+        "! gsutil cp $test_item_2 $BUCKET_URI/$file_2\n",
         "\n",
-        "test_item_1 = BUCKET_NAME + \"/\" + file_1\n",
-        "test_item_2 = BUCKET_NAME + \"/\" + file_2"
+        "test_item_1 = BUCKET_URI + \"/\" + file_1\n",
+        "test_item_2 = BUCKET_URI + \"/\" + file_2"
       ]
     },
     {
@@ -982,7 +995,7 @@
         "\n",
         "import tensorflow as tf\n",
         "\n",
-        "gcs_input_uri = BUCKET_NAME + \"/test.jsonl\"\n",
+        "gcs_input_uri = BUCKET_URI + \"/test.jsonl\"\n",
         "with tf.io.gfile.GFile(gcs_input_uri, \"w\") as f:\n",
         "    data = {\"content\": test_item_1, \"mime_type\": \"image/jpeg\"}\n",
         "    f.write(json.dumps(data) + \"\\n\")\n",
@@ -1018,9 +1031,9 @@
       "outputs": [],
       "source": [
         "batch_predict_job = model.batch_predict(\n",
-        "    job_display_name=\"salads_\" + TIMESTAMP,\n",
+        "    job_display_name=\"salads_\" + UUID,\n",
         "    gcs_source=gcs_input_uri,\n",
-        "    gcs_destination_prefix=BUCKET_NAME,\n",
+        "    gcs_destination_prefix=BUCKET_URI,\n",
         "    sync=False,\n",
         ")\n",
         "\n",
@@ -1378,60 +1391,25 @@
       },
       "outputs": [],
       "source": [
-        "delete_all = True\n",
+        "# Delete the dataset using the Vertex dataset object\n",
         "\n",
-        "if delete_all:\n",
-        "    # Delete the dataset using the Vertex dataset object\n",
-        "    try:\n",
-        "        if \"dataset\" in globals():\n",
-        "            dataset.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
+        "dataset.delete()\n",
         "\n",
-        "    # Delete the model using the Vertex model object\n",
-        "    try:\n",
-        "        if \"model\" in globals():\n",
-        "            model.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
+        "# Delete the model using the Vertex model object\n",
+        "model.delete()\n",
         "\n",
-        "    # Delete the endpoint using the Vertex endpoint object\n",
-        "    try:\n",
-        "        if \"endpoint\" in globals():\n",
-        "            endpoint.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
+        "# Delete the endpoint using the Vertex endpoint object\n",
+        "endpoint.delete()\n",
         "\n",
-        "    # Delete the AutoML or Pipeline trainig job\n",
-        "    try:\n",
-        "        if \"dag\" in globals():\n",
-        "            dag.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
+        "# Delete the AutoML or Pipeline trainig job\n",
         "\n",
-        "    # Delete the custom trainig job\n",
-        "    try:\n",
-        "        if \"job\" in globals():\n",
-        "            job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
+        "dag.delete()\n",
         "\n",
-        "    # Delete the batch prediction job using the Vertex batch prediction object\n",
-        "    try:\n",
-        "        if \"batch_predict_job\" in globals():\n",
-        "            batch_predict_job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
+        "# Delete the batch prediction job using the Vertex batch prediction object\n",
+        "batch_predict_job.delete()\n",
         "\n",
-        "    # Delete the hyperparameter tuning job using the Vertex hyperparameter tuning object\n",
-        "    try:\n",
-        "        if \"hpt_job\" in globals():\n",
-        "            hpt_job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    if \"BUCKET_NAME\" in globals():\n",
-        "        ! gsutil rm -r $BUCKET_NAME"
+        "if \"BUCKET_NAME\" in globals():\n",
+        "    ! gsutil rm -r $BUCKET_URI"
       ]
     }
   ],

notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb

@@ -29,6 +29,8 @@
         "id": "JAPoU8Sm5E6e"
       },
       "source": [
+        "# Vertex AI: Track parameters and metrics for custom training jobs\n",
+        "\n",
         "<table align=\"left\">\n",
         "\n",
         "  <td>\n",
@@ -51,15 +53,6 @@
         "</table>"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "j9gUDU_3vV9d"
-      },
-      "source": [
-        "# Vertex AI: Track parameters and metrics for custom training jobs"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -68,11 +61,15 @@
       "source": [
         "## Overview\n",
         "\n",
-        "This notebook demonstrates how to track metrics and parameters for `Vertex AI` custom training jobs, and how to perform detailed analysis using this data.\n",
-        "\n",
-        "### Dataset\n",
-        "\n",
-        "This example uses the Abalone Dataset. For more information about this dataset please visit: https://archive.ics.uci.edu/ml/datasets/abalone\n",
+        "This notebook demonstrates how to track metrics and parameters for `Vertex AI` custom training jobs, and how to perform detailed analysis using this data."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "37147bd9c3c4"
+      },
+      "source": [
         "### Objective\n",
         "\n",
         "In this notebook, you learn how to use `Vertex ML Metadata` to track training parameters and evaluation metrics.\n",
@@ -85,8 +82,26 @@
         "The steps performed include:\n",
         "\n",
         "- Track parameters and metrics for a `Vertex AI` custom trained model.\n",
-        "- Extract and perform analysis for all parameters and metrics within an Experiment.\n",
+        "- Extract and perform analysis for all parameters and metrics within an Experiment."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "96cb18467417"
+      },
+      "source": [
+        "### Dataset\n",
         "\n",
+        "This example uses the Abalone Dataset. For more information about this dataset please visit: https://archive.ics.uci.edu/ml/datasets/abalone"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "c831245dc1d5"
+      },
+      "source": [
         "### Costs \n",
         "\n",
         "\n",
@@ -285,8 +300,7 @@
       },
       "outputs": [],
       "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
-        "    PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}\n",
+        "if PROJECT_ID == \"[your-project-id]\" or PROJECT_ID == \"\" or PROJECT_ID is None:\n",
         "    # Get your GCP project id from gcloud\n",
         "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
         "    PROJECT_ID = shell_output[0]\n",
@@ -332,7 +346,10 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -368,7 +385,7 @@
         "### Authenticate your Google Cloud account\n",
         "\n",
         "**If you are using Vertex AI Workbench**, your environment is already\n",
-        "authenticated. Skip this step."
+        "authenticated. "
       ]
     },
     {
@@ -664,9 +681,9 @@
         "id": "35QVNhACqcTJ"
       },
       "source": [
-        "### Create a managed tabular dataset from a CSV\n",
+        "### Create a Vertex AI Dataset from a CSV\n",
         "\n",
-        "A Managed dataset can be used to create an AutoML model or a custom model. "
+        "A Vertex AI Dataset can be used to create an AutoML model or a custom model. "
       ]
     },
     {
@@ -679,7 +696,7 @@
       "source": [
         "ds = aiplatform.TabularDataset.create(display_name=\"abalone\", gcs_source=[gcs_csv_path])\n",
         "\n",
-        "ds.resource_name"
+        "print(ds.resource_name)"
       ]
     },
     {
@@ -780,7 +797,11 @@
         "id": "k_QorXXztzPH"
       },
       "source": [
-        "Start a new experiment run to track training parameters and start the training job. Note that this operation will take around 10 mins."
+        "Start a new experiment run to track training parameters and start the training job. \n",
+        "\n",
+        "Prior to executing the training job, you call the `start_run()` method to initialize the start of the experiment, and then use the `log_params()` to log the parameters used in the experiment.\n",
+        "\n",
+        "*Note:* This operation will take around 10 mins."
       ]
     },
     {
@@ -818,7 +839,7 @@
         "id": "O-uCOL3Naap4"
       },
       "source": [
-        "Deploy model to Google Cloud. This operation will take 10-20 mins."
+        "Deploy model to Google Cloud. This operation may take a few minutes."
       ]
     },
     {
@@ -899,7 +920,7 @@
         "id": "_HphZ38obJeB"
       },
       "source": [
-        "Perform online prediction."
+        "### Perform online prediction"
       ]
     },
     {
@@ -911,7 +932,7 @@
       "outputs": [],
       "source": [
         "prediction = endpoint.predict(test_dataset.tolist())\n",
-        "prediction"
+        "print(prediction)"
       ]
     },
     {
@@ -920,7 +941,11 @@
         "id": "TDKiv_O7bNwE"
       },
       "source": [
-        "Calculate and track prediction evaluation metrics."
+        "### Calculate and track prediction evaluation metrics.\n",
+        "\n",
+        "Next, log the evaluation metrics for your experiment.\n",
+        "\n",
+        "Once the experiment is completed, you call the `end_run()` method to indicate the end of tracking for the experiment."
       ]
     },
     {
@@ -934,7 +959,9 @@
         "mse = mean_squared_error(test_labels, prediction.predictions)\n",
         "mae = mean_absolute_error(test_labels, prediction.predictions)\n",
         "\n",
-        "aiplatform.log_metrics({\"mse\": mse, \"mae\": mae})"
+        "aiplatform.log_metrics({\"mse\": mse, \"mae\": mae})\n",
+        "\n",
+        "aiplatform.end_run()"
       ]
     },
     {

notebooks/official/model-registry/bqml-vertexai-model-registry.ipynb

@@ -0,0 +1,921 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ur8xi4C7S06n"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2022 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "JAPoU8Sm5E6e"
+      },
+      "source": [
+        "<table align=\"left\">\n",
+        "\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model-registry/bqml-vertexai-model-registry.ipynb\" target=\"_blank\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model-registry/bqml-vertexai-model-registry.ipynb\" target=\"_blank\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model-registry/bqml-vertexai-model-registry.ipynb\" target=\"_blank\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "  </td>                                                                                               \n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tvgnzT1CKxrO"
+      },
+      "source": [
+        "# Deploy BiqQuery ML Model on Vertex AI Model Registry and Make Predictions\n",
+        "\n",
+        "## Overview\n",
+        "\n",
+        "This tutorial demonstrates how to train a model with BigQuery ML and upload it on Vertex AI model registry, then make batch predictions.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "132a9ee68ba6"
+      },
+      "source": [
+        "### Objective\n",
+        "\n",
+        "In this tutorial, you learn how to use `Vertex AI Model Registry` with `BigQuery ML` and make batch predictions:\n",
+        "\n",
+        "This tutorial uses the following Google Cloud ML services and resources:\n",
+        "\n",
+        "- `Vertex AI Model Registry`\n",
+        "- `Vertex AI Model` resources \n",
+        "- `Vertex AI Endpoint` resources\n",
+        "- `Vertex AI Prediction`\n",
+        "- `BigQuery ML`\n",
+        "\n",
+        "\n",
+        "The steps performed include:\n",
+        "\n",
+        "- Train a model with `BQML`\n",
+        "- Upload the model to `Vertex AI Model Registry` \n",
+        "- Create a `Vertex AI Endpoint` resource\n",
+        "- Deploy the `Model` resource to the `Endpoint` resource\n",
+        "- Make `prediction` requests to the model endpoint\n",
+        "- Run `batch prediction` job on the `Model` resource \n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2de0477b10ce"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the Penguins dataset from <a href=\"https://cloud.google.com/bigquery/public-data\" target=\"_blank\">BigQuery public datasets</a>. This version of the dataset is used to predict the species of penguins from the available features like culmen-length, flipper-depth etc."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "76330e07673b"
+      },
+      "source": [
+        "### Costs \n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* BigQuery ML\n",
+        "\n",
+        "Learn about <a href=\"https://cloud.google.com/vertex-ai/pricing\" target=\"_blank\">Vertex AI\n",
+        "pricing</a> and <a href=\"https://cloud.google.com/bigquery/pricing\" target=\"_blank\">BigQuery pricing</a>, and use the <a href=\"https://cloud.google.com/products/calculator/\" target=\"_blank\">Pricing\n",
+        "Calculator</a>\n",
+        "to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ze4-nDLfK4pw"
+      },
+      "source": [
+        "### Set up your local development environment\n",
+        "\n",
+        "**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets\n",
+        "all the requirements to run this notebook. You can skip this step."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gCuSR8GkAgzl"
+      },
+      "source": [
+        "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
+        "You need the following:\n",
+        "\n",
+        "* The Google Cloud SDK\n",
+        "* Git\n",
+        "* Python 3\n",
+        "* virtualenv\n",
+        "* Jupyter notebook running in a virtual environment with Python 3\n",
+        "\n",
+        "The Google Cloud guide to <a href=\"https://cloud.google.com/python/setup\" target=\"_blank\">Setting up a Python development\n",
+        "environment</a> and the <a href=\"https://jupyter.org/install\" target=\"_blank\">Jupyter\n",
+        "installation guide</a> provide detailed instructions\n",
+        "for meeting these requirements. The following steps provide a condensed set of\n",
+        "instructions:\n",
+        "\n",
+        "1. <a href=\"https://cloud.google.com/sdk/docs/\" target=\"_blank\">Install and initialize the Cloud SDK.</a>\n",
+        "\n",
+        "1. <a href=\"https://cloud.google.com/python/setup#installing_python\" target=\"_blank\">Install Python 3.</a>\n",
+        "\n",
+        "1. <a href=\"https://cloud.google.com/python/setup#installing_and_using_virtualenv\" target=\"_blank\">Install\n",
+        "   virtualenv</a>\n",
+        "   and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
+        "\n",
+        "1. To install Jupyter, run `pip3 install jupyter` on the\n",
+        "command-line in a terminal shell.\n",
+        "\n",
+        "1. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
+        "\n",
+        "1. Open this notebook in the Jupyter Notebook Dashboard."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "i7EUnXsZhAGF"
+      },
+      "source": [
+        "### Install additional packages\n",
+        "\n",
+        "Install the following packages required to execute this notebook. "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2b4ef9b72d43"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
+        "\n",
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "USER_FLAG = \"\"\n",
+        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\"\n",
+        "\n",
+        "! pip3 install --upgrade google-cloud-aiplatform {USER_FLAG} -q\n",
+        "! pip3 install --upgrade google-cloud-bigquery {USER_FLAG} -q"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "hhq5zEbGg0XX"
+      },
+      "source": [
+        "### Restart the kernel\n",
+        "\n",
+        "After you install the additional packages, you need to restart the notebook kernel so it can find the packages."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "EzrelQZ22IZj"
+      },
+      "outputs": [],
+      "source": [
+        "# Automatically restart kernel after installs\n",
+        "import os\n",
+        "\n",
+        "if not os.getenv(\"IS_TESTING\"):\n",
+        "    # Automatically restart kernel after installs\n",
+        "    import IPython\n",
+        "\n",
+        "    app = IPython.Application.instance()\n",
+        "    app.kernel.do_shutdown(True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "lWEdiXsJg0XY"
+      },
+      "source": [
+        "## Before you begin"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "BF1j6f9HApxa"
+      },
+      "source": [
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. <a href=\"https://console.cloud.google.com/cloud-resource-manager\" target=\"_blank\">Select or create a Google Cloud project</a>. When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. <a href=\"https://cloud.google.com/billing/docs/how-to/modify-project\" target=\"_blank\">Make sure that billing is enabled for your project</a>.\n",
+        "\n",
+        "1. <a href=\"https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,bigquery\" target=\"_blank\">Enable the Vertex AI and BigQuery APIs</a>. \n",
+        "\n",
+        "1. If you are running this notebook locally, you will need to install the <a href=\"https://cloud.google.com/sdk\" target=\"_blank\">Cloud SDK</a>.\n",
+        "\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WReHDGG5g0XY"
+      },
+      "source": [
+        "#### Set your project ID\n",
+        "\n",
+        "**If you don't know your project ID**, you can get your project ID using `gcloud`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "o1AuQDpf_hS-"
+      },
+      "outputs": [],
+      "source": [
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "RYbBU1jXAETD"
+      },
+      "outputs": [],
+      "source": [
+        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
+        "    # Get your GCP project id from gcloud\n",
+        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "    PROJECT_ID = shell_output[0]\n",
+        "    print(\"Project ID:\", PROJECT_ID)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "set_gcloud_project_id"
+      },
+      "outputs": [],
+      "source": [
+        "! gcloud config set project $PROJECT_ID"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "region"
+      },
+      "source": [
+        "#### Region\n",
+        "\n",
+        "You can also change the `REGION` variable, which is used for operations\n",
+        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
+        "\n",
+        "- Americas: `us-central1`\n",
+        "- Europe: `europe-west4`\n",
+        "- Asia Pacific: `asia-east1`\n",
+        "\n",
+        "You might not be able to use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
+        "\n",
+        "Learn more about <a href=\"https://cloud.google.com/vertex-ai/docs/general/locations\" target=\"_blank\">Vertex AI regions</a>."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "vO3W8YdN2LuA"
+      },
+      "outputs": [],
+      "source": [
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "29e912d1b106"
+      },
+      "source": [
+        "#### UUID\n",
+        "\n",
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 2,
+      "metadata": {
+        "id": "c704897922c0"
+      },
+      "outputs": [],
+      "source": [
+        "import random\n",
+        "import string\n",
+        "\n",
+        "\n",
+        "# Generate a uuid of a specifed length(default=8)\n",
+        "def generate_uuid(length: int = 8) -> str:\n",
+        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
+        "\n",
+        "\n",
+        "UUID = generate_uuid()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dr--iN2kAylZ"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
+        "authenticated. Skip this step."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "sBCra4QMA2wR"
+      },
+      "source": [
+        "**If you are using Colab**, run the cell below and follow the instructions\n",
+        "when prompted to authenticate your account via oAuth.\n",
+        "\n",
+        "**Otherwise**, follow these steps:\n",
+        "\n",
+        "1. In the Cloud Console, go to the <a href=\"https://console.cloud.google.com/apis/credentials/serviceaccountkey\" target=\"_blank\">Create service account key page</a>.\n",
+        "\n",
+        "2. Click **Create service account**.\n",
+        "\n",
+        "3. In the **Service account name** field, enter a name, and\n",
+        "   click **Create**.\n",
+        "\n",
+        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
+        "into the filter box, and select\n",
+        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "\n",
+        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
+        "local environment.\n",
+        "\n",
+        "6. Enter the path to your service account key as the\n",
+        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "PyQmSRbKA8r-"
+      },
+      "outputs": [],
+      "source": [
+        "# If you are running this notebook in Colab, run this cell and follow the\n",
+        "# instructions to authenticate your GCP account. This provides access to your\n",
+        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
+        "# requests.\n",
+        "\n",
+        "import os\n",
+        "import sys\n",
+        "\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = \"google.colab\" in sys.modules\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
+        "    if \"google.colab\" in sys.modules:\n",
+        "        from google.colab import auth as google_auth\n",
+        "\n",
+        "        google_auth.authenticate_user()\n",
+        "\n",
+        "    # If you are running this notebook locally, replace the string below with the\n",
+        "    # path to your service account key and run this cell to authenticate your GCP\n",
+        "    # account.\n",
+        "    elif not os.getenv(\"IS_TESTING\"):\n",
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "XoEqT2Y4DJmf"
+      },
+      "source": [
+        "### Import libraries"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "pRUOFELefqf1"
+      },
+      "outputs": [],
+      "source": [
+        "import google.cloud.aiplatform as aiplatform\n",
+        "from google.cloud import bigquery"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "init_aip:mbsdk,all"
+      },
+      "source": [
+        "### Initialize Vertex AI and BigQuery SDKs for Python\n",
+        "\n",
+        "Initialize the Vertex AI and Big Query SDKs for Python for your project and corresponding bucket."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "BgaYKz2-2LuC"
+      },
+      "outputs": [],
+      "source": [
+        "aiplatform.init(project=PROJECT_ID, location=REGION)\n",
+        "bqclient = bigquery.Client(project=PROJECT_ID)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "lUEtpimzL17Z"
+      },
+      "source": [
+        "## BigQuery ML introduction\n",
+        "\n",
+        "BigQuery ML (BQML) provides the capability to train ML tabular models, such as classification and regression, in BigQuery using SQL syntax.\n",
+        "\n",
+        "Learn more about <a href=\"https://cloud.google.com/bigquery-ml/docs\" target=\"_blank\">BigQuery ML documentation</a>."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "30adf1b74bf9"
+      },
+      "source": [
+        "### BigQuery table used for training"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3O7qlOGWNEU4"
+      },
+      "outputs": [],
+      "source": [
+        "# Define BigQuery table to be used for training\n",
+        "\n",
+        "BQ_TABLE = \"bigquery-public-data.ml_datasets.penguins\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "PKQD2e0eMg3M"
+      },
+      "source": [
+        "### Create BigQuery dataset resource"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "BnXOpvs2MmzF"
+      },
+      "source": [
+        "First, you create an empty dataset resource in your project."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "luqb-DBiMn-0"
+      },
+      "outputs": [],
+      "source": [
+        "BQ_DATASET_NAME = \"penguins\" + UUID\n",
+        "DATASET_QUERY = f\"\"\"CREATE SCHEMA {BQ_DATASET_NAME}\"\"\"\n",
+        "\n",
+        "job = bqclient.query(DATASET_QUERY)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "59bf85366baf"
+      },
+      "outputs": [],
+      "source": [
+        "job.result()\n",
+        "print(job.state)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "b-_8rZO8NIEb"
+      },
+      "source": [
+        "## Train BigQuery ML model and upload it to Vertex AI Model Registry\n",
+        "Next, you create and train a BQML tabular regression model from the public dataset penguins and store the model in your project `Vertex AI Model Registry` using the `CREATE MODEL` statement. The model configuration is specified in the `OPTIONS` statement as follows:\n",
+        "\n",
+        "- `model_type`: The type and archictecture of tabular model to train, e.g., LOGISTIC_REG.\n",
+        "\n",
+        "- `labels`: The column which are the labels.\n",
+        "\n",
+        "- `model_registry`: To register a BigQuery ML model to Vertex AI Model Registry, you must use `model_registry=\"vertex_ai\"`.\n",
+        "\n",
+        "Learn more about the <a href=\"https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-create\" target=\"_blank\">CREATE MODEL statement</a>.\n",
+        "\n",
+        "Learn more about <a href=\"https://cloud.google.com/bigquery-ml/docs/managing-models-vertex\" target=\"_blank\">Managing BigQuery ML models in the Vertex AI Model Registry</a>."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Q96rlZKRNPjU"
+      },
+      "outputs": [],
+      "source": [
+        "# Write the query to create Big Query ML model\n",
+        "\n",
+        "MODEL_NAME = \"penguins-lr\" + UUID\n",
+        "MODEL_QUERY = f\"\"\"\n",
+        "CREATE OR REPLACE MODEL `{BQ_DATASET_NAME}.{MODEL_NAME}`\n",
+        "OPTIONS(\n",
+        "    model_type='LOGISTIC_REG',\n",
+        "    labels = ['species'],\n",
+        "    model_registry='vertex_ai'\n",
+        "    )\n",
+        "AS\n",
+        "SELECT *\n",
+        "FROM `{BQ_TABLE}`\n",
+        "\"\"\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "eee158e2a375"
+      },
+      "source": [
+        "Create the BigQuery ML model using the query above and the BigQuery client that you created previously:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "LtdieY-BWILs"
+      },
+      "outputs": [],
+      "source": [
+        "# Run the model creation query using BigQuery client\n",
+        "\n",
+        "job = bqclient.query(MODEL_QUERY)\n",
+        "print(f\"Job state: {job.state}\\nJob Error:{job.errors}\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e4b007777e68"
+      },
+      "source": [
+        "Check the job status:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Y4C_3hTEXOE7"
+      },
+      "outputs": [],
+      "source": [
+        "job.result()\n",
+        "print(job.state)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5nHvVttrYfQ8"
+      },
+      "source": [
+        "### Find the model in the Vertex Model Registry\n",
+        "\n",
+        "You can use the `Vertex AI Model()` method with `model_name` parameter to find the automatically registered model."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "E08IwUX5YpAG"
+      },
+      "outputs": [],
+      "source": [
+        "model = aiplatform.Model(model_name=MODEL_NAME)\n",
+        "\n",
+        "print(model.gca_resource)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "zgvXXlaZaYw3"
+      },
+      "source": [
+        "## Deploy Vertex AI Model resource to a Vertex AI Endpoint resource\n",
+        "You must deploy a model to an `endpoint` before that model can be used to serve online predictions; deploying a model associates physical resources with the model so it can serve online predictions with low latency. \n",
+        "\n",
+        "Learn more about <a href=\"https://cloud.google.com/vertex-ai/docs/predictions/deploy-model-api#aiplatform_deploy_model_custom_trained_model_sample-python\" target=\"_blank\">Deploy a model using the Vertex AI API</a>\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "N27z-_by5gti"
+      },
+      "source": [
+        "### Create a Vertex AI Endpoint resource\n",
+        "\n",
+        "If you are deploying a model to an existing endpoint, you can skip this cell.\n",
+        "\n",
+        "- `display_name`: Display name for the endpoint.\n",
+        "- `project`: The project ID on which you are creating an endpoint.\n",
+        "- `location`: The region where you are using Vertex AI."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "XmmyCtW055Ya"
+      },
+      "outputs": [],
+      "source": [
+        "ENDPOINT_DISPLAY_NAME = \"bqml-lr-model-endpoint\" + UUID\n",
+        "\n",
+        "endpoint = aiplatform.Endpoint.create(\n",
+        "    display_name=ENDPOINT_DISPLAY_NAME,\n",
+        "    project=PROJECT_ID,\n",
+        "    location=REGION,\n",
+        ")\n",
+        "\n",
+        "print(endpoint.display_name)\n",
+        "print(endpoint.resource_name)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "xSCIKdFo56YO"
+      },
+      "source": [
+        "### Deploy the Vertex AI Model resource to Vertex AI Endpoint resource"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "f7KfDgALE4aD"
+      },
+      "outputs": [],
+      "source": [
+        "DEPLOYED_NAME = \"bqml-lr-penguins\"\n",
+        "\n",
+        "model.deploy(endpoint=endpoint, deployed_model_display_name=DEPLOYED_NAME)\n",
+        "\n",
+        "print(model.display_name)\n",
+        "print(model.resource_name)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "H3K6SVplJ9Mg"
+      },
+      "source": [
+        "## Send prediction request to the Vertex AI Endpoint resource\n",
+        "\n",
+        "Now that your Vertex AI Model resource is deployed to a Vertex AI `Endpoint` resource, you can do online predictions by sending prediction requests to the `Endpoint` resource.\n",
+        "\n",
+        "Learn more about <a href=\"https://cloud.google.com/vertex-ai/docs/predictions/online-predictions-custom-models\" target=\"_blank\">Get online predictions from custom-trained models</a>"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2j7ioB3VKEtx"
+      },
+      "outputs": [],
+      "source": [
+        "instance = {\n",
+        "    \"island\": \"Dream\",\n",
+        "    \"culmen_length_mm\": 36.6,\n",
+        "    \"culmen_depth_mm\": 18.4,\n",
+        "    \"flipper_length_mm\": 184.0,\n",
+        "    \"body_mass_g\": 3475.0,\n",
+        "    \"sex\": \"FEMALE\",\n",
+        "}\n",
+        "\n",
+        "prediction = endpoint.predict([instance])\n",
+        "print(prediction)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "C39qOaBHZI1G"
+      },
+      "source": [
+        "## Batch Prediction on the BQML model"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "UBffk3GyaPY3"
+      },
+      "source": [
+        "Here you request batch predictions directly from the BigQuery ML model; you don't need to deploy the model to an endpoint. For data types that support both batch and online predictions, use batch predictions when you don't require an immediate response and want to process accumulated data by using a single request.\n",
+        "\n",
+        "Learn more abount <a href=\"https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-predict\" target=\"_blank\">The ML.PREDICT function</a>"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "_QxZb19_o6jx"
+      },
+      "outputs": [],
+      "source": [
+        "sql_ml_predict = f\"\"\"SELECT * FROM ML.PREDICT(MODEL `{PROJECT_ID}.{BQ_DATASET_NAME}.{MODEL_NAME}`, \n",
+        "(SELECT\n",
+        "      *\n",
+        "    FROM\n",
+        "      `{BQ_TABLE}` LIMIT 10))\"\"\"\n",
+        "\n",
+        "job = bqclient.query(sql_ml_predict)\n",
+        "prediction_result = job.result().to_arrow().to_pandas()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "TVpsLI5nrVii"
+      },
+      "outputs": [],
+      "source": [
+        "display(prediction_result.head())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "TpV-iwP9qw9c"
+      },
+      "source": [
+        "## Cleaning up\n",
+        "\n",
+        "To clean up all Google Cloud resources used in this project, you can <a href=\"https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects\" target=\"_blank\">delete the Google Cloud\n",
+        "project</a> you used for the tutorial.\n",
+        "\n",
+        "Learn more about <a href=\"https://cloud.google.com/bigquery-ml/docs/managing-models-vertex\" target=\"_blank\">Deleting BigQuery ML models from Vertex AI Model Registry</a>\n",
+        "\n",
+        "Otherwise, you can delete the individual resources you created in this tutorial."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "sx_vKniMq9ZX"
+      },
+      "outputs": [],
+      "source": [
+        "# Delete the endpoint using the Vertex endpoint object\n",
+        "endpoint.undeploy_all()\n",
+        "endpoint.delete()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3c5c8dc2f597"
+      },
+      "outputs": [],
+      "source": [
+        "# Delete BigQuery ML model\n",
+        "\n",
+        "delete_query = f\"\"\"DROP MODEL `{PROJECT_ID}.{BQ_DATASET_NAME}.{MODEL_NAME}`\"\"\"\n",
+        "job = bqclient.query(delete_query)\n",
+        "job.result()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "85dfc88f5472"
+      },
+      "outputs": [],
+      "source": [
+        "# Delete the created BigQuery dataset\n",
+        "! bq rm -r -f $PROJECT_ID:$BQ_DATASET_NAME"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "bqml-vertexai-model-registry.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/official/model_monitoring/model_monitoring.ipynb

@@ -59,15 +59,8 @@
         "id": "lA32H1oKGgpf"
       },
       "source": [
-        "## Overview"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "t6Cd51FkG09E"
-      },
-      "source": [
+        "## Overview\n",
+        "\n",
         "### What is Vertex AI Model Monitoring?\n",
         "\n",
         "Modern applications rely on a well established set of capabilities to monitor the health of their services. Examples include:\n",
@@ -88,7 +81,15 @@
         "\n",
         "[Vertex Explainable AI](https://cloud.google.com/vertex-ai/docs/explainable-ai/overview) adds another facet to model monitoring, which we call feature attribution monitoring. Explainable AI enables you to understand the relative contribution of each feature to a resulting prediction. In essence, it assesses the magnitude of each feature's influence.\n",
         "\n",
-        "If production traffic differs from  training data, or varies substantially over time, **either in terms of model predictions or feature attributions**, that's likely to impact the quality of the answers your model produces. When that happens, you'd like to be alerted automatically and responsively, so that **you can anticipate problems before they affect your customer experiences or your revenue streams**.\n",
+        "If production traffic differs from  training data, or varies substantially over time, **either in terms of model predictions or feature attributions**, that's likely to impact the quality of the answers your model produces. When that happens, you'd like to be alerted automatically and responsively, so that **you can anticipate problems before they affect your customer experiences or your revenue streams**."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "t6Cd51FkG09E"
+      },
+      "source": [
         "### Objective\n",
         "\n",
         "In this notebook, you learn to use the `Vertex AI Model Monitoring` service to detect drift and anomalies in prediction requests from a deployed `Vertex AI Model` resource. \n",
@@ -107,18 +108,38 @@
         "- Deploy the `Model` resource to the `Endpoint` resource.\n",
         "- Configure the `Endpoint` resource for model monitoring.\n",
         "- Generate synthetic prediction requests.\n",
-        "- Understand how to interpret the statistics, visualizations, other data reported by the model monitoring feature.\n",
+        "- Understand how to interpret the statistics, visualizations, other data reported by the model monitoring feature."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "edba71dc9840"
+      },
+      "source": [
+        "### Model\n",
         "\n",
+        "This tutorial uses a pre-trained model, where the model artifacts are stored in a public Cloud Storage bucket. The model predicts for an online gaming site, the probability that a player will churn, i.e. stop being an active player."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5abcd585354f"
+      },
+      "source": [
         "### Costs \n",
         "\n",
         "This tutorial uses billable components of Google Cloud:\n",
         "\n",
-        "* Vertext AI\n",
+        "* Vertex AI\n",
         "* BigQuery\n",
+        "* Cloud Storage\n",
         "\n",
-        "Learn about [Vertext AI\n",
-        "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
-        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
+        "Learn about [Vertext AI pricing](https://cloud.google.com/vertex-ai/pricing), \n",
+        "[Cloud Storage pricing](https://cloud.google.com/storage/pricing), \n",
+        "and [BigQuery pricing](https://cloud.google.com/bigquery/pricing)\n",
+        "and use the [Pricing\n",
         "Calculator](https://cloud.google.com/products/calculator/)\n",
         "to generate a cost estimate based on your projected usage."
       ]
@@ -129,6 +150,8 @@
         "id": "8yVpQt-JHKPF"
       },
       "source": [
+        "## Before you begin\n",
+        "\n",
         "### Set up your local development environment\n",
         "\n",
         "**If you are using Colab or Vertex AI Workbench notebooks**, your environment already meets\n",
@@ -165,15 +188,6 @@
         "1. Open this notebook in the Jupyter Notebook dashboard."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "ze4-nDLfK4pw"
-      },
-      "source": [
-        "## Before you begin"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -209,12 +223,22 @@
         "if IS_WORKBENCH_NOTEBOOK:\n",
         "    USER_FLAG = \"--user\"\n",
         "\n",
+        "# Don't bother installing tensorflow or explainable_ai_sdk on Colab\n",
+        "extra_pkgs = \"tensorflow explainable_ai_sdk\"\n",
+        "if \"google.colab\" in sys.modules:\n",
+        "  extra_pkgs = \"\"\n",
+        "\n",
         "# Install required packages.\n",
-        "! pip3 install {USER_FLAG} --quiet --upgrade google-cloud-aiplatform\n",
-        "! pip3 install {USER_FLAG} --quiet --upgrade tensorflow\n",
-        "! pip3 install {USER_FLAG} --quiet --upgrade explainable_ai_sdk\n",
-        "! pip3 install {USER_FLAG} --quiet --upgrade google-api-python-client google-auth-oauthlib google-auth-httplib2 oauth2client requests\n",
-        "! pip3 install {USER_FLAG} --quiet --upgrade google-cloud-storage==1.32.0"
+        "! pip3 install {USER_FLAG} \\\n",
+        "    google-cloud-aiplatform \\\n",
+        "    explainable_ai_sdk \\\n",
+        "    $extra_pkgs \\\n",
+        "    google-api-python-client \\\n",
+        "    google-auth-oauthlib \\\n",
+        "    google-auth-httplib2 \\\n",
+        "    oauth2client \\\n",
+        "    requests \\\n",
+        "    google-cloud-storage==1.32.0"
       ]
     },
     {
@@ -301,8 +325,7 @@
       },
       "outputs": [],
       "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
-        "    PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}\n",
+        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
         "    # Get your GCP project id from gcloud\n",
         "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
         "    PROJECT_ID = shell_output[0]\n",
@@ -349,7 +372,41 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "06571eb4063b"
+      },
+      "source": [
+        "#### UUID\n",
+        "\n",
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "4e166d927e36"
+      },
+      "outputs": [],
+      "source": [
+        "import random\n",
+        "import string\n",
+        "\n",
+        "\n",
+        "# Generate a uuid of a specifed length(default=8)\n",
+        "def generate_uuid(length: int = 8) -> str:\n",
+        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
+        "\n",
+        "\n",
+        "UUID = generate_uuid()"
       ]
     },
     {
@@ -472,7 +529,6 @@
       },
       "outputs": [],
       "source": [
-        "import copy\n",
         "# Import required packages.\n",
         "import os\n",
         "import pprint as pp\n",
@@ -480,25 +536,9 @@
         "\n",
         "import google.cloud.aiplatform as aiplatform\n",
         "import matplotlib.pyplot as plt\n",
+        "from google.cloud.aiplatform import model_monitoring\n",
         "from google.cloud.aiplatform.explain.metadata.tf.v2 import \\\n",
-        "    saved_model_metadata_builder\n",
-        "from google.cloud.aiplatform_v1.services.endpoint_service import \\\n",
-        "    EndpointServiceClient\n",
-        "from google.cloud.aiplatform_v1.services.job_service import JobServiceClient\n",
-        "from google.cloud.aiplatform_v1.services.prediction_service import \\\n",
-        "    PredictionServiceClient\n",
-        "from google.cloud.aiplatform_v1.types.io import BigQuerySource\n",
-        "from google.cloud.aiplatform_v1.types.model_deployment_monitoring_job import (\n",
-        "    ModelDeploymentMonitoringJob, ModelDeploymentMonitoringObjectiveConfig,\n",
-        "    ModelDeploymentMonitoringScheduleConfig)\n",
-        "from google.cloud.aiplatform_v1.types.model_monitoring import (\n",
-        "    ModelMonitoringAlertConfig, ModelMonitoringObjectiveConfig,\n",
-        "    SamplingStrategy, ThresholdConfig)\n",
-        "from google.cloud.aiplatform_v1.types.prediction_service import (\n",
-        "    ExplainRequest, PredictRequest)\n",
-        "from google.protobuf import json_format\n",
-        "from google.protobuf.duration_pb2 import Duration\n",
-        "from google.protobuf.struct_pb2 import Value"
+        "    saved_model_metadata_builder"
       ]
     },
     {
@@ -509,16 +549,36 @@
       },
       "outputs": [],
       "source": [
-        "SUFFIX = \"aiplatform.googleapis.com\"\n",
-        "API_ENDPOINT = f\"{REGION}-{SUFFIX}\"\n",
-        "PREDICT_API_ENDPOINT = f\"{REGION}-prediction-{SUFFIX}\"\n",
         "if os.getenv(\"IS_TESTING\"):\n",
-        "    !gcloud --quiet components install beta\n",
-        "    !gcloud --quiet components update\n",
-        "!gcloud config set ai/region $REGION\n",
+        "    ! gcloud --quiet components install beta\n",
+        "    ! gcloud --quiet components update\n",
+        "\n",
+        "! gcloud config set ai/region $REGION\n",
         "os.environ[\"GOOGLE_CLOUD_PROJECT\"] = PROJECT_ID"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "init_aip:mbsdk,region"
+      },
+      "source": [
+        "### Initialize Vertex AI SDK for Python\n",
+        "\n",
+        "Initialize the Vertex AI SDK for Python for your project and corresponding bucket."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "wGa5T9eRR8Mz"
+      },
+      "outputs": [],
+      "source": [
+        "aiplatform.init(project=PROJECT_ID, location=REGION)"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -527,14 +587,14 @@
       "source": [
         "### The example model\n",
         "\n",
-        "The model you'll use in this notebook is based on [this blog post](https://cloud.google.com/blog/topics/developers-practitioners/churn-prediction-game-developers-using-google-analytics-4-ga4-and-bigquery-ml). The idea behind this model is that your company has extensive log data describing how your game users have interacted with the site. The raw data contains the following categories of information:\n",
+        "The model you use in this notebook is based on [this blog post](https://cloud.google.com/blog/topics/developers-practitioners/churn-prediction-game-developers-using-google-analytics-4-ga4-and-bigquery-ml). The idea behind this model is that your company has extensive log data describing how your game users have interacted with the site. The raw data contains the following categories of information:\n",
         "\n",
         "- identity - unique player identitity numbers\n",
         "- demographic features - information about the player, such as the geographic region in which a player is located\n",
         "- behavioral features - counts of the number of times a  player has triggered certain game events, such as reaching a new level\n",
         "- churn propensity - this is the label or target feature, it provides an estimated probability that this player will churn, i.e. stop being an active player.\n",
         "\n",
-        "The blog article referenced above explains how to use BigQuery to store the raw data, pre-process it for use in machine learning, and train a model. Because this notebook focuses on model monitoring, rather than training models, you're going to reuse a pre-trained version of this model, which has been exported to Cloud Storage. In the next section, you will setup your environment and import this model into your own project."
+        "The blog article referenced above explains how to use BigQuery to store the raw data, pre-process the data for machine learning, and train the corresponding model. Because this notebook focuses on model monitoring, rather than training models, you're going to reuse a pre-trained version of this model, which has been exported to Cloud Storage. In the next section, you will setup your environment and import this model into your own project."
       ]
     },
     {
@@ -543,149 +603,9 @@
         "id": "btZeLzqQ7pXc"
       },
       "source": [
-        "### Define some helper functions and data structures\n",
+        "### Define some helper data structures\n",
         "\n",
-        "Run the following cells to define some utility functions and data structures used throughout this notebook. Some highlights:\n",
-        "\n",
-        "* create_monitoring_job - convenience function for requesting a model monitoring job\n",
-        "* send_predict_request - convenience function for sending a prediction request and receiving the response\n",
-        "\n",
-        "Although these functions and data strctures are not critical to understand the main concepts, feel free to expand the  cell if you're curious or want to dive deeper into how some of your API requests are made."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "cellView": "form",
-        "id": "yhDFSB2YDvfT"
-      },
-      "outputs": [],
-      "source": [
-        "# @title Utility functions\n",
-        "DEFAULT_THRESHOLD_VALUE = 0.001\n",
-        "\n",
-        "\n",
-        "def create_monitoring_job(objective_configs):\n",
-        "    # Create sampling configuration.\n",
-        "    random_sampling = SamplingStrategy.RandomSampleConfig(sample_rate=LOG_SAMPLE_RATE)\n",
-        "    sampling_config = SamplingStrategy(random_sample_config=random_sampling)\n",
-        "\n",
-        "    # Create schedule configuration.\n",
-        "    duration = Duration(seconds=MONITOR_INTERVAL)\n",
-        "    schedule_config = ModelDeploymentMonitoringScheduleConfig(monitor_interval=duration)\n",
-        "\n",
-        "    # Create alerting configuration.\n",
-        "    emails = [USER_EMAIL]\n",
-        "    email_config = ModelMonitoringAlertConfig.EmailAlertConfig(user_emails=emails)\n",
-        "    alerting_config = ModelMonitoringAlertConfig(email_alert_config=email_config)\n",
-        "\n",
-        "    # Create the monitoring job.\n",
-        "    endpoint = f\"projects/{PROJECT_ID}/locations/{REGION}/endpoints/{ENDPOINT_ID}\"\n",
-        "    predict_schema = \"\"\n",
-        "    analysis_schema = \"\"\n",
-        "    job = ModelDeploymentMonitoringJob(\n",
-        "        display_name=JOB_NAME,\n",
-        "        endpoint=endpoint,\n",
-        "        model_deployment_monitoring_objective_configs=objective_configs,\n",
-        "        logging_sampling_strategy=sampling_config,\n",
-        "        model_deployment_monitoring_schedule_config=schedule_config,\n",
-        "        model_monitoring_alert_config=alerting_config,\n",
-        "        predict_instance_schema_uri=predict_schema,\n",
-        "        analysis_instance_schema_uri=analysis_schema,\n",
-        "    )\n",
-        "    options = dict(api_endpoint=API_ENDPOINT)\n",
-        "    client = JobServiceClient(client_options=options)\n",
-        "    parent = f\"projects/{PROJECT_ID}/locations/{REGION}\"\n",
-        "    response = client.create_model_deployment_monitoring_job(\n",
-        "        parent=parent, model_deployment_monitoring_job=job\n",
-        "    )\n",
-        "    print(\"Created monitoring job:\")\n",
-        "    print(response)\n",
-        "    return response\n",
-        "\n",
-        "\n",
-        "def get_thresholds(default_thresholds, custom_thresholds):\n",
-        "    thresholds = {}\n",
-        "    default_threshold = ThresholdConfig(value=DEFAULT_THRESHOLD_VALUE)\n",
-        "    for feature in default_thresholds.split(\",\"):\n",
-        "        feature = feature.strip()\n",
-        "        thresholds[feature] = default_threshold\n",
-        "    for custom_threshold in custom_thresholds.split(\",\"):\n",
-        "        pair = custom_threshold.split(\":\")\n",
-        "        if len(pair) != 2:\n",
-        "            print(f\"Invalid custom skew threshold: {custom_threshold}\")\n",
-        "            return\n",
-        "        feature, value = pair\n",
-        "        thresholds[feature] = ThresholdConfig(value=float(value))\n",
-        "    return thresholds\n",
-        "\n",
-        "\n",
-        "def get_deployed_model_ids(endpoint_id):\n",
-        "    client_options = dict(api_endpoint=API_ENDPOINT)\n",
-        "    client = EndpointServiceClient(client_options=client_options)\n",
-        "    parent = f\"projects/{PROJECT_ID}/locations/{REGION}\"\n",
-        "    response = client.get_endpoint(name=f\"{parent}/endpoints/{endpoint_id}\")\n",
-        "    model_ids = []\n",
-        "    for model in response.deployed_models:\n",
-        "        model_ids.append(model.id)\n",
-        "    return model_ids\n",
-        "\n",
-        "\n",
-        "def set_objectives(model_ids, objective_template):\n",
-        "    # Use the same objective config for all models.\n",
-        "    objective_configs = []\n",
-        "    for model_id in model_ids:\n",
-        "        objective_config = copy.deepcopy(objective_template)\n",
-        "        objective_config.deployed_model_id = model_id\n",
-        "        objective_configs.append(objective_config)\n",
-        "    return objective_configs\n",
-        "\n",
-        "\n",
-        "def send_predict_request(endpoint, input, type=\"predict\"):\n",
-        "    client_options = {\"api_endpoint\": PREDICT_API_ENDPOINT}\n",
-        "    client = PredictionServiceClient(client_options=client_options)\n",
-        "    if type == \"predict\":\n",
-        "        obj = PredictRequest\n",
-        "        method = client.predict\n",
-        "    elif type == \"explain\":\n",
-        "        obj = ExplainRequest\n",
-        "        method = client.explain\n",
-        "    else:\n",
-        "        raise Exception(\"unsupported request type:\" + type)\n",
-        "    params = {}\n",
-        "    params = json_format.ParseDict(params, Value())\n",
-        "    request = obj(endpoint=endpoint, parameters=params)\n",
-        "    inputs = [json_format.ParseDict(input, Value())]\n",
-        "    request.instances.extend(inputs)\n",
-        "    response = None\n",
-        "    try:\n",
-        "        response = method(request)\n",
-        "    except Exception as ex:\n",
-        "        print(ex)\n",
-        "    return response\n",
-        "\n",
-        "\n",
-        "def list_monitoring_jobs():\n",
-        "    client_options = dict(api_endpoint=API_ENDPOINT)\n",
-        "    parent = f\"projects/{PROJECT_ID}/locations/us-central1\"\n",
-        "    client = JobServiceClient(client_options=client_options)\n",
-        "    response = client.list_model_deployment_monitoring_jobs(parent=parent)\n",
-        "    print(response)\n",
-        "\n",
-        "\n",
-        "def pause_monitoring_job(job):\n",
-        "    client_options = dict(api_endpoint=API_ENDPOINT)\n",
-        "    client = JobServiceClient(client_options=client_options)\n",
-        "    response = client.pause_model_deployment_monitoring_job(name=job)\n",
-        "    print(response)\n",
-        "\n",
-        "\n",
-        "def delete_monitoring_job(job):\n",
-        "    client_options = dict(api_endpoint=API_ENDPOINT)\n",
-        "    client = JobServiceClient(client_options=client_options)\n",
-        "    response = client.delete_model_deployment_monitoring_job(name=job)\n",
-        "    print(response)"
+        "Run the following cell to define some data structures used throughout this notebook."
       ]
     },
     {
@@ -701,6 +621,7 @@
         "\n",
         "# Sampling distributions for categorical features...\n",
         "DAYOFWEEK = {1: 1040, 2: 1223, 3: 1352, 4: 1217, 5: 1078, 6: 1011, 7: 1110}\n",
+        "\n",
         "LANGUAGE = {\n",
         "    \"en-us\": 4807,\n",
         "    \"en-gb\": 678,\n",
@@ -723,8 +644,11 @@
         "    \"en-nz\": 29,\n",
         "    \"es-es\": 25,\n",
         "}\n",
+        "\n",
         "OS = {\"IOS\": 3980, \"ANDROID\": 3798, \"null\": 253}\n",
+        "\n",
         "MONTH = {6: 3125, 7: 1838, 8: 1276, 9: 1718, 10: 74}\n",
+        "\n",
         "COUNTRY = {\n",
         "    \"United States\": 4395,\n",
         "    \"India\": 486,\n",
@@ -793,7 +717,7 @@
       "source": [
         "### Generate model metadata for Vertex Explainable AI\n",
         "\n",
-        "Run the following cell to extract metadata from the exported model, which is needed for generating the prediction explanations."
+        "Run the following cell to extract metadata from the exported model, which is needed for generating the explanations for a prediction request."
       ]
     },
     {
@@ -805,10 +729,12 @@
       "outputs": [],
       "source": [
         "MODEL_PATH = \"gs://mco-mm/churn\"\n",
+        "\n",
         "params = {\"sampled_shapley_attribution\": {\"path_count\": 10}}\n",
         "EXPLAIN_PARAMS = aiplatform.explain.ExplanationParameters(params)\n",
+        "\n",
         "builder = saved_model_metadata_builder.SavedModelMetadataBuilder(\n",
-        "    MODEL_PATH, outputs_to_explain=[\"churned_probs\"]\n",
+        "    model_path=MODEL_PATH, outputs_to_explain=[\"churned_probs\"]\n",
         ")\n",
         "EXPLAIN_META = builder.get_metadata_protobuf()"
       ]
@@ -837,15 +763,15 @@
         "MODEL_NAME = \"churn\"\n",
         "IMAGE = \"us-docker.pkg.dev/cloud-aiplatform/prediction/tf2-cpu.2-5:latest\"\n",
         "\n",
-        "aiplatform.init(project=PROJECT_ID, location=REGION)\n",
         "model = aiplatform.Model.upload(\n",
         "    display_name=MODEL_NAME,\n",
         "    artifact_uri=MODEL_PATH,\n",
         "    serving_container_image_uri=IMAGE,\n",
         "    explanation_parameters=EXPLAIN_PARAMS,\n",
         "    explanation_metadata=EXPLAIN_META,\n",
+        "    sync=True,\n",
         ")\n",
-        "model.wait()\n",
+        "\n",
         "MODEL_ID = model.resource_name.split(\"/\")[-1]"
       ]
     },
@@ -866,7 +792,7 @@
       "source": [
         "## Deploy your Model resource to an Endpoint resource\n",
         "\n",
-        "Now that you've imported your model into your project, you need to create an endpoint to serve your model. An endpoint can be thought of as a channel through which your model provides prediction services. Once established, you'll be able to make prediction requests on your model via the public internet.  Your endpoint is also serverless, in the sense that Google Cloud ensures high availability by reducing single points of failure, and scalability by dynamically allocating resources to meet the demand for your service. In this way, you are able to focus on your model quality, and freed from adminstrative and infrastructure concerns.\n",
+        "Now that you've imported your model into your project, you need to create an endpoint to serve your model. An endpoint can be thought of as a channel through which your model provides prediction services. Once established, you can make online prediction requests on your model via the public internet.  Your endpoint is also serverless, in the sense that Google Cloud ensures high availability by reducing single points of failure, and scalability by dynamically allocating resources to meet the demand for your service. In this way, you are able to focus on your model quality, and freed from adminstrative and infrastructure concerns.\n",
         "\n",
         "Run the next cell to deploy your model to an endpoint. **This will take about ten minutes to complete.**"
       ]
@@ -918,7 +844,7 @@
       "outputs": [],
       "source": [
         "try:\n",
-        "    resp = send_predict_request(ENDPOINT, DEFAULT_INPUT)\n",
+        "    resp = endpoint.predict([DEFAULT_INPUT])\n",
         "    for i in resp.predictions:\n",
         "        vals = i[\"churned_values\"]\n",
         "        probs = i[\"churned_probs\"]\n",
@@ -937,7 +863,8 @@
       },
       "source": [
         "### Test results\n",
-        "Taking a closer look at the results, we see the following elements:\n",
+        "\n",
+        "Taking a look at the results, you see the following elements:\n",
         "\n",
         "- **churned_values** - a set of possible values (0 and 1) for the target field\n",
         "- **churned_probs** - a corresponding set of probabilities for each possible target field value (5x10^-40 and 1.0, respectively)\n",
@@ -954,7 +881,7 @@
       "source": [
         "## Run an explanation test\n",
         "\n",
-        "We can also run a test of Explainable AI on this endpoint. Run the next cell to send a test explanation request. If everything works as expected, you should receive a response encoding the feature importance of this prediction in a text representation called JSON, along with a bar chart summarizing the results.\n",
+        "You can run a test of Explainable AI on this endpoint. Run the next cell to send a test explanation request. The response you receive encodes the feature importance of this prediction in a text representation called JSON, along with a bar chart summarizing the results.\n",
         "\n",
         "**Try this now by running the next cell.**"
       ]
@@ -970,8 +897,7 @@
         "try:\n",
         "    features = []\n",
         "    scores = []\n",
-        "    resp = send_predict_request(ENDPOINT, DEFAULT_INPUT, type=\"explain\")\n",
-        "    # pp.pprint(resp)\n",
+        "    resp = endpoint.explain([DEFAULT_INPUT])\n",
         "    for i in resp.explanations:\n",
         "        for j in i.attributions:\n",
         "            for k in j.feature_attributions:\n",
@@ -997,7 +923,7 @@
         "\n",
         "Now that you've created an endpoint to serve prediction requests on your model, you're ready to start a monitoring job to keep an eye on model quality and to alert you if and when input begins to deviate in way that may impact your model's prediction quality.\n",
         "\n",
-        "In this section, you will configure and create a model monitoring job based on the churn propensity model you imported from BigQuery ML."
+        "In this section, you configure and create a model monitoring job based on the churn propensity model you imported from BigQuery ML."
       ]
     },
     {
@@ -1039,17 +965,35 @@
         "# Prediction target column name in training dataset.\n",
         "TARGET = \"churned\"\n",
         "\n",
-        "# Skew and drift thresholds.\n",
-        "SKEW_DEFAULT_THRESHOLDS = \"country,cnt_user_engagement\"  # @param {type:\"string\"}\n",
-        "SKEW_CUSTOM_THRESHOLDS = \"cnt_level_start_quickplay:.01\"  # @param {type:\"string\"}\n",
-        "DRIFT_DEFAULT_THRESHOLDS = \"country,cnt_user_engagement\"  # @param {type:\"string\"}\n",
-        "DRIFT_CUSTOM_THRESHOLDS = \"cnt_level_start_quickplay:.01\"  # @param {type:\"string\"}\n",
-        "ATTRIB_SKEW_DEFAULT_THRESHOLDS = \"country,cnt_user_engagement\"  # @param {type:\"string\"}\n",
-        "# fmt: off\n",
-        "ATTRIB_SKEW_CUSTOM_THRESHOLDS = \"cnt_level_start_quickplay:.01\"  # @param {type:\"string\"}\n",
-        "ATTRIB_DRIFT_DEFAULT_THRESHOLDS = \"country,cnt_user_engagement\"  # @param {type:\"string\"}\n",
-        "ATTRIB_DRIFT_CUSTOM_THRESHOLDS = \"cnt_level_start_quickplay:.01\"  # @param {type:\"string\"}\n",
-        "# fmt: on"
+        "# # Skew and drift thresholds.\n",
+        "\n",
+        "DEFAULT_THRESHOLD_VALUE = 0.001\n",
+        "\n",
+        "SKEW_THRESHOLDS = {\n",
+        "    \"country\": DEFAULT_THRESHOLD_VALUE,\n",
+        "    \"cnt_user_engagement\": DEFAULT_THRESHOLD_VALUE,\n",
+        "}\n",
+        "DRIFT_THRESHOLDS = {\n",
+        "    \"country\": DEFAULT_THRESHOLD_VALUE,\n",
+        "    \"cnt_user_engagement\": DEFAULT_THRESHOLD_VALUE,\n",
+        "}\n",
+        "ATTRIB_SKEW_THRESHOLDS = {\n",
+        "    \"country\": DEFAULT_THRESHOLD_VALUE,\n",
+        "    \"cnt_user_engagement\": DEFAULT_THRESHOLD_VALUE,\n",
+        "}\n",
+        "ATTRIB_DRIFT_THRESHOLDS = {\n",
+        "    \"country\": DEFAULT_THRESHOLD_VALUE,\n",
+        "    \"cnt_user_engagement\": DEFAULT_THRESHOLD_VALUE,\n",
+        "}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e10f3d0fa538"
+      },
+      "source": [
+        "You can change the threshold values and the configuration settings, so that you can monitor other features in the model as well."
       ]
     },
     {
@@ -1071,58 +1015,46 @@
       },
       "outputs": [],
       "source": [
-        "skew_thresholds = get_thresholds(SKEW_DEFAULT_THRESHOLDS, SKEW_CUSTOM_THRESHOLDS)\n",
-        "drift_thresholds = get_thresholds(DRIFT_DEFAULT_THRESHOLDS, DRIFT_CUSTOM_THRESHOLDS)\n",
-        "attrib_skew_thresholds = get_thresholds(\n",
-        "    ATTRIB_SKEW_DEFAULT_THRESHOLDS, ATTRIB_SKEW_CUSTOM_THRESHOLDS\n",
-        ")\n",
-        "attrib_drift_thresholds = get_thresholds(\n",
-        "    ATTRIB_DRIFT_DEFAULT_THRESHOLDS, ATTRIB_DRIFT_CUSTOM_THRESHOLDS\n",
+        "skew_config = model_monitoring.SkewDetectionConfig(\n",
+        "    data_source=DATASET_BQ_URI,\n",
+        "    skew_thresholds=SKEW_THRESHOLDS,\n",
+        "    attribute_skew_thresholds=ATTRIB_SKEW_THRESHOLDS,\n",
+        "    target_field=TARGET,\n",
         ")\n",
         "\n",
-        "skew_config = ModelMonitoringObjectiveConfig.TrainingPredictionSkewDetectionConfig(\n",
-        "    skew_thresholds=skew_thresholds,\n",
-        "    attribution_score_skew_thresholds=attrib_skew_thresholds,\n",
-        ")\n",
-        "drift_config = ModelMonitoringObjectiveConfig.PredictionDriftDetectionConfig(\n",
-        "    drift_thresholds=drift_thresholds,\n",
-        "    attribution_score_drift_thresholds=attrib_drift_thresholds,\n",
-        ")\n",
-        "explanation_config = ModelMonitoringObjectiveConfig.ExplanationConfig(\n",
-        "    enable_feature_attributes=True\n",
+        "drift_config = model_monitoring.DriftDetectionConfig(\n",
+        "    drift_thresholds=DRIFT_THRESHOLDS,\n",
+        "    attribute_drift_thresholds=ATTRIB_DRIFT_THRESHOLDS,\n",
         ")\n",
         "\n",
-        "training_dataset = ModelMonitoringObjectiveConfig.TrainingDataset(target_field=TARGET)\n",
-        "training_dataset.bigquery_source = BigQuerySource(input_uri=DATASET_BQ_URI)\n",
-        "objective_config = ModelMonitoringObjectiveConfig(\n",
-        "    training_dataset=training_dataset,\n",
-        "    training_prediction_skew_detection_config=skew_config,\n",
-        "    prediction_drift_detection_config=drift_config,\n",
-        "    explanation_config=explanation_config,\n",
+        "explanation_config = model_monitoring.ExplanationConfig()\n",
+        "objective_config = model_monitoring.ObjectiveConfig(\n",
+        "    skew_config, drift_config, explanation_config\n",
         ")\n",
-        "model_ids = get_deployed_model_ids(ENDPOINT_ID)\n",
-        "objective_template = ModelDeploymentMonitoringObjectiveConfig(\n",
-        "    objective_config=objective_config\n",
-        ")\n",
-        "objective_configs = set_objectives(model_ids, objective_template)\n",
         "\n",
-        "monitoring_job = create_monitoring_job(objective_configs)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "OiwOVR4D_xhl"
-      },
-      "outputs": [],
-      "source": [
-        "# Run a prediction request to generate schema, if necessary.\n",
-        "try:\n",
-        "    _ = send_predict_request(ENDPOINT, DEFAULT_INPUT)\n",
-        "    print(\"prediction succeeded\")\n",
-        "except Exception:\n",
-        "    print(\"prediction failed\")"
+        "# Create sampling configuration\n",
+        "random_sampling = model_monitoring.RandomSampleConfig(sample_rate=LOG_SAMPLE_RATE)\n",
+        "\n",
+        "# Create schedule configuration\n",
+        "schedule_config = model_monitoring.ScheduleConfig(monitor_interval=MONITOR_INTERVAL)\n",
+        "\n",
+        "# Create alerting configuration.\n",
+        "emails = [USER_EMAIL]\n",
+        "alerting_config = model_monitoring.EmailAlertConfig(\n",
+        "    user_emails=emails, enable_logging=True\n",
+        ")\n",
+        "\n",
+        "# Create the monitoring job.\n",
+        "job = aiplatform.ModelDeploymentMonitoringJob.create(\n",
+        "    display_name=JOB_NAME,\n",
+        "    logging_sampling_strategy=random_sampling,\n",
+        "    schedule_config=schedule_config,\n",
+        "    alert_config=alerting_config,\n",
+        "    objective_configs=objective_config,\n",
+        "    project=PROJECT_ID,\n",
+        "    location=REGION,\n",
+        "    endpoint=endpoint,\n",
+        ")"
       ]
     },
     {
@@ -1140,6 +1072,35 @@
         "As your monitoring job collects data, measurements are stored in Cloud Storage and you are free to examine your data at any time. The \"Statistics and Anomalies Root Path\" specifies the location of your measurements in Cloud Storage. Run the following cell to see an example of the layout of these measurements in Cloud Storage. If you substitute the Cloud Storage URL in your job creation email, you can view the structure and content of the data files for your own monitoring job."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6f38e8423bce"
+      },
+      "source": [
+        "### Create the sampling distribution\n",
+        "\n",
+        "Next, you send a first test prediction request. The model monitoring service will analyze the distribution of features and automatically create a baseline to monitor deviations from the baseline.\n",
+        "\n",
+        "*Note:* You need to wait for the email notification before making the first prediction request."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "OiwOVR4D_xhl"
+      },
+      "outputs": [],
+      "source": [
+        "# Run a prediction request to generate schema, if necessary.\n",
+        "try:\n",
+        "    _ = endpoint.predict([DEFAULT_INPUT])\n",
+        "    print(\"prediction succeeded\")\n",
+        "except Exception as e:\n",
+        "    print(f\"prediction failed: {e}\")"
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -1177,7 +1138,7 @@
       "source": [
         "### You can create monitoring jobs with other user interfaces\n",
         "\n",
-        "In the previous cells, you created a monitoring job using the Python client library. You can also use the *gcloud* command line tool or the Cloud Console to create a model monitoring job. \n"
+        "In the previous cells, you created a monitoring job using the Python client library. Alternatively, you can use the *gcloud* command line tool or the Cloud Console to create a model monitoring job. \n"
       ]
     },
     {
@@ -1188,9 +1149,9 @@
       "source": [
         "## Interpret your results\n",
         "\n",
-        "Model Monitoring detects an anomaly when the threshold set for a feature is exceeded. The following cells give you a sense of the alerting and reporting experience after model monitoring anomalies have been detected.\n",
+        "Vertex AI Model Monitoring detects an anomaly when the threshold set for a feature is exceeded. The following cells give you a sense of the alerting and reporting experience after model monitoring anomalies have been detected.\n",
         "\n",
-        "Model Monitoring automatically notifies you of detected anomalies through email, but you can also [set up alerts through Cloud Logging](https://cloud.google.com/vertex-ai/docs/model-monitoring/using-model-monitoring#monitor-job)."
+        "Vertex AI Model Monitoring automatically notifies you of detected anomalies through email, but you can also [set up alerts through Cloud Logging](https://cloud.google.com/vertex-ai/docs/model-monitoring/using-model-monitoring#monitor-job)."
       ]
     },
     {
@@ -1271,20 +1232,15 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "TPP_ImwJDFJf"
+        "id": "d6cc924aa1fb"
       },
       "outputs": [],
       "source": [
-        "# Delete endpoint resource\n",
-        "tmp = ! gcloud ai endpoints describe --region=$REGION $ENDPOINT_ID --quiet \\\n",
-        "        | grep \"  id:\"\n",
-        "DEPLOYED_MODEL_ID = tmp[1][7:-1]\n",
-        "! gcloud ai endpoints undeploy-model --deployed-model-id=$DEPLOYED_MODEL_ID \\\n",
-        "    $ENDPOINT_ID --quiet\n",
-        "! gcloud ai endpoints delete $ENDPOINT --quiet\n",
+        "# Undeploy the model and delete the endpoint\n",
+        "endpoint.undeploy_all()\n",
+        "endpoint.delete()\n",
         "\n",
-        "# Delete model resource\n",
-        "! gcloud ai models delete $MODEL_ID --quiet\n",
+        "model.delete()\n",
         "\n",
         "# Delete BQ table and dataset\n",
         "rmtable = f\"bq rm -f model_deployment_monitoring_{ENDPOINT_ID}.serving_predict\"\n",

notebooks/official/pipelines/automl_tabular_classification_beans.ipynb

@@ -64,22 +64,11 @@
         "\n",
         "This notebook shows how to use the components defined in [`google_cloud_pipeline_components`](https://github.com/kubeflow/pipelines/tree/master/components/google-cloud) to build an AutoML tabular classification workflow on [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines).\n",
         "\n",
-        "You'll build a pipeline that looks like this:\n",
+        "You build a pipeline in this notebook that looks like this:\n",
         "\n",
         "<a href=\"https://storage.googleapis.com/amy-jo/images/mp/beans.png\" target=\"_blank\"><img src=\"https://storage.googleapis.com/amy-jo/images/mp/beans.png\" width=\"95%\"/></a>"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "dataset:beans,lcn"
-      },
-      "source": [
-        "### Dataset\n",
-        "\n",
-        "The dataset used for this tutorial is the UCI Machine Learning ['Dry beans dataset'](https://archive.ics.uci.edu/ml/datasets/Dry+Bean+Dataset), from: KOKLU, M. and OZKAN, I.A., (2020), \"Multiclass Classification of Dry Beans Using Computer Vision and Machine Learning Techniques.\"In Computers and Electronics in Agriculture, 174, 105507. [DOI](https://doi.org/10.1016/j.compag.2020.105507)."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -96,8 +85,8 @@
         "- `Vertex AI Pipelines`\n",
         "- `Google Cloud Pipeline Components`\n",
         "- `Vertex AutoML`\n",
-        "- `Vertex AI Model` resource\n",
-        "- `Vertex AI Endpoint` resource\n",
+        "- `Vertex AI Model`\n",
+        "- `Vertex AI Endpoint`\n",
         "\n",
         "The steps performed include:\n",
         "\n",
@@ -112,6 +101,17 @@
         "The components are [documented here](https://google-cloud-pipeline-components.readthedocs.io/en/latest/google_cloud_pipeline_components.aiplatform.html#module-google_cloud_pipeline_components.aiplatform)."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dataset:beans,lcn"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the UCI Machine Learning ['Dry beans dataset'](https://archive.ics.uci.edu/ml/datasets/Dry+Bean+Dataset), from: KOKLU, M. and OZKAN, I.A., (2020), \"Multiclass Classification of Dry Beans Using Computer Vision and Machine Learning Techniques.\"In Computers and Electronics in Agriculture, 174, 105507. [DOI](https://doi.org/10.1016/j.compag.2020.105507)."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -140,7 +140,7 @@
       "source": [
         "### Set up your local development environment\n",
         "\n",
-        "If you are using Colab or Vertex AI Workbench Notebook, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
+        "If you are using Colab or Vertex AI Workbench Notebook, your environment already meets all the requirements to run this notebook.\n",
         "\n",
         "Otherwise, make sure your environment meets this notebook's requirements. You need the following:\n",
         "\n",
@@ -197,9 +197,10 @@
         "if IS_WORKBENCH_NOTEBOOK:\n",
         "    USER_FLAG = \"--user\"\n",
         "\n",
-        "! pip3 install --upgrade google-cloud-aiplatform {USER_FLAG} -q\n",
-        "! pip3 install -U google-cloud-storage $USER_FLAG -q\n",
-        "! pip3 install $USER kfp google-cloud-pipeline-components --upgrade -q"
+        "! pip3 install --upgrade {USER_FLAG} google-cloud-aiplatform \\\n",
+        "                                    google-cloud-storage \\\n",
+        "                                    kfp \\\n",
+        "                                    google-cloud-pipeline-components -q"
       ]
     },
     {
@@ -237,6 +238,8 @@
         "id": "check_versions"
       },
       "source": [
+        "### Check the package versions\n",
+        "\n",
         "Check the versions of the packages you installed.  The KFP SDK version should be >=1.8."
       ]
     },
@@ -282,6 +285,17 @@
         "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$`."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "af794e75b7e3"
+      },
+      "source": [
+        "#### Set your project ID\n",
+        "\n",
+        "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -290,7 +304,7 @@
       },
       "outputs": [],
       "source": [
-        "PROJECT_ID = \"python-docs-samples-tests\"  # @param {type:\"string\"}"
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -301,7 +315,7 @@
       },
       "outputs": [],
       "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"python-docs-samples-tests\":\n",
+        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
         "    # Get your GCP project id from gcloud\n",
         "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
         "    PROJECT_ID = shell_output[0]\n",
@@ -347,7 +361,10 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -356,9 +373,9 @@
         "id": "timestamp"
       },
       "source": [
-        "#### Timestamp\n",
+        "#### UUID\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
       ]
     },
     {
@@ -369,9 +386,16 @@
       },
       "outputs": [],
       "source": [
-        "from datetime import datetime\n",
+        "import random\n",
+        "import string\n",
         "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+        "\n",
+        "# Generate a uuid of a specifed length(default=8)\n",
+        "def generate_uuid(length: int = 8) -> str:\n",
+        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
+        "\n",
+        "\n",
+        "UUID = generate_uuid()"
       ]
     },
     {
@@ -382,7 +406,7 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Vertex AI Workbench Notebook**, your environment is already authenticated. Skip this step.\n",
+        "**If you are using Vertex AI Workbench Notebook**, your environment is already authenticated.\n",
         "\n",
         "**If you are using Colab**, run the cell below and follow the instructions\n",
         "when prompted to authenticate your account via oAuth.\n",
@@ -477,8 +501,8 @@
       "outputs": [],
       "source": [
         "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
-        "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
+        "    BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
     {
@@ -599,9 +623,6 @@
         "id": "setup_vars"
       },
       "source": [
-        "### Set up variables\n",
-        "\n",
-        "Next, set up some variables used throughout the tutorial.\n",
         "### Import libraries and define constants"
       ]
     },
@@ -613,7 +634,13 @@
       },
       "outputs": [],
       "source": [
-        "import google.cloud.aiplatform as aip"
+        "from typing import NamedTuple\n",
+        "\n",
+        "import kfp\n",
+        "from google.cloud import aiplatform\n",
+        "from kfp.v2 import dsl\n",
+        "from kfp.v2.dsl import (Artifact, ClassificationMetrics, Input, Metrics,\n",
+        "                        Output, component)"
       ]
     },
     {
@@ -624,9 +651,9 @@
       "source": [
         "#### Vertex AI constants\n",
         "\n",
-        "Setup up the following constants for Vertex AI:\n",
-        "\n",
-        "- `API_ENDPOINT`: The Vertex AI API service endpoint for `Dataset`, `Model`, `Job`, `Pipeline` and `Endpoint` services."
+        "Setup up the following constants for Vertex AI Pipeline:\n",
+        "- `PIPELINE_NAME`: Set name for the Pipeline.\n",
+        "- `PIPELINE_ROOT`: Cloud Storage bucket path to store pipeline artifacts."
       ]
     },
     {
@@ -637,65 +664,18 @@
       },
       "outputs": [],
       "source": [
-        "# API service endpoint\n",
-        "API_ENDPOINT = \"{}-aiplatform.googleapis.com\".format(REGION)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "pipeline_constants"
-      },
-      "source": [
-        "#### Vertex AI Pipelines constants\n",
-        "\n",
-        "Setup up the following constants for Vertex AI Pipelines:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "pipeline_constants"
-      },
-      "outputs": [],
-      "source": [
+        "# set path for storing the pipeline artifacts\n",
+        "PIPELINE_NAME = \"automl-tabular-beans-training\"\n",
         "PIPELINE_ROOT = \"{}/pipeline_root/beans\".format(BUCKET_URI)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "additional_imports"
-      },
-      "source": [
-        "Additional imports."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "import_pipelines"
-      },
-      "outputs": [],
-      "source": [
-        "from typing import NamedTuple\n",
-        "\n",
-        "import kfp\n",
-        "from google_cloud_pipeline_components import aiplatform as gcc_aip\n",
-        "from kfp.v2 import dsl\n",
-        "from kfp.v2.dsl import (Artifact, ClassificationMetrics, Input, Metrics,\n",
-        "                        Output, component)"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "init_aip:mbsdk"
       },
       "source": [
-        "## Initialize Vertex AI SDK for Python\n",
+        "### Initialize Vertex AI SDK for Python\n",
         "\n",
         "Initialize the Vertex AI SDK for Python for your project and corresponding bucket."
       ]
@@ -708,7 +688,7 @@
       },
       "outputs": [],
       "source": [
-        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
+        "aiplatform.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
       ]
     },
     {
@@ -751,8 +731,7 @@
         ")\n",
         "def classification_model_eval_metrics(\n",
         "    project: str,\n",
-        "    location: str,  # \"us-central1\",\n",
-        "    api_endpoint: str,  # \"us-central1-aiplatform.googleapis.com\",\n",
+        "    location: str,\n",
         "    thresholds_dict_str: str,\n",
         "    model: Input[Artifact],\n",
         "    metrics: Output[Metrics],\n",
@@ -762,20 +741,21 @@
         "    import json\n",
         "    import logging\n",
         "\n",
-        "    from google.cloud import aiplatform as aip\n",
+        "    from google.cloud import aiplatform\n",
+        "\n",
+        "    aiplatform.init(project=project)\n",
         "\n",
         "    # Fetch model eval info\n",
-        "    def get_eval_info(client, model_name):\n",
-        "        from google.protobuf.json_format import MessageToDict\n",
-        "\n",
-        "        response = client.list_model_evaluations(parent=model_name)\n",
+        "    def get_eval_info(model):\n",
+        "        response = model.list_model_evaluations()\n",
         "        metrics_list = []\n",
         "        metrics_string_list = []\n",
         "        for evaluation in response:\n",
+        "            evaluation = evaluation.to_dict()\n",
         "            print(\"model_evaluation\")\n",
-        "            print(\" name:\", evaluation.name)\n",
-        "            print(\" metrics_schema_uri:\", evaluation.metrics_schema_uri)\n",
-        "            metrics = MessageToDict(evaluation._pb.metrics)\n",
+        "            print(\" name:\", evaluation[\"name\"])\n",
+        "            print(\" metrics_schema_uri:\", evaluation[\"metricsSchemaUri\"])\n",
+        "            metrics = evaluation[\"metrics\"]\n",
         "            for metric in metrics.keys():\n",
         "                logging.info(\"metric: %s, value: %s\", metric, metrics[metric])\n",
         "            metrics_str = json.dumps(metrics)\n",
@@ -783,7 +763,7 @@
         "            metrics_string_list.append(metrics_str)\n",
         "\n",
         "        return (\n",
-        "            evaluation.name,\n",
+        "            evaluation[\"name\"],\n",
         "            metrics_list,\n",
         "            metrics_string_list,\n",
         "        )\n",
@@ -832,20 +812,18 @@
         "            if metric != \"confidenceMetrics\":\n",
         "                val_string = json.dumps(metrics_list[0][metric])\n",
         "                metrics.log_metric(metric, val_string)\n",
-        "        # metrics.metadata[\"model_type\"] = \"AutoML Tabular classification\"\n",
         "\n",
         "    logging.getLogger().setLevel(logging.INFO)\n",
-        "    aip.init(project=project)\n",
+        "\n",
         "    # extract the model resource name from the input Model Artifact\n",
         "    model_resource_path = model.metadata[\"resourceName\"]\n",
         "    logging.info(\"model path: %s\", model_resource_path)\n",
         "\n",
-        "    client_options = {\"api_endpoint\": api_endpoint}\n",
-        "    # Initialize client that will be used to create and send requests.\n",
-        "    client = aip.gapic.ModelServiceClient(client_options=client_options)\n",
-        "    eval_name, metrics_list, metrics_str_list = get_eval_info(\n",
-        "        client, model_resource_path\n",
-        "    )\n",
+        "    # Get the trained model resource\n",
+        "    model = aiplatform.Model(model_resource_path)\n",
+        "\n",
+        "    # Get model evaluation metrics from the the trained model\n",
+        "    eval_name, metrics_list, metrics_str_list = get_eval_info(model)\n",
         "    logging.info(\"got evaluation name: %s\", eval_name)\n",
         "    logging.info(\"got metrics list: %s\", metrics_list)\n",
         "    log_metrics(metrics_list, metricsc)\n",
@@ -867,7 +845,9 @@
         "id": "define_pipeline:gcpc,beans,lcn"
       },
       "source": [
-        "## Define an AutoML tabular classification pipeline that uses components from `google_cloud_pipeline_components`"
+        "## Define pipeline \n",
+        "\n",
+        "Define the pipeline for AutoML tabular classification using the components from `google_cloud_pipeline_components`."
       ]
     },
     {
@@ -878,30 +858,34 @@
       },
       "outputs": [],
       "source": [
-        "DISPLAY_NAME = \"automl-beans{}\".format(TIMESTAMP)\n",
-        "PIPELINE_NAME = \"automl-tabular-beans-training-v2\"\n",
-        "MACHINE_TYPE = \"n1-standard-4\"\n",
-        "\n",
-        "\n",
         "@kfp.dsl.pipeline(name=PIPELINE_NAME, pipeline_root=PIPELINE_ROOT)\n",
         "def pipeline(\n",
-        "    bq_source: str = \"bq://aju-dev-demos.beans.beans1\",\n",
-        "    display_name: str = DISPLAY_NAME,\n",
-        "    project: str = PROJECT_ID,\n",
-        "    gcp_region: str = REGION,\n",
-        "    api_endpoint: str = API_ENDPOINT,\n",
-        "    thresholds_dict_str: str = '{\"auRoc\": 0.95}',\n",
+        "    bq_source: str,\n",
+        "    DATASET_DISPLAY_NAME: str,\n",
+        "    TRAINING_DISPLAY_NAME: str,\n",
+        "    MODEL_DISPLAY_NAME: str,\n",
+        "    ENDPOINT_DISPLAY_NAME: str,\n",
+        "    MACHINE_TYPE: str,\n",
+        "    project: str,\n",
+        "    gcp_region: str,\n",
+        "    thresholds_dict_str: str,\n",
         "):\n",
-        "    dataset_create_op = gcc_aip.TabularDatasetCreateOp(\n",
-        "        project=project, display_name=display_name, bq_source=bq_source\n",
+        "\n",
+        "    from google_cloud_pipeline_components.aiplatform import (\n",
+        "        AutoMLTabularTrainingJobRunOp, EndpointCreateOp, ModelDeployOp,\n",
+        "        TabularDatasetCreateOp)\n",
+        "\n",
+        "    dataset_create_op = TabularDatasetCreateOp(\n",
+        "        project=project, display_name=DATASET_DISPLAY_NAME, bq_source=bq_source\n",
         "    )\n",
         "\n",
-        "    training_op = gcc_aip.AutoMLTabularTrainingJobRunOp(\n",
+        "    training_op = AutoMLTabularTrainingJobRunOp(\n",
         "        project=project,\n",
-        "        display_name=display_name,\n",
+        "        display_name=TRAINING_DISPLAY_NAME,\n",
         "        optimization_prediction_type=\"classification\",\n",
         "        optimization_objective=\"minimize-log-loss\",\n",
         "        budget_milli_node_hours=1000,\n",
+        "        model_display_name=MODEL_DISPLAY_NAME,\n",
         "        column_specs={\n",
         "            \"Area\": \"numeric\",\n",
         "            \"Perimeter\": \"numeric\",\n",
@@ -924,10 +908,10 @@
         "        dataset=dataset_create_op.outputs[\"dataset\"],\n",
         "        target_column=\"Class\",\n",
         "    )\n",
+        "\n",
         "    model_eval_task = classification_model_eval_metrics(\n",
         "        project,\n",
         "        gcp_region,\n",
-        "        api_endpoint,\n",
         "        thresholds_dict_str,\n",
         "        training_op.outputs[\"model\"],\n",
         "    )\n",
@@ -937,13 +921,13 @@
         "        name=\"deploy_decision\",\n",
         "    ):\n",
         "\n",
-        "        endpoint_op = gcc_aip.EndpointCreateOp(\n",
+        "        endpoint_op = EndpointCreateOp(\n",
         "            project=project,\n",
         "            location=gcp_region,\n",
-        "            display_name=\"train-automl-beans\",\n",
+        "            display_name=ENDPOINT_DISPLAY_NAME,\n",
         "        )\n",
         "\n",
-        "        gcc_aip.ModelDeployOp(\n",
+        "        ModelDeployOp(\n",
         "            model=training_op.outputs[\"model\"],\n",
         "            endpoint=endpoint_op.outputs[\"endpoint\"],\n",
         "            dedicated_resources_min_replica_count=1,\n",
@@ -960,7 +944,7 @@
       "source": [
         "## Compile the pipeline\n",
         "\n",
-        "Next, compile the pipeline."
+        "Next, compile the pipeline to the specified json file."
       ]
     },
     {
@@ -971,11 +955,11 @@
       },
       "outputs": [],
       "source": [
-        "from kfp.v2 import compiler  # noqa: F811\n",
+        "from kfp.v2 import compiler\n",
         "\n",
         "compiler.Compiler().compile(\n",
         "    pipeline_func=pipeline,\n",
-        "    package_path=\"tabular classification_pipeline.json\".replace(\" \", \"_\"),\n",
+        "    package_path=\"tabular_classification_pipeline.json\",\n",
         ")"
       ]
     },
@@ -987,7 +971,53 @@
       "source": [
         "## Run the pipeline\n",
         "\n",
-        "Next, run the pipeline."
+        "Next, pass the input parameters required for the pipeline and run it. The defined pipeline takes the following parameters:\n",
+        "\n",
+        "- `bq_source`: BigQuery source for the tabular dataset.\n",
+        "- `DATASET_DISPLAY_NAME`: Display name for the Vertex AI managed dataset.\n",
+        "- `TRAINIG_DISPLAY_NAME`: Display name for the AutoML Training job.\n",
+        "- `MODEL_DISPLAY_NAME`: Display name for the Vertex AI Model generated as a result of the training job.\n",
+        "- `ENDPOINT_DISPLAY_NAME`: Display name for the Vertex AI Endpoint where the model is deployed.\n",
+        "- `MACHINE_TYPE`: Machine type for the serving container.\n",
+        "- `project`: Project-id where the pipeline is run.\n",
+        "- `gcp_region`: Region for setting the pipeline location.\n",
+        "- `thresholds_dict_str`: dictionary of thresholds based on which the model deployment is conditioned.\n",
+        "- `pipeline_root`:  To override the pipeline root path specified in the pipeline job's definition, specify a path that your pipeline job can access, such as a Cloud Storage bucket URI."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "1adf9b056954"
+      },
+      "outputs": [],
+      "source": [
+        "# Set the display-names for Vertex AI resources\n",
+        "PIPELINE_DISPLAY_NAME = \"[your-pipeline-display-name]\"  # @param {type:\"string\"}\n",
+        "DATASET_DISPLAY_NAME = \"[your-dataset-display-name]\"  # @param {type:\"string\"}\n",
+        "MODEL_DISPLAY_NAME = \"[your-model-display-name]\"  # @param {type:\"string\"}\n",
+        "TRAINING_DISPLAY_NAME = \"[your-training-job-display-name]\"  # @param {type:\"string\"}\n",
+        "ENDPOINT_DISPLAY_NAME = \"[your-endpoint-display-name]\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# Otherwise, use the default display-names\n",
+        "if PIPELINE_DISPLAY_NAME == \"[your-pipeline-display-name]\":\n",
+        "    PIPELINE_DISPLAY_NAME = f\"pipeline_beans_{UUID}\"\n",
+        "\n",
+        "if DATASET_DISPLAY_NAME == \"[your-dataset-display-name]\":\n",
+        "    DATASET_DISPLAY_NAME = f\"dataset_beans_{UUID}\"\n",
+        "\n",
+        "if MODEL_DISPLAY_NAME == \"[your-model-display-name]\":\n",
+        "    MODEL_DISPLAY_NAME = f\"model_beans_{UUID}\"\n",
+        "\n",
+        "if TRAINING_DISPLAY_NAME == \"[your-training-job-display-name]\":\n",
+        "    TRAINING_DISPLAY_NAME = f\"automl_training_beans_{UUID}\"\n",
+        "\n",
+        "if ENDPOINT_DISPLAY_NAME == \"[your-endpoint-display-name]\":\n",
+        "    ENDPOINT_DISPLAY_NAME = f\"endpoint_beans_{UUID}\"\n",
+        "\n",
+        "# Set machine type\n",
+        "MACHINE_TYPE = \"n1-standard-4\""
       ]
     },
     {
@@ -998,19 +1028,24 @@
       },
       "outputs": [],
       "source": [
-        "DISPLAY_NAME = \"beans_\" + TIMESTAMP\n",
-        "\n",
-        "job = aip.PipelineJob(\n",
-        "    display_name=DISPLAY_NAME,\n",
-        "    template_path=\"tabular classification_pipeline.json\".replace(\" \", \"_\"),\n",
+        "# Configure the pipeline\n",
+        "job = aiplatform.PipelineJob(\n",
+        "    display_name=PIPELINE_DISPLAY_NAME,\n",
+        "    template_path=\"tabular_classification_pipeline.json\",\n",
         "    pipeline_root=PIPELINE_ROOT,\n",
-        "    parameter_values={\"project\": PROJECT_ID, \"display_name\": DISPLAY_NAME},\n",
+        "    parameter_values={\n",
+        "        \"project\": PROJECT_ID,\n",
+        "        \"gcp_region\": REGION,\n",
+        "        \"bq_source\": \"bq://aju-dev-demos.beans.beans1\",\n",
+        "        \"thresholds_dict_str\": '{\"auRoc\": 0.95}',\n",
+        "        \"DATASET_DISPLAY_NAME\": DATASET_DISPLAY_NAME,\n",
+        "        \"TRAINING_DISPLAY_NAME\": TRAINING_DISPLAY_NAME,\n",
+        "        \"MODEL_DISPLAY_NAME\": MODEL_DISPLAY_NAME,\n",
+        "        \"ENDPOINT_DISPLAY_NAME\": ENDPOINT_DISPLAY_NAME,\n",
+        "        \"MACHINE_TYPE\": MACHINE_TYPE,\n",
+        "    },\n",
         "    enable_caching=False,\n",
-        ")\n",
-        "\n",
-        "job.submit()\n",
-        "\n",
-        "! rm tabular_classification_pipeline.json"
+        ")"
       ]
     },
     {
@@ -1019,11 +1054,19 @@
         "id": "view_pipeline_run:model"
       },
       "source": [
-        "Click on the generated link to see your run in the Cloud Console.\n",
-        "\n",
-        "<!-- It should look something like this as it is running:\n",
-        "\n",
-        "<a href=\"https://storage.googleapis.com/amy-jo/images/mp/automl_tabular_classif.png\" target=\"_blank\"><img src=\"https://storage.googleapis.com/amy-jo/images/mp/automl_tabular_classif.png\" width=\"40%\"/></a> -->"
+        "Run the pipeline job. Click on the generated link to see your run in the Cloud Console."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "114ab8ff24ac"
+      },
+      "outputs": [],
+      "source": [
+        "# Run the job\n",
+        "job.run()"
       ]
     },
     {
@@ -1045,7 +1088,7 @@
       },
       "outputs": [],
       "source": [
-        "pipeline_df = aip.get_pipeline_df(pipeline=PIPELINE_NAME)\n",
+        "pipeline_df = aiplatform.get_pipeline_df(pipeline=PIPELINE_NAME)\n",
         "print(pipeline_df.head(2))"
       ]
     },
@@ -1055,21 +1098,14 @@
         "id": "cleanup:pipelines"
       },
       "source": [
-        "# Cleaning up\n",
+        "## Cleaning up\n",
         "\n",
         "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
         "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
         "\n",
-        "Otherwise, you can delete the individual resources you created in this tutorial -- *Note:* this is auto-generated and not all resources may be applicable for this tutorial:\n",
+        "Otherwise, you can delete the individual resources you created in this tutorial:\n",
         "\n",
-        "- Dataset\n",
-        "- Pipeline\n",
-        "- Model\n",
-        "- Endpoint\n",
-        "- Batch Job\n",
-        "- Custom Job\n",
-        "- Hyperparameter Tuning Job\n",
-        "- Cloud Storage Bucket"
+        "(Set `delete_bucket` to **True** to delete the Cloud Storage bucket.)"
       ]
     },
     {
@@ -1080,94 +1116,37 @@
       },
       "outputs": [],
       "source": [
-        "delete_dataset = True\n",
-        "delete_pipeline = True\n",
-        "delete_model = True\n",
-        "delete_endpoint = True\n",
-        "delete_batchjob = True\n",
-        "delete_customjob = True\n",
-        "delete_hptjob = True\n",
-        "delete_bucket = True\n",
+        "delete_bucket = False\n",
         "\n",
-        "try:\n",
-        "    if delete_model and \"DISPLAY_NAME\" in globals():\n",
-        "        models = aip.Model.list(\n",
-        "            filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
-        "        )\n",
-        "        model = models[0]\n",
-        "        aip.Model.delete(model)\n",
-        "        print(\"Deleted model:\", model)\n",
-        "except Exception as e:\n",
-        "    print(e)\n",
+        "# Delete the Vertex AI Pipeline Job\n",
+        "job.delete()\n",
         "\n",
-        "try:\n",
-        "    if delete_endpoint and \"DISPLAY_NAME\" in globals():\n",
-        "        endpoints = aip.Endpoint.list(\n",
-        "            filter=f\"display_name={DISPLAY_NAME}_endpoint\", order_by=\"create_time\"\n",
-        "        )\n",
-        "        endpoint = endpoints[0]\n",
-        "        endpoint.undeploy_all()\n",
-        "        aip.Endpoint.delete(endpoint.resource_name)\n",
-        "        print(\"Deleted endpoint:\", endpoint)\n",
-        "except Exception as e:\n",
-        "    print(e)\n",
+        "# Delete the Vertex AI Endpoint\n",
+        "endpoints = aiplatform.Endpoint.list(\n",
+        "    filter=f\"display_name={ENDPOINT_DISPLAY_NAME}\", order_by=\"create_time\"\n",
+        ")\n",
         "\n",
-        "if delete_dataset and \"DISPLAY_NAME\" in globals():\n",
-        "    if \"tabular\" == \"tabular\":\n",
-        "        try:\n",
-        "            datasets = aip.TabularDataset.list(\n",
-        "                filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
-        "            )\n",
-        "            dataset = datasets[0]\n",
-        "            aip.TabularDataset.delete(dataset.resource_name)\n",
-        "            print(\"Deleted dataset:\", dataset)\n",
-        "        except Exception as e:\n",
-        "            print(e)\n",
+        "if len(endpoints) > 0:\n",
+        "    endpoint = endpoints[0]\n",
+        "    endpoint.delete(force=True)\n",
         "\n",
-        "    if \"tabular\" == \"image\":\n",
-        "        try:\n",
-        "            datasets = aip.ImageDataset.list(\n",
-        "                filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
-        "            )\n",
-        "            dataset = datasets[0]\n",
-        "            aip.ImageDataset.delete(dataset.resource_name)\n",
-        "            print(\"Deleted dataset:\", dataset)\n",
-        "        except Exception as e:\n",
-        "            print(e)\n",
+        "# Delete the Vertex AI model\n",
+        "models = aiplatform.Model.list(\n",
+        "    filter=f\"display_name={MODEL_DISPLAY_NAME}\", order_by=\"create_time\"\n",
+        ")\n",
+        "if len(models) > 0:\n",
+        "    model = models[0]\n",
+        "    model.delete()\n",
         "\n",
-        "    if \"tabular\" == \"text\":\n",
-        "        try:\n",
-        "            datasets = aip.TextDataset.list(\n",
-        "                filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
-        "            )\n",
-        "            dataset = datasets[0]\n",
-        "            aip.TextDataset.delete(dataset.resource_name)\n",
-        "            print(\"Deleted dataset:\", dataset)\n",
-        "        except Exception as e:\n",
-        "            print(e)\n",
-        "\n",
-        "    if \"tabular\" == \"video\":\n",
-        "        try:\n",
-        "            datasets = aip.VideoDataset.list(\n",
-        "                filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
-        "            )\n",
-        "            dataset = datasets[0]\n",
-        "            aip.VideoDataset.delete(dataset.resource_name)\n",
-        "            print(\"Deleted dataset:\", dataset)\n",
-        "        except Exception as e:\n",
-        "            print(e)\n",
-        "\n",
-        "try:\n",
-        "    if delete_pipeline and \"DISPLAY_NAME\" in globals():\n",
-        "        pipelines = aip.PipelineJob.list(\n",
-        "            filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
-        "        )\n",
-        "        pipeline = pipelines[0]\n",
-        "        aip.PipelineJob.delete(pipeline.resource_name)\n",
-        "        print(\"Deleted pipeline:\", pipeline)\n",
-        "except Exception as e:\n",
-        "    print(e)\n",
+        "# Delete the Vertex AI Dataset\n",
+        "datasets = aiplatform.TabularDataset.list(\n",
+        "    filter=f\"display_name={DATASET_DISPLAY_NAME}\", order_by=\"create_time\"\n",
+        ")\n",
+        "if len(datasets) > 0:\n",
+        "    dataset = datasets[0]\n",
+        "    dataset.delete()\n",
         "\n",
+        "# Delete the Cloud Storage bucket\n",
         "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
         "    ! gsutil rm -r $BUCKET_URI"
       ]

notebooks/official/pipelines/custom_model_training_and_batch_prediction.ipynb

@@ -2,7 +2,7 @@
   "cells": [
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 1,
       "metadata": {
         "id": "b0b4f2bf"
       },
@@ -32,6 +32,7 @@
         "# Vertex AI Pipelines: Custom training with pre-built Google Cloud Pipeline Components\n",
         "\n",
         "<table align=\"left\">\n",
+        "\n",
         "  <td>\n",
         "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/custom_model_training_and_batch_prediction.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
@@ -44,11 +45,11 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/custom_model_training_and_batch_prediction.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooksofficial/pipelines/custom_model_training_and_batch_prediction.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
-        "     </a>\n",
-        "  </td>\n",
+        "    </a>\n",
+        "  </td>                                                                                               \n",
         "</table>\n",
         "<br/><br/><br/>"
       ]
@@ -65,17 +66,6 @@
         "This tutorial demonstrates how to use Vertex AI Pipelines with pre-built Google Cloud Pipeline Components for custom training."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "57139e75264f"
-      },
-      "source": [
-        "### Dataset\n",
-        "\n",
-        "The dataset used for this tutorial is the [CIFAR10 dataset](https://www.tensorflow.org/datasets/catalog/cifar10) from [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/overview). The version of the dataset you will use is built into TensorFlow. The trained model predicts which type of class an image is from ten classes: airplane, automobile, bird, cat, deer, dog, frog, horse, ship, or truck."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -107,6 +97,17 @@
         "Learn more about [Google Cloud Pipeline Components](https://cloud.google.com/vertex-ai/docs/pipelines/build-pipeline)."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "57139e75264f"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the [CIFAR10 dataset](https://www.tensorflow.org/datasets/catalog/cifar10) from [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/overview). The version of the dataset you will use is built into TensorFlow. The trained model predicts which type of class an image is from ten classes: airplane, automobile, bird, cat, deer, dog, frog, horse, ship, or truck."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -196,8 +197,8 @@
         "! pip3 install -U google-cloud-storage {USER_FLAG} -q\n",
         "! pip3 install {USER_FLAG} kfp google-cloud-pipeline-components --upgrade -q\n",
         "\n",
-        "if os.getenv(\"IS_TESTING\"):\n",
-        "    ! pip3 install --upgrade --force-reinstall $USER_FLAG tensorflow==2.5 kfp google-cloud-aiplatform google-cloud-storage google-cloud-pipeline-components"
+        "\n",
+        "! pip3 install --upgrade --force-reinstall $USER_FLAG tensorflow kfp google-cloud-aiplatform google-cloud-storage google-cloud-pipeline-components -q"
       ]
     },
     {
@@ -213,7 +214,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 3,
       "metadata": {
         "id": "restart"
       },
@@ -261,7 +262,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 1,
       "metadata": {
         "id": "set_project_id"
       },
@@ -318,13 +319,16 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 1,
       "metadata": {
         "id": "region"
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -333,22 +337,29 @@
         "id": "timestamp"
       },
       "source": [
-        "#### Timestamp\n",
+        "#### UUID\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 6,
       "metadata": {
         "id": "wJjft8z1IA81"
       },
       "outputs": [],
       "source": [
-        "from datetime import datetime\n",
+        "import random\n",
+        "import string\n",
         "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+        "\n",
+        "# Generate a uuid of a specifed length(default=8)\n",
+        "def generate_uuid(length: int = 8) -> str:\n",
+        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
+        "\n",
+        "\n",
+        "UUID = generate_uuid()"
       ]
     },
     {
@@ -435,7 +446,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 7,
       "metadata": {
         "id": "bucket"
       },
@@ -447,14 +458,14 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 8,
       "metadata": {
         "id": "autoset_bucket"
       },
       "outputs": [],
       "source": [
         "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
         "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
       ]
     },
@@ -489,7 +500,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 10,
       "metadata": {
         "id": "validate_bucket"
       },
@@ -511,7 +522,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 11,
       "metadata": {
         "id": "set_service_account"
       },
@@ -559,7 +570,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 15,
       "metadata": {
         "id": "set_service_account:pipelines"
       },
@@ -591,7 +602,6 @@
       "outputs": [],
       "source": [
         "import google.cloud.aiplatform as aip\n",
-        "import tensorflow as tf\n",
         "from google_cloud_pipeline_components.experimental.custom_job import utils\n",
         "from kfp.v2 import compiler, dsl\n",
         "from kfp.v2.dsl import component"
@@ -610,7 +620,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 17,
       "metadata": {
         "id": "pipeline_constants"
       },
@@ -632,7 +642,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 18,
       "metadata": {
         "id": "yX-aimhGRRRl"
       },
@@ -665,7 +675,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 19,
       "metadata": {
         "id": "ipG9uBUDRRRm"
       },
@@ -835,12 +845,12 @@
         "\n",
         "#### Package Assembly\n",
         "\n",
-        "In the following cells, you will assemble the training package."
+        "In the following cells, you assemble the training package."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 22,
       "metadata": {
         "id": "YpA6MFcLRRRn"
       },
@@ -880,7 +890,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 23,
       "metadata": {
         "id": "2etUCVVMRRRo"
       },
@@ -1096,12 +1106,14 @@
       "source": [
         "### Convert the component to a Vertex AI Custom Job\n",
         "\n",
-        "Next, use the `create_custom_training_job_op_from_component` method to convert the custom component into a Vertex AI Custom Job pre-built component."
+        "Next, use the `create_custom_training_job_op_from_component` method to convert the custom component into a Vertex AI Custom Job pre-built component.\n",
+        "\n",
+        "**replica_count :**  The  number of machine replicas the batch operation may be scaled to. Only used if machine_type is set. Default is 10."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 24,
       "metadata": {
         "id": "deb6c1cc"
       },
@@ -1129,7 +1141,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 25,
       "metadata": {
         "id": "5R23d2J_HJr-"
       },
@@ -1219,7 +1231,7 @@
         "    pipeline_func=pipeline, package_path=\"custom_model_training_spec.json\"\n",
         ")\n",
         "\n",
-        "DISPLAY_NAME = \"cifar10_\" + TIMESTAMP\n",
+        "DISPLAY_NAME = \"cifar10_\" + UUID\n",
         "\n",
         "job = aip.PipelineJob(\n",
         "    display_name=DISPLAY_NAME,\n",
@@ -1240,7 +1252,7 @@
       "source": [
         "### View custom training pipeline results\n",
         "\n",
-        "Finally, you will view the artifact outputs of each task in the pipeline."
+        "Finally, you view the artifact outputs of each task in the pipeline."
       ]
     },
     {
@@ -1253,6 +1265,8 @@
       "source": [
         "import json\n",
         "\n",
+        "import tensorflow as tf\n",
+        "\n",
         "PROJECT_NUMBER = job.gca_resource.name.split(\"/\")[1]\n",
         "print(PROJECT_NUMBER)\n",
         "\n",
@@ -1358,6 +1372,7 @@
         "batch_job = aip.BatchPredictionJob(batch_job_id)\n",
         "batch_job.delete()\n",
         "\n",
+        "\n",
         "delete_bucket = False\n",
         "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
         "    ! gsutil rm -rf {BUCKET_URI}"

notebooks/official/pipelines/google_cloud_pipeline_components_automl_tabular.ipynb

@@ -64,19 +64,6 @@
         "This notebook shows how to use the components defined in [`google_cloud_pipeline_components`](https://github.com/kubeflow/pipelines/tree/master/components/google-cloud) to build an AutoML tabular regression workflow on [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines)."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "dataset:cal_housing,lrg"
-      },
-      "source": [
-        "### Dataset\n",
-        "\n",
-        "The dataset used for this tutorial is the [California Housing dataset from the 1990 Census](https://developers.google.com/machine-learning/crash-course/california-housing-data-description)\n",
-        "\n",
-        "The dataset predicts the median house price."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -109,6 +96,19 @@
         "The components are [documented here](https://google-cloud-pipeline-components.readthedocs.io/en/latest/google_cloud_pipeline_components.aiplatform.html#module-google_cloud_pipeline_components.aiplatform)."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "77de5c53ac91"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the [California Housing dataset from the 1990 Census](https://developers.google.com/machine-learning/crash-course/california-housing-data-description)\n",
+        "\n",
+        "The dataset predicts the median house price."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -234,6 +234,8 @@
         "id": "check_versions"
       },
       "source": [
+        "### Check installed package versions\n",
+        "\n",
         "Check the versions of the packages you installed.  The KFP SDK version should be >=1.6."
       ]
     },
@@ -344,7 +346,10 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -353,9 +358,9 @@
         "id": "timestamp"
       },
       "source": [
-        "#### Timestamp\n",
+        "#### UUID\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append the uuid onto the name of resources you create in this tutorial."
       ]
     },
     {
@@ -366,9 +371,16 @@
       },
       "outputs": [],
       "source": [
-        "from datetime import datetime\n",
+        "import random\n",
+        "import string\n",
         "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+        "\n",
+        "# Generate a uuid of length 8\n",
+        "def generate_uuid():\n",
+        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=8))\n",
+        "\n",
+        "\n",
+        "UUID = generate_uuid()"
       ]
     },
     {
@@ -474,7 +486,7 @@
       "outputs": [],
       "source": [
         "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
         "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
       ]
     },
@@ -718,10 +730,10 @@
         "    endpoint_op = EndpointCreateOp(\n",
         "        project=project,\n",
         "        location=region,\n",
-        "        display_name=\"train-automl-flowers\",\n",
+        "        display_name=\"train-automl-cal_housing_endpoint\",\n",
         "    )\n",
         "\n",
-        "    ModelDeployOp(\n",
+        "    _ = ModelDeployOp(\n",
         "        model=training_op.outputs[\"model\"],\n",
         "        endpoint=endpoint_op.outputs[\"endpoint\"],\n",
         "        dedicated_resources_machine_type=\"n1-standard-4\",\n",
@@ -753,7 +765,7 @@
         "\n",
         "compiler.Compiler().compile(\n",
         "    pipeline_func=pipeline,\n",
-        "    package_path=\"tabular regression_pipeline.json\".replace(\" \", \"_\"),\n",
+        "    package_path=\"tabular_regression_pipeline.json\",\n",
         ")"
       ]
     },
@@ -776,11 +788,11 @@
       },
       "outputs": [],
       "source": [
-        "DISPLAY_NAME = \"cal_housing_\" + TIMESTAMP\n",
+        "DISPLAY_NAME = \"cal_housing_\" + UUID\n",
         "\n",
         "job = aip.PipelineJob(\n",
         "    display_name=DISPLAY_NAME,\n",
-        "    template_path=\"tabular regression_pipeline.json\".replace(\" \", \"_\"),\n",
+        "    template_path=\"tabular_regression_pipeline.json\",\n",
         "    pipeline_root=PIPELINE_ROOT,\n",
         "    enable_caching=False,\n",
         ")\n",
@@ -818,16 +830,7 @@
         "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
         "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
         "\n",
-        "Otherwise, you can delete the individual resources you created in this tutorial -- *Note:* this is auto-generated and not all resources may be applicable for this tutorial:\n",
-        "\n",
-        "- Dataset\n",
-        "- Pipeline\n",
-        "- Model\n",
-        "- Endpoint\n",
-        "- Batch Job\n",
-        "- Custom Job\n",
-        "- Hyperparameter Tuning Job\n",
-        "- Cloud Storage Bucket"
+        "Otherwise, you can delete the individual resources you created in this tutorial "
       ]
     },
     {
@@ -842,89 +845,73 @@
         "delete_pipeline = True\n",
         "delete_model = True\n",
         "delete_endpoint = True\n",
-        "delete_batchjob = True\n",
-        "delete_customjob = True\n",
-        "delete_hptjob = True\n",
         "delete_bucket = True\n",
         "\n",
-        "try:\n",
-        "    if delete_model and \"DISPLAY_NAME\" in globals():\n",
-        "        models = aip.Model.list(\n",
-        "            filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
-        "        )\n",
-        "        model = models[0]\n",
-        "        aip.Model.delete(model)\n",
-        "        print(\"Deleted model:\", model)\n",
-        "except Exception as e:\n",
-        "    print(e)\n",
+        "dataset_display_name = \"housing\"\n",
+        "pipeline_display_name = \"automl-tab-training-v2\"\n",
+        "model_display_name = \"train-automl-cal_housing\"\n",
+        "endpoint_display_name = \"train-automl-cal_housing_endpoint\"\n",
         "\n",
-        "try:\n",
-        "    if delete_endpoint and \"DISPLAY_NAME\" in globals():\n",
-        "        endpoints = aip.Endpoint.list(\n",
-        "            filter=f\"display_name={DISPLAY_NAME}_endpoint\", order_by=\"create_time\"\n",
-        "        )\n",
+        "\n",
+        "if delete_endpoint:\n",
+        "    endpoints = aip.Endpoint.list(\n",
+        "        filter=f\"display_name={endpoint_display_name}\", order_by=\"create_time\"\n",
+        "    )\n",
+        "    if endpoints:\n",
         "        endpoint = endpoints[0]\n",
         "        endpoint.undeploy_all()\n",
-        "        aip.Endpoint.delete(endpoint.resource_name)\n",
+        "        endpoint.delete()\n",
         "        print(\"Deleted endpoint:\", endpoint)\n",
-        "except Exception as e:\n",
-        "    print(e)\n",
         "\n",
-        "if delete_dataset and \"DISPLAY_NAME\" in globals():\n",
+        "if delete_model:\n",
+        "    models = aip.Model.list(\n",
+        "        filter=f\"display_name={model_display_name}\", order_by=\"create_time\"\n",
+        "    )\n",
+        "    if models:\n",
+        "        model = models[0]\n",
+        "        model.delete()\n",
+        "        print(\"Deleted model:\", model)\n",
+        "\n",
+        "if delete_dataset:\n",
         "    if \"tabular\" == \"tabular\":\n",
-        "        try:\n",
-        "            datasets = aip.TabularDataset.list(\n",
-        "                filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
-        "            )\n",
+        "        datasets = aip.TabularDataset.list(\n",
+        "            filter=f\"display_name={dataset_display_name}\", order_by=\"create_time\"\n",
+        "        )\n",
+        "        if datasets:\n",
         "            dataset = datasets[0]\n",
-        "            aip.TabularDataset.delete(dataset.resource_name)\n",
+        "            dataset.delete()\n",
         "            print(\"Deleted dataset:\", dataset)\n",
-        "        except Exception as e:\n",
-        "            print(e)\n",
         "\n",
         "    if \"tabular\" == \"image\":\n",
-        "        try:\n",
-        "            datasets = aip.ImageDataset.list(\n",
-        "                filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
-        "            )\n",
+        "        datasets = aip.ImageDataset.list(\n",
+        "            filter=f\"display_name={dataset_display_name}\", order_by=\"create_time\"\n",
+        "        )\n",
+        "        if datasets:\n",
         "            dataset = datasets[0]\n",
-        "            aip.ImageDataset.delete(dataset.resource_name)\n",
+        "            dataset.delete()\n",
         "            print(\"Deleted dataset:\", dataset)\n",
-        "        except Exception as e:\n",
-        "            print(e)\n",
         "\n",
         "    if \"tabular\" == \"text\":\n",
-        "        try:\n",
-        "            datasets = aip.TextDataset.list(\n",
-        "                filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
-        "            )\n",
+        "        datasets = aip.TextDataset.list(\n",
+        "            filter=f\"display_name={dataset_display_name}\", order_by=\"create_time\"\n",
+        "        )\n",
+        "        if datasets:\n",
         "            dataset = datasets[0]\n",
-        "            aip.TextDataset.delete(dataset.resource_name)\n",
+        "            dataset.delete()\n",
         "            print(\"Deleted dataset:\", dataset)\n",
-        "        except Exception as e:\n",
-        "            print(e)\n",
         "\n",
         "    if \"tabular\" == \"video\":\n",
-        "        try:\n",
-        "            datasets = aip.VideoDataset.list(\n",
-        "                filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
-        "            )\n",
-        "            dataset = datasets[0]\n",
-        "            aip.VideoDataset.delete(dataset.resource_name)\n",
-        "            print(\"Deleted dataset:\", dataset)\n",
-        "        except Exception as e:\n",
-        "            print(e)\n",
-        "\n",
-        "try:\n",
-        "    if delete_pipeline and \"DISPLAY_NAME\" in globals():\n",
-        "        pipelines = aip.PipelineJob.list(\n",
-        "            filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
+        "        datasets = aip.VideoDataset.list(\n",
+        "            filter=f\"display_name={dataset_display_name}\", order_by=\"create_time\"\n",
         "        )\n",
-        "        pipeline = pipelines[0]\n",
-        "        aip.PipelineJob.delete(pipeline.resource_name)\n",
-        "        print(\"Deleted pipeline:\", pipeline)\n",
-        "except Exception as e:\n",
-        "    print(e)\n",
+        "        if datasets:\n",
+        "            dataset = datasets[0]\n",
+        "            dataset.delete()\n",
+        "            print(\"Deleted dataset:\", dataset)\n",
+        "\n",
+        "if delete_pipeline:\n",
+        "    job.delete()\n",
+        "\n",
         "\n",
         "if delete_bucket and os.getenv(\"IS_TESTING\"):\n",
         "    ! gsutil rm -r $BUCKET_URI"

notebooks/official/pipelines/google_cloud_pipeline_components_bqml_text.ipynb

@@ -29,21 +29,24 @@
         "id": "JAPoU8Sm5E6e"
       },
       "source": [
+        "# Training an acquisition-prediction model using Swivel, BigQuery ML and Vertex AI Pipelines\n",
+        "\n",
         "<table align=\"left\">\n",
         "\n",
         "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/official/pipelines/google_cloud_pipeline_components_bqml_text.ipynb\"\">\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/google_cloud_pipeline_components_bqml_text.ipynb\"\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/notebooks/official/pipelines/google_cloud_pipeline_components_bqml_text.ipynb\">\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/google_cloud_pipeline_components_bqml_text.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/notebooks/official/pipelines/google_cloud_pipeline_components_bqml_text.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/google_cloud_pipeline_components_bqml_text.ipynb\">\n",
+        "    <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
         "  </td>\n",
@@ -55,33 +58,67 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "tvgnzT1CKxrO"
+        "id": "991ab00f3d75"
       },
       "source": [
         "## Overview\n",
         "\n",
-        "This notebooks shows the DataflowPythonJobOp and the main BQML components in a Text Categorization Vertex AI Pipeline. \n",
+        "This notebook demonstrates the usage of `DataflowPythonJobOp` and BigQuery ML components through buidling a Text Categorization model and running it on Vertex AI Pipelines. \n",
         "\n",
-        "The pipeline will \n",
+        "The steps performed include:\n",
         "\n",
-        "1. Read raw text (HTML) documents stored in Google Cloud Storage\n",
-        "2. Extract title, content and topic of (HTML) documents using Dataflow and ingest into BigQuery\n",
-        "3. Apply the Swivel model to generate embeddings of our document’s content\n",
+        "1. Read raw text (HTML) documents stored in Google Cloud Storage.\n",
+        "2. Extract title, content and topic of (HTML) documents using Dataflow and ingest into BigQuery.\n",
+        "3. Apply the Swivel model to generate embeddings of your document’s content.\n",
         "4. Train a Logistic regression model to classify if an article is about corporate acquisitions (`acq` category). \n",
-        "5. Evaluate the model \n",
-        "6. Apply the model to a dataset in order to generate predictions\n",
-        "\n",
-        "### Dataset\n",
-        "\n",
-        "The dataset is [Reuters-21578 Text Categorization Collection Data Set](https://archive.ics.uci.edu/ml/datasets/reuters-21578+text+categorization+collection).\n",
-        "\n",
-        "The dataset is a collection of publicly available news articles appeared on the Reuters newswire in 1987. They were assembled and indexed with categories by personnel from Reuters Ltd. and Carnegie Group, Inc. in 1987.\n",
-        "\n",
+        "5. Evaluate the model.\n",
+        "6. Apply the model to a dataset in order to generate predictions."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "acc98a3361cc"
+      },
+      "source": [
         "### Objective\n",
         "\n",
-        "In this notebook, you will learn how to build a simple BigQuery ML pipeline on Vertex AI pipeline in order to calculate text embeddings of articles' content and classify them\n",
-        "depending the *corporate acquisitions* category.\n",
+        "In this notebook, you learn how to build a simple BigQuery ML pipeline using Vertex AI pipelines in order to calculate text embeddings of content from articles and classify them\n",
+        "into the *corporate acquisitions* category.\n",
         "\n",
+        "\n",
+        "This tutorial uses the following Google Cloud ML services and resources:\n",
+        "\n",
+        "- Vertex AI Pipelines\n",
+        "- BigQuery ML\n",
+        "\n",
+        "\n",
+        "The steps performed include:\n",
+        "\n",
+        "- Creating a component for Dataflow job that ingests data to BigQuery.\n",
+        "- Creating a component for preprocessing steps to run on the data in BigQuery.\n",
+        "- Creating a component for training a logistic regression model using BigQuery ML.\n",
+        "- Building and configuring a Kubeflow DSL pipeline with all the created components.\n",
+        "- Compiling and running the pipeline in Vertex AI Pipelines."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9bc6d52899ba"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used in this notebook is the [Reuters-21578 Text Categorization Collection Data Set](https://archive.ics.uci.edu/ml/datasets/reuters-21578+text+categorization+collection). This dataset is a collection of publicly available news articles appeared on the Reuters newswire in 1987. They were assembled and indexed with categories by personnel from Reuters Ltd. and Carnegie Group, Inc. in 1987."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tvgnzT1CKxrO"
+      },
+      "source": [
         "### Costs \n",
         "\n",
         "This tutorial uses billable components of Google Cloud:\n",
@@ -89,19 +126,15 @@
         "* Vertex AI\n",
         "* Cloud Storage\n",
         "* BigQuery\n",
-        "* Dataflow"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "ze4-nDLfK4pw"
-      },
-      "source": [
-        "### Set up your local development environment\n",
+        "* Dataflow\n",
         "\n",
-        "**If you are using Colab or Google Cloud Notebooks**, your environment already meets\n",
-        "all the requirements to run this notebook. You can skip this step."
+        "Learn about [Vertex AI\n",
+        "pricing](https://cloud.google.com/vertex-ai/pricing), [Cloud Storage\n",
+        "pricing](https://cloud.google.com/storage/pricing), [BigQuery\n",
+        "pricing](https://cloud.google.com/bigquery/pricing), [Dataflow\n",
+        "pricing](https://cloud.google.com/dataflow/pricing) and use the [Pricing\n",
+        "Calculator](https://cloud.google.com/products/calculator/)\n",
+        "to generate a cost estimate based on your projected usage."
       ]
     },
     {
@@ -110,6 +143,11 @@
         "id": "gCuSR8GkAgzl"
       },
       "source": [
+        "### Set up your local development environment\n",
+        "\n",
+        "**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets\n",
+        "all the requirements to run this notebook. You can skip this step.\n",
+        "\n",
         "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
         "You need the following:\n",
         "\n",
@@ -147,7 +185,7 @@
         "id": "i7EUnXsZhAGF"
       },
       "source": [
-        "### Install additional packages\n",
+        "## Install additional packages\n",
         "\n",
         "Install additional package dependencies not installed in your notebook environment, such as Vertex AI SDK. Use the latest major GA version of each package."
       ]
@@ -162,43 +200,26 @@
       "source": [
         "import os\n",
         "\n",
-        "# The Google Cloud Notebook product has specific requirements\n",
-        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
         "\n",
-        "# Google Cloud Notebook requires dependencies to be installed with '--user'\n",
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
         "USER_FLAG = \"\"\n",
-        "if IS_GOOGLE_CLOUD_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\""
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "e6d33c55a3c5"
-      },
-      "outputs": [],
-      "source": [
-        "if os.getenv(\"IS_TESTING\"):\n",
-        "    ! touch /builder/home/.local/lib/python3.9/site-packages/google_api_core-2.7.1.dist-info/METADATA"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "wyy5Lbnzg5fi"
-      },
-      "outputs": [],
-      "source": [
-        "! pip3 install {USER_FLAG} --upgrade \"apache-beam[gcp]==2.36.0\"\n",
-        "! pip3 install {USER_FLAG} --upgrade \"bs4==0.0.1\"\n",
-        "! pip3 install {USER_FLAG} --upgrade \"nltk==3.7\"\n",
-        "! pip3 install {USER_FLAG} --upgrade \"tensorflow<2.8.0\"\n",
-        "! pip3 install {USER_FLAG} --upgrade \"tensorflow-hub==0.12.0\"\n",
-        "! pip3 install {USER_FLAG} --upgrade \"kfp==1.8.2\"\n",
-        "! pip3 install {USER_FLAG} --upgrade \"google-cloud-aiplatform==1.10.0\"\n",
-        "! pip3 install {USER_FLAG} --upgrade \"google_cloud_pipeline_components==1.0.1\""
+        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\"\n",
+        "     \n",
+        "# Install dependencies\n",
+        "! pip3 install {USER_FLAG} --upgrade google-cloud-aiplatform==1.10.0 \\\n",
+        "                                    google_cloud_pipeline_components==1.0.1 \\\n",
+        "                                    google-api-core==2.8.2 \\\n",
+        "                                    google-auth==1.35.0 -q\n",
+        "\n",
+        "! pip3 install {USER_FLAG} --upgrade tensorflow==2.8.0 \\\n",
+        "                                    tensorflow-hub==0.12.0 \\\n",
+        "                                    kfp==1.8.9 -q"
       ]
     },
     {
@@ -254,9 +275,9 @@
         "\n",
         "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
         "\n",
-        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com)\n",
+        "1. [Enable the required APIs](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,bigquery.googleapis.com,dataflow.googleapis.com,storage-component.googleapis.com).\n",
         "\n",
-        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
         "\n",
         "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
         "Cloud SDK uses the right project for all the commands in this notebook.\n",
@@ -279,40 +300,26 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "oM1iC_MfAts1"
+        "id": "cd85f5c794e5"
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
-        "PROJECT_ID = \"\"\n",
-        "\n",
-        "# Get your Google Cloud project ID from gcloud\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    shell_output = !gcloud config list --format 'value(core.project)' 2>/dev/null\n",
-        "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID: \", PROJECT_ID)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "qJYoRfYng0XZ"
-      },
-      "source": [
-        "Otherwise, set your project ID here."
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "riG_qUokg0XZ"
+        "id": "oM1iC_MfAts1"
       },
       "outputs": [],
       "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
-        "    PROJECT_ID = \"\"  # @param {type:\"string\"}"
+        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
+        "    # Get your GCP project id from gcloud\n",
+        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "    PROJECT_ID = shell_output[0]\n",
+        "    print(\"Project ID:\", PROJECT_ID)"
       ]
     },
     {
@@ -323,7 +330,41 @@
       },
       "outputs": [],
       "source": [
-        "!gcloud config set project $PROJECT_ID"
+        "! gcloud config set project $PROJECT_ID"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "264543a144ad"
+      },
+      "source": [
+        "#### Region\n",
+        "\n",
+        "You can also change the `REGION` variable, which is used for operations\n",
+        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. It is recommended that you choose the region closest to you.\n",
+        "\n",
+        "- Americas: `us-central1`\n",
+        "- Europe: `europe-west4`\n",
+        "- Asia Pacific: `asia-east1`\n",
+        "\n",
+        "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
+        "\n",
+        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3281bedf6d3c"
+      },
+      "outputs": [],
+      "source": [
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -332,9 +373,9 @@
         "id": "06571eb4063b"
       },
       "source": [
-        "#### Timestamp\n",
+        "#### UUID\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append it onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
       ]
     },
     {
@@ -345,9 +386,16 @@
       },
       "outputs": [],
       "source": [
-        "from datetime import datetime\n",
+        "import random\n",
+        "import string\n",
         "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+        "\n",
+        "# Generate a uuid of length 8\n",
+        "def generate_uuid():\n",
+        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=8))\n",
+        "\n",
+        "\n",
+        "UUID = generate_uuid()"
       ]
     },
     {
@@ -358,16 +406,9 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Google Cloud Notebooks**, your environment is already\n",
-        "authenticated. Skip this step."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "sBCra4QMA2wR"
-      },
-      "source": [
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
+        "authenticated.\n",
+        "\n",
         "**If you are using Colab**, run the cell below and follow the instructions\n",
         "when prompted to authenticate your account via oAuth.\n",
         "\n",
@@ -400,19 +441,19 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "import sys\n",
-        "\n",
         "# If you are running this notebook in Colab, run this cell and follow the\n",
         "# instructions to authenticate your GCP account. This provides access to your\n",
         "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
         "# requests.\n",
         "\n",
-        "# The Google Cloud Notebook product has specific requirements\n",
-        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
+        "import os\n",
+        "import sys\n",
         "\n",
-        "# If on Google Cloud Notebooks, then don't execute this code\n",
-        "if not IS_GOOGLE_CLOUD_NOTEBOOK:\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = \"google.colab\" in sys.modules\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
         "    if \"google.colab\" in sys.modules:\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
@@ -435,12 +476,10 @@
         "\n",
         "**The following steps are required, regardless of your notebook environment.**\n",
         "\n",
-        "Set the name of your Cloud Storage bucket below. It must be unique across all\n",
-        "Cloud Storage buckets.\n",
+        "When you use services like Vertex AI and Dataflow, you need to specify Cloud Storage bucket paths as staging locations. Cloud Storage bucket is used to save the artifacts that are required or that are generated while using the services.\n",
         "\n",
-        "You may also change the `REGION` variable, which is used for operations\n",
-        "throughout the rest of this notebook. We suggest that you [choose a region where Vertex AI services are\n",
-        "available](https://cloud.google.com/vertex-ai/docs/general/locations#available_regions)."
+        "Set the name of your Cloud Storage bucket below. It must be unique across all\n",
+        "Cloud Storage buckets."
       ]
     },
     {
@@ -451,8 +490,8 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_URI = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "REGION = \"[your-region]\"  # @param {type:\"string\"}"
+        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
     {
@@ -463,11 +502,9 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"-aip-\" + TIMESTAMP\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
+        "    BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
     {
@@ -510,6 +547,78 @@
         "! gsutil ls -al $BUCKET_URI"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3708bd0b1855"
+      },
+      "source": [
+        "#### Service Account\n",
+        "\n",
+        "You use a service account to create Vertex AI Pipeline jobs. If you do not want to use your project's Compute Engine service account, set `SERVICE_ACCOUNT` to another service account ID."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "199a32a35466"
+      },
+      "outputs": [],
+      "source": [
+        "SERVICE_ACCOUNT = \"[your-service-account]\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "abb872bb98c1"
+      },
+      "outputs": [],
+      "source": [
+        "if (\n",
+        "    SERVICE_ACCOUNT == \"\"\n",
+        "    or SERVICE_ACCOUNT is None\n",
+        "    or SERVICE_ACCOUNT == \"[your-service-account]\"\n",
+        "):\n",
+        "    # Get your service account from gcloud\n",
+        "    if not IS_COLAB:\n",
+        "        shell_output = !gcloud auth list 2>/dev/null\n",
+        "        SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
+        "\n",
+        "    else:  # IS_COLAB:\n",
+        "        shell_output = ! gcloud projects describe  $PROJECT_ID\n",
+        "        project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n",
+        "        SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n",
+        "\n",
+        "    print(\"Service Account:\", SERVICE_ACCOUNT)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "4a63f2d1cd52"
+      },
+      "source": [
+        "#### Set service account access for Vertex AI Pipelines\n",
+        "\n",
+        "Run the following commands to grant your service account access to read and write pipeline artifacts in the bucket that you created in the previous step. You only need to run this step once per service account."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "749c598c5f5d"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
+        "\n",
+        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -552,11 +661,11 @@
       "source": [
         "### Prepare input data\n",
         "\n",
-        "In the following code, you will \n",
+        "In the following cell, you:\n",
         "\n",
-        "1) Get dataset from UCI archive.\n",
-        "2) Untar the dataset\n",
-        "3) Copy the dataset to a Cloud Storage location."
+        "1) Get the dataset from UCI archive.\n",
+        "2) Untar the dataset.\n",
+        "3) Copy the dataset to the Cloud Storage location."
       ]
     },
     {
@@ -601,18 +710,16 @@
       },
       "outputs": [],
       "source": [
-        "import random\n",
         "from pathlib import Path as path\n",
         "from urllib.parse import urlparse\n",
         "\n",
         "import tensorflow_hub as hub\n",
-        "\n",
-        "os.environ[\"TFHUB_MODEL_LOAD_FORMAT\"] = \"UNCOMPRESSED\"\n",
-        "\n",
-        "import google.cloud.aiplatform as vertex_ai\n",
+        "from google.cloud import aiplatform as vertex_ai\n",
         "from kfp import dsl\n",
         "from kfp.v2 import compiler\n",
-        "from kfp.v2.dsl import component"
+        "from kfp.v2.dsl import component\n",
+        "\n",
+        "os.environ[\"TFHUB_MODEL_LOAD_FORMAT\"] = \"UNCOMPRESSED\""
       ]
     },
     {
@@ -623,7 +730,7 @@
       "source": [
         "### Define constants\n",
         "\n",
-        "About the model we are going to use in preprocessing, we use the [Swivel](https://tfhub.dev/google/tf2-preview/gnews-swivel-20dim/1) embedding which was trained on English Google News 130GB corpus and has 20 dimensions."
+        "About the model you are going to use in preprocessing, you use the [Swivel](https://tfhub.dev/google/tf2-preview/gnews-swivel-20dim/1) embedding which was trained on English Google News 130GB corpus and has 20 dimensions."
       ]
     },
     {
@@ -634,7 +741,7 @@
       },
       "outputs": [],
       "source": [
-        "JOB_NAME = f\"reuters-ingest-{TIMESTAMP}\"\n",
+        "JOB_NAME = f\"reuters-ingest-{UUID}\"\n",
         "SETUP_FILE_URI = urlparse(BUCKET_URI)._replace(path=\"setup.py\").geturl()\n",
         "RUNNER = \"DataflowRunner\"\n",
         "STAGING_LOCATION_URI = urlparse(BUCKET_URI)._replace(path=\"staging\").geturl()\n",
@@ -643,13 +750,13 @@
         "BQ_DATASET = \"mlops_bqml_text_analyisis\"\n",
         "BQ_TABLE = \"reuters_ingested\"\n",
         "MODEL_NAME = \"swivel_text_embedding_model\"\n",
-        "EMBEDDINGS_TABLE = f\"reuters_text_embeddings_{TIMESTAMP}\"\n",
+        "EMBEDDINGS_TABLE = f\"reuters_text_embeddings_{UUID}\"\n",
         "MODEL_PATH = (\n",
         "    f'{hub.resolve(\"https://tfhub.dev/google/tf2-preview/gnews-swivel-20dim/1\")}/*'\n",
         ")\n",
-        "PREPROCESSED_TABLE = f\"reuters_text_preprocessed_{TIMESTAMP}\"\n",
+        "PREPROCESSED_TABLE = f\"reuters_text_preprocessed_{UUID}\"\n",
         "CLASSIFICATION_MODEL_NAME = \"logistic_reg\"\n",
-        "PREDICT_TABLE = f\"reuters_text_predict_{TIMESTAMP}\""
+        "PREDICT_TABLE = f\"reuters_text_predict_{UUID}\""
       ]
     },
     {
@@ -658,7 +765,7 @@
         "id": "NlyOjKrjCXsI"
       },
       "source": [
-        "### Initialize client"
+        "### Initialize Vertex AI SDK client"
       ]
     },
     {
@@ -678,7 +785,9 @@
         "id": "ZrgOD30o7HcL"
       },
       "source": [
-        "## Pipeline formalization"
+        "## Pipeline formalization\n",
+        "\n",
+        "In this step, you create various components for the pipeline and build the final pipeline."
       ]
     },
     {
@@ -757,7 +866,7 @@
         "from apache_beam.options.pipeline_options import SetupOptions\n",
         "\n",
         "\n",
-        "# Helpers -------------------------------------------------------- -------------\n",
+        "# Helpers ---------------------------------------------------------------------\n",
         "\n",
         "def get_args():\n",
         "    \"\"\"\n",
@@ -949,7 +1058,7 @@
         "apache-beam[gcp]==2.36.0\n",
         "bs4==0.0.1\n",
         "nltk==3.7\n",
-        "tensorflow<2.8.0"
+        "tensorflow==2.8.0"
       ]
     },
     {
@@ -993,7 +1102,7 @@
         "REQUIRED_PACKAGES = [\n",
         "    'bs4==0.0.1',\n",
         "    'nltk==3.7',\n",
-        "    'tensorflow<2.8.0']\n",
+        "    'tensorflow==2.8.0']\n",
         "\n",
         "setuptools.setup(\n",
         "    name='ingest',\n",
@@ -1010,7 +1119,7 @@
         "id": "5Nc7ByK1AEe8"
       },
       "source": [
-        "#### Copy the setup, the python module and requirements file to Cloud Storage\n",
+        "#### Copy the setup, module and requirements files to Cloud Storage\n",
         "\n",
         "Finally, copy the Python module, requirements and setup file to your Cloud Storage bucket."
       ]
@@ -1035,15 +1144,15 @@
         "id": "MI_wYYwdAZZs"
       },
       "source": [
-        "### BQML components\n",
+        "### BigQuery ML components\n",
         "\n",
-        "To build the next steps of our pipelines, we define a set of queries to:\n",
+        "For the next steps in building the pipeline, you define a set of queries to:\n",
         "\n",
         "1) Create the BigQuery dataset schema.\n",
-        "2) Preprocess our text data and generate the embeddings using Swevel model\n",
+        "2) Preprocess your text data and generate the embeddings using Swivel model.\n",
         "2) Train the BigQuery ML Logistic Regression model.\n",
         "3) Evaluate the model.\n",
-        "4) Run a batch prediction\n"
+        "4) Run a batch prediction.\n"
       ]
     },
     {
@@ -1066,9 +1175,9 @@
         "id": "8tYtZMRiepKe"
       },
       "source": [
-        "#### Create BQ Dataset query\n",
+        "#### Create BigQuery Dataset query\n",
         "\n",
-        "With this query, we create the Bigquery dataset schema we are going to use to train our model."
+        "With this query, you create the Bigquery dataset schema that you are going to use to train your model."
       ]
     },
     {
@@ -1096,9 +1205,9 @@
         "id": "T2_kuUSZUBDY"
       },
       "source": [
-        "#### Create BQ Preprocess query\n",
+        "#### Create BigQuery Preprocess query\n",
         "\n",
-        "The following query use the TFHub Swevel model to generate the embedding of our text data and split the dataset for training and serving purposes."
+        "The following query uses the TFHub Swivel model to generate embeddings for your text data and splits the dataset for training and serving purposes."
       ]
     },
     {
@@ -1187,9 +1296,9 @@
         "id": "yF9W5x4HgUQb"
       },
       "source": [
-        "#### Create BQ Model query\n",
+        "#### Create BigQuery Model query\n",
         "\n",
-        "Below you have a simple query to build a BigQuery ML Logistic Classifier model for topic's articles classification."
+        "Below, you have a simple query to build a BigQuery ML Logistic Classifier model for topic's articles classification."
       ]
     },
     {
@@ -1224,9 +1333,9 @@
         "id": "RlLTcuUhdzFU"
       },
       "source": [
-        "#### Create BQ Prediction query\n",
+        "#### Create BigQuery Prediction query\n",
         "\n",
-        "With the following query, we run a prediction job using the table with the preprocessing query."
+        "With the following query, you run a prediction job using the table with the preprocessing query."
       ]
     },
     {
@@ -1252,7 +1361,14 @@
         "id": "OxemUVCxAiSo"
       },
       "source": [
-        "### Build Pipeline"
+        "### Build the pipeline\n",
+        "\n",
+        "In this step, you build the pipeline using the individual components.\n",
+        "\n",
+        "Define the `JOB_NAME` and `JOB_CONFIG` below. `JOB_CONFIG` consists of the following parameters for the destination table:\n",
+        "- `PROJECT_ID`: Id of the project.\n",
+        "- `BQ_DATASET`: Id of the BigQuery dataset.\n",
+        "- `PREDICT_TABLE`: Id of the BigQuery table where predictions are stored."
       ]
     },
     {
@@ -1264,7 +1380,7 @@
       "outputs": [],
       "source": [
         "ID = random.randint(1, 10000)\n",
-        "JOB_NAME = f\"reuters-preprocess-{TIMESTAMP}-{ID}\"\n",
+        "JOB_NAME = f\"reuters-preprocess-{UUID}-{ID}\"\n",
         "JOB_CONFIG = {\n",
         "    \"destinationTable\": {\n",
         "        \"projectId\": PROJECT_ID,\n",
@@ -1280,7 +1396,9 @@
         "id": "mdO8st_gLKBZ"
       },
       "source": [
-        "#### Create a custom component to pass `DataflowPythonJobOp` arguments"
+        "#### Create a custom component for arguments\n",
+        "\n",
+        "Next, you create a component to pass arguments to the `DataflowPythonJobOp` component."
       ]
     },
     {
@@ -1323,7 +1441,26 @@
         "id": "pcSL1FHk69KT"
       },
       "source": [
-        "#### Create the pipeline"
+        "#### Create the pipeline\n",
+        "\n",
+        "Define the workflow of the pipeline and build the pipeline. The parameters passed to the pipeline include:\n",
+        "\n",
+        "- `create_bq_dataset_query`: SQL query to create the dataset in BigQuery.\n",
+        "- `job_name`: Name of the Cloud Dataflow job to be configured in `PipelineOptions`.\n",
+        "- `inputs_uri`: A directory location of input data.\n",
+        "-  `bq_dataset`: Dataset name used in BigQuery.\n",
+        "-  `bq_table`: Table name used in BigQuery for ingestion.\n",
+        "-  `requirements_file_path`: The GCS path to the pip requirements file.\n",
+        "-  `python_file_path`: The GCS path to the python file to run.\n",
+        "-  `setup_file_uri`: Path to a Python setup file containing package dependencies.\n",
+        "-  `temp_location`:  GCS path for Dataflow to stage temporary job files created during the execution of the pipeline.\n",
+        "-  `runner`: Pipeline runner used to execute the workflow.\n",
+        "-  `create_bq_preprocess_query`: SQL query to preprocess the data in BigQuery.\n",
+        "-  `create_bq_model_query`: SQL query to create the BigQuery ML model.\n",
+        "-  `create_bq_prediction_query`: SQL query for prediction.\n",
+        "-  `job_config`: A json formatted string describing the job configuration. For more information, vist this [page]( https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery).\n",
+        "-  `project`: Project ID.\n",
+        "-  `region`: Selected region to run the Dataflow job."
       ]
     },
     {
@@ -1371,7 +1508,7 @@
         "        project=project,\n",
         "        location=\"US\",\n",
         "    )\n",
-        "    # instanciate dataflow args\n",
+        "    # instantiate dataflow args\n",
         "    build_dataflow_args_op = build_dataflow_args(\n",
         "        job_name=job_name,\n",
         "        inputs_uri=inputs_uri,\n",
@@ -1431,7 +1568,9 @@
         "id": "nghLONQX7JNg"
       },
       "source": [
-        "## Compile and Run the pipeline"
+        "## Compile and Run the pipeline\n",
+        "\n",
+        "Pass the necessary constants and parameters to the pipeline and compile it to a json file."
       ]
     },
     {
@@ -1451,6 +1590,15 @@
         "compiler.Compiler().compile(pipeline_func=pipeline, package_path=PIPELINE_PACKAGE)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "b5c1b7b8b290"
+      },
+      "source": [
+        "Using the compiled json file, create Vertex AI Pipeline Job and run it by passing the `SERVICE_ACCOUNT` details configured earlier."
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -1460,7 +1608,7 @@
       "outputs": [],
       "source": [
         "pipeline = vertex_ai.PipelineJob(\n",
-        "    display_name=f\"data_preprocess_{TIMESTAMP}\",\n",
+        "    display_name=f\"data_preprocess_{UUID}\",\n",
         "    template_path=PIPELINE_PACKAGE,\n",
         "    pipeline_root=PIPELINE_ROOT,\n",
         "    parameter_values={\n",
@@ -1482,7 +1630,27 @@
         "    enable_caching=False,\n",
         ")\n",
         "\n",
-        "pipeline.run()"
+        "pipeline.run(service_account=SERVICE_ACCOUNT)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "bcb582e65740"
+      },
+      "source": [
+        "Once the pipeline job gets finished successfully, the trained model can be found created in the BigQuery dataset. Run the following cell to see the model listed in the output."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "c0d194c006ae"
+      },
+      "outputs": [],
+      "source": [
+        "! bq ls $PROJECT_ID:$BQ_DATASET"
       ]
     },
     {
@@ -1494,7 +1662,9 @@
         "## Cleaning up\n",
         "\n",
         "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
-        "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n"
+        "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
+        "\n",
+        "Otherwise, you can delete the individual resources you created in this tutorial in the following cell. Set `delete_bucket` and `delete_dataset` to **True** to delete the Cloud Storage bucket and the Bigquery dataset used in this notebook respectively."
       ]
     },
     {
@@ -1505,11 +1675,19 @@
       },
       "outputs": [],
       "source": [
+        "# delete the pipeline job\n",
+        "pipeline.delete()\n",
+        "\n",
+        "delete_bucket = False\n",
+        "delete_dataset = False\n",
+        "\n",
         "# delete bucket\n",
-        "! gsutil -m rm -r $BUCKET_URI\n",
+        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "    ! gsutil -m rm -r $BUCKET_URI\n",
         "\n",
         "# delete dataset\n",
-        "! bq rm -r -f -d $PROJECT_ID:$BQ_DATASET"
+        "if delete_dataset or os.getenv(\"IS_TESTING\"):\n",
+        "    ! bq rm -r -f -d $PROJECT_ID:$BQ_DATASET"
       ]
     }
   ],

notebooks/official/pipelines/google_cloud_pipeline_components_dataproc_tabular.ipynb

@@ -29,7 +29,7 @@
         "id": "JAPoU8Sm5E6e"
       },
       "source": [
-        "# Vertex AI Pipelines: Loan eligibility prediction using google-cloud-pipeline-components and Spark ML\n",
+        "# Vertex AI Pipelines: Loan eligibility prediction using `google-cloud-pipeline-components` and Spark ML\n",
         "\n",
         "<table align=\"left\">\n",
         "\n",
@@ -206,7 +206,7 @@
         "    \n",
         "!pip3 install {USER_FLAG} --upgrade google-cloud-aiplatform==1.11.0 \\\n",
         "                                    kfp==1.8.11 \\\n",
-        "                                    google-cloud-pipeline-components==1.0.1 --quiet --no-warn-conflicts"
+        "                                    google-cloud-pipeline-components==1.0.18 --quiet --no-warn-conflicts"
       ]
     },
     {
@@ -425,9 +425,19 @@
         "3. In the **Service account name** field, enter a name, and\n",
         "   click **Create**.\n",
         "\n",
-        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
-        "into the filter box, and select\n",
-        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type the following role and select them\n",
+        "\n",
+        "      - Artifact Registry Administrator\n",
+        "      - Artifact Registry Repository Administrator\n",
+        "      - Cloud Build Editor\n",
+        "      - Compute Network Admin\n",
+        "      - Dataproc Administrator\n",
+        "      - Dataproc Worker\n",
+        "      - Service Account User\n",
+        "      - Storage Admin\n",
+        "      - Storage Object Admin\n",
+        "      - Vertex AI Administrator\n",
+        "\n",
         "\n",
         "5. Click *Create*. A JSON file that contains your key downloads to your\n",
         "local environment.\n",
@@ -639,15 +649,10 @@
       },
       "outputs": [],
       "source": [
-        "from pathlib import Path as path\n",
-        "\n",
-        "DATA_PATH = path(\"content/path/\")\n",
         "PUBLIC_DATA_URI = \"gs://cloud-samples-data/vertex-ai/dataset-management/datasets/loan_eligibilty/data.csv\"\n",
         "FEATURES_TRAIN_URI = f\"{BUCKET_URI}/data/features/snapshots/{UUID}\"\n",
         "\n",
-        "!mkdir -m 777 -p $DATA_PATH\n",
-        "!gsutil cp -r $PUBLIC_DATA_URI $DATA_PATH\n",
-        "!gsutil cp -r $DATA_PATH $FEATURES_TRAIN_URI"
+        "!gsutil cp -r $PUBLIC_DATA_URI $FEATURES_TRAIN_URI"
       ]
     },
     {
@@ -728,14 +733,10 @@
         "from pathlib import Path as path\n",
         "from typing import NamedTuple\n",
         "\n",
-        "# Part 1 - ML Training\n",
         "from google.cloud import aiplatform as vertex_ai\n",
-        "from google_cloud_pipeline_components import aiplatform as vertex_ai_components\n",
         "from kfp.v2 import compiler, dsl\n",
         "from kfp.v2.dsl import (Artifact, ClassificationMetrics, Condition, Input,\n",
-        "                        Metrics, Output, component)\n",
-        "\n",
-        "ID = random.randint(1, 10000)"
+        "                        Metrics, Output, component)"
       ]
     },
     {
@@ -760,14 +761,14 @@
         "PIPELINE_ROOT = f\"{BUCKET_URI}/pipelines\"\n",
         "PIPELINE_PACKAGE_PATH = str(BUILD_PATH / f\"pipeline_{UUID}.json\")\n",
         "RUNTIME_CONTAINER_IMAGE = f\"gcr.io/{PROJECT_ID}/{RUNTIME_IMAGE}:{IMAGE_TAG}\"\n",
-        "ML_APPLICATION = \"spark\"\n",
-        "TASK = \"classifier\"\n",
+        "ML_APPLICATION = \"loan-eligibility\"\n",
+        "TASK = \"sparkml\"\n",
         "MODEL_TYPE = \"rfor\"\n",
         "VERSION = \"1.0.0\"\n",
         "MODEL_NAME = f\"{ML_APPLICATION}-{TASK}-{MODEL_TYPE}-{VERSION}\"\n",
+        "ARTIFACT_URI = f\"{BUCKET_URI}/deliverables/bundle/{UUID}\"\n",
         "\n",
         "# Preprocessing\n",
-        "PREPROCESSING_BATCH_ID = f\"data-preprocessing-{ID}\"\n",
         "PREPROCESSING_PYTHON_FILE_URI = f\"{BUCKET_URI}/src/data_preprocessing.py\"\n",
         "PROCESSED_DATA_URI = f\"{BUCKET_URI}/data/processed\"\n",
         "PREPROCESSING_ARGS = [\n",
@@ -778,11 +779,10 @@
         "]\n",
         "\n",
         "# Dataset\n",
-        "DATASET_NAME = f\"preprocessed-dataset-{ID}\"\n",
+        "DATASET_NAME = f\"preprocessed-dataset-{UUID}\"\n",
         "GCS_PREPROCESSED_URI = f\"{PROCESSED_DATA_URI}/*/?.csv\"\n",
         "\n",
         "# Training\n",
-        "TRAINING_BATCH_ID = f\"model-training-{ID}\"\n",
         "TRAINING_PYTHON_FILE_URI = f\"{BUCKET_URI}/src/model_training.py\"\n",
         "MODEL_URI = f\"{BUCKET_URI}/deliverables/model/rfor/{UUID}/train_model\"\n",
         "METRICS_URI = f\"{BUCKET_URI}/deliverables/metrics/rfor/{UUID}/train_metrics.json\"\n",
@@ -797,10 +797,9 @@
         "\n",
         "# Condition\n",
         "AUPR_THRESHOLD = 0.5\n",
-        "AUPR_HYPERTUNE_CONDITION = \"[AUPR_HYPERTUNE]\"\n",
+        "AUPR_HYPERTUNE_CONDITION = \"hypertune\"\n",
         "\n",
         "# Hypertuning\n",
-        "HPT_TRAINING_BATCH_ID = f\"hyper-tuning-{ID}\"\n",
         "HPT_PYTHON_FILE_URI = f\"{BUCKET_URI}/src/hp_tuning.py\"\n",
         "HPT_MODEL_URI = f\"{BUCKET_URI}/deliverables/model/rfor/{UUID}/model\"\n",
         "HPT_METRICS_URI = f\"{BUCKET_URI}/deliverables/metrics/rfor/{UUID}/metrics.json\"\n",
@@ -811,7 +810,24 @@
         "    HPT_MODEL_URI,\n",
         "    \"--metrics-path\",\n",
         "    HPT_METRICS_URI,\n",
-        "]"
+        "]\n",
+        "HPT_BUNDLE_URI = f\"{ARTIFACT_URI}/model.zip\"\n",
+        "HPT_ARGS = [\n",
+        "    \"--train-path\",\n",
+        "    PROCESSED_DATA_URI,\n",
+        "    \"--model-path\",\n",
+        "    HPT_MODEL_URI,\n",
+        "    \"--metrics-path\",\n",
+        "    HPT_METRICS_URI,\n",
+        "    \"--bundle-path\",\n",
+        "    HPT_BUNDLE_URI,\n",
+        "]\n",
+        "HPT_RUNTIME_PROPERTIES = {\n",
+        "    \"spark.jars.packages\": \"ml.combust.mleap:mleap-spark-base_2.12:0.20.0,ml.combust.mleap:mleap-spark_2.12:0.20.0\"\n",
+        "}\n",
+        "\n",
+        "# Deploy\n",
+        "SERVING_IMAGE_URI = f\"{REGION}-docker.pkg.dev/{PROJECT_ID}/{REPO_NAME}/spark-ml-serving\""
       ]
     },
     {
@@ -840,7 +856,7 @@
         "id": "LB2aM7VyRyZG"
       },
       "source": [
-        "## PART I - Build the Vertex Pipeline to train and deploy a Spark model\n",
+        "## Build the Vertex Pipeline to train and deploy a Spark model\n",
         "\n",
         "In this case, the ML pipeline includes the following steps:\n",
         "\n",
@@ -848,10 +864,16 @@
         "2.   Train an `RandomForestClassifier` with `DataprocPySparkBatchOp`\n",
         "3.   Run a custom component in order to evaluate the model\n",
         "\n",
-        "If the model respects the performance condition, then\n",
+        "If the model respects the performance condition, then:\n",
         "\n",
         "4.   Hypertune the `RandomForestClassifier` with `DataprocPySparkBatchOp`\n",
-        "5.   Register the model in the Vertex AI Model Registry\n"
+        "5.   Serializes the model to MLeap format to use the model outside of Spark.\n",
+        "\n",
+        "If the `deploy_model` pipeline parameter is set to `True`:\n",
+        "\n",
+        "6.   Upload the model to Vertex AI Model Registry.\n",
+        "7.   Creates a Vertex AI endpoint.\n",
+        "8.   Deploys the model to the Vertex AI endpoint for serving online prediction requests.\n"
       ]
     },
     {
@@ -909,6 +931,7 @@
         "\n",
         "\"\"\"\n",
         "data_preprocessing.py is the module for\n",
+        "\n",
         "  - ingest data\n",
         "  - do simple preprocessing tasks\n",
         "  - upload processed data to gcs\n",
@@ -1021,8 +1044,7 @@
         "\n",
         "    spark = (SparkSession.builder\n",
         "             .master(\"local[*]\")\n",
-        "             .appName(\"spark go live\")\n",
-        "             .config('spark.ui.port', '4050')\n",
+        "             .appName(\"loan eligibility\")\n",
         "             .getOrCreate())\n",
         "    try:\n",
         "        logger.info(f'spark version: {spark.sparkContext.version}')\n",
@@ -1041,12 +1063,7 @@
         "        training_data_raw_df.show(truncate=False)\n",
         "\n",
         "        logger.info(f'load prepared data to {output_data_path}.')\n",
-        "        if output_data_path.startswith('gs://'):\n",
-        "            training_data_raw_df.write.mode('overwrite').csv(str(output_data_path), header=True)\n",
-        "        else:\n",
-        "            output_file_path = Path(output_data_path)\n",
-        "            output_file_path.mkdir(parents=True, exist_ok=True)\n",
-        "            training_data_raw_df.write.mode('overwrite').csv(str(output_file_path), header=True)\n",
+        "        training_data_raw_df.write.mode('overwrite').csv(str(output_data_path), header=True)\n",
         "    except RuntimeError as main_error:\n",
         "        logger.error(main_error)\n",
         "    else:\n",
@@ -1376,8 +1393,7 @@
         "        logger.info('start spark session.')\n",
         "        spark = (SparkSession.builder\n",
         "                 .master(\"local[*]\")\n",
-        "                 .appName(\"spark go live\")\n",
-        "                 .config('spark.ui.port', '4050')\n",
+        "                 .appName(\"loan eligibility\")\n",
         "                 .getOrCreate())\n",
         "        logger.info(f'spark version: {spark.sparkContext.version}')\n",
         "        logger.info('start bulding pipeline.')\n",
@@ -1402,23 +1418,12 @@
         "\n",
         "        logger.info(f'load model pipeline in {model_path}.')\n",
         "        pipeline.write().overwrite().save(model_path)\n",
-        "        if model_path.startswith('gs://'):\n",
-        "            pipeline.write().overwrite().save(model_path)\n",
-        "        else:\n",
-        "            path(model_path).mkdir(parents=True, exist_ok=True)\n",
-        "            pipeline.write().overwrite().save(model_path)\n",
         "\n",
-        "        logger.info(f'Upload metrics under {metrics_path}.')\n",
-        "        if metrics_path.startswith('gs://'):\n",
-        "            bucket = urlparse(model_path).netloc\n",
-        "            metrics_file_path = urlparse(metrics_path).path.strip('/')\n",
-        "            write_metrics(bucket, metrics, metrics_file_path)\n",
-        "        else:\n",
-        "            metrics_version_path = path(metrics_path).parents[0]\n",
-        "            metrics_version_path.mkdir(parents=True, exist_ok=True)\n",
-        "            with open(metrics_path, 'w') as json_file:\n",
-        "                json.dump(metrics, json_file)\n",
-        "            json_file.close()\n",
+        "        logger.info(f'Upload metrics under {metrics_path}.')     \n",
+        "        bucket = urlparse(model_path).netloc\n",
+        "        metrics_file_path = urlparse(metrics_path).path.strip('/')\n",
+        "        write_metrics(bucket, metrics, metrics_file_path)\n",
+        "        \n",
         "    except RuntimeError as main_error:\n",
         "        logger.error(main_error)\n",
         "    else:\n",
@@ -1444,7 +1449,9 @@
         "\n",
         "- `--train-path`: The GCS path of the training sample.\n",
         "- `--model-path`: The GCS path to store the trained model.\n",
-        "- `--metrics-path`: The GCS path to store the metrics of model."
+        "- `--metrics-path`: The GCS path to store the metrics of model.\n",
+        "\n",
+        "The hyperparameter tuning job will also serialize the best performing model to an MLeap bundle, which can be imported to Vertex AI as a model for serving predictions - see the *Serve your model in Vertex AI* section further below."
       ]
     },
     {
@@ -1481,6 +1488,9 @@
         "except ImportError as e:\n",
         "    print('WARN: Something wrong with pyspark library. Please check configuration settings!')\n",
         "    print(e)\n",
+        "    \n",
+        "import mleap.pyspark\n",
+        "from mleap.pyspark.spark_support import SimpleSparkSerializer\n",
         "\n",
         "from pyspark.sql.types import StructType, DoubleType, StringType\n",
         "from pyspark.sql.functions import col, udf\n",
@@ -1586,6 +1596,16 @@
         "        ''',\n",
         "        type=str,\n",
         "        required=True)\n",
+        "    args_parser.add_argument(\n",
+        "        '--bundle-path',\n",
+        "        help='''\n",
+        "        The GCS path to store the exported MLeap bundle. \n",
+        "        Format: \n",
+        "        - locally: /path/to/dir\n",
+        "        - cloud: gs://bucket/path\n",
+        "        ''',\n",
+        "        type=str,\n",
+        "        required=True)\n",
         "    return args_parser.parse_args()\n",
         "\n",
         "\n",
@@ -1742,18 +1762,14 @@
         "    train_path = args.train_path\n",
         "    model_path = args.model_path\n",
         "    metrics_path = args.metrics_path\n",
+        "    bundle_path = args.bundle_path\n",
         "\n",
         "    try:\n",
         "        logger.info('initializing pipeline training.')\n",
         "        logger.info('start spark session.')\n",
         "        spark = (SparkSession.builder\n",
         "                 .master(\"local[*]\")\n",
-        "                 .appName(\"spark go live\")\n",
-        "                 .config('spark.ui.port', '4050')\n",
-        "                 .config('spark.jars.packages', 'ml.combust.mleap:mleap-runtime_2.12:0.19.0')\n",
-        "                 .config('spark.jars.packages', 'ml.combust.mleap:mleap-base_2.12:0.19.0')\n",
-        "                 .config('spark.jars.packages', 'ml.combust.mleap:mleap-spark_2.12:0.19.0')\n",
-        "                 .config('spark.jars.packages', 'ml.combust.mleap:mleap-spark-extension_2.12:0.19.0')\n",
+        "                 .appName(\"loan eligibility\")\n",
         "                 .getOrCreate())\n",
         "        logger.info(f'spark version: {spark.sparkContext.version}')\n",
         "        logger.info('start building pipeline.')\n",
@@ -1763,12 +1779,7 @@
         "        pipeline_cross_validator = build_hp_pipeline(preprocessing_stages, feature_engineering_stages,\n",
         "                                                     model_training_stage)\n",
         "        logger.info(f'load train data from {train_path}.')\n",
-        "        if train_path.startswith('bq://'):\n",
-        "            raw_data = spark.read.format('bigquery') \\\n",
-        "                .option('table', train_path.replace('bq://', '')) \\\n",
-        "                .load()\n",
-        "        else:\n",
-        "            raw_data = (spark.read.format('csv')\n",
+        "        raw_data = (spark.read.format('csv')\n",
         "                        .option(\"header\", \"true\")\n",
         "                        .schema(DATA_SCHEMA)\n",
         "                        .load(train_path))\n",
@@ -1781,23 +1792,22 @@
         "            print(f'{m}: {v}')\n",
         "\n",
         "        logger.info(f'load model pipeline in {model_path}.')\n",
-        "        if model_path.startswith('gs://'):\n",
-        "            pipeline_model.write().overwrite().save(model_path)\n",
-        "        else:\n",
-        "            path(model_path).mkdir(parents=True, exist_ok=True)\n",
-        "            pipeline_model.write().overwrite().save(model_path)\n",
+        "        pipeline_model.write().overwrite().save(model_path)\n",
         "\n",
-        "        logger.info(f'Upload metrics under {metrics_path}.')\n",
-        "        if metrics_path.startswith('gs://'):\n",
-        "            bucket = urlparse(model_path).netloc\n",
-        "            metrics_file_path = urlparse(metrics_path).path.strip('/')\n",
-        "            write_metrics(bucket, metrics, metrics_file_path)\n",
-        "        else:\n",
-        "            metrics_version_path = path(metrics_path).parents[0]\n",
-        "            metrics_version_path.mkdir(parents=True, exist_ok=True)\n",
-        "            with open(metrics_path, 'w') as json_file:\n",
-        "                json.dump(metrics, json_file)\n",
-        "            json_file.close()\n",
+        "        logger.info(f'upload metrics under {metrics_path}.')\n",
+        "        bucket = urlparse(model_path).netloc\n",
+        "        metrics_file_path = urlparse(metrics_path).path.strip('/')\n",
+        "        write_metrics(bucket, metrics, metrics_file_path)\n",
+        "        \n",
+        "        logger.info('export MLeap bundle to temporary location')\n",
+        "        pipeline_model.bestModel.serializeToBundle(f'jar:file:/tmp/bundle.zip', predictions)\n",
+        "        \n",
+        "        logger.info(f'upload MLeap bundle to {bundle_path}')\n",
+        "        bundle_file_path = urlparse(bundle_path).path.strip('/')\n",
+        "        bucket = urlparse(bundle_path).netloc\n",
+        "        logger.info(f'Copying /tmp/bundle.zip to bucket {bucket} using object name {bundle_file_path} ...')\n",
+        "        upload_file(bucket, '/tmp/bundle.zip', bundle_file_path)\n",
+        "        \n",
         "    except RuntimeError as main_error:\n",
         "        logger.error(main_error)\n",
         "    else:\n",
@@ -1842,11 +1852,11 @@
         "id": "68nYBB5GS9TB"
       },
       "source": [
-        "### Build a custom dataproc serverless image\n",
+        "### Build a custom Dataproc Serverless container image\n",
         "\n",
-        "The `DataprocPySparkBatchOp` allows you to pass custom image that you use when the [provided Dataproc Serverless runtime versions](https://cloud.google.com/dataproc-serverless/docs/concepts/versions/spark-runtime-versions) does not respect your requirements. \n",
+        "Dataproc Serverless provides [default runtime images](https://cloud.google.com/dataproc-serverless/docs/concepts/versions/spark-runtime-versions). You can also use custom container images for your Dataproc Serverless workloads. \n",
         "\n",
-        "**Note:** This step is optional and is included here for general awareness."
+        "The steps in this section builds a custom container image that includes additional dependencies. The custom container image can be specified when using the `DataprocPySparkBatchOp` component to launch the workload within a pipeline."
       ]
     },
     {
@@ -1855,7 +1865,7 @@
         "id": "GF9_5IGYqLAX"
       },
       "source": [
-        "#### Define the Dataproc serverless custom runtime image"
+        "#### Define the Dataproc Serverless custom runtime image"
       ]
     },
     {
@@ -1927,7 +1937,7 @@
         "      scikit-image \\\n",
         "      scikit-learn \\\n",
         "      scipy \\\n",
-        "      mleap \n",
+        "      mleap\n",
         "\n",
         "# (Required) Create the 'spark' group/user.\n",
         "# The GID and UID must be 1099. Home directory is required.\n",
@@ -1963,7 +1973,9 @@
         "id": "ZXzI2xInqb3V"
       },
       "source": [
-        "#### Build the Dataproc serverless custom runtime using Google Cloud Build"
+        "#### Build the Dataproc Serverless custom runtime using Google Cloud Build\n",
+        "\n",
+        "**Note:** this step may take approximately 5 to 10 minutes to complete."
       ]
     },
     {
@@ -2140,12 +2152,12 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "1-Ccx4uLDz4N"
+        "id": "28f3d22dd97f"
       },
       "source": [
-        "#### Model registration custom component\n",
+        "#### Create component for passing args to hyperparameter tuning component\n",
         "\n",
-        "Define a component to create a model resource for the trained model on Vertex AI Model registry."
+        "The following component passes the args `--train-path`, `--model-path` and `--metrics-path`, and `--bundle-path` in the required format for the hyperparamter tuning function defined earlier."
       ]
     },
     {
@@ -2156,22 +2168,230 @@
       },
       "outputs": [],
       "source": [
-        "# TODO: Build a custom compiler using Spark docker image to compile the Mleap bundle\n",
-        "\n",
-        "\n",
         "@component(base_image=\"python:3.8-slim\")\n",
-        "def register_model(\n",
-        "    artifact_uri: str,\n",
-        "    model: Output[Artifact],\n",
-        ") -> NamedTuple(\"Outputs\", [(\"uri\", str)]):\n",
+        "def build_hpt_args(\n",
+        "    dataset_uri: Input[Artifact],\n",
+        "    train_path: str,\n",
+        "    model_path: str,\n",
+        "    metrics_path: str,\n",
+        "    bundle_path: str,\n",
+        ") -> list:\n",
+        "    return [\n",
+        "        \"--train-path\",\n",
+        "        train_path,\n",
+        "        \"--model-path\",\n",
+        "        model_path,\n",
+        "        \"--metrics-path\",\n",
+        "        metrics_path,\n",
+        "        \"--bundle-path\",\n",
+        "        bundle_path,\n",
+        "    ]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0db73fff95b3"
+      },
+      "source": [
+        "### (Optional) Serve your model using Vertex AI\n",
         "\n",
-        "    component_outputs = NamedTuple(\n",
-        "        \"Outputs\",\n",
-        "        [\n",
-        "            (\"uri\", str),\n",
-        "        ],\n",
-        "    )\n",
-        "    return component_outputs(artifact_uri)"
+        "The hyperparameter tuning task exports the best performing model as an MLeap bundle. The MLeap bundle can be imported into the Vertex AI Model Registry and used for prediction serving. See [Serving Spark ML model using Vertex AI](https://cloud.google.com/architecture/spark-ml-model-with-vertexai) for more information.\n",
+        "\n",
+        "Enable import of the MLeap bundle into the Vertex AI Model Registry and online prediction serving."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2d7e7c8fc21b"
+      },
+      "outputs": [],
+      "source": [
+        "# Set DEPLOY_MODEL to True\n",
+        "DEPLOY_MODEL = False"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0bb792622d9f"
+      },
+      "source": [
+        "### Build the model serving container image\n",
+        "\n",
+        "A *serving container image* is required to import your model into the Model Registry. The serving container image provides the model serving implementation for the model. The following replicates the instructions from [Serving Spark ML model using Vertex AI](https://cloud.google.com/architecture/spark-ml-model-with-vertexai) to build the serving container image.\n",
+        "\n",
+        "**Note:** this step may take approximately 5 to 10 minutes to complete."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "4703a0f969a3"
+      },
+      "outputs": [],
+      "source": [
+        "DEPLOY_MODEL_CONDITION = 'deploy'\n",
+        "\n",
+        "if DEPLOY_MODEL:\n",
+        "\n",
+        "    import os\n",
+        "    \n",
+        "    CWD = os.getcwd()\n",
+        "\n",
+        "    # Clone and build the scala-sbt cloud builder\n",
+        "    ! git clone https://github.com/GoogleCloudPlatform/cloud-builders-community.git\n",
+        "    ! cd ${CWD}/cloud-builders-community/scala-sbt && \\\n",
+        "        gcloud builds submit .\n",
+        "\n",
+        "    # Clone and build the serving container code\n",
+        "    ! cd {CWD} && git clone https://github.com/GoogleCloudPlatform/vertex-ai-spark-ml-serving.git\n",
+        "    ! cd {CWD}/vertex-ai-spark-ml-serving && \\\n",
+        "        gcloud builds submit --config=cloudbuild.yaml \\\n",
+        "            --substitutions=\"_LOCATION={REGION},_REPOSITORY={REPO_NAME},_IMAGE=spark-ml-serving\" ."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "947b51adc087"
+      },
+      "source": [
+        "### Create component for importing a model artifact into a pipeline\n",
+        "\n",
+        "The pipeline uses the `ModelImportOp` component to import (upload) a model to Vertex AI Model Registry.\n",
+        "\n",
+        "The `import_model_artifact` python component creates a model artifact that can be passed to the `ModelImportOp` component."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2ed96e7ad046"
+      },
+      "outputs": [],
+      "source": [
+        "@dsl.component(\n",
+        "    base_image=\"python:3.8-slim\",\n",
+        "    packages_to_install=[\"google-cloud-aiplatform\"],\n",
+        ")\n",
+        "def import_model_artifact(\n",
+        "    model: dsl.Output[dsl.Artifact], artifact_uri: str, serving_image_uri: str\n",
+        "):\n",
+        "    model.metadata[\"containerSpec\"] = {\n",
+        "        \"imageUri\": serving_image_uri,\n",
+        "        \"healthRoute\": \"/health\",\n",
+        "        \"predictRoute\": \"/predict\",\n",
+        "    }\n",
+        "    model.uri = artifact_uri"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0d83d9e80923"
+      },
+      "source": [
+        "The serving container requires the model schema in JSON format, which is read during container startup. See [Provide the model schema](https://cloud.google.com/architecture/spark-ml-model-with-vertexai#provide_the_model_schema) for more information.\n",
+        "\n",
+        "Write the model schema file:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "521d2f4d7992"
+      },
+      "outputs": [],
+      "source": [
+        "%%writefile $SRC/schema.json\n",
+        "{\n",
+        "  \"input\": [\n",
+        "    {\n",
+        "      \"name\": \"loan_amount\",\n",
+        "      \"type\": \"DOUBLE\"\n",
+        "    },\n",
+        "    {\n",
+        "      \"name\": \"loan_term\",\n",
+        "      \"type\": \"STRING\"\n",
+        "    },\n",
+        "    {\n",
+        "      \"name\": \"property_area\",\n",
+        "      \"type\": \"STRING\"\n",
+        "    },\n",
+        "    {\n",
+        "      \"name\": \"feature_7\",\n",
+        "      \"type\": \"DOUBLE\"\n",
+        "    },\n",
+        "    {\n",
+        "      \"name\": \"feature_3\",\n",
+        "      \"type\": \"DOUBLE\"\n",
+        "    },\n",
+        "    {\n",
+        "      \"name\": \"feature_1\",\n",
+        "      \"type\": \"DOUBLE\"\n",
+        "    },\n",
+        "    {\n",
+        "      \"name\": \"feature_9\",\n",
+        "      \"type\": \"DOUBLE\"\n",
+        "    },\n",
+        "    {\n",
+        "      \"name\": \"feature_5\",\n",
+        "      \"type\": \"DOUBLE\"\n",
+        "    },\n",
+        "    {\n",
+        "      \"name\": \"feature_0\",\n",
+        "      \"type\": \"DOUBLE\"\n",
+        "    },\n",
+        "    {\n",
+        "      \"name\": \"feature_8\",\n",
+        "      \"type\": \"DOUBLE\"\n",
+        "    },\n",
+        "    {\n",
+        "      \"name\": \"feature_4\",\n",
+        "      \"type\": \"DOUBLE\"\n",
+        "    },\n",
+        "    {\n",
+        "      \"name\": \"feature_2\",\n",
+        "      \"type\": \"DOUBLE\"\n",
+        "    },\n",
+        "    {\n",
+        "      \"name\": \"feature_6\",\n",
+        "      \"type\": \"DOUBLE\"\n",
+        "    }\n",
+        "  ],\n",
+        "  \"output\": [\n",
+        "    {\n",
+        "      \"name\": \"prediction\",\n",
+        "      \"type\": \"DOUBLE\"\n",
+        "    }\n",
+        "  ]\n",
+        "}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d0b88e26570a"
+      },
+      "source": [
+        "Copy the model schema configuration file to GCS. The serving container reads the model schema file location from the `AIP_STORAGE_URI` environment at startup. See [Import the model into Vertex AI](https://cloud.google.com/architecture/spark-ml-model-with-vertexai#import-the-model-into-vertex-ai) for more information."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b7763bb558f3"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil cp $SRC/schema.json $ARTIFACT_URI/schema.json"
       ]
     },
     {
@@ -2195,30 +2415,35 @@
       "source": [
         "@dsl.pipeline(name=PIPELINE_NAME, description=\"A pipeline to train a PySpark model.\")\n",
         "def pipeline(\n",
-        "    preprocessing_batch_id: str = PREPROCESSING_BATCH_ID,\n",
         "    preprocessing_main_python_file_uri: str = PREPROCESSING_PYTHON_FILE_URI,\n",
         "    train_data_path: str = FEATURES_TRAIN_URI,\n",
         "    preprocessed_data_path: str = PROCESSED_DATA_URI,\n",
         "    dataset_name: str = DATASET_NAME,\n",
         "    dataset_uri: str = GCS_PREPROCESSED_URI,\n",
-        "    training_batch_id: str = TRAINING_BATCH_ID,\n",
         "    training_main_python_file_uri: str = TRAINING_PYTHON_FILE_URI,\n",
         "    train_path: str = PROCESSED_DATA_URI,\n",
         "    model_path: str = MODEL_URI,\n",
         "    metrics_path: str = METRICS_URI,\n",
         "    threshold: float = AUPR_THRESHOLD,\n",
-        "    hpt_batch_id: str = HPT_TRAINING_BATCH_ID,\n",
         "    hpt_main_python_file_uri: str = HPT_PYTHON_FILE_URI,\n",
         "    hpt_model_path: str = HPT_MODEL_URI,\n",
         "    hpt_metrics_path: str = HPT_METRICS_URI,\n",
+        "    hpt_bundle_path: str = HPT_BUNDLE_URI,\n",
         "    custom_container_image: str = RUNTIME_CONTAINER_IMAGE,\n",
         "    model_name: str = MODEL_NAME,\n",
         "    project_id: str = PROJECT_ID,\n",
         "    location: str = REGION,\n",
+        "    deploy_model: bool = DEPLOY_MODEL,\n",
+        "    artifact_uri: str = ARTIFACT_URI,\n",
+        "    serving_image_uri: str = SERVING_IMAGE_URI,\n",
         "):\n",
-        "\n",
-        "    from google_cloud_pipeline_components.experimental.dataproc import \\\n",
+        "    from google_cloud_pipeline_components.v1.dataproc import \\\n",
         "        DataprocPySparkBatchOp\n",
+        "    from google_cloud_pipeline_components.v1.dataset import \\\n",
+        "        TabularDatasetCreateOp\n",
+        "    from google_cloud_pipeline_components.v1.endpoint import (EndpointCreateOp,\n",
+        "                                                              ModelDeployOp)\n",
+        "    from google_cloud_pipeline_components.v1.model import ModelUploadOp\n",
         "\n",
         "    # build preprocessed data args\n",
         "    build_preprocessing_args_op = build_preprocessing_args(\n",
@@ -2230,13 +2455,12 @@
         "        project=project_id,\n",
         "        location=location,\n",
         "        container_image=custom_container_image,\n",
-        "        batch_id=preprocessing_batch_id,\n",
         "        main_python_file_uri=preprocessing_main_python_file_uri,\n",
         "        args=build_preprocessing_args_op.output,\n",
         "    ).after(build_preprocessing_args_op)\n",
         "\n",
         "    # create dataset\n",
-        "    create_dataset_op = vertex_ai_components.TabularDatasetCreateOp(\n",
+        "    create_dataset_op = TabularDatasetCreateOp(\n",
         "        display_name=dataset_name,\n",
         "        gcs_source=dataset_uri,\n",
         "        project=project_id,\n",
@@ -2256,7 +2480,6 @@
         "        project=project_id,\n",
         "        location=location,\n",
         "        container_image=custom_container_image,\n",
-        "        batch_id=training_batch_id,\n",
         "        main_python_file_uri=training_main_python_file_uri,\n",
         "        args=build_training_args_op.output,\n",
         "    ).after(build_training_args_op)\n",
@@ -2269,11 +2492,12 @@
         "        name=AUPR_HYPERTUNE_CONDITION,\n",
         "    ):\n",
         "\n",
-        "        build_hpt_args_op = build_training_args(\n",
+        "        build_hpt_args_op = build_hpt_args(\n",
         "            dataset_uri=create_dataset_op.output,\n",
         "            train_path=train_path,\n",
         "            model_path=hpt_model_path,\n",
         "            metrics_path=hpt_metrics_path,\n",
+        "            bundle_path=hpt_bundle_path,\n",
         "        ).after(evaluate_model_op)\n",
         "\n",
         "        # hyperparameter tuning\n",
@@ -2281,13 +2505,46 @@
         "            project=project_id,\n",
         "            location=location,\n",
         "            container_image=custom_container_image,\n",
-        "            batch_id=hpt_batch_id,\n",
         "            main_python_file_uri=hpt_main_python_file_uri,\n",
         "            args=build_hpt_args_op.output,\n",
+        "            runtime_config_properties=HPT_RUNTIME_PROPERTIES,\n",
         "        ).after(model_traning_op)\n",
         "\n",
-        "        # upload model\n",
-        "        register_model(artifact_uri=hpt_model_path).after(hyperparameter_tuning_op)"
+        "        # evaluate condition to upload and deploy model to Vertex AI\n",
+        "        with Condition(\n",
+        "            # kfp casts `bool` parameter to `str`\n",
+        "            deploy_model == \"True\",\n",
+        "            name=DEPLOY_MODEL_CONDITION,\n",
+        "        ):\n",
+        "            # import the model into the pipeline as a kfp model artifact\n",
+        "            import_model_artifact_op = import_model_artifact(\n",
+        "                artifact_uri=artifact_uri,\n",
+        "                serving_image_uri=serving_image_uri,\n",
+        "            )\n",
+        "\n",
+        "            # upload model to Vertex AI\n",
+        "            model_upload_op = ModelUploadOp(\n",
+        "                project=project_id,\n",
+        "                location=location,\n",
+        "                display_name=model_name,\n",
+        "                unmanaged_container_model=import_model_artifact_op.outputs[\"model\"],\n",
+        "            ).after(hyperparameter_tuning_op)\n",
+        "\n",
+        "            # create a serving endpoint\n",
+        "            endpoint_op = EndpointCreateOp(\n",
+        "                project=project_id,\n",
+        "                location=location,\n",
+        "                display_name=model_name,\n",
+        "            ).after(model_upload_op)\n",
+        "\n",
+        "            # deploy  model to the serving endpoint\n",
+        "            _ = ModelDeployOp(\n",
+        "                model=model_upload_op.outputs[\"model\"],\n",
+        "                endpoint=endpoint_op.outputs[\"endpoint\"],\n",
+        "                dedicated_resources_machine_type=\"n1-standard-2\",\n",
+        "                dedicated_resources_min_replica_count=1,\n",
+        "                dedicated_resources_max_replica_count=1,\n",
+        "            ).after(endpoint_op)"
       ]
     },
     {
@@ -2363,6 +2620,65 @@
         "pipeline.wait()"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "b584afa5a1b1"
+      },
+      "source": [
+        "### (Optional) Get online predictions from the deployed model\n",
+        "\n",
+        "You can request online predictions if the model was deployed to a Vertex AI endpoint. Use the `google-cloud-aiplatform` client library to request predictions, or use `curl` as per below:\n",
+        "\n",
+        "Create the prediction request payload with the instances that you want to predict:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "12d068e1877c"
+      },
+      "outputs": [],
+      "source": [
+        "%%writefile instances.json\n",
+        "{\n",
+        "    \"instances\": [\n",
+        "        [214.0, \"360\", \"Rural\", 2.13, 2.21, 0.0, 0.0, 2.31, 2.01, 0.0, 0.0, 0.0, 0.0],\n",
+        "        [213.0, \"360\", \"Semiurban\", 2.03, 2.11, 0.0, 0.0, 2.13, 2.02, 0.0, 0.0, 0.0, 0.0]\n",
+        "    ]\n",
+        "}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "b7cbfec4537d"
+      },
+      "source": [
+        "Use `curl` to send the prediction request to the Vertex AI endpoint. The response contains the predicted label (`0 == not eligible`, `1 == eligible`) for each instance sent in the request payload."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b1617d6e8a3d"
+      },
+      "outputs": [],
+      "source": [
+        "ENDPOINT_ID=!(gcloud ai endpoints list \\\n",
+        "              --region={REGION} \\\n",
+        "              --filter=display_name={MODEL_NAME} \\\n",
+        "              --format='value(name)')\n",
+        "\n",
+        "!curl -X POST \\\n",
+        "   -H \"Authorization: Bearer $(gcloud auth print-access-token)\" \\\n",
+        "   -H \"Content-Type: application/json\" \\\n",
+        "   https://{REGION}-aiplatform.googleapis.com/v1/projects/{PROJECT_ID}/locations/us-central1/endpoints/{ENDPOINT_ID[-1]}:predict \\\n",
+        "   -d \"@instances.json\""
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -2388,8 +2704,14 @@
         "# Delete pipeline\n",
         "pipeline.delete()\n",
         "\n",
+        "# Delete endpoints\n",
+        "endpoint_list = vertex_ai.Endpoint.list(filter=f'display_name=\"{MODEL_NAME}\"')\n",
+        "for endpoint in endpoint_list:\n",
+        "    endpoint.undeploy_all()\n",
+        "    endpoint.delete()\n",
+        "\n",
         "# Delete model\n",
-        "model_list = vertex_ai.TabularDataset.list(filter=f'display_name=\"{MODEL_NAME}\"')\n",
+        "model_list = vertex_ai.Model.list(filter=f'display_name=\"{MODEL_NAME}\"')\n",
         "for model in model_list:\n",
         "    model.delete()\n",
         "\n",
@@ -2433,6 +2755,17 @@
         "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
         "    ! gsutil -m rm -r $BUCKET_URI"
       ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "bc7f1247f0ec"
+      },
+      "outputs": [],
+      "source": [
+        "!rm -rf $SRC $BUILD_PATH"
+      ]
     }
   ],
   "metadata": {

notebooks/official/pipelines/metrics_viz_run_compare_kfp.ipynb

@@ -64,30 +64,6 @@
         "This notebook shows how to use [the Kubeflow Pipelines (KFP) SDK](https://www.kubeflow.org/docs/components/pipelines/) to build [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines) that generate model metrics and metrics visualizations, and comparing pipeline runs."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "dataset:wine,lcn,sklearn"
-      },
-      "source": [
-        "### Datasets\n",
-        "\n",
-        "The dataset used for this tutorial is the [Wine dataset](https://archive.ics.uci.edu/ml/datasets/wine) from [Scikit-learn builtin datasets](https://scikit-learn.org/stable/datasets.html).\n",
-        "\n",
-        "The dataset predicts the origin of a wine."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "dataset:iris,lcn,sklearn"
-      },
-      "source": [
-        "The dataset used for this tutorial is the [Iris dataset](https://scikit-learn.org/stable/auto_examples/datasets/plot_iris_dataset.html) from [Scikit-learn builtin datasets](https://scikit-learn.org/stable/datasets.html).\n",
-        "\n",
-        "The dataset predicts the type of Iris flower species from a class of three species: setosa, virginica, or versicolor."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -112,6 +88,30 @@
         "- Compare metrics across pipeline runs"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dataset:wine,lcn,sklearn"
+      },
+      "source": [
+        "### Datasets\n",
+        "\n",
+        "The dataset used for this tutorial is the [Wine dataset](https://archive.ics.uci.edu/ml/datasets/wine) from [Scikit-learn builtin datasets](https://scikit-learn.org/stable/datasets.html).\n",
+        "\n",
+        "The dataset predicts the origin of a wine."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dataset:iris,lcn,sklearn"
+      },
+      "source": [
+        "The dataset used for this tutorial is the [Iris dataset](https://scikit-learn.org/stable/auto_examples/datasets/plot_iris_dataset.html) from [Scikit-learn builtin datasets](https://scikit-learn.org/stable/datasets.html).\n",
+        "\n",
+        "The dataset predicts the type of Iris flower species from a class of three species: setosa, virginica, or versicolor."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -202,7 +202,7 @@
         "! pip3 install {USER_FLAG} kfp google-cloud-pipeline-components --upgrade -q\n",
         "\n",
         "if os.getenv(\"IS_TESTING\"):\n",
-        "    ! pip3 install --upgrade matplotlib $USER_FLAG"
+        "    ! pip3 install --upgrade matplotlib $USER_FLAG -q"
       ]
     },
     {
@@ -240,6 +240,8 @@
         "id": "check_versions"
       },
       "source": [
+        "### KFP SDK version\n",
+        "\n",
         "Check the versions of the packages you installed.  The KFP SDK version should be >=1.6."
       ]
     },
@@ -349,7 +351,10 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -358,9 +363,9 @@
         "id": "timestamp"
       },
       "source": [
-        "#### Timestamp\n",
+        "#### UUID\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
       ]
     },
     {
@@ -371,9 +376,16 @@
       },
       "outputs": [],
       "source": [
-        "from datetime import datetime\n",
+        "import random\n",
+        "import string\n",
         "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+        "\n",
+        "# Generate a uuid of a specifed length(default=8)\n",
+        "def generate_uuid(length: int = 8) -> str:\n",
+        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
+        "\n",
+        "\n",
+        "UUID = generate_uuid()"
       ]
     },
     {
@@ -384,7 +396,7 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Vertex AI Workbench Notebook**, your environment is already authenticated. Skip this step.\n",
+        "**If you are using Vertex AI Workbench Notebook**, your environment is already authenticated.\n",
         "\n",
         "**If you are using Colab**, run the cell below and follow the instructions\n",
         "when prompted to authenticate your account via oAuth.\n",
@@ -479,7 +491,7 @@
       "outputs": [],
       "source": [
         "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
         "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
       ]
     },
@@ -553,6 +565,10 @@
       },
       "outputs": [],
       "source": [
+        "import sys\n",
+        "\n",
+        "IS_COLAB = \"google.colab\" in sys.modules\n",
+        "\n",
         "if (\n",
         "    SERVICE_ACCOUNT == \"\"\n",
         "    or SERVICE_ACCOUNT is None\n",
@@ -912,12 +928,12 @@
       },
       "outputs": [],
       "source": [
-        "DISPLAY_NAME = \"iris_\" + TIMESTAMP\n",
+        "DISPLAY_NAME = \"iris_\" + UUID\n",
         "\n",
         "job = aip.PipelineJob(\n",
         "    display_name=DISPLAY_NAME,\n",
         "    template_path=\"tabular classification_pipeline.json\".replace(\" \", \"_\"),\n",
-        "    job_id=f\"tabular classification-v2{TIMESTAMP}-1\".replace(\" \", \"\"),\n",
+        "    job_id=f\"tabular classification-v2{UUID}-1\".replace(\" \", \"\"),\n",
         "    pipeline_root=PIPELINE_ROOT,\n",
         "    parameter_values={\"seed\": 7, \"splits\": 10},\n",
         ")\n",
@@ -950,7 +966,18 @@
         "\n",
         "Next, generate another pipeline run that uses a different `seed` and `split` for the `iris_logregression` step.\n",
         "\n",
-        "Submit the new pipeline run:"
+        "Submit the new pipeline run:\n",
+        "\n",
+        "\n",
+        "**pipeline_root :**   Specify a Cloud Storage URI that your pipelines service account can access. The artifacts of your pipeline runs are stored within the pipeline root. \n",
+        "\n",
+        "**display_name :**  The name of the pipeline, this will show up in the Google Cloud console. \n",
+        "\n",
+        "**parameter_values :**  The pipeline parameters to pass to this run. For example, create a dict() with the parameter names as the dictionary keys and the parameter values as the dictionary values. \n",
+        "\n",
+        "**job_id :**  A unique identifier for this pipeline run. If the job ID is not specified, Vertex AI Pipelines creates a job ID for you using the pipeline name and the timestamp of when the pipeline run was started. \n",
+        "\n",
+        "**template_path :**  complete pipeline path"
       ]
     },
     {
@@ -962,9 +989,9 @@
       "outputs": [],
       "source": [
         "job = aip.PipelineJob(\n",
-        "    display_name=\"iris_\" + TIMESTAMP,\n",
+        "    display_name=\"iris_\" + UUID,\n",
         "    template_path=\"tabular classification_pipeline.json\".replace(\" \", \"_\"),\n",
-        "    job_id=f\"tabular classification-pipeline-v2{TIMESTAMP}-2\".replace(\" \", \"\"),\n",
+        "    job_id=f\"tabular classification-pipeline-v2{UUID}-2\".replace(\" \", \"\"),\n",
         "    pipeline_root=PIPELINE_ROOT,\n",
         "    parameter_values={\"seed\": 5, \"splits\": 7},\n",
         ")\n",
@@ -1081,16 +1108,7 @@
         "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
         "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
         "\n",
-        "Otherwise, you can delete the individual resources you created in this tutorial -- *Note:* this is auto-generated and not all resources may be applicable for this tutorial:\n",
-        "\n",
-        "- Dataset\n",
-        "- Pipeline\n",
-        "- Model\n",
-        "- Endpoint\n",
-        "- Batch Job\n",
-        "- Custom Job\n",
-        "- Hyperparameter Tuning Job\n",
-        "- Cloud Storage Bucket"
+        "Otherwise, you can delete the individual resources you created in this tutorial -- *Note:* this is auto-generated and not all resources may be applicable for this tutorial:"
       ]
     },
     {
@@ -1101,94 +1119,9 @@
       },
       "outputs": [],
       "source": [
-        "delete_dataset = True\n",
-        "delete_pipeline = True\n",
-        "delete_model = True\n",
-        "delete_endpoint = True\n",
-        "delete_batchjob = True\n",
-        "delete_customjob = True\n",
-        "delete_hptjob = True\n",
-        "delete_bucket = True\n",
-        "\n",
-        "try:\n",
-        "    if delete_model and \"DISPLAY_NAME\" in globals():\n",
-        "        models = aip.Model.list(\n",
-        "            filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
-        "        )\n",
-        "        model = models[0]\n",
-        "        aip.Model.delete(model)\n",
-        "        print(\"Deleted model:\", model)\n",
-        "except Exception as e:\n",
-        "    print(e)\n",
-        "\n",
-        "try:\n",
-        "    if delete_endpoint and \"DISPLAY_NAME\" in globals():\n",
-        "        endpoints = aip.Endpoint.list(\n",
-        "            filter=f\"display_name={DISPLAY_NAME}_endpoint\", order_by=\"create_time\"\n",
-        "        )\n",
-        "        endpoint = endpoints[0]\n",
-        "        endpoint.undeploy_all()\n",
-        "        aip.Endpoint.delete(endpoint.resource_name)\n",
-        "        print(\"Deleted endpoint:\", endpoint)\n",
-        "except Exception as e:\n",
-        "    print(e)\n",
-        "\n",
-        "if delete_dataset and \"DISPLAY_NAME\" in globals():\n",
-        "    if \"tabular\" == \"tabular\":\n",
-        "        try:\n",
-        "            datasets = aip.TabularDataset.list(\n",
-        "                filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
-        "            )\n",
-        "            dataset = datasets[0]\n",
-        "            aip.TabularDataset.delete(dataset.resource_name)\n",
-        "            print(\"Deleted dataset:\", dataset)\n",
-        "        except Exception as e:\n",
-        "            print(e)\n",
-        "\n",
-        "    if \"tabular\" == \"image\":\n",
-        "        try:\n",
-        "            datasets = aip.ImageDataset.list(\n",
-        "                filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
-        "            )\n",
-        "            dataset = datasets[0]\n",
-        "            aip.ImageDataset.delete(dataset.resource_name)\n",
-        "            print(\"Deleted dataset:\", dataset)\n",
-        "        except Exception as e:\n",
-        "            print(e)\n",
-        "\n",
-        "    if \"tabular\" == \"text\":\n",
-        "        try:\n",
-        "            datasets = aip.TextDataset.list(\n",
-        "                filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
-        "            )\n",
-        "            dataset = datasets[0]\n",
-        "            aip.TextDataset.delete(dataset.resource_name)\n",
-        "            print(\"Deleted dataset:\", dataset)\n",
-        "        except Exception as e:\n",
-        "            print(e)\n",
-        "\n",
-        "    if \"tabular\" == \"video\":\n",
-        "        try:\n",
-        "            datasets = aip.VideoDataset.list(\n",
-        "                filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
-        "            )\n",
-        "            dataset = datasets[0]\n",
-        "            aip.VideoDataset.delete(dataset.resource_name)\n",
-        "            print(\"Deleted dataset:\", dataset)\n",
-        "        except Exception as e:\n",
-        "            print(e)\n",
-        "\n",
-        "try:\n",
-        "    if delete_pipeline and \"DISPLAY_NAME\" in globals():\n",
-        "        pipelines = aip.PipelineJob.list(\n",
-        "            filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
-        "        )\n",
-        "        pipeline = pipelines[0]\n",
-        "        aip.PipelineJob.delete(pipeline.resource_name)\n",
-        "        print(\"Deleted pipeline:\", pipeline)\n",
-        "except Exception as e:\n",
-        "    print(e)\n",
+        "delete_bucket = False\n",
         "\n",
+        "job.delete()\n",
         "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
         "    ! gsutil rm -r $BUCKET_URI"
       ]

notebooks/official/pipelines/pipelines_intro_kfp.ipynb

@@ -81,7 +81,6 @@
         "The steps performed include:\n",
         "\n",
         "- Define and compile a `Vertex AI` pipeline.\n",
-        "- Schedule a recurring pipeline run.\n",
         "- Specify which service account to use for a pipeline run."
       ]
     },
@@ -97,13 +96,9 @@
         "\n",
         "* Vertex AI\n",
         "* Cloud Storage\n",
-        "* Cloud Functions\n",
-        "* Cloud Scheduler\n",
         "\n",
         "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing),\n",
         "[Cloud Storage pricing](https://cloud.google.com/storage/pricing),\n",
-        "[Cloud Functions pricing](ttps://cloud.google.com/functions/pricing), and\n",
-        "[Clould Scheduler pricing]((https://cloud.google.com/scheduler/pricing)),\n",
         "and use the [Pricing\n",
         "Calculator](https://cloud.google.com/products/calculator/)\n",
         "to generate a cost estimate based on your projected usage."
@@ -926,69 +921,6 @@
         "job.delete()"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "schedule_pipeline_run"
-      },
-      "source": [
-        "## Recurring pipeline runs: create a scheduled pipeline job\n",
-        "\n",
-        "This section shows how to create a **scheduled pipeline job**.  You do this using the pipeline you already defined.\n",
-        "\n",
-        "Under the hood, the scheduled jobs are supported by the Cloud Scheduler and a Cloud Functions function.  Check first that the APIs for both of these services are enabled.\n",
-        "You will need to first enable the [enable the Cloud Scheduler API](http://console.cloud.google.com/apis/library/cloudscheduler.googleapis.com) and the [Cloud Functions and Cloud Build APIs](https://console.cloud.google.com/flows/enableapi?apiid=cloudfunctions,cloudbuild.googleapis.com) if you have not already done so.\n",
-        "Note:you need to [create an App Engine app for your project](https://cloud.google.com/scheduler/docs/quickstart) if one does not already exist.\n",
-        "\n",
-        "\n",
-        "See the [Cloud Scheduler](https://cloud.google.com/scheduler/docs/configuring/cron-job-schedules) documentation for more on the cron syntax.\n",
-        "\n",
-        "Create a scheduled pipeline job, passing as an argument the job specification file that you compiled above.\n",
-        "\n",
-        "*Note:* You can pass a `parameter_values` dict that specifies the pipeline input parameters you want to use."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "Ty5hDoNX2Ou8"
-      },
-      "outputs": [],
-      "source": [
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    from kfp.v2.google.client import AIPlatformClient  # noqa: F811\n",
-        "\n",
-        "    api_client = AIPlatformClient(project_id=PROJECT_ID, region=REGION)\n",
-        "\n",
-        "    # adjust time zone and cron schedule as necessary\n",
-        "    response = api_client.create_schedule_from_job_spec(\n",
-        "        job_spec_path=\"intro_pipeline.json\",\n",
-        "        schedule=\"2 * * * *\",\n",
-        "        time_zone=\"America/Los_Angeles\",  # change this as necessary\n",
-        "        parameter_values={\"text\": \"Hello world!\"},\n",
-        "        # pipeline_root=PIPELINE_ROOT  # this argument is necessary if you did not specify PIPELINE_ROOT as part of the pipeline definition.\n",
-        "    )"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "J8AP1viy2Ou8"
-      },
-      "source": [
-        "Once the scheduled job is created, you can see it listed in the [Cloud Scheduler](https://console.cloud.google.com/cloudscheduler/) panel in the Console.\n",
-        "\n",
-        "<a href=\"https://storage.googleapis.com/amy-jo/images/kf-pls/pipelines_scheduler.png\" target=\"_blank\"><img src=\"https://storage.googleapis.com/amy-jo/images/kf-pls/pipelines_scheduler.png\" width=\"95%\"/></a>\n",
-        "\n",
-        "You can test the setup from the Cloud Scheduler panel by clicking 'RUN NOW'.\n",
-        "\n",
-        "> **Note**: The implementation is using a Cloud Functions function, which you can see listed in the [Cloud Functions](https://console.cloud.google.com/functions/list) panel in the console as `templated_http_request-v1`.\n",
-        "Don't delete this function, as it will prevent the Cloud Scheduler jobs from actually kicking off the pipeline run.  If you do delete it, create a new scheduled job in order to recreate the function.\n",
-        "\n",
-        "When you're done experimenting, you probably want to **PAUSE** your scheduled job from the Cloud Scheduler panel, so that the recurrent jobs do not keep running."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -1222,7 +1154,7 @@
       "outputs": [],
       "source": [
         "delete_pipeline = True\n",
-        "delete_bucket = True\n",
+        "delete_bucket = False\n",
         "\n",
         "try:\n",
         "    if delete_pipeline and \"DISPLAY_NAME\" in globals():\n",

notebooks/official/workbench/fraud_detection/fraud-detection-model.ipynb

@@ -1,14 +1,5 @@
 {
   "cells": [
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "c4b363e1330b"
-      },
-      "source": [
-        "# Build a fraud detection model on Vertex AI"
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -38,6 +29,8 @@
         "id": "05c670d35496"
       },
       "source": [
+        "# Build a fraud detection model on Vertex AI\n",
+        "\n",
         "<table align=\"left\">\n",
         "\n",
         "  <td>\n",
@@ -60,28 +53,6 @@
         "</table>"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "4c5fb7f2090f"
-      },
-      "source": [
-        "## Table of contents\n",
-        "\n",
-        "* [Overview](#section-1)\n",
-        "* [Dataset](#section-2)\n",
-        "* [Objective](#section-3)\n",
-        "* [Costs](#section-4)\n",
-        "* [Analyze the dataset](#section-5)\n",
-        "* [Fit a random forest model](#section-6)\n",
-        "* [Analyzing results](#section-7)\n",
-        "* [Save the model to a Cloud Storagae path](#section-8)\n",
-        "* [Create a model in Vertex AI](#section-9)\n",
-        "* [Create an Endpoint](#section-10)  \n",
-        "* [What-If Tool ](#section-11)\n",
-        "* [Clean up](#section-12)"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -94,19 +65,6 @@
         "This tutorial shows you how to build, deploy, and analyze predictions from a simple [random forest](https://en.wikipedia.org/wiki/Random_forest) model using tools like scikit-learn, Vertex AI, and the [What-IF Tool (WIT)](https://cloud.google.com/ai-platform/prediction/docs/using-what-if-tool) on a synthetic fraud transaction dataset to solve a financial fraud detection problem.\n"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "9625185ccee9"
-      },
-      "source": [
-        "## Dataset\n",
-        "<a name=\"section-2\"></a>\n",
-        "\n",
-        "\n",
-        "The dataset used in this tutorial is publicly available at Kaggle. See [Synthetic Financial Datasets For Fraud Detection](https://www.kaggle.com/ealaxi/paysim1)."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -118,6 +76,13 @@
         "\n",
         "This tutorial demonstrates data analysis and model-building using a synthetic financial dataset. The model is trained on identifying fraudulent cases among the transactions. Then, the trained model is deployed on a Vertex AI Endpoint and analyzed using the What-If Tool. The steps taken in this tutorial are as follows: \n",
         "\n",
+        "This tutorial uses the following Google Cloud ML services and resources:\n",
+        "\n",
+        "- Vertex AI Model\n",
+        "- Vertex AI Endpoint\n",
+        "\n",
+        "The steps performed include:\n",
+        "\n",
         "- Installation of required libraries\n",
         "- Reading the dataset from a Cloud Storage bucket\n",
         "- Performing exploratory analysis on the dataset\n",
@@ -129,6 +94,19 @@
         "- Un-deploying the model and cleaning up the model resources"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3037523e7523"
+      },
+      "source": [
+        "## Dataset\n",
+        "<a name=\"section-2\"></a>\n",
+        "\n",
+        "\n",
+        "The dataset used in this tutorial is publicly available at Kaggle. See [Synthetic Financial Datasets For Fraud Detection](https://www.kaggle.com/ealaxi/paysim1)."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -154,21 +132,15 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "1ba37fa1511f"
+        "id": "cd1bc75a1cb2"
       },
       "source": [
         "### Set up your local development environment\n",
         "\n",
         "**If you are using Colab or Google Cloud Notebooks**, your environment already meets\n",
-        "all the requirements to run this notebook. You can skip this step."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "cd1bc75a1cb2"
-      },
-      "source": [
+        "all the requirements to run this notebook. You can skip this step.\n",
+        "\n",
+        "\n",
         "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
         "You need the following:\n",
         "\n",
@@ -211,142 +183,44 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 1,
       "metadata": {
         "id": "172533a994ad"
       },
-      "outputs": [],
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
+            "flake8 4.0.1 requires importlib-metadata<4.3; python_version < \"3.8\", but you have importlib-metadata 4.12.0 which is incompatible.\u001b[0m\u001b[31m\n",
+            "\u001b[0m"
+          ]
+        }
+      ],
       "source": [
         "import os\n",
         "\n",
-        "import google.auth\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
         "\n",
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
         "USER_FLAG = \"\"\n",
-        "# Google Cloud Notebook requires dependencies to be installed with '--user'\n",
-        "if \"default\" in dir(google.auth):\n",
-        "    USER_FLAG = \"--user\""
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "a465cf9367de"
-      },
-      "source": [
-        "Install the latest version of the Vertex AI client library.\n",
+        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\"\n",
         "\n",
-        "Run the following command in your notebook environment to install the Vertex SDK for Python:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "6380f7ee5f54"
-      },
-      "outputs": [],
-      "source": [
-        "! pip install {USER_FLAG} --upgrade google-cloud-aiplatform"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "1969a1cc46cf"
-      },
-      "source": [
-        "Run the following command in your notebook environment to install witwidget:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "8b10e59b0911"
-      },
-      "outputs": [],
-      "source": [
-        "! pip install {USER_FLAG} witwidget"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "4099ce79705a"
-      },
-      "source": [
-        "Run the following command in your notebook environment to install joblib:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "1e56d524753a"
-      },
-      "outputs": [],
-      "source": [
-        "! pip install {USER_FLAG} joblib"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "b87ee3041f7d"
-      },
-      "source": [
-        "Run the following command in your notebook environment to install scikit-learn:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "c3ebecd9bd72"
-      },
-      "outputs": [],
-      "source": [
-        "! pip install {USER_FLAG} scikit-learn"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "b624b5163531"
-      },
-      "source": [
-        "Run the following command in your notebook environment to install fsspec:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "79c7a64b04de"
-      },
-      "outputs": [],
-      "source": [
-        "! pip install {USER_FLAG} fsspec"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "5593090dcf0a"
-      },
-      "source": [
-        "Run the following command in your notebook environment to install gcsfs:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "7bf981bc5bf6"
-      },
-      "outputs": [],
-      "source": [
-        "! pip install {USER_FLAG} gcsfs"
+        "# Install the latest version of the Vertex AI client library.\n",
+        "! pip3 install --upgrade google-cloud-aiplatform {USER_FLAG} -q\n",
+        "\n",
+        "# Install additional libraries\n",
+        "! pip3 install {USER_FLAG} witwidget -q\n",
+        "! pip3 install {USER_FLAG} joblib -q\n",
+        "! pip3 install {USER_FLAG} scikit-learn -q\n",
+        "! pip3 install {USER_FLAG} fsspec -q\n",
+        "! pip3 install {USER_FLAG} gcsfs -q"
       ]
     },
     {
@@ -378,21 +252,14 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "2d9b3731b3e0"
-      },
-      "source": [
-        "## Before you begin"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "7a5cb1df1ef7"
       },
       "source": [
+        "## Before you begin\n",
+        "\n",
         "### Set up your Google Cloud project\n",
         "\n",
         "**The following steps are required, regardless of your notebook environment.**\n",
@@ -426,19 +293,26 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "b27f37ed1ccf"
+        "id": "dcdfccf50581"
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
-        "PROJECT_ID = \"\"\n",
-        "\n",
-        "# Get your Google Cloud project ID from gcloud\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    shell_output = !gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "5bf9979b96ff"
+      },
+      "outputs": [],
+      "source": [
+        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
+        "    # Get your GCP project id from gcloud\n",
+        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
         "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID: \", PROJECT_ID)"
+        "    print(\"Project ID:\", PROJECT_ID)"
       ]
     },
     {
@@ -450,18 +324,6 @@
         "Otherwise, set your project ID here."
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "3dbdf6a5c539"
-      },
-      "outputs": [],
-      "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
-        "    PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -473,15 +335,49 @@
         "! gcloud config set project $PROJECT_ID"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "264543a144ad"
+      },
+      "source": [
+        "#### Region\n",
+        "\n",
+        "You can also change the `REGION` variable, which is used for operations\n",
+        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. It is recommended that you choose the region closest to you.\n",
+        "\n",
+        "- Americas: `us-central1`\n",
+        "- Europe: `europe-west4`\n",
+        "- Asia Pacific: `asia-east1`\n",
+        "\n",
+        "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
+        "\n",
+        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3281bedf6d3c"
+      },
+      "outputs": [],
+      "source": [
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "e663bd062c6f"
       },
       "source": [
-        "#### Timestamp\n",
+        "#### UUID\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append it onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
       ]
     },
     {
@@ -492,21 +388,16 @@
       },
       "outputs": [],
       "source": [
-        "from datetime import datetime\n",
+        "import random\n",
+        "import string\n",
         "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "0c7f603fcdcf"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Google Cloud Notebooks**, your environment is already\n",
-        "authenticated. Skip this step."
+        "# Generate a uuid of length 8\n",
+        "def generate_uuid():\n",
+        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=8))\n",
+        "\n",
+        "\n",
+        "UUID = generate_uuid()"
       ]
     },
     {
@@ -515,6 +406,11 @@
         "id": "72bf8f7c9ab3"
       },
       "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
+        "authenticated. Skip this step.\n",
+        "\n",
         "**If you are using Colab**, run the cell below and follow the instructions\n",
         "when prompted to authenticate your account via oAuth.\n",
         "\n",
@@ -547,19 +443,19 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "import sys\n",
-        "\n",
         "# If you are running this notebook in Colab, run this cell and follow the\n",
         "# instructions to authenticate your GCP account. This provides access to your\n",
         "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
         "# requests.\n",
         "\n",
-        "# The Google Cloud Notebook product has specific requirements\n",
-        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
+        "import os\n",
+        "import sys\n",
         "\n",
-        "# If on Google Cloud Notebooks, then don't execute this code\n",
-        "if not IS_GOOGLE_CLOUD_NOTEBOOK:\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = \"google.colab\" in sys.modules\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
         "    if \"google.colab\" in sys.modules:\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
@@ -601,27 +497,25 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "f56c52ba662c"
+        "id": "5e9a782f5608"
       },
       "outputs": [],
       "source": [
         "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "REGION = \"[your-region]\"  # @param {type:\"string\"}"
+        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "68d1f4908641"
+        "id": "6d0729c4ae94"
       },
       "outputs": [],
       "source": [
         "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"-vertex-ai-\" + TIMESTAMP\n",
-        "BUCKET_URI = f\"gs://{BUCKET_NAME}\"\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
+        "    BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
     {
@@ -697,6 +591,7 @@
         "import numpy as np\n",
         "import pandas as pd\n",
         "from google.cloud import aiplatform, storage\n",
+        "from IPython.display import display\n",
         "from sklearn.ensemble import RandomForestClassifier\n",
         "from sklearn.metrics import (average_precision_score, classification_report,\n",
         "                             confusion_matrix, f1_score)\n",
@@ -706,6 +601,15 @@
         "warnings.filterwarnings(\"ignore\")"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "fdcb614c716f"
+      },
+      "source": [
+        "## Load dataset"
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -714,7 +618,6 @@
       },
       "outputs": [],
       "source": [
-        "# Load dataset\n",
         "df = pd.read_csv(\n",
         "    \"gs://cloud-samples-data/vertex-ai/managed_notebooks/fraud_detection/fraud_detection_data.csv\"\n",
         ")"
@@ -1031,6 +934,8 @@
         "\n",
         "# Upload the saved model file to Cloud Storage\n",
         "BLOB_PATH = \"[your-blob-path]\"\n",
+        "if BLOB_PATH == \"[your-blob-path]\":\n",
+        "    BLOB_PATH = \"fraud-detection-model-path\"\n",
         "BLOB_NAME = os.path.join(BLOB_PATH, FILE_NAME)\n",
         "\n",
         "bucket = storage.Client(PROJECT_ID).bucket(BUCKET_NAME)\n",
@@ -1057,6 +962,8 @@
       "outputs": [],
       "source": [
         "MODEL_DISPLAY_NAME = \"[your-model-display-name]\"\n",
+        "if MODEL_DISPLAY_NAME == \"[your-model-display-name]\":\n",
+        "    MODEL_DISPLAY_NAME = \"fraud-detection-model-display-name\"\n",
         "ARTIFACT_GCS_PATH = f\"{BUCKET_URI}/{BLOB_PATH}\"\n",
         "SERVING_CONTAINER_IMAGE_URI = (\n",
         "    \"us-docker.pkg.dev/vertex-ai/prediction/sklearn-cpu.1-0:latest\"\n",
@@ -1105,7 +1012,9 @@
       },
       "outputs": [],
       "source": [
-        "ENDPOINT_DISPLAY_NAME = \"[your-endpoint-display-name]\""
+        "ENDPOINT_DISPLAY_NAME = \"[your-endpoint-display-name]\"\n",
+        "if ENDPOINT_DISPLAY_NAME == \"[your-endpoint-display-name]\":\n",
+        "    ENDPOINT_DISPLAY_NAME = \"fraud-detection-endpoint\""
       ]
     },
     {
@@ -1143,6 +1052,8 @@
       "outputs": [],
       "source": [
         "DEPLOYED_MODEL_NAME = \"[your-deployed-model-name]\"\n",
+        "if DEPLOYED_MODEL_NAME == \"[your-deployed-model-name]\":\n",
+        "    DEPLOYED_MODEL_NAME = \"fraud-detection-deployed-model\"\n",
         "MACHINE_TYPE = \"n1-standard-2\""
       ]
     },
@@ -1224,34 +1135,33 @@
       },
       "outputs": [],
       "source": [
-        "# define target and labels\n",
-        "TARGET_FEATURE = \"isFraud\"\n",
-        "LABEL_VOCAB = [\"not-fraud\", \"fraud\"]\n",
+        "if not IS_COLAB:\n",
+        "    # define target and labels\n",
+        "    TARGET_FEATURE = \"isFraud\"\n",
+        "    LABEL_VOCAB = [\"not-fraud\", \"fraud\"]\n",
         "\n",
-        "# define the function to adjust the predictions\n",
+        "    # define the function to adjust the predictions\n",
         "\n",
+        "    def adjust_prediction(pred):\n",
+        "        return [1 - pred, pred]\n",
         "\n",
-        "def adjust_prediction(pred):\n",
-        "    return [1 - pred, pred]\n",
-        "\n",
-        "\n",
-        "# Combine the features and labels into one array for the What-If Tool\n",
-        "test_examples = np.hstack(\n",
-        "    (test_samples_X.to_numpy(), test_samples_y.to_numpy().reshape(-1, 1))\n",
-        ")\n",
-        "\n",
-        "# Configure the WIT to run on the locally trained model\n",
-        "config_builder = (\n",
-        "    WitConfigBuilder(\n",
-        "        test_examples.tolist(), test_samples_X.columns.tolist() + [\"isFraud\"]\n",
+        "    # Combine the features and labels into one array for the What-If Tool\n",
+        "    test_examples = np.hstack(\n",
+        "        (test_samples_X.to_numpy(), test_samples_y.to_numpy().reshape(-1, 1))\n",
         "    )\n",
-        "    .set_custom_predict_fn(forest.predict_proba)\n",
-        "    .set_target_feature(TARGET_FEATURE)\n",
-        "    .set_label_vocab(LABEL_VOCAB)\n",
-        ")\n",
         "\n",
-        "# display the WIT widget\n",
-        "WitWidget(config_builder, height=600)"
+        "    # Configure the WIT to run on the locally trained model\n",
+        "    config_builder = (\n",
+        "        WitConfigBuilder(\n",
+        "            test_examples.tolist(), test_samples_X.columns.tolist() + [\"isFraud\"]\n",
+        "        )\n",
+        "        .set_custom_predict_fn(forest.predict_proba)\n",
+        "        .set_target_feature(TARGET_FEATURE)\n",
+        "        .set_label_vocab(LABEL_VOCAB)\n",
+        "    )\n",
+        "\n",
+        "    # display the WIT widget\n",
+        "    display(WitWidget(config_builder, height=600))"
       ]
     },
     {
@@ -1271,36 +1181,35 @@
       },
       "outputs": [],
       "source": [
-        "# configure the target and class-labels\n",
-        "TARGET_FEATURE = \"isFraud\"\n",
-        "LABEL_VOCAB = [\"not-fraud\", \"fraud\"]\n",
+        "if not IS_COLAB:\n",
+        "    # configure the target and class-labels\n",
+        "    TARGET_FEATURE = \"isFraud\"\n",
+        "    LABEL_VOCAB = [\"not-fraud\", \"fraud\"]\n",
         "\n",
-        "# function to return predictions from the deployed Model\n",
+        "    # function to return predictions from the deployed Model\n",
         "\n",
+        "    def endpoint_predict_sample(instances: list):\n",
+        "        prediction = endpoint.predict(instances=instances)\n",
+        "        preds = [[1 - i, i] for i in prediction.predictions]\n",
+        "        return preds\n",
         "\n",
-        "def endpoint_predict_sample(instances: list):\n",
-        "    prediction = endpoint.predict(instances=instances)\n",
-        "    preds = [[1 - i, i] for i in prediction.predictions]\n",
-        "    return preds\n",
-        "\n",
-        "\n",
-        "# Combine the features and labels into one array for the What-If Tool\n",
-        "test_examples = np.hstack(\n",
-        "    (test_samples_X.to_numpy(), test_samples_y.to_numpy().reshape(-1, 1))\n",
-        ")\n",
-        "\n",
-        "# Configure the WIT with the prediction function\n",
-        "config_builder = (\n",
-        "    WitConfigBuilder(\n",
-        "        test_examples.tolist(), test_samples_X.columns.tolist() + [\"isFraud\"]\n",
+        "    # Combine the features and labels into one array for the What-If Tool\n",
+        "    test_examples = np.hstack(\n",
+        "        (test_samples_X.to_numpy(), test_samples_y.to_numpy().reshape(-1, 1))\n",
         "    )\n",
-        "    .set_custom_predict_fn(endpoint_predict_sample)\n",
-        "    .set_target_feature(TARGET_FEATURE)\n",
-        "    .set_label_vocab(LABEL_VOCAB)\n",
-        ")\n",
         "\n",
-        "# run the WIT-widget\n",
-        "WitWidget(config_builder, height=400)"
+        "    # Configure the WIT with the prediction function\n",
+        "    config_builder = (\n",
+        "        WitConfigBuilder(\n",
+        "            test_examples.tolist(), test_samples_X.columns.tolist() + [\"isFraud\"]\n",
+        "        )\n",
+        "        .set_custom_predict_fn(endpoint_predict_sample)\n",
+        "        .set_target_feature(TARGET_FEATURE)\n",
+        "        .set_label_vocab(LABEL_VOCAB)\n",
+        "    )\n",
+        "\n",
+        "    # run the WIT-widget\n",
+        "    display(WitWidget(config_builder, height=400))"
       ]
     },
     {

notebooks/official/workbench/predictive_maintainance/predictive_maintenance_usecase.ipynb

@@ -1,17 +1,62 @@
 {
   "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "18ebbd838e32"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2022 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "aef73cfa8725"
       },
       "source": [
-        "# Predictive Maintenance \n",
+        "# Predictive Maintenance using Vertex AI\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/predictive_maintainance/predictive_maintenance_usecase.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "    <td>\n",
+        "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/predictive_maintainance/predictive_maintenance_usecase.ipynb\">\n",
+        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\\\" alt=\"Colab logo\"> Run in Colab\n",
+        "        </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/predictive_maintainance/predictive_maintenance_usecase.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "  </td>\n",
+        "</table>\n",
+        "<br/><br/><br/>\n",
+        "\n",
         "\n",
         "## Table of contents\n",
         "* [Overview](#section-1)\n",
-        "* [Dataset](#section-2)\n",
-        "* [Objective](#section-3)\n",
+        "* [Objective](#section-2)\n",
+        "* [Dataset](#section-3)\n",
         "* [Costs](#section-4)\n",
         "* [Data analysis](#section-5)\n",
         "* [Fit a regression model](#section-6)\n",
@@ -22,24 +67,32 @@
         "  * [Create an endpoint](#section-11)\n",
         "  * [Deploy the model to the created endpoint](#section-12)\n",
         "  * [Test calling the endpoint](#section-13)\n",
-        "* [Clean up](#section-14)\n",
-        "\n",
-        "\n",
+        "* [Clean up](#section-14)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e10c5167a061"
+      },
+      "source": [
         "## Overview\n",
         "<a name=\"section-1\"></a>\n",
         "\n",
-        "This notebook demonstrates how to perform predictive maintenance on industrial data using machine learning techniques, deploy the machine learning model on Vertex AI, and automate the workflow using the executor feature of Vertex AI Workbench.\n",
+        "In this notebook, you go through a predictive maintenance usecase on industrial data using machine learning techniques, deploy the machine learning model on Vertex AI, and automate the workflow using the executor feature of Vertex AI Workbench.\n",
         "\n",
-        "*Note: This notebook file was developed to run in a [Vertex AI Workbench managed notebooks](https://console.cloud.google.com/vertex-ai/workbench/list/managed) instance using the XGBoost (Local) kernel. Some components of this notebook may not work in other notebook environments.*\n",
-        "\n",
-        "## Dataset\n",
+        "*Note: This notebook file is developed to run in a [Vertex AI Workbench managed notebooks](https://console.cloud.google.com/vertex-ai/workbench/list/managed) instance using the XGBoost (Local) kernel. Some components of this notebook may not work in other notebook environments.*"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "fead9e83ebd7"
+      },
+      "source": [
+        "### Objective\n",
         "<a name=\"section-2\"></a>\n",
         "\n",
-        "The dataset used in this notebook is a part of the [NASA Turbofan Engine Degradation Simulation dataset](https://ti.arc.nasa.gov/tech/dash/groups/pcoe/prognostic-data-repository/), which consists of simulated time-series data for four sets of fleet engines under different combinations of operational conditions and fault modes. In this notebook, only one of the engine's simulated data (FD001) has been used to analyze and train a model that can predict the engine's remaining useful life.\n",
-        "\n",
-        "## Objectives\n",
-        "<a name=\"section-3\"></a>\n",
-        "\n",
         "The objectives of this notebook include:\n",
         "\n",
         "- Loading the required dataset from a Cloud Storage bucket.\n",
@@ -49,9 +102,28 @@
         "- Evaluating the model.\n",
         "- Running the notebook end-to-end as a training job using Executor.\n",
         "- Deploying the model on Vertex AI.\n",
-        "- Clean up.\n",
+        "- Clean up."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "a71f4d96bf80"
+      },
+      "source": [
+        "### Dataset\n",
+        "<a name=\"section-3\"></a>\n",
         "\n",
-        "## Costs\n",
+        "The dataset used in this notebook is a part of the [NASA Turbofan Engine Degradation Simulation dataset](https://ti.arc.nasa.gov/tech/dash/groups/pcoe/prognostic-data-repository/), which consists of simulated time-series data for four sets of fleet engines under different combinations of operational conditions and fault modes. A version of this dataset which is saved to a public Cloud Storage bucket is used in this notebook. In this notebook, one of the engine's simulated data (FD001) is used to analyze and train a model that can predict the engine's remaining useful life."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "36c53c95b4b9"
+      },
+      "source": [
+        "### Costs\n",
         "<a name=\"section-4\"></a>\n",
         "\n",
         "This tutorial uses the following billable components of Google Cloud:\n",
@@ -69,24 +141,126 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "5b15a97278df"
+        "id": "629f52f6efe1"
       },
       "source": [
-        "## Before you begin\n",
-        "\n",
         "### Kernel selection\n",
-        "Select <b>XGBoost</b> kernel while running this notebook on Vertex AI Workbench managed notebooks instances or ensure that the following libraries are installed in the environment where this notebook is being run.\n",
+        "Select <b>XGBoost</b> kernel while running this notebook on Vertex AI Workbench's managed instances. Otherwise, ensure that the following libraries are installed in the environment where this notebook is being run.\n",
         "- XGBoost\n",
         "- Pandas\n",
         "- Seaborn\n",
         "- Sklearn\n",
         "\n",
-        "Along with the above libraries, the following google-cloud libraries are also used in this notebook.\n",
+        "Along with the above libraries, th`e following google-cloud libraries are also used in this notebook.\n",
         "\n",
         "- google.cloud.aiplatform\n",
-        "- google.cloud.storage\n",
+        "- google.cloud.storage"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "16bee0754628"
+      },
+      "source": [
+        "## Installation\n",
         "\n",
-        "### Set your project ID\n",
+        "Install the following packages to run this notebook outside Vertex AI Workbench's managed instances."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "69520a67e54c"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
+        "\n",
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "USER_FLAG = \"\"\n",
+        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\"\n",
+        "    \n",
+        "! pip3 install {USER_FLAG} --upgrade google-cloud-aiplatform \\\n",
+        "                                    google-cloud-storage \\\n",
+        "                                    xgboost \\\n",
+        "                                    seaborn \\\n",
+        "                                    sklearn \\\n",
+        "                                    fsspec \\\n",
+        "                                    gcsfs \\\n",
+        "                                    pandas -q"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "eda79cca981d"
+      },
+      "source": [
+        "### Restart the kernel\n",
+        "\n",
+        "Once you've installed the additional packages, you need to restart the notebook kernel so it can find the packages."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "e200999cabe5"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "if not os.getenv(\"IS_TESTING\"):\n",
+        "    # Automatically restart kernel after installs\n",
+        "    import IPython\n",
+        "\n",
+        "    app = IPython.Application.instance()\n",
+        "    app.kernel.do_shutdown(True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5b15a97278df"
+      },
+      "source": [
+        "## Before you begin \n",
+        "\n",
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com).\n",
+        "\n",
+        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5aee4379e8e5"
+      },
+      "source": [
+        "#### Set your project ID\n",
         "\n",
         "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
       ]
@@ -99,36 +273,67 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
-        "PROJECT_ID = \"\"\n",
-        "\n",
-        "# Get your Google Cloud project ID from gcloud\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    shell_output = !gcloud config list --format 'value(core.project)' 2>/dev/null\n",
-        "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID: \", PROJECT_ID)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "750bf2883c2d"
-      },
-      "source": [
-        "Otherwise, set your project ID here."
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "3c6db1ca88b9"
+        "id": "5bf9979b96ff"
       },
       "outputs": [],
       "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
-        "    PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
+        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
+        "    # Get your GCP project id from gcloud\n",
+        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "    PROJECT_ID = shell_output[0]\n",
+        "    print(\"Project ID:\", PROJECT_ID)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "09021c90b34c"
+      },
+      "outputs": [],
+      "source": [
+        "! gcloud config set project $PROJECT_ID"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9658ecf524b1"
+      },
+      "source": [
+        "#### Region\n",
+        "\n",
+        "You can also change the `REGION` variable, which is used for operations\n",
+        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. It is recommended that you choose the region closest to you.\n",
+        "\n",
+        "- Americas: `us-central1`\n",
+        "- Europe: `europe-west4`\n",
+        "- Asia Pacific: `asia-east1`\n",
+        "\n",
+        "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
+        "\n",
+        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "5c615e53149f"
+      },
+      "outputs": [],
+      "source": [
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -137,9 +342,9 @@
         "id": "f66f96816fd0"
       },
       "source": [
-        "#### Timestamp\n",
+        "#### UUID\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append it onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
       ]
     },
     {
@@ -150,9 +355,84 @@
       },
       "outputs": [],
       "source": [
-        "from datetime import datetime\n",
+        "import random\n",
+        "import string\n",
         "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+        "\n",
+        "# Generate a uuid of a specifed length(default=8)\n",
+        "def generate_uuid(length: int = 8) -> str:\n",
+        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
+        "\n",
+        "\n",
+        "UUID = generate_uuid()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "df899ce9999c"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
+        "authenticated.\n",
+        "\n",
+        "**If you are using Colab**, run the cell below and follow the instructions\n",
+        "when prompted to authenticate your account via oAuth.\n",
+        "\n",
+        "**Otherwise**, follow these steps:\n",
+        "\n",
+        "1. In the Cloud Console, go to the [**Create service account key**\n",
+        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
+        "\n",
+        "2. Click **Create service account**.\n",
+        "\n",
+        "3. In the **Service account name** field, enter a name, and\n",
+        "   click **Create**.\n",
+        "\n",
+        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
+        "into the filter box, and select\n",
+        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "\n",
+        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
+        "local environment.\n",
+        "\n",
+        "6. Enter the path to your service account key as the\n",
+        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "201e8e760d22"
+      },
+      "outputs": [],
+      "source": [
+        "# If you are running this notebook in Colab, run this cell and follow the\n",
+        "# instructions to authenticate your GCP account. This provides access to your\n",
+        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
+        "# requests.\n",
+        "\n",
+        "import os\n",
+        "import sys\n",
+        "\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = \"google.colab\" in sys.modules\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
+        "    if \"google.colab\" in sys.modules:\n",
+        "        from google.colab import auth as google_auth\n",
+        "\n",
+        "        google_auth.authenticate_user()\n",
+        "\n",
+        "    # If you are running this notebook locally, replace the string below with the\n",
+        "    # path to your service account key and run this cell to authenticate your GCP\n",
+        "    # account.\n",
+        "    elif not os.getenv(\"IS_TESTING\"):\n",
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
       ]
     },
     {
@@ -161,11 +441,18 @@
         "id": "ea53caa30628"
       },
       "source": [
-        "## Select or Create a Cloud Storage Bucket for storing the model\n",
+        "### Create a Cloud Storage bucket\n",
         "\n",
-        "When you create a model resource on Vertex AI using the Cloud SDK, you need to give a Cloud Storage bucket URI of the model where the model is stored. Using the model saved, you can then create Vertex AI model and endpoint resources in order to serve online predictions.\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
         "\n",
-        "Set the name of your Cloud Storage bucket below. It must be unique across all Cloud Storage buckets. You may also change the `REGION` variable, which is used for operations throughout the rest of this notebook. Make sure to choose a region where Vertex AI services are available."
+        "\n",
+        "When you create a model in Vertex AI using the Cloud SDK, you give a Cloud Storage path where the trained model is saved. \n",
+        "In this tutorial, Vertex AI saves the trained model to a Cloud Storage bucket. Using this model artifact, you can then\n",
+        "create Vertex AI model and endpoint resources in order to serve\n",
+        "online predictions.\n",
+        "\n",
+        "Set the name of your Cloud Storage bucket below. It must be unique across all\n",
+        "Cloud Storage buckets."
       ]
     },
     {
@@ -176,9 +463,8 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"[your-bucket-name]\"\n",
-        "BUCKET_URI = f\"gs://{BUCKET_NAME}\"\n",
-        "REGION = \"us-central1\""
+        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
     {
@@ -189,13 +475,9 @@
       },
       "outputs": [],
       "source": [
-        "# Set a default bucketname in case bucket name is not given\n",
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None:\n",
-        "    from datetime import datetime\n",
-        "\n",
-        "    TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
-        "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
+        "    BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
     {
@@ -215,7 +497,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_NAME"
+        "! gsutil mb -l $REGION $BUCKET_URI"
       ]
     },
     {
@@ -235,7 +517,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_NAME"
+        "! gsutil ls -al $BUCKET_URI"
       ]
     },
     {
@@ -244,7 +526,7 @@
         "id": "4c0f6aac282a"
       },
       "source": [
-        "## Import the required libraries"
+        "### Import the required libraries"
       ]
     },
     {
@@ -287,7 +569,7 @@
       "outputs": [],
       "source": [
         "# load the data from the source\n",
-        "INPUT_PATH = \"gs://vertex_ai_managed_services_demo/mfg_predictive_maintenance/train_FD001.txt\"  # data source\n",
+        "INPUT_PATH = \"gs://cloud-samples-data/ai-platform-unified/datasets/tabular/predictive_maintenance.csv\"  # data source\n",
         "raw_data = pd.read_csv(INPUT_PATH, sep=\" \", header=None)\n",
         "# check the data\n",
         "print(raw_data.shape)\n",
@@ -492,7 +774,7 @@
         "id": "8197cdef2cff"
       },
       "source": [
-        "As the current objective is to predict the remaining useful life (RUL) of each unit (ID), the target variable needs to be identified. Since we're dealing with a timeseries data that represents the lifetime of a unit, remaining useful life of a unit can be calculated by subtracting the current cycle from the maximum cycle of that unit.\n",
+        "As the current objective is to predict the remaining useful life (RUL) of each unit (ID), the target variable needs to be identified. Since you're dealing with a timeseries data that represents the lifetime of a unit, remaining useful life of a unit can be calculated by subtracting the current cycle from the maximum cycle of that unit.\n",
         "\n",
         "\t\t\t\t\tRUL = Max. Cycle - Current Cycle    \n",
         "## RUL calculation and Feature selection"
@@ -810,6 +1092,7 @@
         "## Running a notebook end-to-end using executor\n",
         "<a name=\"section-9\"></a>\n",
         "\n",
+        "**Note:** This section can only be considered when running this notebook on Managed instances from Vertex AI Workbench.\n",
         "### Automating the notebook execution\n",
         "All the steps followed until now can be run as a training job without using any additional code using the Vertex AI Workbench executor. The executor can help you run a notebook file from start to end, with your choice of the environment, machine type, input parameters, and other characteristics. After setting up an execution, the notebook is executed as a job in Vertex AI custom training. Your jobs can be monitored from the Executor pane in the left sidebar.\n",
         "\n",
@@ -817,13 +1100,13 @@
         "\n",
         "The executor also lets you choose the environment and machine type while automating the runs similar to Vertex AI training jobs without switching to the training jobs UI. Apart from the custom container that replicates the existing kernel by default, pre-built environments like TensorFlow Enterprise, PyTorch, and others can also be selected to run the notebook. The required compute power can be specified by choosing from the list of machine types available, including GPUs.\n",
         "\n",
-        "## Scheduled runs on executor\n",
+        "### Scheduled runs on executor\n",
         "\n",
         "Notebook runs can also be scheduled recurringly with the executor. To do so, select Schedule-based recurring executions as the run type instead of One-time execution. The frequency of the job and the time when it executes is provided when you create the execution.\n",
         "\n",
         "<img src=\"https://storage.googleapis.com/gweb-cloudblog-publish/images/7_Vertex_AI_Workbench.max-1100x1100.jpg\">\n",
         "\n",
-        "## Parameterizing the variables\n",
+        "### Parameterizing the variables\n",
         "\n",
         "The executor lets you run a notebook with different sets of input parameters. If you’ve added parameter tags to any of your notebook cells, you can pass in your parameter values to the executor. More about how to use this feature can be found on this [blog](https://cloud.google.com/blog/products/ai-machine-learning/schedule-and-execute-notebooks-with-vertex-ai-workbench).\n",
         "\n",
@@ -855,6 +1138,37 @@
         "ARTIFACT_GCS_PATH = f\"gs://{BUCKET_NAME}/{BLOB_PATH}\""
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "1aa75b3d4616"
+      },
+      "source": [
+        "Give a display name to the Vertex AI model resource."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "02ca350dba6c"
+      },
+      "outputs": [],
+      "source": [
+        "# Set the model-dsiplay-name\n",
+        "MODEL_DISPLAY_NAME = \"[your-model-display-name]\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# Otherwise, use the default name\n",
+        "if (\n",
+        "    MODEL_DISPLAY_NAME == \"[your-model-display-name]\"\n",
+        "    or MODEL_DISPLAY_NAME is None\n",
+        "    or MODEL_DISPLAY_NAME == \"\"\n",
+        "):\n",
+        "    MODEL_DISPLAY_NAME = \"pred_maint_model_\" + UUID\n",
+        "\n",
+        "print(MODEL_DISPLAY_NAME)"
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -891,6 +1205,28 @@
         "Next, create an endpoint resource for deploying the model."
       ]
     },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "e1e0cd571992"
+      },
+      "outputs": [],
+      "source": [
+        "# Set the endpoint-dsiplay-name\n",
+        "ENDPOINT_DISPLAY_NAME = \"[your-endpoint-display-name]\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# Otherwise, use the default name\n",
+        "if (\n",
+        "    ENDPOINT_DISPLAY_NAME == \"[your-endpoint-display-name]\"\n",
+        "    or ENDPOINT_DISPLAY_NAME is None\n",
+        "    or ENDPOINT_DISPLAY_NAME == \"\"\n",
+        "):\n",
+        "    ENDPOINT_DISPLAY_NAME = \"pred_maint_endpoint_\" + UUID\n",
+        "\n",
+        "print(ENDPOINT_DISPLAY_NAME)"
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -899,6 +1235,7 @@
       },
       "outputs": [],
       "source": [
+        "# Create the Endpoint resource\n",
         "endpoint = aiplatform.Endpoint.create(display_name=ENDPOINT_DISPLAY_NAME)\n",
         "\n",
         "print(endpoint.display_name)\n",
@@ -915,18 +1252,11 @@
         "<a name=\"section-12\"></a>\n",
         "\n",
         "\n",
-        "Configure the deployment name, machine type, and other parameters for the deployment and deploy the model to the created endpoint."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "ca41cac871d6"
-      },
-      "outputs": [],
-      "source": [
-        "MACHINE_TYPE = \"n1-standard-2\""
+        "Configure the following parameters and deploy the model to the created endpoint.\n",
+        "\n",
+        "- `endpoint`: The `Endpoint` object created using Vertex AI SDK.\n",
+        "- `deployed_model_display_name`: A display-name for the deployment.\n",
+        "- `machine_type`: Type of the machine required for the deployment environment. See [here](https://cloud.google.com/vertex-ai/docs/predictions/configure-compute) for references."
       ]
     },
     {
@@ -940,8 +1270,8 @@
         "# deploy the model to the endpoint\n",
         "model.deploy(\n",
         "    endpoint=endpoint,\n",
-        "    deployed_model_display_name=DEPLOYED_MODEL_NAME,\n",
-        "    machine_type=MACHINE_TYPE,\n",
+        "    deployed_model_display_name=MODEL_DISPLAY_NAME + \"_deployment\",\n",
+        "    machine_type=\"n1-standard-2\",\n",
         ")\n",
         "\n",
         "model.wait()\n",
@@ -984,7 +1314,15 @@
         "## Clean up\n",
         "<a name=\"section-14\"></a>\n",
         "\n",
-        "Undeploy the model from the endpoint."
+        "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
+        "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
+        "\n",
+        "Otherwise, you can delete the individual resources you created in this tutorial:\n",
+        "* Vertex AI Model\n",
+        "* Vertex AI Endpoint\n",
+        "* Cloud Storage bucket\n",
+        "\n",
+        "Set `delete_bucket` to **True** to delete the Cloud Storage bucket."
       ]
     },
     {
@@ -995,68 +1333,19 @@
       },
       "outputs": [],
       "source": [
-        "DEPLOYED_MODEL_ID = \"\"\n",
-        "endpoint.undeploy(deployed_model_id=DEPLOYED_MODEL_ID)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "96e427b77791"
-      },
-      "source": [
-        "Delete the endpoint."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "ace028ac23ea"
-      },
-      "outputs": [],
-      "source": [
-        "endpoint.delete()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "4b77998d0512"
-      },
-      "source": [
-        "Delete the model."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "e034150a4c94"
-      },
-      "outputs": [],
-      "source": [
-        "model.delete()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "23cb2deb122d"
-      },
-      "source": [
-        "Remove the contents of the Cloud Storage bucket."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "98aaac27d85d"
-      },
-      "outputs": [],
-      "source": [
-        "! gsutil -m rm -r $BUCKET_URI"
+        "# Undeploy all the models from the endpoint\n",
+        "endpoint.undeploy_all()\n",
+        "\n",
+        "# Delete the endpoint resource\n",
+        "endpoint.delete()\n",
+        "\n",
+        "# Delete the model resource\n",
+        "model.delete()\n",
+        "\n",
+        "# Delete the Cloud Storage bucket\n",
+        "delete_bucket = False\n",
+        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "    ! gsutil -m rm -r $BUCKET_URI"
       ]
     }
   ],