Merge branch 'main' into mlops_model_monitoring

community-content/CODEOWNERS

@@ -1,6 +1,5 @@
 *  @vertex-ai-samples-contributors  @GoogleCloudPlatform/cloudml-samples-owners
 /tf_agents_bandits_movie_recommendation_with_kfp_and_vertex_sdk	@yinghsienwu
-/pytorch_pre_built_images_deployment @googleapis/vertex-prediction-team
 /pytorch_text_classification_using_vertex_sdk_and_gcloud  @RajeshThallam
 /pytorch_text_classification_using_vertex_sdk_and_gcloud		@RajeshThallam @ultrons
 /sklearn_text_classification_from_script_using_vertex_sdk       @maxhardt

community-content/pytorch_pre_built_images_deployment/README.md

@@ -1,30 +0,0 @@
-# PyTorch Deployment on Google Cloud: Text Classification
-
-**This is an Experimental release**, covered by the Pre-GA Offerings Terms of your Google Cloud Platform [Terms of Service](https://cloud.google.com/terms).
-
-Experiments are focused on validating a prototype and are not guaranteed to be released. They are not intended for production use or covered by any SLA, support obligation, or deprecation policy and might be subject to backward-incompatible changes.
-
-**Kindly drop us a note before you run any scale tests.**
-
-**Do not hesitate to contact vertexai-prediction-preview-feedback@google.com if you have any questions or run into any issues.**
-
-The projects need to be added to the allowlist in order to deploy PyTorch models using Vertex AI Prediction pre-built PyTorch images. If you are interested in the feature, please send an email to vertexai-prediction-preview-feedback@google.com to provide your project numbers OR project ids.
-
-## Overview
-
-In the PyTorch on Google Cloud series of blog posts, we aim to share how to deploy PyTorch models at scale on [Vertex AI](https://cloud.google.com/vertex-ai).
-
-This tutorial on text classification shows how to deploy a PyTorch based text classification model on [Vertex AI](https://cloud.google.com/vertex-ai/docs/start/client-libraries#python) using Vertex SDK and [`gcloud ai`](https://cloud.google.com/sdk/gcloud/reference/beta/ai).
-
-## Notebooks
-
-| <h4>Notebook</h4> | <h4>Description</h4>                       |
-| :-------- | :------- |
-| [pytorch-text-classification-vertex-ai-deploy.ipynb](./pytorch-text-classification-vertex-ai-deploy.ipynb) | Notebook to show deploying a PyTorch model on Vertex AI |
-
-## Folders
-
-
-| <h4>Folder Name</h4> | <h4>Description</h4>                       |
-| :-------- | :------- |
-| [`predictor`](./predictor) | Folder with custom prediction handler to deploy a PyTorch model to Vertex Prediction. In the [notebook](./pytorch-text-classification-vertex-ai-deploy.ipynb), this folder is used for deploying a PyTorch model on Vertex AI using Vertex Prediction pre-built PyTorch images |

community-content/pytorch_pre_built_images_deployment/predictor/custom_handler.py

@@ -1,91 +0,0 @@
-
-import os
-import json
-import logging
-
-import torch
-from transformers import AutoModelForSequenceClassification, AutoTokenizer
-from ts.torch_handler.base_handler import BaseHandler
-
-logger = logging.getLogger(__name__)
-
-
-class TransformersClassifierHandler(BaseHandler):
-    """
-    The handler takes an input string and returns the classification text 
-    based on the serialized transformers checkpoint.
-    """
-    def __init__(self):
-        super(TransformersClassifierHandler, self).__init__()
-        self.initialized = False
-
-    def initialize(self, ctx):
-        """ Loads the model.pt file and initialized the model object.
-        Instantiates Tokenizer for preprocessor to use
-        Loads labels to name mapping file for post-processing inference response
-        """
-        self.manifest = ctx.manifest
-
-        properties = ctx.system_properties
-        model_dir = properties.get("model_dir")
-        self.device = torch.device("cuda:" + str(properties.get("gpu_id")) if torch.cuda.is_available() else "cpu")
-
-        # Read model serialize/pt file
-        serialized_file = self.manifest["model"]["serializedFile"]
-        model_pt_path = os.path.join(model_dir, serialized_file)
-        if not os.path.isfile(model_pt_path):
-            raise RuntimeError("Missing the model.pt or pytorch_model.bin file")
-        
-        # Load model
-        self.model = AutoModelForSequenceClassification.from_pretrained(model_dir)
-        self.model.to(self.device)
-        self.model.eval()
-        logger.debug('Transformer model from path {0} loaded successfully'.format(model_dir))
-        
-        # Ensure to use the same tokenizer used during training
-        self.tokenizer = AutoTokenizer.from_pretrained('bert-base-cased')
-
-        # Read the mapping file, index to object name
-        mapping_file_path = os.path.join(model_dir, "index_to_name.json")
-
-        if os.path.isfile(mapping_file_path):
-            with open(mapping_file_path) as f:
-                self.mapping = json.load(f)
-        else:
-            logger.warning('Missing the index_to_name.json file. Inference output will default.')
-            self.mapping = {"0": "Negative",  "1": "Positive"}
-
-        self.initialized = True
-
-    def preprocess(self, data):
-        """ Preprocessing input request by tokenizing
-            Extend with your own preprocessing steps as needed
-        """
-        text = data[0].get("data")
-        if text is None:
-            text = data[0].get("body")
-        sentences = text.decode('utf-8')
-        logger.info("Received text: '%s'", sentences)
-
-        # Tokenize the texts
-        tokenizer_args = ((sentences,))
-        inputs = self.tokenizer(*tokenizer_args,
-                                padding='max_length',
-                                max_length=128,
-                                truncation=True,
-                                return_tensors = "pt")
-        return inputs
-
-    def inference(self, inputs):
-        """ Predict the class of a text using a trained transformer model.
-        """
-        prediction = self.model(inputs['input_ids'].to(self.device))[0].argmax().item()
-
-        if self.mapping:
-            prediction = self.mapping[str(prediction)]
-
-        logger.info("Model predicted: '%s'", prediction)
-        return [prediction]
-
-    def postprocess(self, inference_output):
-        return inference_output

community-content/pytorch_pre_built_images_deployment/predictor/index_to_name.json

@@ -1,5 +0,0 @@
-
-{
-    "0": "Negative", 
-    "1": "Positive"
-}

community-content/pytorch_text_classification_using_vertex_sdk_and_gcloud/pytorch-text-classification-vertex-ai-train-tune-deploy.ipynb

@@ -658,8 +658,8 @@
       },
       "outputs": [],
       "source": [
-        "dataset = load_dataset(\"imdb\")\n",
-        "dataset"
+        "datasets = load_dataset(\"imdb\")\n",
+        "datasets"
       ]
     },
     {
@@ -668,7 +668,7 @@
         "id": "RzfPtOMoIrIu"
       },
       "source": [
-        "The `dataset` object itself is [`DatasetDict`](https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasetdict), which contains one key for the training, validation and test set."
+        "The `datasets` object itself is [`DatasetDict`](https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasetdict), which contains one key for the training, validation and test set."
       ]
     },
     {
@@ -681,12 +681,12 @@
       "source": [
         "print(\n",
         "    \"Total # of rows in training dataset {} and size {:5.2f} MB\".format(\n",
-        "        dataset[\"train\"].shape[0], dataset[\"train\"].size_in_bytes / (1024 * 1024)\n",
+        "        datasets[\"train\"].shape[0], datasets[\"train\"].size_in_bytes / (1024 * 1024)\n",
         "    )\n",
         ")\n",
         "print(\n",
         "    \"Total # of rows in test dataset {} and size {:5.2f} MB\".format(\n",
-        "        dataset[\"test\"].shape[0], dataset[\"test\"].size_in_bytes / (1024 * 1024)\n",
+        "        datasets[\"test\"].shape[0], datasets[\"test\"].size_in_bytes / (1024 * 1024)\n",
         "    )\n",
         ")"
       ]
@@ -708,7 +708,7 @@
       },
       "outputs": [],
       "source": [
-        "dataset[\"train\"][0]"
+        "datasets[\"train\"][0]"
       ]
     },
     {
@@ -728,7 +728,7 @@
       },
       "outputs": [],
       "source": [
-        "label_list = dataset[\"train\"].unique(\"label\")\n",
+        "label_list = datasets[\"train\"].unique(\"label\")\n",
         "label_list"
       ]
     },
@@ -779,7 +779,7 @@
       },
       "outputs": [],
       "source": [
-        "show_random_elements(dataset[\"train\"])"
+        "show_random_elements(datasets[\"train\"])"
       ]
     },
     {
@@ -883,7 +883,7 @@
       },
       "outputs": [],
       "source": [
-        "example = dataset[\"train\"][4]\n",
+        "example = datasets[\"train\"][4]\n",
         "print(example)"
       ]
     },
@@ -920,7 +920,7 @@
       "source": [
         "# Dataset loading repeated here to make this cell idempotent\n",
         "# Since we are over-writing datasets variable\n",
-        "dataset = load_dataset(\"imdb\")\n",
+        "datasets = load_dataset(\"imdb\")\n",
         "\n",
         "# Mapping labels to ids\n",
         "# NOTE: We can extract this automatically but the `Unique` method of the datasets\n",
@@ -948,7 +948,7 @@
         "\n",
         "\n",
         "# apply preprocessing function to input examples\n",
-        "dataset = dataset.map(preprocess_function, batched=True, load_from_cache_file=True)"
+        "datasets = datasets.map(preprocess_function, batched=True, load_from_cache_file=True)"
       ]
     },
     {
@@ -1091,8 +1091,8 @@
         "trainer = Trainer(\n",
         "    model,\n",
         "    args,\n",
-        "    train_dataset=dataset[\"train\"],\n",
-        "    eval_dataset=dataset[\"test\"],\n",
+        "    train_dataset=datasets[\"train\"],\n",
+        "    eval_dataset=datasets[\"test\"],\n",
         "    data_collator=default_data_collator,\n",
         "    tokenizer=tokenizer,\n",
         "    compute_metrics=compute_metrics,\n",

notebooks/community/ml_ops/stage5/get_started_with_vertex_endpoints.ipynb

@@ -340,7 +340,7 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. \n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
         "\n",
         "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
         "\n",
@@ -376,11 +376,12 @@
         "import sys\n",
         "\n",
         "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = \"google.colab\" in sys.modules\n",
+        "IS_COLAB = False\n",
         "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
         "    \"DL_ANACONDA_HOME\"\n",
         "):\n",
         "    if \"google.colab\" in sys.modules:\n",
+        "        IS_COLAB = True\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
         "        google_auth.authenticate_user()\n",
@@ -427,9 +428,8 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
-        "    BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
       ]
     },
     {
@@ -785,7 +785,7 @@
       },
       "outputs": [],
       "source": [
-        "print(endpoint.gca_resource)"
+        "endpoint.gca_resource"
       ]
     },
     {
@@ -908,7 +908,7 @@
       },
       "outputs": [],
       "source": [
-        "print(endpoint.gca_resource.deployed_models[0])"
+        "endpoint.gca_resource.deployed_models[0]"
       ]
     },
     {
@@ -1203,10 +1203,12 @@
         "\n",
         "In this pipeline, you create an `Endpoint` resource, and then you deploy a `Model` resource to the `Endpoint` resource. The `Model` resource  to deploy is your existing TFHub model which you previously imported as a `Model` resource. The steps are:\n",
         "\n",
-        "- For pipeline parameters, pass the resource name for the existing `Model` resource.\n",
-        "- Use the `GetVertexModelOp()` component to create a `VertexModel` pipeline artifact for the model.\n",
+        "- For pipeline parameters, pass the resource name and resource URI for the existing `Model` resource.\n",
+        "- Use the `importer_node()` component to create a `VertexModel` pipeline artifact for the model.\n",
         "- Create an `Endpoint` resource.\n",
-        "- Using the `VertexModel` pipeline artifact, deploy the `Model` resource to the `Endpoint` resource."
+        "- Using the `VertexModel` pipeline artifact, deploy the `Model` resource to the `Endpoint` resource.\n",
+        "\n",
+        "*Note:* This example currently blocked by internal issue: b/219835305"
       ]
     },
     {
@@ -1223,6 +1225,20 @@
         "\n",
         "PIPELINE_ROOT = \"{}/pipeline_root/endpoint_example\".format(BUCKET_URI)\n",
         "\n",
+        "# (WORKAROUND  b/219835305)\n",
+        "@component(\n",
+        "    base_image=\"python:3.9\",\n",
+        "    packages_to_install=[\"google-cloud-aiplatform\"],\n",
+        ")\n",
+        "def return_unmanaged_model(\n",
+        "    serving_image: str, artifact_uri: str, resource_name: str, model: Output[Artifact]\n",
+        "):\n",
+        "    model.metadata[\"containerSpec\"] = {\"imageUri\": serving_image}\n",
+        "\n",
+        "    model.metadata[\"resourceName\"] = resource_name\n",
+        "\n",
+        "    model.uri = artifact_uri\n",
+        "\n",
         "\n",
         "@dsl.pipeline(\n",
         "    name=\"create-endpoint-deploy-model\",\n",
@@ -1230,16 +1246,34 @@
         ")\n",
         "def pipeline(\n",
         "    display_name: str,\n",
+        "    resource_uri: str,\n",
         "    resource_name: str,\n",
+        "    # Model properties (WORKAROUND  b/219835305)\n",
+        "    serving_image: str,\n",
+        "    artifact_uri: str,\n",
         "    project: str = PROJECT_ID,\n",
         "    region: str = REGION,\n",
         "):\n",
-        "    from google_cloud_pipeline_components.experimental.evaluation import \\\n",
-        "        GetVertexModelOp\n",
+        "    from google_cloud_pipeline_components.types import artifact_types\n",
         "    from google_cloud_pipeline_components.v1.endpoint import (EndpointCreateOp,\n",
         "                                                              ModelDeployOp)\n",
+        "    from kfp.v2.components import importer_node\n",
         "\n",
-        "    model = GetVertexModelOp(model_resource_name=resource_name)\n",
+        "    # Desired sequence: blocked by b/219835305\n",
+        "    \"\"\"\n",
+        "    model = importer_node.importer(\n",
+        "        artifact_uri=resource_uri,\n",
+        "        artifact_class=artifact_types.VertexModel,\n",
+        "        metadata={\"resourceName\": resource_name},\n",
+        "    )\n",
+        "    \"\"\"\n",
+        "\n",
+        "    # (WORKAROUND  b/219835305)\n",
+        "    model = return_unmanaged_model(\n",
+        "        serving_image=serving_image,\n",
+        "        artifact_uri=artifact_uri,\n",
+        "        resource_name=resource_name,\n",
+        "    )\n",
         "\n",
         "    endpoint_op = EndpointCreateOp(\n",
         "        project=project,\n",
@@ -1247,7 +1281,7 @@
         "        display_name=display_name,\n",
         "    )\n",
         "\n",
-        "    _ = ModelDeployOp(\n",
+        "    deploy_op = ModelDeployOp(\n",
         "        model=model.outputs[\"model\"],\n",
         "        endpoint=endpoint_op.outputs[\"endpoint\"],\n",
         "        dedicated_resources_min_replica_count=1,\n",
@@ -1276,6 +1310,7 @@
         "\n",
         "- `display_name`: The display name for the generated Vertex AI resources.\n",
         "- `resource_name`: The resource name of the existing `Model` resource.\n",
+        "- `resource_uri`: The resource uri of the existing `Model` resource.\n",
         "- `project`: The project ID.\n",
         "- `region`: The region."
       ]
@@ -1288,6 +1323,10 @@
       },
       "outputs": [],
       "source": [
+        "# Model properties (WORKAROUND  b/219835305)\n",
+        "SERVING_CONTAINER_URI = model.gca_resource.container_spec.image_uri\n",
+        "ARTIFACT_URI = model.gca_resource.artifact_uri\n",
+        "\n",
         "try:\n",
         "    pipeline = aip.PipelineJob(\n",
         "        display_name=\"create-endpoint-deploy-pipeline\",\n",
@@ -1296,6 +1335,11 @@
         "        parameter_values={\n",
         "            \"display_name\": \"create_endpoint_and_deploy_model_\" + TIMESTAMP,\n",
         "            \"resource_name\": model.resource_name,\n",
+        "            \"resource_uri\": \"https://us-central1-aiplatform.googleapis.com/v1/\"\n",
+        "            + model.resource_name,\n",
+        "            # Model properties (WORKAROUND  b/219835305)\n",
+        "            \"serving_image\": SERVING_CONTAINER_URI,\n",
+        "            \"artifact_uri\": ARTIFACT_URI,\n",
         "            \"project\": PROJECT_ID,\n",
         "            \"region\": REGION,\n",
         "        },\n",
@@ -1444,7 +1488,7 @@
         "\n",
         "- For pipeline parameters, pass the resource names and resource URIs for the existing `Model` and `Endpoint` resource.\n",
         "- Use the `importer_node()` component to create a `VertexModel` pipeline artifact for the model.\n",
-        "- Use the `GetVertexModelOp()` component to create a `VertexModel` pipeline artifact for the model.\n",
+        "- Use the `importer_node()` component to create a `VertexEndpoint` pipeline artifact for the endpoint.\n",
         "- Using the `VertexModel` and `VertexEndpoint` pipeline artifacts, deploy the `Model` resource to the `Endpoint` resource.\n",
         "\n",
         "*Note:* This example currently blocked by internal issue: b/219835305"
@@ -1460,7 +1504,6 @@
       "source": [
         "PIPELINE_ROOT = \"{}/pipeline_root/endpoint_example_2\".format(BUCKET_URI)\n",
         "\n",
-        "\n",
         "# (WORKAROUND  b/219835305)\n",
         "@component(\n",
         "    base_image=\"python:3.9\",\n",
@@ -1477,23 +1520,35 @@
         ")\n",
         "def pipeline(\n",
         "    display_name: str,\n",
+        "    model_resource_uri: str,\n",
         "    model_resource_name: str,\n",
         "    endpoint_resource_uri: str,\n",
         "    endpoint_resource_name: str,\n",
+        "    # Model properties (WORKAROUND  b/219835305)\n",
+        "    serving_image: str,\n",
+        "    artifact_uri: str,\n",
         "    project: str = PROJECT_ID,\n",
         "    region: str = REGION,\n",
         "):\n",
-        "    from google_cloud_pipeline_components.experimental.evaluation import \\\n",
-        "        GetVertexModelOp\n",
+        "    from google_cloud_pipeline_components.types import artifact_types\n",
         "    from google_cloud_pipeline_components.v1.endpoint import ModelDeployOp\n",
+        "    from kfp.v2.components import importer_node\n",
         "\n",
         "    # Desired sequence: blocked by b/219835305\n",
         "    \"\"\"\n",
-        "    from kfp.v2.components import importer_node\n",
-        "    from google_cloud_pipeline_components.types import artifact_types\n",
+        "    model = importer_node.importer(\n",
+        "        artifact_uri=resource_uri,\n",
+        "        artifact_class=artifact_types.VertexModel,\n",
+        "        metadata={\"resourceName\": resource_name},\n",
+        "    )\n",
         "    \"\"\"\n",
         "\n",
-        "    model = GetVertexModelOp(model_resource_name=model_resource_name)\n",
+        "    # (WORKAROUND  b/219835305)\n",
+        "    model = return_unmanaged_model(\n",
+        "        serving_image=serving_image,\n",
+        "        artifact_uri=artifact_uri,\n",
+        "        resource_name=model_resource_name,\n",
+        "    )\n",
         "\n",
         "    # Desired sequence: blocked by b/219835305\n",
         "    \"\"\"\n",
@@ -1507,7 +1562,7 @@
         "    # (WORKAROUND  b/219835305)\n",
         "    endpoint = return_unmanaged_endpoint(resource_name=endpoint_resource_name)\n",
         "\n",
-        "    _ = ModelDeployOp(\n",
+        "    deploy_op = ModelDeployOp(\n",
         "        model=model.outputs[\"model\"],\n",
         "        endpoint=endpoint.outputs[\"endpoint\"],\n",
         "        dedicated_resources_min_replica_count=1,\n",
@@ -1536,6 +1591,7 @@
         "\n",
         "- `display_name`: The display name for the generated Vertex AI resources.\n",
         "- `model_resource_name`: The resource name of the existing `Model` resource.\n",
+        "- `model_resource_uri`: The resource uri of the existing `Model` resource.\n",
         "- `endpoint_resource_name`: The resource name of the existing `Endpoint` resource.\n",
         "- `endpoint_resource_uri`: The resource uri of the existing `Endpoint` resource.\n",
         "- `project`: The project ID.\n",
@@ -1558,9 +1614,14 @@
         "        parameter_values={\n",
         "            \"display_name\": \"deploy_model_existing_endpoint_\" + TIMESTAMP,\n",
         "            \"model_resource_name\": model.resource_name,\n",
+        "            \"model_resource_uri\": \"https://us-central1-aiplatform.googleapis.com/v1/\"\n",
+        "            + model.resource_name,\n",
         "            \"endpoint_resource_name\": endpoint.resource_name,\n",
         "            \"endpoint_resource_uri\": \"https://us-central1-aiplatform.googleapis.com/v1/\"\n",
         "            + endpoint.resource_name,\n",
+        "            # Model properties (WORKAROUND  b/219835305)\n",
+        "            \"serving_image\": SERVING_CONTAINER_URI,\n",
+        "            \"artifact_uri\": ARTIFACT_URI,\n",
         "            \"project\": PROJECT_ID,\n",
         "            \"region\": REGION,\n",
         "        },\n",

notebooks/community/ml_ops/stage5/get_started_with_vertex_private_endpoints.ipynb

@@ -385,11 +385,12 @@
         "import sys\n",
         "\n",
         "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = \"google.colab\" in sys.modules\n",
+        "IS_COLAB = False\n",
         "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
         "    \"DL_ANACONDA_HOME\"\n",
         "):\n",
         "    if \"google.colab\" in sys.modules:\n",
+        "        IS_COLAB = True\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
         "        google_auth.authenticate_user()\n",
@@ -1067,6 +1068,7 @@
         "- `model`: The `Model` resource.\n",
         "- `deployed_model_displayed_name`: The human readable name for the deployed model instance.\n",
         "- `machine_type`: The machine type for each VM instance.\n",
+        "- `traffic_split`: Set to `{}` to indicate no traffic split.\n",
         "\n",
         "Do to the requirements to provision the resource, this may take upto a few minutes."
       ]
@@ -1083,6 +1085,7 @@
         "    model=model,\n",
         "    deployed_model_display_name=\"example_\" + TIMESTAMP,\n",
         "    machine_type=DEPLOY_COMPUTE,\n",
+        "    traffic_split={},  # no traffic split\n",
         ")\n",
         "\n",
         "print(endpoint)"
@@ -1184,6 +1187,62 @@
         "    f.write(json.dumps({\"instances\": [{serving_input: {\"b64\": b64str}}]}))"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "23e995c35fd6"
+      },
+      "source": [
+        "#### Construct the `Private Endpoint` URI\n",
+        "\n",
+        "Next, you construct the URI for the `Private Endpoint`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "97b248b2efb5"
+      },
+      "outputs": [],
+      "source": [
+        "endpoint_id = endpoint.resource_name\n",
+        "\n",
+        "ENDPOINT_URL = ! gcloud beta ai endpoints describe {endpoint_id} \\\n",
+        "  --region={REGION} \\\n",
+        "  --format=\"value(deployedModels.privateEndpoints.predictHttpUri)\"\n",
+        "\n",
+        "private_url = ENDPOINT_URL[1]\n",
+        "print(private_url)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "27605b5f0c3a"
+      },
+      "source": [
+        "### Make the prediction request using curl\n",
+        "\n",
+        "Use `curl` to make the prediction request to the private URI."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6cb568e6bb49"
+      },
+      "outputs": [],
+      "source": [
+        "output = ! curl -X POST -d@instances.json $private_url\n",
+        "\n",
+        "predictions = output[5]\n",
+        "print(predictions)\n",
+        "\n",
+        "! rm test.jpg instances.json"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -1192,7 +1251,7 @@
       "source": [
         "### Make the prediction request using SDK\n",
         "\n",
-        "Next, use the `Vertex AI SDK` to make a prediction request."
+        "Finally, use the `Vertex AI SDK` to make a prediction request."
       ]
     },
     {

notebooks/community/ml_ops/stage6/get_started_with_tf_serving.ipynb

@@ -702,7 +702,7 @@
         "        + f\"/{PRIVATE_REPO}\"\n",
         "        + \"/tf_serving:gpu\"\n",
         "    )\n",
-        "    TF_IMAGE = \"tensorflow/serving:2.5.4-gpu\"\n",
+        "    TF_IMAGE = \"tensorflow/serving:latest-gpu\"\n",
         "else:\n",
         "    DEPLOY_IMAGE = (\n",
         "        f\"{REGION}-docker.pkg.dev/\"\n",
@@ -710,15 +710,15 @@
         "        + f\"/{PRIVATE_REPO}\"\n",
         "        + \"/tf_serving:cpu\"\n",
         "    )\n",
-        "    TF_IMAGE = \"tensorflow/serving:2.5.4\"\n",
+        "    TF_IMAGE = \"tensorflow/serving:latest\"\n",
         "\n",
         "if not IS_COLAB:\n",
         "    if DEPLOY_GPU:\n",
-        "        ! sudo docker pull tensorflow/serving:2.5.4-gpu\n",
+        "        ! sudo docker pull tensorflow/serving:latest-gpu\n",
         "    else:\n",
-        "        ! sudo docker pull tensorflow/serving:2.5.4\n",
+        "        ! sudo docker pull tensorflow/serving:latest\n",
         "\n",
-        "    ! docker tag $TF_IMAGE $DEPLOY_IMAGE\n",
+        "    ! docker tag tensorflow/serving $DEPLOY_IMAGE\n",
         "    ! docker push $DEPLOY_IMAGE\n",
         "else:\n",
         "    # install docker daemon\n",
@@ -1214,20 +1214,6 @@
         "     [1.0,3.0,\"cat1\"],\n",
         "     [2.0,4.0,\"cat2\"]\n",
         "    ]}\n",
-        "    \n",
-        "**BigQuery**\n",
-        "\n",
-        "Each row is converted to a JSON array. For example:\n",
-        "\n",
-        "    [1.0,3.0,\"cat1\"]\n",
-        "    [2.0,4.0,\"cat2\"]\n",
-        "    \n",
-        "The batch server generates the pivot data with the same format. The generated pivot data is then wrapped into a payload request:\n",
-        "\n",
-        "    {\"instances\": [\n",
-        "     [1.0,3.0,\"cat1\"],\n",
-        "     [2.0,4.0,\"cat2\"]\n",
-        "    ]}\n",
         "\n",
         "**TFRecords**\n",
         "\n",
@@ -1434,7 +1420,7 @@
       },
       "outputs": [],
       "source": [
-        "delete_bucket = True\n",
+        "delete_bucket = False\n",
         "delete_model = True\n",
         "delete_endpoint = True\n",
         "delete_batch_job = True\n",

notebooks/community/ml_ops/stage7/model_monitoring_custom.ipynb

@@ -88,9 +88,8 @@
         "- Deploy the `Model` resource to the `Endpoint` resource.\n",
         "- Configure the `Endpoint` resource for model monitoring.\n",
         "- Generate synthetic prediction requests for skew.\n",
-        "- Wait for email alert notification.\n",
         "- Generate synthetic prediction requests for drift.\n",
-        "- Wait for email alert notification.\n",
+        "- Interpret the monitored data.\n",
         "\n",
         "Learn more about [Introduction to Vertex AI Model Monitoring](https://cloud.google.com/vertex-ai/docs/model-monitoring/overview)."
       ]
@@ -125,7 +124,6 @@
         "\n",
         "* Vertex AI\n",
         "* BigQuery\n",
-        "* Cloud Storage\n",
         "\n",
         "Learn about [Vertext AI\n",
         "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
@@ -503,86 +501,6 @@
         "- For the CSV data source, grant both service accounts the \"Storage Object Viewer\" role."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "bucket:custom"
-      },
-      "source": [
-        "### Create a Cloud Storage bucket\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "Set the name of your Cloud Storage bucket below, which you use in this tutorial to upload the `input schema` for the monitoring service.\n",
-        "\n",
-        "Bucket names must be globally unique across all Google Cloud projects, including those outside of your organization."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "bucket"
-      },
-      "outputs": [],
-      "source": [
-        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "autoset_bucket"
-      },
-      "outputs": [],
-      "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
-        "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "create_bucket"
-      },
-      "source": [
-        "**Only if your bucket doesn't already exist**: Run the following cell to create your Cloud Storage bucket."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "create_bucket"
-      },
-      "outputs": [],
-      "source": [
-        "! gsutil mb -l $REGION $BUCKET_URI"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "validate_bucket"
-      },
-      "source": [
-        "Finally, validate access to your Cloud Storage bucket by examining its contents:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "validate_bucket"
-      },
-      "outputs": [],
-      "source": [
-        "! gsutil ls -al $BUCKET_URI"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -676,11 +594,7 @@
       },
       "outputs": [],
       "source": [
-        "GPU = False\n",
-        "if GPU:\n",
-        "    DEPLOY_GPU, DEPLOY_NGPU = (aiplatform.gapic.AcceleratorType.NVIDIA_TESLA_K80, 1)\n",
-        "else:\n",
-        "    DEPLOY_GPU, DEPLOY_NGPU = (None, None)"
+        "DEPLOY_GPU, DEPLOY_NGPU = (aiplatform.gapic.AcceleratorType.NVIDIA_TESLA_K80, 1)"
       ]
     },
     {
@@ -704,10 +618,7 @@
       },
       "outputs": [],
       "source": [
-        "if GPU:\n",
-        "    DEPLOY_VERSION = \"tf2-gpu.2-5\"\n",
-        "else:\n",
-        "    DEPLOY_VERSION = \"tf2-cpu.2-5\"\n",
+        "DEPLOY_VERSION = \"tf2-gpu.2-5\"\n",
         "\n",
         "DEPLOY_IMAGE = \"{}-docker.pkg.dev/vertex-ai/prediction/{}:latest\".format(\n",
         "    REGION.split(\"-\")[0], DEPLOY_VERSION\n",
@@ -765,28 +676,22 @@
       "source": [
         "## Introduction to Vertex AI Model Monitoring\n",
         "\n",
-        "Vertex AI Model Monitoring is supported for AutoML tabular models and custom tabular models. You can monitor for skew and drift detection of the features in the inbound prediction requests or skew and drift detection of the feature attributions (Explainable AI) in the outbound prediction response -- that is, the distribution of the attributions on how they contributed to the output (predictions).\n",
+        "Vertex AI Model Monitoring is supported for AutoML tabular models and custom tabular model. You can monitor for skew and drift detection of the features in the inbound prediction requests or skew and drift detection of the feature attributions (Explainable AI) in the outbound prediction response -- that is, the distribution of the attributions on how they contributed to the output (predictions).\n",
         "\n",
         "The following are the basic steps to enable model monitoring:\n",
         "\n",
-        "1. Deploy a `Vertex AI` AutoML or custom tabular model to an `Vertex AI Endpoint`.\n",
+        "1. Deploy a `Vertex AI` tabular model to an `Vertex AI Endpoint`.\n",
         "2. Configure a model monitoring specification.\n",
-        "3. Upload the model monitoring specification to the `Vertex AI Endpoint`.\n",
-        "4. Upload or automatic generation of the `input schema` for parsing.\n",
-        "5. For feature skew detection, upload the training data for automatic generation of the feature distribution.\n",
-        "6. For feature attributions, upload corresponding `Vertex AI Explainability` specification.\n",
+        "3. Configure the alerts.\n",
+        "4. Upload the model monitoring specification to the `Vertex AI Endpoint`.\n",
         "\n",
         "Once configured, you can enable/disable monitoring, change alerts and update the model monitoring configuration. \n",
         "\n",
-        "When model monitoring is enabled, the sampled incoming prediction requests are logged into a BigQuery table. The input feature values contained in the logged requests are then analyzed for skew or drift on an specified interval basis. You set a sampling rate to monitor a subset of the production inputs to a model, and the monitoring interval.\n",
+        "When model monitoring is enabled, incoming prediction requests are logged in a BigQuery table. The input feature values contained in the logged requests are then analyzed for skew or drift. You can set a sampling rate to monitor a subset of the production inputs to a model.\n",
         "\n",
-        "The model monitoring service needs to know how to parse the feature values, which is referred to as the input schema. For AutoML tabular models, the input schema is automatically generated. For custom tabular models, the service will attempt to automatically derive the input schema from the first 1000 prediction requests. Alternatively, one can upload the input schema.\n",
+        "The model monitoring service needs to know how to parse the feature values, which is referred to as the input schema. For AutoML tabular models, the input schema is automatically provided. For custom tabular models, the service will attempt to automatically derive the input schema from the first 1000 prediction requests. Alternatively, one can upload the input schema.\n",
         "\n",
-        "For skew detection, the monitoring service requires a baseline for the statistical distribution of values in the training data. For AutoML tabular models this is automatically derived. For custom tabular models, you upload the training data to the service, and have the service automatically derive the distribution.\n",
-        "\n",
-        "For feature attribution skew and drift detection, requires enabling your deployed model for `Vertex AI Explainability` for custom tabular models. For AutoML models, `Vertex AI Explainability` is automatically enabled.\n",
-        "\n",
-        "Learn more about [Introduction to Vertex AI Model Monitoring](https://cloud.google.com/vertex-ai/docs/model-monitoring/overview)."
+        "For skew detection, requires a baseline for the statistical distribution of values in the training data. For AutoML tabular models this is automatically derived. For custom tabular models, you upload the training data to the service, and have the service automatically derived the distribution."
       ]
     },
     {
@@ -854,22 +759,14 @@
         "MIN_NODES = 1\n",
         "MAX_NODES = 1\n",
         "\n",
-        "if GPU:\n",
-        "    endpoint = model.deploy(\n",
-        "        deployed_model_display_name=\"churn_\" + UUID,\n",
-        "        machine_type=DEPLOY_COMPUTE,\n",
-        "        min_replica_count=MIN_NODES,\n",
-        "        max_replica_count=MAX_NODES,\n",
-        "        accelerator_type=DEPLOY_GPU.name,\n",
-        "        accelerator_count=DEPLOY_NGPU,\n",
-        "    )\n",
-        "else:\n",
-        "    endpoint = model.deploy(\n",
-        "        deployed_model_display_name=\"churn_\" + UUID,\n",
-        "        machine_type=DEPLOY_COMPUTE,\n",
-        "        min_replica_count=MIN_NODES,\n",
-        "        max_replica_count=MAX_NODES,\n",
-        "    )"
+        "endpoint = model.deploy(\n",
+        "    deployed_model_display_name=\"churn_\" + UUID,\n",
+        "    machine_type=DEPLOY_COMPUTE,\n",
+        "    min_replica_count=MIN_NODES,\n",
+        "    max_replica_count=MAX_NODES,\n",
+        "    accelerator_type=DEPLOY_GPU.name,\n",
+        "    accelerator_count=DEPLOY_NGPU,\n",
+        ")"
       ]
     },
     {
@@ -955,7 +852,7 @@
         "\n",
         "Next, you configure the `logging_sampling_strategy` specification with the following settings:\n",
         "\n",
-        "- `sample_rate`: The rate as a percentage (between 0 and 1) to randomly sample prediction requests for monitoring. Selected samples are logged to a BigQuery table."
+        "- `sample_rate`: The rate as a percentage (between 0 and 1) to randomly sample predictions for monitoring. Select samples are logged to a BigQuery table.\n"
       ]
     },
     {
@@ -996,11 +893,11 @@
       },
       "outputs": [],
       "source": [
-        "DRIFT_THRESHOLD_VALUE = 0.05\n",
+        "DEFAULT_THRESHOLD_VALUE = 0.05\n",
         "\n",
         "DRIFT_THRESHOLDS = {\n",
-        "    \"country\": DRIFT_THRESHOLD_VALUE,\n",
-        "    \"cnt_user_engagement\": DRIFT_THRESHOLD_VALUE,\n",
+        "    \"country\": DEFAULT_THRESHOLD_VALUE,\n",
+        "    \"cnt_user_engagement\": DEFAULT_THRESHOLD_VALUE,\n",
         "}\n",
         "\n",
         "drift_config = model_monitoring.DriftDetectionConfig(drift_thresholds=DRIFT_THRESHOLDS)"
@@ -1039,11 +936,9 @@
         "# Prediction target column name in training dataset.\n",
         "TARGET = \"churned\"\n",
         "\n",
-        "SKEW_THRESHOLD_VALUE = 0.5\n",
-        "\n",
         "SKEW_THRESHOLDS = {\n",
-        "    \"country\": SKEW_THRESHOLD_VALUE,\n",
-        "    \"cnt_user_engagement\": SKEW_THRESHOLD_VALUE,\n",
+        "    \"country\": DEFAULT_THRESHOLD_VALUE,\n",
+        "    \"cnt_user_engagement\": DEFAULT_THRESHOLD_VALUE,\n",
         "}\n",
         "\n",
         "skew_config = model_monitoring.SkewDetectionConfig(\n",
@@ -1081,72 +976,6 @@
         ")"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "8ac30ffa72b5"
-      },
-      "source": [
-        "### Create the input schema\n",
-        "\n",
-        "The monitoring service needs to know the features and data types for the the feature inputs to the model, which is referred to as the `input schema`. The `input schema` can either be \n",
-        " - Preloaded to the monitoring service.\n",
-        " - Automatically generated by the monitoring service after receiving first 1000 prediction instances.\n",
-        " \n",
-        "In this tutorial, you preload the `input schema`.\n",
-        "\n",
-        "#### Create the predefined input schema\n",
-        "\n",
-        "The predefined `input schema` is specified as a YAML file. In this example, you retrieve the BigQuery schema for the training data, which includes the feature names and data types, to generate the YAML specification. The predefined `input schema` must be loade to a Cloud Storage location.\n",
-        "\n",
-        "Learn more about [Custom instance schemas for parsing input](https://cloud.google.com/vertex-ai/docs/model-monitoring/overview#custom-input-schemas)."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "59becb56ad34"
-      },
-      "outputs": [],
-      "source": [
-        "# Get the BQ table\n",
-        "\n",
-        "table = bigquery.TableReference.from_string(DATASET_BQ_URI[5:])\n",
-        "bq_table = bqclient.get_table(table)\n",
-        "\n",
-        "yaml = \"\"\"type: object\n",
-        "properties:\n",
-        "\"\"\"\n",
-        "\n",
-        "schema = bq_table.schema\n",
-        "for feature in schema:\n",
-        "    if feature.name == TARGET:\n",
-        "        continue\n",
-        "    if feature.field_type == \"STRING\":\n",
-        "        f_type = \"string\"\n",
-        "    else:\n",
-        "        f_type = \"integer\"\n",
-        "    yaml += f\"\"\"  {feature.name}:\n",
-        "    type: {f_type}\n",
-        "\"\"\"\n",
-        "\n",
-        "yaml += \"\"\"required:\n",
-        "\"\"\"\n",
-        "for feature in schema:\n",
-        "    if feature.name == TARGET:\n",
-        "        continue\n",
-        "    yaml += f\"\"\"- {feature.name}\n",
-        "\"\"\"\n",
-        "\n",
-        "print(yaml)\n",
-        "\n",
-        "with open(\"schema.yaml\", \"w\") as f:\n",
-        "    f.write(yaml)\n",
-        "\n",
-        "! gsutil cp schema.yaml {BUCKET_URI}/schema.yaml"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -1164,15 +993,14 @@
         "- `logging_sampling_strategy`: The specification for the sampling configuration.\n",
         "- `schedule_config`: The specification for the scheduling configuration.\n",
         "- `alert_config`: The specification for the alerting configuration.\n",
-        "- `objective_configs`: The specification for the objectives configuration.\n",
-        "- `analysis_instance_schema_uri`: The location of the YAML file containing the `input schema`."
+        "- `objective_configs`: The specification for the objectives configuration."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "4b8dd381c5c3"
+        "id": "86f76670f439"
       },
       "outputs": [],
       "source": [
@@ -1185,10 +1013,9 @@
         "    schedule_config=schedule_config,\n",
         "    alert_config=alerting_config,\n",
         "    objective_configs=objective_config,\n",
-        "    analysis_instance_schema_uri=f\"{BUCKET_URI}/schema.yaml\",\n",
         ")\n",
         "\n",
-        "print(monitoring_job)"
+        "print(monitoring_job.gca_resource)"
       ]
     },
     {
@@ -1197,11 +1024,11 @@
         "id": "3c6d3b620264"
       },
       "source": [
-        "#### Email notification of the monitoring job.\n",
+        "#### Wait for the notification of the monitoring job.\n",
         "\n",
         "An email notification is sent to the email address in the alerting configuration, notifying that the model monitoring job is now enabled.\n",
         "\n",
-        "The contents will appear like:\n",
+        "Before proceeding, wait for the email notification. The contents will appear like:\n",
         "\n",
         "<blockquote>\n",
         "Hello Vertex AI Customer,\n",
@@ -1212,43 +1039,6 @@
         "</blockquote>"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "dcc4aae9e20f"
-      },
-      "source": [
-        "#### Monitoring Job State\n",
-        "\n",
-        "After you start the `Vertex AI Model Monitoring` job, it will be in a `PENDING` state until `skew distribution baseline` is calculated. The monitoring service will initiate a batch job to generate the distribution baseline from the training data. \n",
-        "\n",
-        "Once the baseline distribution is generated, then the monitoring job will enter `OFFLINE` state. On the per interval basis -- e.g., once an hour, the monitoring job will enter `RUNNING` state while analyzing the sampled data. Once completed, it will return to an `OFFLINE` state while awaiting the next scheduled analysis."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "f640fb7f10cd"
-      },
-      "outputs": [],
-      "source": [
-        "jobs = monitoring_job.list(filter=f\"display_name=churn_{UUID}\")\n",
-        "job = jobs[0]\n",
-        "print(job.state)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "e385d103aba6"
-      },
-      "source": [
-        "### Automatic generation of the baseline distribution\n",
-        "\n",
-        "Next, the monitoring service creates a batch job to analyze the training data to generate the baseline distribution. Once completed, the monitoring service will starting monitoring on the specified interval."
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -1259,22 +1049,46 @@
       "source": [
         "import time\n",
         "\n",
-        "# Pause a bit for the baseline distribution to be calculated\n",
-        "if os.getenv(\"IS_TESTING\"):\n",
-        "    time.sleep(180)"
+        "while True:\n",
+        "    time.sleep(60)\n",
+        "    jobs = monitoring_job.list(filter=f\"display_name=churn_{UUID}\")\n",
+        "    job = jobs[0]\n",
+        "    print(job.state)\n",
+        "    if job.state == aiplatform.gapic.JobState.JOB_STATE_PENDING:\n",
+        "        continue\n",
+        "    break"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "26bac245f6c5"
+        "id": "08f499f7adae"
       },
       "source": [
-        "### Generate synthetic prediction requests for skew detection\n",
+        "#### The location of the BigQuery table for monitoring\n",
         "\n",
-        "Next, you extract the first 1000 instances from the BigQuery training table to use for prediction requests. You modify the data (synthetic) to trigger the skew detection in the prediction requests from the training distribution versus serving distribution, as follows:\n",
+        "The BigQuery table for logging the sampled requests is located at:\n",
         "\n",
-        "- `country`: Set all values to Canada"
+        "    `<PROJECT_ID>.model_deployment_monitoring_<ENDPOINT_ID>`.serving_predict, \n",
+        "    \n",
+        "Where <ENDPOINT_ID> is the numerical identifier for the `Vertex AI Endpoint` resource."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3960076190ab"
+      },
+      "source": [
+        "## Initialize the parsing for automatically generating the input schema\n",
+        "\n",
+        "After your `Endpoint` receives a 1000 prediction requests, the modeling service will automatically parse and create the `input schema`.\n",
+        "\n",
+        "### Create the 1000 instance data\n",
+        "\n",
+        "In this example, the first 1000 entries in the BigQuery training data are used as the first 1000 prediction requests. \n",
+        "\n",
+        "*Note:* In this context, each instance is a prediction request. In otherwords, sending 1000 prediction requests of a single instance is the same as sending a single prediction request with 1000 instances."
       ]
     },
     {
@@ -1290,6 +1104,14 @@
         "\n",
         "rows = bqclient.list_rows(table, max_results=1000)\n",
         "\n",
+        "num_cols = [\n",
+        "    \"cnt_challenge_a_friend\",\n",
+        "    \"cnt_ad_reward\",\n",
+        "    \"cnt_completed_5_levels\",\n",
+        "    \"cnt_level_reset_quickplay\",\n",
+        "    \"cnt_spend_virtual_currency\",\n",
+        "]\n",
+        "\n",
         "instances = []\n",
         "for row in rows:\n",
         "    instance = {}\n",
@@ -1297,9 +1119,10 @@
         "        if key == TARGET:\n",
         "            continue\n",
         "        if value is None:\n",
-        "            value = \"\"\n",
-        "        if key == \"country\":\n",
-        "            value = \"Canada\"\n",
+        "            if key in num_cols:\n",
+        "                value = 0.0\n",
+        "            else:\n",
+        "                value = \"\"\n",
         "        instance[key] = value\n",
         "    instances.append(instance)\n",
         "\n",
@@ -1312,9 +1135,9 @@
         "id": "6d002569dadc"
       },
       "source": [
-        "### Make the prediction requests\n",
+        "### Make the initial prediction request\n",
         "\n",
-        "Next, you send the the 1000 prediction requests to your `Vertex AI Endpoint` resource using the `predict()` method."
+        "Next, you send the the 1000 prediction request to your `Vertex AI Endpoint` resource using the `predict()` method."
       ]
     },
     {
@@ -1325,8 +1148,7 @@
       },
       "outputs": [],
       "source": [
-        "for instance in instances:\n",
-        "    response = endpoint.predict(instances=[instance])\n",
+        "response = endpoint.predict(instances=instances)\n",
         "\n",
         "prediction = response[0]\n",
         "\n",
@@ -1337,228 +1159,24 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "2b5859ea4ae9"
+        "id": "69147b259cbf"
       },
       "source": [
-        "### Logging sampled requests\n",
+        "### Pause the monitoring job\n",
         "\n",
-        "Once the monitoring service has started, the sampled prediction requests will be logged to Cloud Storage. On the next monitoring interval, the sampled predictions are then copied over to the BigQuery logging table. Once the entries are in the BigQuery table, the monitoring service will analyze the sampled data.\n",
-        "\n",
-        "Next, you wait for the first logged entres to appear in the BigQuery table used for logging prediction samples. Since you sent 1000 prediction requests, with 50% sampling, you should see around 500 entries."
+        "You can pause and resume a monitoring job with the methods `pause()` and `resume()`, respectively."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "bd177a8decbb"
+        "id": "2f4e99831237"
       },
       "outputs": [],
       "source": [
-        "while True:\n",
-        "    time.sleep(180)\n",
-        "\n",
-        "    ENDPOINT_ID = endpoint.resource_name.split(\"/\")[-1]\n",
-        "\n",
-        "    table = bigquery.TableReference.from_string(\n",
-        "        f\"{PROJECT_ID}.model_deployment_monitoring_{ENDPOINT_ID}.serving_predict\"\n",
-        "    )\n",
-        "    rows = bqclient.list_rows(table)\n",
-        "    print(rows.total_rows)\n",
-        "    if rows.total_rows > 0:\n",
-        "        break"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "aeaea3a7a194"
-      },
-      "source": [
-        "### Skew detection during monitoring\n",
-        "\n",
-        "The feature input skew detection will occur at the next monitoring interval. In this tutorial, you set the monitoring interval to one hour. So, in about an hour your monitoring job will go from `OFFLINE` to `RUNNING`. While running, it will analyze the logged sampled tables from the predictions during this interval and compare them to the baseline distribution.\n",
-        "\n",
-        "Once the analysis is completed, the monitoring job will send email notifications on the detected skew, in this case `country`, and the monitoring job will go into `OFFLINE` state until the next interval.\n",
-        "\n",
-        "#### Wait for monitoring interval\n",
-        "\n",
-        "It can take upwards of 40 minutes from when the analyis occurred on the monitoring interval to when you receive an email alert.\n",
-        "\n",
-        "The contents will appear like\n",
-        "\n",
-        "<blockquote>\n",
-        "   Hello Vertex AI Customer,\n",
-        "\n",
-        "You are receiving this mail because you are subscribing to the Vertex AI Model Monitoring service.\n",
-        "This mail is just to inform you that there are some anomalies detected in your deployed models and may need your attention.\n",
-        "\n",
-        "\n",
-        "Basic Information:\n",
-        "\n",
-        "Endpoint Name: projects/[your-project-id]/locations/us-central1/endpoints/3315907167046860800\n",
-        "Monitoring Job: projects/[your-project-id]/locations/us-central1/modelDeploymentMonitoringJobs/8672170640054157312\n",
-        "Statistics and Anomalies Root Path(Google Cloud Storage): gs://cloud-ai-platform-773884b1-2a32-48d6-8b83-c03cde416b68/model_monitoring/job-8672170640054157312\n",
-        "BigQuery Command: SELECT * FROM `bq://[your-project-id].model_deployment_monitoring_3315907167046860800.serving_predict`\n",
-        "\n",
-        "\n",
-        "Training Prediction Skew Anomalies (Raw Feature):\n",
-        "\n",
-        "Anomalies Report Path(Google Cloud Storage): gs://cloud-ai-platform-773884b1-2a32-48d6-8b83-c03cde416b68/model_monitoring/job-8672170640054157312/serving/2022-08-25T00:00/stats_and_anomalies/<deployed-model-id>/anomalies/training_prediction_skew_anomalies\n",
-        "\n",
-        "For more information about the alert, please visit the model monitoring alert page.\n",
-        "\n",
-        "Deployed model id: <deployed-model-id>\n",
-        "\n",
-        "Feature name\tAnomaly short description\tAnomaly long description\n",
-        "country\tHigh Linfty distance between training and serving\tThe Linfty distance between training and serving is 0.947563 (up to six significant digits), above the threshold 0.5. The feature value with maximum difference is: Canada\n",
-        "<blockquote>"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "b91a0e19ff8b"
-      },
-      "outputs": [],
-      "source": [
-        "if os.getenv(\"IS_TESTING\"):\n",
-        "    time.sleep(60 * 45)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "555642c341e1"
-      },
-      "source": [
-        "### Generate synthetic prediction requests for drift detection\n",
-        "\n",
-        "Next, you extract the same first 1000 instances from the BigQuery training table to use for prediction requests. You modify the data (synthetic) to trigger the drift detection in the prediction requests from the training distribution versus serving distribution, as follows:\n",
-        "\n",
-        "- `cnt_user_engagement`: increase the value 4x."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "cb26c3dea306"
-      },
-      "outputs": [],
-      "source": [
-        "# Download the table.\n",
-        "table = bigquery.TableReference.from_string(DATASET_BQ_URI[5:])\n",
-        "\n",
-        "rows = bqclient.list_rows(table, max_results=1000)\n",
-        "\n",
-        "instances = []\n",
-        "for row in rows:\n",
-        "    instance = {}\n",
-        "    for key, value in row.items():\n",
-        "        if key == TARGET:\n",
-        "            continue\n",
-        "        if value is None:\n",
-        "            value = \"\"\n",
-        "        elif key == \"cnt_user_engagement\":\n",
-        "            value = int(value * 4)\n",
-        "        instance[key] = value\n",
-        "    instances.append(instance)\n",
-        "\n",
-        "print(len(instances))"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "6d002569dadc"
-      },
-      "source": [
-        "### Make the prediction requests\n",
-        "\n",
-        "Next, you send the the 1000 prediction requests to your `Vertex AI Endpoint` resource using the `predict()` method."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "b2d69d89cd66"
-      },
-      "outputs": [],
-      "source": [
-        "for instance in instances:\n",
-        "    response = endpoint.predict(instances=[instance])\n",
-        "\n",
-        "prediction = response[0]\n",
-        "\n",
-        "# print the prediction for the first instance\n",
-        "print(prediction[0])"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "5184c68e4c99"
-      },
-      "source": [
-        "### Logging sampled requests\n",
-        "\n",
-        "On the next monitoring interval, the sampled predictions are then copied over to the BigQuery logging table. Once the entries are in the BigQuery table, the monitoring service will analyze the sampled data.\n",
-        "\n",
-        "Next, you wait for the first logged entres to appear in the BigQuery table used for logging prediction samples. Since you sent 1000 prediction requests, with 50% sampling, you should see around 1000 entries."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "d9a2f9342b06"
-      },
-      "outputs": [],
-      "source": [
-        "while True:\n",
-        "    time.sleep(180)\n",
-        "\n",
-        "    ENDPOINT_ID = endpoint.resource_name.split(\"/\")[-1]\n",
-        "\n",
-        "    table = bigquery.TableReference.from_string(\n",
-        "        f\"{PROJECT_ID}.model_deployment_monitoring_{ENDPOINT_ID}.serving_predict\"\n",
-        "    )\n",
-        "    rows = bqclient.list_rows(table)\n",
-        "    print(rows.total_rows)\n",
-        "    if rows.total_rows > 550:\n",
-        "        break"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "683ed0ba4ccd"
-      },
-      "source": [
-        "### Drift detection during monitoring\n",
-        "\n",
-        "The feature input drift detection will occur at the next monitoring interval. In this tutorial, you set the monitoring interval to one hour. So, in about an hour your monitoring job will go from `OFFLINE` to `RUNNING`. While running, it will analyze the logged sampled tables from the predictions during this interval and compare them to the previous monitoring interva distribution.\n",
-        "\n",
-        "Once the analysis is completed, the monitoring job will send email notifications on the detected drift, in this case `cnt_user_engagement`, and the monitoring job will go into `OFFLINE` state until the next interval.\n",
-        "\n",
-        "#### Wait for monitoring interval\n",
-        "\n",
-        "It can take upwards of 40 minutes from when the analyis occurred on the monitoring interval to when you receive an email alert."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "2e64ffaae2de"
-      },
-      "outputs": [],
-      "source": [
-        "if os.getenv(\"IS_TESTING\"):\n",
-        "    time.sleep(60 * 45)"
+        "monitoring_job.pause()\n",
+        "# monitoring_job.resume()"
       ]
     },
     {
@@ -1569,21 +1187,45 @@
       "source": [
         "### Delete the monitoring job\n",
         "\n",
-        "You can delete the monitoring job using the `delete()` method. "
+        "You can delete the monitoring job using the `delete()` method. \n",
+        "\n",
+        "*Note:* You cannot delete a monitoring job when in a state of RUNNING. You must pause the job first."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "ef1ddc1d6017"
+        "id": "bdc44ad0471a"
       },
       "outputs": [],
       "source": [
-        "monitoring_job.pause()\n",
         "monitoring_job.delete()"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "313b543dc772"
+      },
+      "source": [
+        "### Delete the logged sampled data"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9a638ce66806"
+      },
+      "outputs": [],
+      "source": [
+        "# Delete the temporary BigQuery dataset\n",
+        "ENDPOINT_ID = endpoint.resource_name.split(\"/\")[-1]\n",
+        "\n",
+        "! bq rm -r -f {PROJECT_ID}:model_deployment_monitoring_{ENDPOINT_ID}"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -1610,33 +1252,32 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "18889460bd33"
+        "id": "448f3698d50f"
       },
       "source": [
-        "## Cleaning up\n",
+        "#### Delete the `Vertex AI Model` resource\n",
         "\n",
-        "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
-        "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
-        "\n",
-        "Otherwise, you can delete the individual resources you created in this tutorial."
+        "Your `Vertex AI Model` resource can be deleted using the `delete()` method."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "c736a6bf1428"
+        "id": "0feab0a0b5d7"
       },
       "outputs": [],
       "source": [
-        "delete_bucket = False\n",
-        "\n",
-        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
-        "    ! gsutil rm -rf {BUCKET_URI}\n",
-        "\n",
-        "! rm -f schema.yaml\n",
-        "\n",
-        "! bq rm -f {PROJECT_ID}.model_deployment_monitoring_{ENDPOINT_ID}"
+        "model.delete()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "18889460bd33"
+      },
+      "source": [
+        "### Cleanup"
       ]
     }
   ],

notebooks/community/prediction/custom_prediction_routines/SDK_Custom_Preprocess.ipynb

@@ -673,7 +673,7 @@
         "Before you can deploy your model for serving, Vertex AI needs access to the following files in Cloud Storage:\n",
         "\n",
         "* `model.joblib` (model artifact)\n",
-        "* `preprocessor.pkl` (preprocessor code)\n",
+        "* `preprocessor.pkl` (model artifact)\n",
         "\n",
         "Run the following commands to upload your files:"
       ]

notebooks/official/CODEOWNERS

@@ -10,7 +10,7 @@
 /tabnet/tabnet_vertex_tutorial.ipynb @longtle
 
 /migration @andrewferlitsch
-/explainabl_ai
+/explainabl_ai   
 /pipelines @andrewferlitsch
 /ml_metadata @andrewferlitsch
 /model_monitoring @andrewferlitsch
@@ -27,7 +27,5 @@
 /pipelines/google_cloud_pipelines_dataproc_tabular @inardini
 /automl/automl_forecasting_bqml_arima_plus_comparison.ipynb @TheMichaelHu
 /automl/automl_tabular_on_vertex_pipelines.ipynb @helinwang
-/custom/custom_training_tensorboard_profiler.ipynb @itseric
-/workbench/spark/spark_sample_notebook.ipynb @bmiro
-/model-registry/bqml-vertexai-model-registry.ipynb @soheilazangeneh
-/workbench/exploratory_data_analysis/explore_data_in_bigquery_with_workbench.ipynb @alokpattani
+/custom/custom_training_tensorboard_profiler.ipynb @gericdong
+/workbench/spark/spark_sample_notebook.ipynb @bradmiro

notebooks/official/automl/automl_forecasting_bqml_arima_plus_comparison.ipynb

@@ -176,10 +176,7 @@
         "else:\n",
         "    USER_FLAG = \"\"\n",
         "\n",
-        "! (pip3 install --upgrade $USER_FLAG \\\n",
-        "    google-cloud-bigquery[pandas]==2.34.4 \\\n",
-        "    google-cloud-aiplatform==1.16.1 \\\n",
-        "    google-cloud-pipeline-components==1.0.18)"
+        "! pip3 install --upgrade google-cloud-bigquery[pandas] google-cloud-aiplatform google-cloud-pipeline-components $USER_FLAG"
       ]
     },
     {

notebooks/official/automl/sdk_automl_image_object_detection_batch.ipynb

@@ -8,7 +8,7 @@
       },
       "outputs": [],
       "source": [
-        "# Copyright 2021 Google LLC\n",
+        "# Copyright 2022 Google LLC\n",
         "#\n",
         "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
         "# you may not use this file except in compliance with the License.\n",
@@ -66,6 +66,17 @@
         "This tutorial demonstrates how to use the Vertex AI SDK to create image object detection models and do batch prediction using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dataset:salads,iod"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the Salads category of the [OpenImages dataset](https://www.tensorflow.org/datasets/catalog/open_images_v4) from [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/overview). This dataset does not require any feature engineering. The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket. The trained model predicts the bounding box locations and the corresponding type of salad items in an image from a class of five items: salad, seafood, tomato, baked goods, or cheese."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -90,17 +101,6 @@
         "* Batch Prediction Service: Does a queued (batch) prediction for the entire set of instances in the background and stores the results in a Cloud Storage bucket when ready."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "dataset:salads,iod"
-      },
-      "source": [
-        "### Dataset\n",
-        "\n",
-        "The dataset used for this tutorial is the Salads category of the [OpenImages dataset](https://www.tensorflow.org/datasets/catalog/open_images_v4) from [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/overview). This dataset does not require any feature engineering. The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket. The trained model predicts the bounding box locations and the corresponding type of salad items in an image from a class of five items: salad, seafood, tomato, baked goods, or cheese."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -201,7 +201,7 @@
       },
       "outputs": [],
       "source": [
-        "! pip3 install -U --upgrade tensorflow google-cloud-storage $USER_FLAG"
+        "! pip3 install -U google-cloud-storage $USER_FLAG"
       ]
     },
     {
@@ -213,6 +213,17 @@
         "Install the latest version of *tensorflow* library."
       ]
     },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "install_tensorflow"
+      },
+      "outputs": [],
+      "source": [
+        "! pip3 install --upgrade tensorflow $USER_FLAG"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -372,9 +383,9 @@
         "id": "timestamp"
       },
       "source": [
-        "#### UUID\n",
+        "#### Timestamp\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
       ]
     },
     {
@@ -385,16 +396,9 @@
       },
       "outputs": [],
       "source": [
-        "import random\n",
-        "import string\n",
+        "from datetime import datetime\n",
         "\n",
-        "\n",
-        "# Generate a uuid of a specifed length(default=8)\n",
-        "def generate_uuid(length: int = 8) -> str:\n",
-        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
-        "\n",
-        "\n",
-        "UUID = generate_uuid()"
+        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
     {
@@ -405,7 +409,7 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Google Cloud Notebooks**, your environment is already authenticated.\n",
+        "**If you are using Google Cloud Notebooks**, your environment is already authenticated. Skip this step.\n",
         "\n",
         "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
         "\n",
@@ -490,7 +494,7 @@
       "outputs": [],
       "source": [
         "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + UUID"
+        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
       ]
     },
     {
@@ -665,7 +669,7 @@
       "outputs": [],
       "source": [
         "dataset = aiplatform.ImageDataset.create(\n",
-        "    display_name=\"Salads\" + \"_\" + UUID,\n",
+        "    display_name=\"Salads\" + \"_\" + TIMESTAMP,\n",
         "    gcs_source=[IMPORT_FILE],\n",
         "    import_schema_uri=aiplatform.schema.dataset.ioformat.image.bounding_box,\n",
         ")\n",
@@ -713,7 +717,7 @@
       "outputs": [],
       "source": [
         "job = aiplatform.AutoMLImageTrainingJob(\n",
-        "    display_name=\"salads_\" + UUID,\n",
+        "    display_name=\"salads_\" + TIMESTAMP,\n",
         "    prediction_type=\"object_detection\",\n",
         "    multi_label=False,\n",
         "    model_type=\"CLOUD\",\n",
@@ -756,7 +760,7 @@
       "source": [
         "model = job.run(\n",
         "    dataset=dataset,\n",
-        "    model_display_name=\"salads_\" + UUID,\n",
+        "    model_display_name=\"salads_\" + TIMESTAMP,\n",
         "    training_fraction_split=0.8,\n",
         "    validation_fraction_split=0.1,\n",
         "    test_fraction_split=0.1,\n",
@@ -786,7 +790,7 @@
       "outputs": [],
       "source": [
         "# Get model resource ID\n",
-        "models = aiplatform.Model.list(filter=\"display_name=salads_\" + UUID)\n",
+        "models = aiplatform.Model.list(filter=\"display_name=salads_\" + TIMESTAMP)\n",
         "\n",
         "# Get a reference to the Model Service client\n",
         "client_options = {\"api_endpoint\": f\"{REGION}-aiplatform.googleapis.com\"}\n",
@@ -957,7 +961,7 @@
       "outputs": [],
       "source": [
         "batch_predict_job = model.batch_predict(\n",
-        "    job_display_name=\"salads_\" + UUID,\n",
+        "    job_display_name=\"salads_\" + TIMESTAMP,\n",
         "    gcs_source=gcs_input_uri,\n",
         "    gcs_destination_prefix=BUCKET_URI,\n",
         "    machine_type=\"n1-standard-4\",\n",

notebooks/official/custom/custom-tabular-bq-managed-dataset.ipynb

@@ -44,7 +44,7 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/custom/custom-tabular-bq-managed-dataset.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/custom/custom-tabular-bq-managed-dataset.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",

notebooks/official/experiments/build_model_experimentation_lineage_with_prebuild_code.ipynb

@@ -8,7 +8,7 @@
       },
       "outputs": [],
       "source": [
-        "# Copyright 2021 Google LLC\n",
+        "# Copyright 2022 Google LLC\n",
         "#\n",
         "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
         "# you may not use this file except in compliance with the License.\n",
@@ -54,31 +54,13 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "d975c5729f18"
+        "id": "tvgnzT1CKxrO"
       },
       "source": [
         "## Overview\n",
         "\n",
-        "As a Data Scientist, you want to be able to reuse code path (data preprocessing, feature engineering etc...) that others within your team have written to simplify and standardize all the complex data wrangling. "
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "3a0f8061b9c1"
-      },
-      "source": [
-        "### Objective\n",
+        "As a Data Scientist, you want to be able to reuse code path (data preprocessing, feature engineering etc...) that others within your team have written to simplify and standardize all the complex data wrangling. \n",
         "\n",
-        "In this notebook, you learn how to integrate preprocessing code in a Vertex AI experiments. Also you build the experiment lineage lets you record, analyze, debug, and audit metadata and artifacts produced along your ML journey."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "tvgnzT1CKxrO"
-      },
-      "source": [
         "### Dataset\n",
         "\n",
         "This dataset is the UCI News Aggregator Data Set which contains 422,937 news collected between March 10th, 2014 and August 10th, 2014. Below are example records from the dataset:\n",
@@ -90,15 +72,13 @@
         "|2  |Fed's Charles Plosser sees high bar for change in pace of tapering   |http://www.livemint.com/Politics/H2EvwJSK2VE6OF7iK1g3PP/Feds-Charles-Plosser-sees-high-bar-for-change-in-pace-of-ta.html   |Livemint         |b       |ddUyU0VZz0BRneMioxUPQVP6sIxvM|www.livemint.com   |1394470371207|\n",
         "|3  |US open: Stocks fall after Fed official hints at accelerated tapering|http://www.ifamagazine.com/news/us-open-stocks-fall-after-fed-official-hints-at-accelerated-tapering-294436                |IFA Magazine     |b       |ddUyU0VZz0BRneMioxUPQVP6sIxvM|www.ifamagazine.com|1394470371550|\n",
         "|4  |Fed risks falling 'behind the curve', Charles Plosser says           |http://www.ifamagazine.com/news/fed-risks-falling-behind-the-curve-charles-plosser-says-294430                             |IFA Magazine     |b       |ddUyU0VZz0BRneMioxUPQVP6sIxvM|www.ifamagazine.com|1394470371793|\n",
-        "|5  |Fed's Plosser: Nasty Weather Has Curbed Job Growth                   |http://www.moneynews.com/Economy/federal-reserve-charles-plosser-weather-job-growth/2014/03/10/id/557011                   |Moneynews        |b       |ddUyU0VZz0BRneMioxUPQVP6sIxvM|www.moneynews.com  |1394470372027|"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "5e2eba58ad71"
-      },
-      "source": [
+        "|5  |Fed's Plosser: Nasty Weather Has Curbed Job Growth                   |http://www.moneynews.com/Economy/federal-reserve-charles-plosser-weather-job-growth/2014/03/10/id/557011                   |Moneynews        |b       |ddUyU0VZz0BRneMioxUPQVP6sIxvM|www.moneynews.com  |1394470372027|\n",
+        "\n",
+        "\n",
+        "### Objective\n",
+        "\n",
+        "In this notebook, you learn how to integrate preprocessing code in a Vertex AI experiments. Also you will build the experiment lineage lets you record, analyze, debug, and audit metadata and artifacts produced along your ML journey.\n",
+        "\n",
         "### Costs \n",
         "\n",
         "This tutorial uses billable components of Google Cloud:\n",
@@ -170,7 +150,7 @@
       "source": [
         "### Install additional packages\n",
         "\n",
-        "Install additional package dependencies not installed in your notebook environment,TensorFlow or Vertex AI SDK. Use the latest major GA version of each package."
+        "Install additional package dependencies not installed in your notebook environment, such as TensorFlow or Vertex AI SDK. Use the latest major GA version of each package."
       ]
     },
     {
@@ -195,7 +175,7 @@
         "    USER_FLAG = \"--user\"\n",
         "\n",
         "! pip3 install {USER_FLAG} --upgrade joblib fsspec gcsfs scikit-learn -q\n",
-        "! pip install {USER_FLAG} --upgrade google-cloud-aiplatform -q"
+        "! pip3 install {USER_FLAG} --force-reinstall 'google-cloud-aiplatform>=1.15' -q"
       ]
     },
     {
@@ -228,14 +208,21 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "lWEdiXsJg0XY"
+      },
+      "source": [
+        "## Before you begin"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "BF1j6f9HApxa"
       },
       "source": [
-        "## Before you begin\n",
-        "\n",
         "### Set up your Google Cloud project\n",
         "\n",
         "**The following steps are required, regardless of your notebook environment.**\n",
@@ -246,7 +233,7 @@
         "\n",
         "1. [Enable APIs](https://console.cloud.google.com/flows/enableapi?apiid=cloudresourcemanager.googleapis.com,aiplatform.googleapis.com).\n",
         "\n",
-        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
         "\n",
         "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
         "Cloud SDK uses the right project for all the commands in this notebook.\n",
@@ -356,9 +343,9 @@
         "id": "06571eb4063b"
       },
       "source": [
-        "#### UUID\n",
+        "#### Timestamp\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append it onto the name of resources you create in this tutorial."
       ]
     },
     {
@@ -369,16 +356,9 @@
       },
       "outputs": [],
       "source": [
-        "import random\n",
-        "import string\n",
+        "from datetime import datetime\n",
         "\n",
-        "\n",
-        "# Generate a uuid of a specifed length(default=8)\n",
-        "def generate_uuid(length: int = 8) -> str:\n",
-        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
-        "\n",
-        "\n",
-        "UUID = generate_uuid()"
+        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
     {
@@ -390,7 +370,7 @@
         "### Authenticate your Google Cloud account\n",
         "\n",
         "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
-        "authenticated."
+        "authenticated. Skip this step."
       ]
     },
     {
@@ -504,7 +484,7 @@
       "outputs": [],
       "source": [
         "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
         "    BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
@@ -557,6 +537,17 @@
         "### Set project folder"
       ]
     },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "AARD6Fsr-DSi"
+      },
+      "outputs": [],
+      "source": [
+        "DATA_PATH = \"data\""
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -565,7 +556,6 @@
       },
       "outputs": [],
       "source": [
-        "DATA_PATH = \"data\"\n",
         "!mkdir -m 777 -p {DATA_PATH}"
       ]
     },
@@ -578,6 +568,17 @@
         "### Get the data"
       ]
     },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3V6W2nIo9FtL"
+      },
+      "outputs": [],
+      "source": [
+        "DATASET_URL = \"https://archive.ics.uci.edu/ml/machine-learning-databases/00359/NewsAggregatorDataset.zip\""
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -586,7 +587,6 @@
       },
       "outputs": [],
       "source": [
-        "DATASET_URL = \"https://archive.ics.uci.edu/ml/machine-learning-databases/00359/NewsAggregatorDataset.zip\"\n",
         "!wget --no-parent {DATASET_URL} --directory-prefix={DATA_PATH}\n",
         "!mkdir -m 777 -p {DATA_PATH}/temp {DATA_PATH}/raw\n",
         "!unzip {DATA_PATH}/*.zip -d {DATA_PATH}/temp\n",
@@ -662,7 +662,7 @@
         "# Experiments\n",
         "TASK = \"classification\"\n",
         "MODEL_TYPE = \"naivebayes\"\n",
-        "EXPERIMENT_NAME = f\"{TASK}-{MODEL_TYPE}-{UUID}\"\n",
+        "EXPERIMENT_NAME = f\"{TASK}-{MODEL_TYPE}-{TIMESTAMP}\"\n",
         "EXPERIMENT_RUN_NAME = \"run-1\"\n",
         "\n",
         "# Preprocessing\n",
@@ -690,7 +690,7 @@
         "FEATURES = \"title\"\n",
         "TEST_SIZE = 0.2\n",
         "SEED = 8\n",
-        "TRAINED_MODEL_URI = f\"{BUCKET_URI}/deliverables/{UUID}\"\n",
+        "TRAINED_MODEL_URI = f\"{BUCKET_URI}/deliverables/{TIMESTAMP}\"\n",
         "MODEL_NAME = f\"{EXPERIMENT_NAME}-model\""
       ]
     },
@@ -800,7 +800,7 @@
       "source": [
         "#### Create a Dataset Metadata Artifact\n",
         "\n",
-        "First you create the Dataset artifact to track the dataset resource in the Vertex ML Metadata and create the experiment lineage."
+        "First you create the Dataset artifact to track the dataset resource in the Vertex AI ML Metadata and create the experiment lineage."
       ]
     },
     {
@@ -839,6 +839,7 @@
         "Preprocess module\n",
         "\"\"\"\n",
         "\n",
+        "import string\n",
         "\n",
         "import pandas as pd\n",
         "\n",
@@ -868,10 +869,7 @@
       "source": [
         "#### Add the `preprocessing` Execution\n",
         "\n",
-        "Vertex AI Experiments supports tracking both executions and artifacts. Executions are steps in an ML workflow that can include but are not limited to data preprocessing, training, and model evaluation. Executions can consume artifacts such as datasets and produce artifacts such as models.\n",
-        "\n",
-        "You add the preprocessing step to track its execution in the lineage associated to Vertex AI Experiment. \n",
-        "For Vertex AI, the parameters are passed inside the message field which we see in the logs. These structures of the logs are predefined."
+        "You add the preprocessing step to track its execution in the lineage associated to Vertex AI Experiment. "
       ]
     },
     {
@@ -945,16 +943,7 @@
       "source": [
         "#### Create model training module\n",
         "\n",
-        "Below the training module.\n",
-        "\n",
-        "**get_training_split :** It takes parameters like  x(The data to be split), y(The labels to be split), test_size(The proportion of the data to be reserved for testing) and random_state(The seed used by the random number generator).\n",
-        "This function return training data, testing data , The training labels and The testing labels.\n",
-        "\n",
-        "**get_pipeline :** It return's the model.\n",
-        "\n",
-        "**train_pipeline :** It train the model by using model, training data, training lables and return's the trained model.\n",
-        "\n",
-        "**evaluate_model :** It evaluate the model and return the accuracy of the model.\n"
+        "Below the training module."
       ]
     },
     {
@@ -1162,15 +1151,6 @@
         "    exc.assign_output_artifacts([model])"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "e595c893de8d"
-      },
-      "source": [
-        "### Stop Experiment run"
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -1190,7 +1170,7 @@
       "source": [
         "### Visualize Experiment Lineage\n",
         "\n",
-        "Below you get the link to Vertex AI Metadata UI in the console that show the experiment lineage."
+        "Below you will get the link to Vertex AI Metadata UI in the console that will show the experiment lineage."
       ]
     },
     {
@@ -1228,8 +1208,17 @@
       "source": [
         "# Delete experiment\n",
         "exp = vertex_ai.Experiment(EXPERIMENT_NAME)\n",
-        "exp.delete()\n",
-        "\n",
+        "exp.delete()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "gW8Ddbr8xaKp"
+      },
+      "outputs": [],
+      "source": [
         "# Delete model\n",
         "model_list = vertex_ai.Model.list(filter=f'display_name=\"{MODEL_NAME}\"')\n",
         "for model in model_list:\n",
@@ -1241,15 +1230,22 @@
         "        filter=f'display_name=\"{dataset_name}\"'\n",
         "    )\n",
         "    for dataset in dataset_list:\n",
-        "        dataset.delete()\n",
-        "\n",
+        "        dataset.delete()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "sx_vKniMq9ZX"
+      },
+      "outputs": [],
+      "source": [
         "# Delete Cloud Storage objects that were created\n",
         "delete_bucket = True\n",
         "\n",
         "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
-        "    ! gsutil -m rm -r $BUCKET_URI\n",
-        "\n",
-        "!rm -Rf {DATA_PATH}"
+        "    ! gsutil -m rm -r $BUCKET_URI"
       ]
     },
     {

notebooks/official/experiments/comparing_pipeline_runs.ipynb

@@ -55,7 +55,7 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/experiments/comparing_pipeline_runs.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/experiments/comparing_pipeline_runs.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
@@ -70,8 +70,6 @@
         "id": "tvgnzT1CKxrO"
       },
       "source": [
-        "# Compare pipeline runs with Vertex AI Experiments\n",
-        "\n",
         "## Overview\n",
         "\n",
         "Depending on the model life cycle of your data science team, you would like to experiment and track training Pipeline runs and its associated parameters. Then, you would to compare runs of these Pipelines to each others in order to figure out which is the best configuration generates the model you will register in the Vertex AI Model Registry."
@@ -209,8 +207,10 @@
         "if IS_WORKBENCH_NOTEBOOK:\n",
         "    USER_FLAG = \"--user\"\n",
         "\n",
-        "!pip3 install {USER_FLAG} --force-reinstall 'google-cloud-aiplatform>=1.15' -q --no-warn-conflicts\n",
-        "!pip3 install {USER_FLAG} kfp -q --no-warn-conflicts"
+        "! pip install {USER_FLAG} --upgrade google-cloud-aiplatform \\\n",
+        "                                    google-cloud-storage \\\n",
+        "                                    google-auth \\\n",
+        "                                    kfp -q"
       ]
     },
     {
@@ -270,7 +270,7 @@
         "\n",
         "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
         "\n",
-        "1. [Enable APIs](https://console.cloud.google.com/flows/enableapi?apiid=cloudresourcemanager.googleapis.com,aiplatform.googleapis.com).\n",
+        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com).\n",
         "\n",
         "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
         "\n",
@@ -451,14 +451,9 @@
         "3. In the **Service account name** field, enter a name, and\n",
         "   click **Create**.\n",
         "\n",
-        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type and select\n",
-        "the following role into the filter box:\n",
-        "\n",
-        "    *   Storage Admin\n",
-        "    *   Storage Object Admin\n",
-        "    *   Service Account User\n",
-        "    *   Vertex AI Administrator\n",
-        "\n",
+        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
+        "into the filter box, and select\n",
+        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
         "\n",
         "5. Click *Create*. A JSON file that contains your key downloads to your\n",
         "local environment.\n",
@@ -725,7 +720,6 @@
         "import kfp.v2.dsl as dsl\n",
         "# Vertex AI\n",
         "from google.cloud import aiplatform as vertex_ai\n",
-        "from google.cloud.aiplatform_v1.types.pipeline_state import PipelineState\n",
         "from kfp.v2.dsl import Metrics, Model, Output, component"
       ]
     },
@@ -744,7 +738,6 @@
         "EXPERIMENT_NAME = f\"{PROJECT_ID}-{TASK}-{MODEL_TYPE}-{UUID}\"\n",
         "\n",
         "# Pipeline\n",
-        "PIPELINE_TEMPLATE_FILE = \"pipeline.json\"\n",
         "PIPELINE_URI = f\"{BUCKET_URI}/pipelines\"\n",
         "TRAIN_URI = f\"{BUCKET_URI}/iris/iris_data.csv\"\n",
         "LABEL_URI = f\"{BUCKET_URI}/iris/iris_target.csv\"\n",
@@ -826,7 +819,9 @@
       "source": [
         "Before you start running your pipeline experiments, you have to formalize your training as pipeline component.\n",
         "\n",
-        "To do that, you build the pipeline by using the `kfp.v2.dsl.component` decorator to convert your training task into a pipeline component. "
+        "To do that, you will use the `kfp.v2.dsl.component` decorator to convert your training task into a pipeline component.\n",
+        "\n",
+        "Training code will import required libraries to train,evaluate and save a model with mentioned features. "
       ]
     },
     {
@@ -1080,7 +1075,7 @@
         "\n",
         "    job = vertex_ai.PipelineJob(\n",
         "        display_name=f\"{EXPERIMENT_NAME}-pipeline-run-{i}\",\n",
-        "        template_path=PIPELINE_TEMPLATE_FILE,\n",
+        "        template_path=\"pipeline.json\",\n",
         "        pipeline_root=PIPELINE_URI,\n",
         "        parameter_values={\n",
         "            \"train_uri\": TRAIN_URI,\n",
@@ -1165,9 +1160,10 @@
       "source": [
         "# Get the PipelineJob resource using the experiment run name\n",
         "pipeline_experiments_df = vertex_ai.get_experiment_df(EXPERIMENT_NAME)\n",
-        "job = vertex_ai.PipelineJob.get(pipeline_experiments_df.run_name[0])\n",
-        "print(\"Pipeline job name: \", job.resource_name)\n",
-        "print(\"Pipeline Run UI link: \", job._dashboard_uri())"
+        "for i in range(5):\n",
+        "    job = vertex_ai.PipelineJob.get(pipeline_experiments_df.run_name[i])\n",
+        "    print(job.resource_name)\n",
+        "    print(job._dashboard_uri())"
       ]
     },
     {
@@ -1195,16 +1191,13 @@
       "outputs": [],
       "source": [
         "# Delete the pipeline\n",
-        "while True:\n",
-        "    for i in range(0, len(runs)):\n",
-        "        pipeline_job = vertex_ai.PipelineJob.get(pipeline_experiments_df.run_name[i])\n",
-        "        if pipeline_job.state != PipelineState.PIPELINE_STATE_SUCCEEDED:\n",
-        "            print(\"Pipeline job is still running...\")\n",
-        "            time.sleep(60)\n",
-        "        else:\n",
-        "            print(\"Pipeline job is complete.\")\n",
-        "            pipeline_job.delete()\n",
-        "    break\n",
+        "# Get the PipelineJob resource using the experiment run name\n",
+        "pipeline_experiments_df = vertex_ai.get_experiment_df(EXPERIMENT_NAME)\n",
+        "for i in range(5):\n",
+        "    job = vertex_ai.PipelineJob.get(pipeline_experiments_df.run_name[i])\n",
+        "    print(job.resource_name)\n",
+        "    print(job._dashboard_uri())\n",
+        "    job.delete()\n",
         "\n",
         "# Delete experiment\n",
         "exp = vertex_ai.Experiment(EXPERIMENT_NAME)\n",
@@ -1213,11 +1206,7 @@
         "# Delete bucket\n",
         "delete_bucket = False\n",
         "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
-        "    ! gsutil rm -rf {BUCKET_URI}\n",
-        "\n",
-        "# Remove local files\n",
-        "\n",
-        "!rm {PIPELINE_TEMPLATE_FILE}"
+        "    ! gsutil rm -rf {BUCKET_URI}"
       ]
     }
   ],

notebooks/official/explainable_ai/sdk_automl_tabular_binary_classification_batch_explain.ipynb

@@ -29,7 +29,7 @@
         "id": "title"
       },
       "source": [
-        "# Vertex AI SDK: AutoML training tabular binary classification model for batch explanation\n",
+        "# Vertex SDK: AutoML training tabular binary classification model for batch explanation\n",
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
@@ -65,6 +65,17 @@
         "This tutorial demonstrates how to use the Vertex AI SDK to create tabular binary classification models and do batch prediction with explanation using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dataset:bank,lbn"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the [Bank Marketing](https://pantheon.corp.google.com/storage/browser/_details/cloud-ml-tables-data/bank-marketing.csv) . This dataset does not require any feature engineering. The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -97,17 +108,6 @@
         "* Batch Prediction Service: Does a queued (batch) prediction for the entire set of instances in the background and stores the results in a Cloud Storage bucket when ready."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "7a4881cf39a4"
-      },
-      "source": [
-        "### Dataset\n",
-        "\n",
-        "The dataset used for this tutorial is the Bank Marketing. This dataset does not require any feature engineering. The version of the dataset you use in this tutorial is stored in a public Cloud Storage bucket."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -136,7 +136,7 @@
       "source": [
         "### Set up your local development environment\n",
         "\n",
-        "If you are using Colab or Vertex AI Workbench Notebook, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
+        "If you are using Colab or Vertex Workbench Notebook, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
         "\n",
         "Otherwise, make sure your environment meets this notebook's requirements. You need the following:\n",
         "\n",
@@ -322,10 +322,7 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
       ]
     },
     {
@@ -334,8 +331,9 @@
         "id": "timestamp"
       },
       "source": [
-        "#### UUID\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append the uuid onto the name of resources you create in this tutorial."
+        "#### Timestamp\n",
+        "\n",
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
       ]
     },
     {
@@ -346,16 +344,9 @@
       },
       "outputs": [],
       "source": [
-        "import random\n",
-        "import string\n",
+        "from datetime import datetime\n",
         "\n",
-        "\n",
-        "# Generate a uuid of length 8\n",
-        "def generate_uuid():\n",
-        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=8))\n",
-        "\n",
-        "\n",
-        "UUID = generate_uuid()"
+        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
     {
@@ -366,7 +357,7 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
+        "**If you are using Workbench AI Notebooks**, your environment is already\n",
         "authenticated. Skip this step.\n",
         "\n",
         "**If you are using Colab**, run the cell below and follow the instructions\n",
@@ -449,7 +440,7 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
         "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
@@ -462,7 +453,7 @@
       "outputs": [],
       "source": [
         "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
         "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
       ]
     },
@@ -592,7 +583,7 @@
       "source": [
         "#### Quick peek at your data\n",
         "\n",
-        "You use a version of the Bank Marketing dataset that is stored in a public Cloud Storage bucket, using a CSV index file.\n",
+        "You will use a version of the Bank Marketing dataset that is stored in a public Cloud Storage bucket, using a CSV index file.\n",
         "\n",
         "Start by doing a quick peek at the data. You count the number of examples by counting the number of rows in the CSV index file  (`wc -l`) and then peek at the first few rows.\n",
         "\n",
@@ -646,7 +637,7 @@
       "outputs": [],
       "source": [
         "dataset = aip.TabularDataset.create(\n",
-        "    display_name=\"Bank Marketing\" + \"_\" + UUID, gcs_source=[IMPORT_FILE]\n",
+        "    display_name=\"Bank Marketing\" + \"_\" + TIMESTAMP, gcs_source=[IMPORT_FILE]\n",
         ")\n",
         "\n",
         "print(dataset.resource_name)"
@@ -697,7 +688,7 @@
       "outputs": [],
       "source": [
         "dag = aip.AutoMLTabularTrainingJob(\n",
-        "    display_name=\"bank_\" + UUID,\n",
+        "    display_name=\"bank_\" + TIMESTAMP,\n",
         "    optimization_prediction_type=\"classification\",\n",
         "    optimization_objective=\"minimize-log-loss\",\n",
         ")\n",
@@ -739,7 +730,7 @@
       "source": [
         "model = dag.run(\n",
         "    dataset=dataset,\n",
-        "    model_display_name=\"bank_\" + UUID,\n",
+        "    model_display_name=\"bank_\" + TIMESTAMP,\n",
         "    training_fraction_split=0.6,\n",
         "    validation_fraction_split=0.2,\n",
         "    test_fraction_split=0.2,\n",
@@ -793,7 +784,7 @@
       "source": [
         "### Make test items\n",
         "\n",
-        "You use synthetic data as a test data items. Don't be concerned that we are using synthetic data."
+        "You will use synthetic data as a test data items. Don't be concerned that we are using synthetic data -- we just want to demonstrate how to make a prediction."
       ]
     },
     {
@@ -861,11 +852,11 @@
       "outputs": [],
       "source": [
         "batch_predict_job = model.batch_predict(\n",
-        "    job_display_name=\"bank_\" + UUID,\n",
+        "    job_display_name=\"bank_\" + TIMESTAMP,\n",
         "    gcs_source=gcs_input_uri,\n",
         "    gcs_destination_prefix=BUCKET_URI,\n",
         "    instances_format=\"csv\",\n",
-        "    predictions_format=\"jsonl\",\n",
+        "    predictions_format=\"csv\",\n",
         "    generate_explanation=True,\n",
         "    sync=False,\n",
         ")\n",
@@ -961,7 +952,6 @@
         "# Set this to true only if you'd like to delete your bucket\n",
         "delete_bucket = False\n",
         "\n",
-        "dataset.delete()\n",
         "model.delete()\n",
         "batch_predict_job.delete()\n",
         "\n",

notebooks/official/feature_store/gapic-feature-store.ipynb

@@ -292,37 +292,6 @@
         "    PROJECT_ID = \"python-docs-samples-tests\"  # @param {type:\"string\"}"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "d9f118b92c74"
-      },
-      "source": [
-        "#### UUID\n",
-        "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "3ee72715c0fd"
-      },
-      "outputs": [],
-      "source": [
-        "import random\n",
-        "import string\n",
-        "\n",
-        "\n",
-        "# Generate a uuid of a specifed length(default=8)\n",
-        "def generate_uuid(length: int = 8) -> str:\n",
-        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
-        "\n",
-        "\n",
-        "UUID = generate_uuid()"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -509,6 +478,7 @@
         "from google.cloud.aiplatform_v1.types import \\\n",
         "    featurestore_service as featurestore_service_pb2\n",
         "from google.cloud.aiplatform_v1.types import io as io_pb2\n",
+        "from google.protobuf.duration_pb2 import Duration\n",
         "\n",
         "# Create admin_client for CRUD and data_client for reading feature values.\n",
         "admin_client = FeaturestoreServiceClient(client_options={\"api_endpoint\": API_ENDPOINT})\n",
@@ -572,7 +542,7 @@
       },
       "outputs": [],
       "source": [
-        "FEATURESTORE_ID = f\"movie_prediction_{UUID}\"\n",
+        "FEATURESTORE_ID = \"movie_prediction\"\n",
         "try:\n",
         "    create_lro = admin_client.create_featurestore(\n",
         "        featurestore_service_pb2.CreateFeaturestoreRequest(\n",
@@ -597,7 +567,7 @@
         "id": "ag8pCQ7rNjVf"
       },
       "source": [
-        "You can use [GetFeaturestore](https://cloud.google.com/vertex-ai/docs/reference/rpc/google.cloud.aiplatform.v1#google.cloud.aiplatform.v1.FeaturestoreService.GetFeaturestore) or [ListFeaturestores](https://cloud.google.com/vertex-ai/docs/reference/rpc/google.cloud.aiplatform.v1#google.cloud.aiplatform.v1.FeaturestoreService.ListFeaturestores) to check if the Featurestore was successfully created. The following example gets the details of the Featurestore.\n"
+        "You can use [GetFeaturestore](https://cloud.google.com/vertex-ai/docs/reference/rpc/google.cloud.aiplatform.v1beta1#google.cloud.aiplatform.v1beta1.FeaturestoreService.GetFeaturestore) or [ListFeaturestores](https://cloud.google.com/vertex-ai/docs/reference/rpc/google.cloud.aiplatform.v1beta1#google.cloud.aiplatform.v1beta1.FeaturestoreService.ListFeaturestores) to check if the Featurestore was successfully created. The following example gets the details of the Featurestore.\n"
       ]
     },
     {
@@ -619,7 +589,7 @@
         "id": "018ab19d934f"
       },
       "source": [
-        "Auto scaling is available in v1 since v1.11. Below is the example for the `CreateFeaturestoreRequest` with auto-scaling, use it with `aiplatform_v1.FeaturestoreServiceClient` to create Featurestore:"
+        "Auto scaling is available in v1beta1 since v1.11. Below is the example for the `CreateFeaturestoreRequest` with auto-scaling, use it with `aiplatform_v1beta1.FeaturestoreServiceClient` to create Featurestore:"
       ]
     },
     {
@@ -630,17 +600,17 @@
       },
       "outputs": [],
       "source": [
-        "from google.cloud.aiplatform_v1.types import \\\n",
-        "    featurestore as v1_featurestore_pb2\n",
-        "from google.cloud.aiplatform_v1.types import \\\n",
-        "    featurestore_service as v1_featurestore_service_pb2\n",
+        "from google.cloud.aiplatform_v1beta1.types import \\\n",
+        "    featurestore as v1beta1_featurestore_pb2\n",
+        "from google.cloud.aiplatform_v1beta1.types import \\\n",
+        "    featurestore_service as v1beta1_featurestore_service_pb2\n",
         "\n",
-        "create_featurestore_request = v1_featurestore_service_pb2.CreateFeaturestoreRequest(\n",
+        "create_featurestore_request = v1beta1_featurestore_service_pb2.CreateFeaturestoreRequest(\n",
         "    parent=BASE_RESOURCE_PATH,\n",
         "    featurestore_id=FEATURESTORE_ID,\n",
-        "    featurestore=v1_featurestore_pb2.Featurestore(\n",
-        "        online_serving_config=v1_featurestore_pb2.Featurestore.OnlineServingConfig(\n",
-        "            scaling=v1_featurestore_pb2.Featurestore.OnlineServingConfig.Scaling(\n",
+        "    featurestore=v1beta1_featurestore_pb2.Featurestore(\n",
+        "        online_serving_config=v1beta1_featurestore_pb2.Featurestore.OnlineServingConfig(\n",
+        "            scaling=v1beta1_featurestore_pb2.Featurestore.OnlineServingConfig.Scaling(\n",
         "                min_node_count=1, max_node_count=5\n",
         "            )\n",
         "        ),\n",
@@ -711,7 +681,7 @@
         "id": "dPkT7KDuEvWv"
       },
       "source": [
-        "Feature [monitoring](https://cloud.google.com/vertex-ai/docs/featurestore/monitoring) is in preview, so you need to use v1 Python. Import feature analysis is only available through SDK for now."
+        "Feature [monitoring](https://cloud.google.com/vertex-ai/docs/featurestore/monitoring) is in preview, so you need to use v1beta1 Python. Import feature analysis is only available through SDK for now."
       ]
     },
     {
@@ -722,35 +692,36 @@
       },
       "outputs": [],
       "source": [
-        "from google.cloud.aiplatform_v1 import \\\n",
-        "    FeaturestoreServiceClient as v1_FeaturestoreServiceClient\n",
-        "from google.cloud.aiplatform_v1.types import entity_type as v1_entity_type_pb2\n",
-        "from google.cloud.aiplatform_v1.types import \\\n",
-        "    featurestore_monitoring as v1_featurestore_monitoring_pb2\n",
-        "from google.cloud.aiplatform_v1.types import \\\n",
-        "    featurestore_service as v1_featurestore_service_pb2\n",
+        "from google.cloud.aiplatform_v1beta1 import \\\n",
+        "    FeaturestoreServiceClient as v1beta1_FeaturestoreServiceClient\n",
+        "from google.cloud.aiplatform_v1beta1.types import \\\n",
+        "    entity_type as v1beta1_entity_type_pb2\n",
+        "from google.cloud.aiplatform_v1beta1.types import \\\n",
+        "    featurestore_monitoring as v1beta1_featurestore_monitoring_pb2\n",
+        "from google.cloud.aiplatform_v1beta1.types import \\\n",
+        "    featurestore_service as v1beta1_featurestore_service_pb2\n",
         "\n",
-        "v1_admin_client = v1_FeaturestoreServiceClient(\n",
+        "v1beta1_admin_client = v1beta1_FeaturestoreServiceClient(\n",
         "    client_options={\"api_endpoint\": API_ENDPOINT}\n",
         ")\n",
         "\n",
         "# Enable import feature analysis for users entity type.\n",
         "# All Features belonging to this EntityType will by default inherit the monitoring config.\n",
-        "v1_admin_client.update_entity_type(\n",
-        "    v1_featurestore_service_pb2.UpdateEntityTypeRequest(\n",
-        "        entity_type=v1_entity_type_pb2.EntityType(\n",
+        "v1beta1_admin_client.update_entity_type(\n",
+        "    v1beta1_featurestore_service_pb2.UpdateEntityTypeRequest(\n",
+        "        entity_type=v1beta1_entity_type_pb2.EntityType(\n",
         "            name=admin_client.entity_type_path(\n",
         "                PROJECT_ID, REGION, FEATURESTORE_ID, \"users\"\n",
         "            ),\n",
-        "            monitoring_config=v1_featurestore_monitoring_pb2.FeaturestoreMonitoringConfig(\n",
-        "                import_features_analysis=v1_featurestore_monitoring_pb2.FeaturestoreMonitoringConfig.ImportFeaturesAnalysis(\n",
-        "                    anomaly_detection_baseline=v1_featurestore_monitoring_pb2.FeaturestoreMonitoringConfig.ImportFeaturesAnalysis.Baseline.LATEST_STATS,\n",
-        "                    state=v1_featurestore_monitoring_pb2.FeaturestoreMonitoringConfig.ImportFeaturesAnalysis.State.ENABLED,\n",
+        "            monitoring_config=v1beta1_featurestore_monitoring_pb2.FeaturestoreMonitoringConfig(\n",
+        "                import_features_analysis=v1beta1_featurestore_monitoring_pb2.FeaturestoreMonitoringConfig.ImportFeaturesAnalysis(\n",
+        "                    anomaly_detection_baseline=v1beta1_featurestore_monitoring_pb2.FeaturestoreMonitoringConfig.ImportFeaturesAnalysis.Baseline.LATEST_STATS,\n",
+        "                    state=v1beta1_featurestore_monitoring_pb2.FeaturestoreMonitoringConfig.ImportFeaturesAnalysis.State.ENABLED,\n",
         "                ),\n",
-        "                numerical_threshold_config=v1_featurestore_monitoring_pb2.FeaturestoreMonitoringConfig.ThresholdConfig(\n",
+        "                numerical_threshold_config=v1beta1_featurestore_monitoring_pb2.FeaturestoreMonitoringConfig.ThresholdConfig(\n",
         "                    value=0.001,\n",
         "                ),\n",
-        "                categorical_threshold_config=v1_featurestore_monitoring_pb2.FeaturestoreMonitoringConfig.ThresholdConfig(\n",
+        "                categorical_threshold_config=v1beta1_featurestore_monitoring_pb2.FeaturestoreMonitoringConfig.ThresholdConfig(\n",
         "                    value=0.001,\n",
         "                ),\n",
         "            ),\n",
@@ -765,7 +736,7 @@
         "id": "85b1f59fbf6d"
       },
       "source": [
-        "The easiest way to set up snapshot analysis for now is using [console UI](https://console.cloud.google.com/vertex-ai/features). For completeness, below is example to do this using v1 SDK.\n",
+        "The easiest way to set up snapshot analysis for now is using [console UI](https://console.cloud.google.com/vertex-ai/features). For completeness, below is example to do this using v1beta1 SDK.\n",
         "\n",
         "You can view monitoring statistics on [console UI](https://console.cloud.google.com/vertex-ai/features)."
       ]
@@ -778,35 +749,36 @@
       },
       "outputs": [],
       "source": [
-        "from google.cloud.aiplatform_v1 import \\\n",
-        "    FeaturestoreServiceClient as v1_FeaturestoreServiceClient\n",
-        "from google.cloud.aiplatform_v1.types import entity_type as v1_entity_type_pb2\n",
-        "from google.cloud.aiplatform_v1.types import \\\n",
-        "    featurestore_monitoring as v1_featurestore_monitoring_pb2\n",
-        "from google.cloud.aiplatform_v1.types import \\\n",
-        "    featurestore_service as v1_featurestore_service_pb2\n",
+        "from google.cloud.aiplatform_v1beta1 import \\\n",
+        "    FeaturestoreServiceClient as v1beta1_FeaturestoreServiceClient\n",
+        "from google.cloud.aiplatform_v1beta1.types import \\\n",
+        "    entity_type as v1beta1_entity_type_pb2\n",
+        "from google.cloud.aiplatform_v1beta1.types import \\\n",
+        "    featurestore_monitoring as v1beta1_featurestore_monitoring_pb2\n",
+        "from google.cloud.aiplatform_v1beta1.types import \\\n",
+        "    featurestore_service as v1beta1_featurestore_service_pb2\n",
         "\n",
-        "v1_admin_client = v1_FeaturestoreServiceClient(\n",
+        "v1beta1_admin_client = v1beta1_FeaturestoreServiceClient(\n",
         "    client_options={\"api_endpoint\": API_ENDPOINT}\n",
         ")\n",
         "\n",
         "# Enable snapshot analysis for users entity type.\n",
         "# All Features belonging to this EntityType will by default inherit the monitoring config.\n",
-        "v1_admin_client.update_entity_type(\n",
-        "    v1_featurestore_service_pb2.UpdateEntityTypeRequest(\n",
-        "        entity_type=v1_entity_type_pb2.EntityType(\n",
+        "v1beta1_admin_client.update_entity_type(\n",
+        "    v1beta1_featurestore_service_pb2.UpdateEntityTypeRequest(\n",
+        "        entity_type=v1beta1_entity_type_pb2.EntityType(\n",
         "            name=admin_client.entity_type_path(\n",
         "                PROJECT_ID, REGION, FEATURESTORE_ID, \"users\"\n",
         "            ),\n",
-        "            monitoring_config=v1_featurestore_monitoring_pb2.FeaturestoreMonitoringConfig(\n",
-        "                snapshot_analysis=v1_featurestore_monitoring_pb2.FeaturestoreMonitoringConfig.SnapshotAnalysis(\n",
-        "                    monitoring_interval_days=1,  # 1 day\n",
+        "            monitoring_config=v1beta1_featurestore_monitoring_pb2.FeaturestoreMonitoringConfig(\n",
+        "                snapshot_analysis=v1beta1_featurestore_monitoring_pb2.FeaturestoreMonitoringConfig.SnapshotAnalysis(\n",
+        "                    monitoring_interval=Duration(seconds=86400),  # 1 day\n",
         "                    staleness_days=30,\n",
         "                ),\n",
-        "                numerical_threshold_config=v1_featurestore_monitoring_pb2.FeaturestoreMonitoringConfig.ThresholdConfig(\n",
+        "                numerical_threshold_config=v1beta1_featurestore_monitoring_pb2.FeaturestoreMonitoringConfig.ThresholdConfig(\n",
         "                    value=0.001,\n",
         "                ),\n",
-        "                categorical_threshold_config=v1_featurestore_monitoring_pb2.FeaturestoreMonitoringConfig.ThresholdConfig(\n",
+        "                categorical_threshold_config=v1beta1_featurestore_monitoring_pb2.FeaturestoreMonitoringConfig.ThresholdConfig(\n",
         "                    value=0.001,\n",
         "                ),\n",
         "            ),\n",
@@ -919,8 +891,8 @@
       "source": [
         "## Search created features\n",
         "\n",
-        "While the [ListFeatures](https://cloud.google.com/vertex-ai/docs/reference/rpc/google.cloud.aiplatform.v1#google.cloud.aiplatform.v1.FeaturestoreService.ListFeatures) method allows you to easily view all features of a single\n",
-        "entity type, the [SearchFeatures](https://cloud.google.com/vertex-ai/docs/reference/rpc/google.cloud.aiplatform.v1#google.cloud.aiplatform.v1.FeaturestoreService.SearchFeatures) method searches across all featurestores\n",
+        "While the [ListFeatures](https://cloud.google.com/vertex-ai/docs/reference/rpc/google.cloud.aiplatform.v1beta1#google.cloud.aiplatform.v1beta1.FeaturestoreService.ListFeatures) method allows you to easily view all features of a single\n",
+        "entity type, the [SearchFeatures](https://cloud.google.com/vertex-ai/docs/reference/rpc/google.cloud.aiplatform.v1beta1#google.cloud.aiplatform.v1beta1.FeaturestoreService.SearchFeatures) method searches across all featurestores\n",
         "and entity types in a given location (such as `us-central1`). This can help you discover features that were created by someone else.\n",
         "\n",
         "You can query based on feature properties including feature ID, entity type ID,\n",
@@ -1234,7 +1206,7 @@
       },
       "source": [
         "The\n",
-        "[Online Serving APIs](https://cloud.google.com/vertex-ai/docs/reference/rpc/google.cloud.aiplatform.v1#featurestoreonlineservingservice)\n",
+        "[Online Serving APIs](https://cloud.google.com/vertex-ai/docs/reference/rpc/google.cloud.aiplatform.v1beta1#featurestoreonlineservingservice)\n",
         "lets you serve feature values for small batches of entities. It's designed for latency-sensitive service, such as online model prediction. For example, for a movie service, you might want to quickly shows movies that the current user would most likely watch by using online predictions."
       ]
     },

notebooks/official/feature_store/sdk-feature-store.ipynb

@@ -29,8 +29,6 @@
         "id": "JAPoU8Sm5E6e"
       },
       "source": [
-        "# Online and Batch predictions using Vertex AI Feature Store\n",
-        "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
         "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/feature_store/sdk-feature-store.ipynb\">\n",
@@ -53,22 +51,19 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "c4aaea3bab5e"
+        "id": "tvgnzT1CKxrO"
       },
       "source": [
         "## Overview\n",
         "\n",
         "This notebook introduces Vertex AI Feature Store, a managed cloud service for machine learning engineers and data scientists to store, serve, manage and share machine learning features at a large scale.\n",
         "\n",
-        "This notebook assumes that you understand basic Google Cloud concepts such as [Project](https://cloud.google.com/storage/docs/projects), [Storage](https://cloud.google.com/storage) and [Vertex AI](https://cloud.google.com/vertex-ai/docs). Some machine learning knowledge is also helpful but not required.\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "71779c8088bf"
-      },
-      "source": [
+        "This notebook assumes that you understand basic Google Cloud concepts such as [Project](https://cloud.google.com/storage/docs/projects), [Storage](https://cloud.google.com/storage) and [Vertex AI](https://cloud.google.com/vertex-ai/docs). Some machine learning knowledge is also helpful but not required.\n",
+        "\n",
+        "### Dataset\n",
+        "\n",
+        "This notebook uses a movie recommendation dataset as an example throughout all the sessions. The task is to train a model to predict if a user is going to watch a movie and serve this model online. \n",
+        "\n",
         "### Objective\n",
         "\n",
         "In this notebook, you will learn how to use `Vertex AI Feature Store` to import feature data, and to access the feature data for both online serving and offline tasks, such as training.\n",
@@ -84,26 +79,8 @@
         "- Create featurestore, entity type, and feature resources.\n",
         "- Import feature data into `Vertex AI Feature Store` resource.\n",
         "- Serve online prediction requests using the imported features.\n",
-        "- Access imported features in offline jobs, such as training jobs."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "55e01a856f57"
-      },
-      "source": [
-        "### Dataset\n",
+        "- Access imported features in offline jobs, such as training jobs.\n",
         "\n",
-        "This notebook uses a movie recommendation dataset as an example throughout all the sessions. The task is to train a model to predict if a user is going to watch a movie and serve this model online."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "tvgnzT1CKxrO"
-      },
-      "source": [
         "### Costs \n",
         "\n",
         "This tutorial uses billable components of Google Cloud:\n",
@@ -285,15 +262,7 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
-        "PROJECT_ID = \"\"\n",
-        "\n",
-        "# Get your Google Cloud project ID from gcloud\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    shell_output = !gcloud config list --format 'value(core.project)' 2>/dev/null\n",
-        "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID: \", PROJECT_ID)"
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -306,7 +275,10 @@
       "source": [
         "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
         "    PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}\n",
-        "print(\"Project ID: \", PROJECT_ID)"
+        "    # Get your GCP project id from gcloud\n",
+        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "    PROJECT_ID = shell_output[0]\n",
+        "    print(\"Project ID:\", PROJECT_ID)"
       ]
     },
     {
@@ -348,9 +320,7 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"[your-region]\"  # @param {type:\"string\"}\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
       ]
     },
     {
@@ -359,9 +329,9 @@
         "id": "timestamp"
       },
       "source": [
-        "#### UUID\n",
+        "#### Timestamp\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
       ]
     },
     {
@@ -372,16 +342,9 @@
       },
       "outputs": [],
       "source": [
-        "import random\n",
-        "import string\n",
+        "from datetime import datetime\n",
         "\n",
-        "\n",
-        "# Generate a uuid of a specifed length(default=8)\n",
-        "def generate_uuid(length: int = 8) -> str:\n",
-        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
-        "\n",
-        "\n",
-        "UUID = generate_uuid()"
+        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
     {
@@ -478,7 +441,7 @@
       "source": [
         "from google.cloud.aiplatform import Feature, Featurestore\n",
         "\n",
-        "FEATURESTORE_ID = \"movie_prediction\" + UUID\n",
+        "FEATURESTORE_ID = \"movie_prediction\"\n",
         "INPUT_CSV_FILE = \"gs://cloud-samples-data-us-central1/vertex-ai/feature-store/datasets/movie_prediction.csv\"\n",
         "ONLINE_STORE_FIXED_NODE_COUNT = 1"
       ]

notebooks/official/migration/UJ4 Vertex SDK AutoML Tabular Binary Classification.ipynb

@@ -55,7 +55,7 @@
       "source": [
         "### Dataset\n",
         "\n",
-        "The dataset used for this tutorial is the Bank Marketing. This dataset does not require any feature engineering. The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket."
+        "The dataset used for this tutorial is the [Bank Marketing](https://pantheon.corp.google.com/storage/browser/_details/cloud-ml-tables-data/bank-marketing.csv) . This dataset does not require any feature engineering. The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket."
       ]
     },
     {

notebooks/official/migration/UJ5 Vertex SDK AutoML Image Object Detection.ipynb

@@ -32,26 +32,18 @@
         "# Vertex AI: Vertex AI Migration: AutoML Image Object Detection\n",
         "\n",
         "<table align=\"left\">\n",
-        "\n",
         "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ5 Vertex SDK AutoML Image Object Detection.ipynb\">\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/ai-platform-samples/blob/master/vertex-ai-samples/tree/master/notebooks/official/migration/UJ5%20Vertex%20SDK%20AutoML%20Image%20Object%20Detection.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ5 Vertex SDK AutoML Image Object Detection.ipynb\">\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/ai-platform-samples/blob/master/vertex-ai-samples/tree/master/notebooks/official/migration/UJ5%20Vertex%20SDK%20AutoML%20Image%20Object%20Detection.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ5 Vertex SDK AutoML Image Object Detection.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
-        "    </a>\n",
-        "  </td>                                                                                               \n",
         "</table>\n",
-        "\n",
         "<br/><br/><br/>"
       ]
     },
@@ -127,7 +119,7 @@
       "source": [
         "## Installation\n",
         "\n",
-        "Install the latest version of Vertex AI SDK for Python."
+        "Install the latest version of Vertex SDK for Python."
       ]
     },
     {
@@ -146,7 +138,7 @@
         "else:\n",
         "    USER_FLAG = \"\"\n",
         "\n",
-        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG -q"
+        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG"
       ]
     },
     {
@@ -158,6 +150,17 @@
         "Install the latest GA version of *google-cloud-storage* library as well."
       ]
     },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "install_storage"
+      },
+      "outputs": [],
+      "source": [
+        "! pip3 install -U google-cloud-storage $USER_FLAG"
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -166,9 +169,8 @@
       },
       "outputs": [],
       "source": [
-        "! pip3 install -U google-cloud-storage $USER_FLAG -q\n",
-        "\n",
-        "! pip3 install --upgrade tensorflow $USER_FLAG -q"
+        "if os.getenv(\"IS_TESTING\"):\n",
+        "    ! pip3 install --upgrade tensorflow $USER_FLAG"
       ]
     },
     {
@@ -295,10 +297,7 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
       ]
     },
     {
@@ -307,9 +306,9 @@
         "id": "timestamp"
       },
       "source": [
-        "#### UUID\n",
+        "#### Timestamp\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
       ]
     },
     {
@@ -320,16 +319,9 @@
       },
       "outputs": [],
       "source": [
-        "import random\n",
-        "import string\n",
+        "from datetime import datetime\n",
         "\n",
-        "\n",
-        "# Generate a uuid of a specifed length(default=8)\n",
-        "def generate_uuid(length: int = 8) -> str:\n",
-        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
-        "\n",
-        "\n",
-        "UUID = generate_uuid()"
+        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
     {
@@ -340,7 +332,7 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
+        "**If you are using Google Cloud Notebooks**, your environment is already authenticated. Skip this step.\n",
         "\n",
         "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
         "\n",
@@ -375,11 +367,8 @@
         "import os\n",
         "import sys\n",
         "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = \"google.colab\" in sys.modules\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
+        "# If on Google Cloud Notebook, then don't execute this code\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
         "    if \"google.colab\" in sys.modules:\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
@@ -415,8 +404,7 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -427,9 +415,8 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
-        "    BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
       ]
     },
     {
@@ -449,7 +436,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
+        "! gsutil mb -l $REGION $BUCKET_NAME"
       ]
     },
     {
@@ -469,7 +456,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_URI"
+        "! gsutil ls -al $BUCKET_NAME"
       ]
     },
     {
@@ -514,7 +501,7 @@
       },
       "outputs": [],
       "source": [
-        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
+        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_NAME)"
       ]
     },
     {
@@ -616,7 +603,7 @@
       "outputs": [],
       "source": [
         "dataset = aip.ImageDataset.create(\n",
-        "    display_name=\"Salads\" + \"_\" + UUID,\n",
+        "    display_name=\"Salads\" + \"_\" + TIMESTAMP,\n",
         "    gcs_source=[IMPORT_FILE],\n",
         "    import_schema_uri=aip.schema.dataset.ioformat.image.bounding_box,\n",
         ")\n",
@@ -701,7 +688,7 @@
       "outputs": [],
       "source": [
         "dag = aip.AutoMLImageTrainingJob(\n",
-        "    display_name=\"salads_\" + UUID,\n",
+        "    display_name=\"salads_\" + TIMESTAMP,\n",
         "    prediction_type=\"object_detection\",\n",
         "    multi_label=False,\n",
         "    model_type=\"CLOUD\",\n",
@@ -755,7 +742,7 @@
       "source": [
         "model = dag.run(\n",
         "    dataset=dataset,\n",
-        "    model_display_name=\"salads_\" + UUID,\n",
+        "    model_display_name=\"salads_\" + TIMESTAMP,\n",
         "    training_fraction_split=0.8,\n",
         "    validation_fraction_split=0.1,\n",
         "    test_fraction_split=0.1,\n",
@@ -828,7 +815,7 @@
       "outputs": [],
       "source": [
         "# Get model resource ID\n",
-        "models = aip.Model.list(filter=\"display_name=salads_\" + UUID)\n",
+        "models = aip.Model.list(filter=\"display_name=salads_\" + TIMESTAMP)\n",
         "\n",
         "# Get a reference to the Model Service client\n",
         "client_options = {\"api_endpoint\": f\"{REGION}-aiplatform.googleapis.com\"}\n",
@@ -958,11 +945,11 @@
         "file_1 = test_item_1.split(\"/\")[-1]\n",
         "file_2 = test_item_2.split(\"/\")[-1]\n",
         "\n",
-        "! gsutil cp $test_item_1 $BUCKET_URI/$file_1\n",
-        "! gsutil cp $test_item_2 $BUCKET_URI/$file_2\n",
+        "! gsutil cp $test_item_1 $BUCKET_NAME/$file_1\n",
+        "! gsutil cp $test_item_2 $BUCKET_NAME/$file_2\n",
         "\n",
-        "test_item_1 = BUCKET_URI + \"/\" + file_1\n",
-        "test_item_2 = BUCKET_URI + \"/\" + file_2"
+        "test_item_1 = BUCKET_NAME + \"/\" + file_1\n",
+        "test_item_2 = BUCKET_NAME + \"/\" + file_2"
       ]
     },
     {
@@ -995,7 +982,7 @@
         "\n",
         "import tensorflow as tf\n",
         "\n",
-        "gcs_input_uri = BUCKET_URI + \"/test.jsonl\"\n",
+        "gcs_input_uri = BUCKET_NAME + \"/test.jsonl\"\n",
         "with tf.io.gfile.GFile(gcs_input_uri, \"w\") as f:\n",
         "    data = {\"content\": test_item_1, \"mime_type\": \"image/jpeg\"}\n",
         "    f.write(json.dumps(data) + \"\\n\")\n",
@@ -1031,9 +1018,9 @@
       "outputs": [],
       "source": [
         "batch_predict_job = model.batch_predict(\n",
-        "    job_display_name=\"salads_\" + UUID,\n",
+        "    job_display_name=\"salads_\" + TIMESTAMP,\n",
         "    gcs_source=gcs_input_uri,\n",
-        "    gcs_destination_prefix=BUCKET_URI,\n",
+        "    gcs_destination_prefix=BUCKET_NAME,\n",
         "    sync=False,\n",
         ")\n",
         "\n",
@@ -1391,25 +1378,60 @@
       },
       "outputs": [],
       "source": [
-        "# Delete the dataset using the Vertex dataset object\n",
+        "delete_all = True\n",
         "\n",
-        "dataset.delete()\n",
+        "if delete_all:\n",
+        "    # Delete the dataset using the Vertex dataset object\n",
+        "    try:\n",
+        "        if \"dataset\" in globals():\n",
+        "            dataset.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
         "\n",
-        "# Delete the model using the Vertex model object\n",
-        "model.delete()\n",
+        "    # Delete the model using the Vertex model object\n",
+        "    try:\n",
+        "        if \"model\" in globals():\n",
+        "            model.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
         "\n",
-        "# Delete the endpoint using the Vertex endpoint object\n",
-        "endpoint.delete()\n",
+        "    # Delete the endpoint using the Vertex endpoint object\n",
+        "    try:\n",
+        "        if \"endpoint\" in globals():\n",
+        "            endpoint.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
         "\n",
-        "# Delete the AutoML or Pipeline trainig job\n",
+        "    # Delete the AutoML or Pipeline trainig job\n",
+        "    try:\n",
+        "        if \"dag\" in globals():\n",
+        "            dag.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
         "\n",
-        "dag.delete()\n",
+        "    # Delete the custom trainig job\n",
+        "    try:\n",
+        "        if \"job\" in globals():\n",
+        "            job.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
         "\n",
-        "# Delete the batch prediction job using the Vertex batch prediction object\n",
-        "batch_predict_job.delete()\n",
+        "    # Delete the batch prediction job using the Vertex batch prediction object\n",
+        "    try:\n",
+        "        if \"batch_predict_job\" in globals():\n",
+        "            batch_predict_job.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
         "\n",
-        "if \"BUCKET_NAME\" in globals():\n",
-        "    ! gsutil rm -r $BUCKET_URI"
+        "    # Delete the hyperparameter tuning job using the Vertex hyperparameter tuning object\n",
+        "    try:\n",
+        "        if \"hpt_job\" in globals():\n",
+        "            hpt_job.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    if \"BUCKET_NAME\" in globals():\n",
+        "        ! gsutil rm -r $BUCKET_NAME"
       ]
     }
   ],

notebooks/official/model-registry/bqml-vertexai-model-registry.ipynb

@@ -1,921 +0,0 @@
-{
-  "cells": [
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "ur8xi4C7S06n"
-      },
-      "outputs": [],
-      "source": [
-        "# Copyright 2022 Google LLC\n",
-        "#\n",
-        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
-        "# you may not use this file except in compliance with the License.\n",
-        "# You may obtain a copy of the License at\n",
-        "#\n",
-        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
-        "#\n",
-        "# Unless required by applicable law or agreed to in writing, software\n",
-        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
-        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
-        "# See the License for the specific language governing permissions and\n",
-        "# limitations under the License."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "JAPoU8Sm5E6e"
-      },
-      "source": [
-        "<table align=\"left\">\n",
-        "\n",
-        "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/bigquery-ml/bqml-vertexai-model-registry.ipynb\" target=\"_blank\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
-        "    </a>\n",
-        "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/bigquery-ml/bqml-vertexai-model-registry.ipynb\" target=\"_blank\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
-        "      View on GitHub\n",
-        "    </a>\n",
-        "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/bigquery-ml/bqml-vertexai-model-registry.ipynb\" target=\"_blank\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
-        "    </a>\n",
-        "  </td>                                                                                               \n",
-        "</table>"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "tvgnzT1CKxrO"
-      },
-      "source": [
-        "# Deploy BiqQuery ML Model on Vertex AI Model Registry and Make Predictions\n",
-        "\n",
-        "## Overview\n",
-        "\n",
-        "This tutorial demonstrates how to train a model with BigQuery ML and upload it on Vertex AI model registry, then make batch predictions.\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "132a9ee68ba6"
-      },
-      "source": [
-        "### Objective\n",
-        "\n",
-        "In this tutorial, you learn how to use `Vertex AI Model Registry` with `BigQuery ML` and make batch predictions:\n",
-        "\n",
-        "This tutorial uses the following Google Cloud ML services and resources:\n",
-        "\n",
-        "- `Vertex AI Model Registry`\n",
-        "- `Vertex AI Model` resources \n",
-        "- `Vertex AI Endpoint` resources\n",
-        "- `Vertex AI Prediction`\n",
-        "- `BigQuery ML`\n",
-        "\n",
-        "\n",
-        "The steps performed include:\n",
-        "\n",
-        "- Train a model with `BQML`\n",
-        "- Upload the model to `Vertex AI Model Registry` \n",
-        "- Create a `Vertex AI Endpoint` resource\n",
-        "- Deploy the `Model` resource to the `Endpoint` resource\n",
-        "- Make `prediction` requests to the model endpoint\n",
-        "- Run `batch prediction` job on the `Model` resource \n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "2de0477b10ce"
-      },
-      "source": [
-        "### Dataset\n",
-        "\n",
-        "The dataset used for this tutorial is the Penguins dataset from <a href=\"https://cloud.google.com/bigquery/public-data\" target=\"_blank\">BigQuery public datasets</a>. This version of the dataset is used to predict the species of penguins from the available features like culmen-length, flipper-depth etc."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "76330e07673b"
-      },
-      "source": [
-        "### Costs \n",
-        "\n",
-        "This tutorial uses billable components of Google Cloud:\n",
-        "\n",
-        "* Vertex AI\n",
-        "* BigQuery ML\n",
-        "\n",
-        "Learn about <a href=\"https://cloud.google.com/vertex-ai/pricing\" target=\"_blank\">Vertex AI\n",
-        "pricing</a> and <a href=\"https://cloud.google.com/bigquery/pricing\" target=\"_blank\">BigQuery pricing</a>, and use the <a href=\"https://cloud.google.com/products/calculator/\" target=\"_blank\">Pricing\n",
-        "Calculator</a>\n",
-        "to generate a cost estimate based on your projected usage."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "ze4-nDLfK4pw"
-      },
-      "source": [
-        "### Set up your local development environment\n",
-        "\n",
-        "**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets\n",
-        "all the requirements to run this notebook. You can skip this step."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "gCuSR8GkAgzl"
-      },
-      "source": [
-        "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
-        "You need the following:\n",
-        "\n",
-        "* The Google Cloud SDK\n",
-        "* Git\n",
-        "* Python 3\n",
-        "* virtualenv\n",
-        "* Jupyter notebook running in a virtual environment with Python 3\n",
-        "\n",
-        "The Google Cloud guide to <a href=\"https://cloud.google.com/python/setup\" target=\"_blank\">Setting up a Python development\n",
-        "environment</a> and the <a href=\"https://jupyter.org/install\" target=\"_blank\">Jupyter\n",
-        "installation guide</a> provide detailed instructions\n",
-        "for meeting these requirements. The following steps provide a condensed set of\n",
-        "instructions:\n",
-        "\n",
-        "1. <a href=\"https://cloud.google.com/sdk/docs/\" target=\"_blank\">Install and initialize the Cloud SDK.</a>\n",
-        "\n",
-        "1. <a href=\"https://cloud.google.com/python/setup#installing_python\" target=\"_blank\">Install Python 3.</a>\n",
-        "\n",
-        "1. <a href=\"https://cloud.google.com/python/setup#installing_and_using_virtualenv\" target=\"_blank\">Install\n",
-        "   virtualenv</a>\n",
-        "   and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
-        "\n",
-        "1. To install Jupyter, run `pip3 install jupyter` on the\n",
-        "command-line in a terminal shell.\n",
-        "\n",
-        "1. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
-        "\n",
-        "1. Open this notebook in the Jupyter Notebook Dashboard."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "i7EUnXsZhAGF"
-      },
-      "source": [
-        "### Install additional packages\n",
-        "\n",
-        "Install the following packages required to execute this notebook. "
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "2b4ef9b72d43"
-      },
-      "outputs": [],
-      "source": [
-        "import os\n",
-        "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
-        "! pip3 install --upgrade google-cloud-aiplatform {USER_FLAG} -q\n",
-        "! pip3 install --upgrade google-cloud-bigquery {USER_FLAG} -q"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "hhq5zEbGg0XX"
-      },
-      "source": [
-        "### Restart the kernel\n",
-        "\n",
-        "After you install the additional packages, you need to restart the notebook kernel so it can find the packages."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "EzrelQZ22IZj"
-      },
-      "outputs": [],
-      "source": [
-        "# Automatically restart kernel after installs\n",
-        "import os\n",
-        "\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    # Automatically restart kernel after installs\n",
-        "    import IPython\n",
-        "\n",
-        "    app = IPython.Application.instance()\n",
-        "    app.kernel.do_shutdown(True)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "lWEdiXsJg0XY"
-      },
-      "source": [
-        "## Before you begin"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "BF1j6f9HApxa"
-      },
-      "source": [
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. <a href=\"https://console.cloud.google.com/cloud-resource-manager\" target=\"_blank\">Select or create a Google Cloud project</a>. When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "1. <a href=\"https://cloud.google.com/billing/docs/how-to/modify-project\" target=\"_blank\">Make sure that billing is enabled for your project</a>.\n",
-        "\n",
-        "1. <a href=\"https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,bigquery\" target=\"_blank\">Enable the Vertex AI and BigQuery APIs</a>. \n",
-        "\n",
-        "1. If you are running this notebook locally, you will need to install the <a href=\"https://cloud.google.com/sdk\" target=\"_blank\">Cloud SDK</a>.\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "WReHDGG5g0XY"
-      },
-      "source": [
-        "#### Set your project ID\n",
-        "\n",
-        "**If you don't know your project ID**, you can get your project ID using `gcloud`."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "o1AuQDpf_hS-"
-      },
-      "outputs": [],
-      "source": [
-        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "RYbBU1jXAETD"
-      },
-      "outputs": [],
-      "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
-        "    # Get your GCP project id from gcloud\n",
-        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
-        "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID:\", PROJECT_ID)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "set_gcloud_project_id"
-      },
-      "outputs": [],
-      "source": [
-        "! gcloud config set project $PROJECT_ID"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "region"
-      },
-      "source": [
-        "#### Region\n",
-        "\n",
-        "You can also change the `REGION` variable, which is used for operations\n",
-        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
-        "\n",
-        "- Americas: `us-central1`\n",
-        "- Europe: `europe-west4`\n",
-        "- Asia Pacific: `asia-east1`\n",
-        "\n",
-        "You might not be able to use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
-        "\n",
-        "Learn more about <a href=\"https://cloud.google.com/vertex-ai/docs/general/locations\" target=\"_blank\">Vertex AI regions</a>."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "vO3W8YdN2LuA"
-      },
-      "outputs": [],
-      "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "29e912d1b106"
-      },
-      "source": [
-        "#### UUID\n",
-        "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 2,
-      "metadata": {
-        "id": "c704897922c0"
-      },
-      "outputs": [],
-      "source": [
-        "import random\n",
-        "import string\n",
-        "\n",
-        "\n",
-        "# Generate a uuid of a specifed length(default=8)\n",
-        "def generate_uuid(length: int = 8) -> str:\n",
-        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
-        "\n",
-        "\n",
-        "UUID = generate_uuid()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "dr--iN2kAylZ"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
-        "authenticated. Skip this step."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "sBCra4QMA2wR"
-      },
-      "source": [
-        "**If you are using Colab**, run the cell below and follow the instructions\n",
-        "when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "1. In the Cloud Console, go to the <a href=\"https://console.cloud.google.com/apis/credentials/serviceaccountkey\" target=\"_blank\">Create service account key page</a>.\n",
-        "\n",
-        "2. Click **Create service account**.\n",
-        "\n",
-        "3. In the **Service account name** field, enter a name, and\n",
-        "   click **Create**.\n",
-        "\n",
-        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
-        "into the filter box, and select\n",
-        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
-        "local environment.\n",
-        "\n",
-        "6. Enter the path to your service account key as the\n",
-        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "PyQmSRbKA8r-"
-      },
-      "outputs": [],
-      "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = \"google.colab\" in sys.modules\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "XoEqT2Y4DJmf"
-      },
-      "source": [
-        "### Import libraries"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "pRUOFELefqf1"
-      },
-      "outputs": [],
-      "source": [
-        "import google.cloud.aiplatform as aiplatform\n",
-        "from google.cloud import bigquery"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "init_aip:mbsdk,all"
-      },
-      "source": [
-        "### Initialize Vertex AI and BigQuery SDKs for Python\n",
-        "\n",
-        "Initialize the Vertex AI and Big Query SDKs for Python for your project and corresponding bucket."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "BgaYKz2-2LuC"
-      },
-      "outputs": [],
-      "source": [
-        "aiplatform.init(project=PROJECT_ID, location=REGION)\n",
-        "bqclient = bigquery.Client(project=PROJECT_ID)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "lUEtpimzL17Z"
-      },
-      "source": [
-        "## BigQuery ML introduction\n",
-        "\n",
-        "BigQuery ML (BQML) provides the capability to train ML tabular models, such as classification and regression, in BigQuery using SQL syntax.\n",
-        "\n",
-        "Learn more about <a href=\"https://cloud.google.com/bigquery-ml/docs\" target=\"_blank\">BigQuery ML documentation</a>."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "30adf1b74bf9"
-      },
-      "source": [
-        "### BigQuery table used for training"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "3O7qlOGWNEU4"
-      },
-      "outputs": [],
-      "source": [
-        "# Define BigQuery table to be used for training\n",
-        "\n",
-        "BQ_TABLE = \"bigquery-public-data.ml_datasets.penguins\""
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "PKQD2e0eMg3M"
-      },
-      "source": [
-        "### Create BigQuery dataset resource"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "BnXOpvs2MmzF"
-      },
-      "source": [
-        "First, you create an empty dataset resource in your project."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "luqb-DBiMn-0"
-      },
-      "outputs": [],
-      "source": [
-        "BQ_DATASET_NAME = \"penguins\" + UUID\n",
-        "DATASET_QUERY = f\"\"\"CREATE SCHEMA {BQ_DATASET_NAME}\"\"\"\n",
-        "\n",
-        "job = bqclient.query(DATASET_QUERY)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "59bf85366baf"
-      },
-      "outputs": [],
-      "source": [
-        "job.result()\n",
-        "print(job.state)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "b-_8rZO8NIEb"
-      },
-      "source": [
-        "## Train BigQuery ML model and upload it to Vertex AI Model Registry\n",
-        "Next, you create and train a BQML tabular regression model from the public dataset penguins and store the model in your project `Vertex AI Model Registry` using the `CREATE MODEL` statement. The model configuration is specified in the `OPTIONS` statement as follows:\n",
-        "\n",
-        "- `model_type`: The type and archictecture of tabular model to train, e.g., LOGISTIC_REG.\n",
-        "\n",
-        "- `labels`: The column which are the labels.\n",
-        "\n",
-        "- `model_registry`: To register a BigQuery ML model to Vertex AI Model Registry, you must use `model_registry=\"vertex_ai\"`.\n",
-        "\n",
-        "Learn more about the <a href=\"https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-create\" target=\"_blank\">CREATE MODEL statement</a>.\n",
-        "\n",
-        "Learn more about <a href=\"https://cloud.google.com/bigquery-ml/docs/managing-models-vertex\" target=\"_blank\">Managing BigQuery ML models in the Vertex AI Model Registry</a>."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "Q96rlZKRNPjU"
-      },
-      "outputs": [],
-      "source": [
-        "# Write the query to create Big Query ML model\n",
-        "\n",
-        "MODEL_NAME = \"penguins-lr\" + UUID\n",
-        "MODEL_QUERY = f\"\"\"\n",
-        "CREATE OR REPLACE MODEL `{BQ_DATASET_NAME}.{MODEL_NAME}`\n",
-        "OPTIONS(\n",
-        "    model_type='LOGISTIC_REG',\n",
-        "    labels = ['species'],\n",
-        "    model_registry='vertex_ai'\n",
-        "    )\n",
-        "AS\n",
-        "SELECT *\n",
-        "FROM `{BQ_TABLE}`\n",
-        "\"\"\""
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "eee158e2a375"
-      },
-      "source": [
-        "Create the BigQuery ML model using the query above and the BigQuery client that you created previously:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "LtdieY-BWILs"
-      },
-      "outputs": [],
-      "source": [
-        "# Run the model creation query using BigQuery client\n",
-        "\n",
-        "job = bqclient.query(MODEL_QUERY)\n",
-        "print(f\"Job state: {job.state}\\nJob Error:{job.errors}\")"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "e4b007777e68"
-      },
-      "source": [
-        "Check the job status:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "Y4C_3hTEXOE7"
-      },
-      "outputs": [],
-      "source": [
-        "job.result()\n",
-        "print(job.state)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "5nHvVttrYfQ8"
-      },
-      "source": [
-        "### Find the model in the Vertex Model Registry\n",
-        "\n",
-        "You can use the `Vertex AI Model list()` method with a filter query to find the automatically registered model."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "E08IwUX5YpAG"
-      },
-      "outputs": [],
-      "source": [
-        "model = aiplatform.Model(model_name=MODEL_NAME)\n",
-        "\n",
-        "print(model.gca_resource)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "zgvXXlaZaYw3"
-      },
-      "source": [
-        "## Deploy Vertex AI Model resource to a Vertex AI Endpoint resource\n",
-        "You must deploy a model to an `endpoint` before that model can be used to serve online predictions; deploying a model associates physical resources with the model so it can serve online predictions with low latency. \n",
-        "\n",
-        "Learn more about <a href=\"https://cloud.google.com/vertex-ai/docs/predictions/deploy-model-api#aiplatform_deploy_model_custom_trained_model_sample-python\" target=\"_blank\">Deploy a model using the Vertex AI API</a>\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "N27z-_by5gti"
-      },
-      "source": [
-        "### Create a Vertex AI Endpoint resource\n",
-        "\n",
-        "If you are deploying a model to an existing endpoint, you can skip this cell.\n",
-        "\n",
-        "- `display_name`: Display name for the endpoint.\n",
-        "- `project`: The project ID on which you are creating an endpoint.\n",
-        "- `location`: The region where you are using Vertex AI."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "XmmyCtW055Ya"
-      },
-      "outputs": [],
-      "source": [
-        "ENDPOINT_DISPLAY_NAME = \"bqml-lr-model-endpoint\" + UUID\n",
-        "\n",
-        "endpoint = aiplatform.Endpoint.create(\n",
-        "    display_name=ENDPOINT_DISPLAY_NAME,\n",
-        "    project=PROJECT_ID,\n",
-        "    location=REGION,\n",
-        ")\n",
-        "\n",
-        "print(endpoint.display_name)\n",
-        "print(endpoint.resource_name)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "xSCIKdFo56YO"
-      },
-      "source": [
-        "### Deploy the Vertex AI Model resource to Vertex AI Endpoint resource"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "f7KfDgALE4aD"
-      },
-      "outputs": [],
-      "source": [
-        "DEPLOYED_NAME = \"bqml-lr-penguins\"\n",
-        "\n",
-        "model.deploy(endpoint=endpoint, deployed_model_display_name=DEPLOYED_NAME)\n",
-        "\n",
-        "print(model.display_name)\n",
-        "print(model.resource_name)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "H3K6SVplJ9Mg"
-      },
-      "source": [
-        "## Send prediction request to the Vertex AI Endpoint resource\n",
-        "\n",
-        "Now that your Vertex AI Model resource is deployed to a Vertex AI `Endpoint` resource, you can do online predictions by sending prediction requests to the `Endpoint` resource.\n",
-        "\n",
-        "Learn more about <a href=\"https://cloud.google.com/vertex-ai/docs/predictions/online-predictions-custom-models\" target=\"_blank\">Get online predictions from custom-trained models</a>"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "2j7ioB3VKEtx"
-      },
-      "outputs": [],
-      "source": [
-        "instance = {\n",
-        "    \"island\": \"Dream\",\n",
-        "    \"culmen_length_mm\": 36.6,\n",
-        "    \"culmen_depth_mm\": 18.4,\n",
-        "    \"flipper_length_mm\": 184.0,\n",
-        "    \"body_mass_g\": 3475.0,\n",
-        "    \"sex\": \"FEMALE\",\n",
-        "}\n",
-        "\n",
-        "prediction = endpoint.predict([instance])\n",
-        "print(prediction)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "C39qOaBHZI1G"
-      },
-      "source": [
-        "## Batch Prediction on the BQML model"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "UBffk3GyaPY3"
-      },
-      "source": [
-        "Here you request batch predictions directly from the BigQuery ML model; you don't need to deploy the model to an endpoint. For data types that support both batch and online predictions, use batch predictions when you don't require an immediate response and want to process accumulated data by using a single request.\n",
-        "\n",
-        "Learn more abount <a href=\"https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-predict\" target=\"_blank\">The ML.PREDICT function</a>"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "_QxZb19_o6jx"
-      },
-      "outputs": [],
-      "source": [
-        "sql_ml_predict = f\"\"\"SELECT * FROM ML.PREDICT(MODEL `{PROJECT_ID}.{BQ_DATASET_NAME}.{MODEL_NAME}`, \n",
-        "(SELECT\n",
-        "      *\n",
-        "    FROM\n",
-        "      `{BQ_TABLE}` LIMIT 10))\"\"\"\n",
-        "\n",
-        "job = bqclient.query(sql_ml_predict)\n",
-        "prediction_result = job.result().to_arrow().to_pandas()"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "TVpsLI5nrVii"
-      },
-      "outputs": [],
-      "source": [
-        "display(prediction_result.head())"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "TpV-iwP9qw9c"
-      },
-      "source": [
-        "## Cleaning up\n",
-        "\n",
-        "To clean up all Google Cloud resources used in this project, you can <a href=\"https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects\" target=\"_blank\">delete the Google Cloud\n",
-        "project</a> you used for the tutorial.\n",
-        "\n",
-        "Learn more about <a href=\"https://cloud.google.com/bigquery-ml/docs/managing-models-vertex\" target=\"_blank\">Deleting BigQuery ML models from Vertex AI Model Registry</a>\n",
-        "\n",
-        "Otherwise, you can delete the individual resources you created in this tutorial."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "sx_vKniMq9ZX"
-      },
-      "outputs": [],
-      "source": [
-        "# Delete the endpoint using the Vertex endpoint object\n",
-        "endpoint.undeploy_all()\n",
-        "endpoint.delete()"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "3c5c8dc2f597"
-      },
-      "outputs": [],
-      "source": [
-        "# Delete BigQuery ML model\n",
-        "\n",
-        "delete_query = f\"\"\"DROP MODEL `{PROJECT_ID}.{BQ_DATASET_NAME}.{MODEL_NAME}`\"\"\"\n",
-        "job = bqclient.query(delete_query)\n",
-        "job.result()"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "85dfc88f5472"
-      },
-      "outputs": [],
-      "source": [
-        "# Delete the created BigQuery dataset\n",
-        "! bq rm -r -f $PROJECT_ID:$BQ_DATASET_NAME"
-      ]
-    }
-  ],
-  "metadata": {
-    "colab": {
-      "collapsed_sections": [],
-      "name": "bqml-vertexai-model-registry.ipynb",
-      "toc_visible": true
-    },
-    "kernelspec": {
-      "display_name": "Python 3",
-      "name": "python3"
-    }
-  },
-  "nbformat": 4,
-  "nbformat_minor": 0
-}

notebooks/official/pipelines/automl_tabular_classification_beans.ipynb

@@ -64,11 +64,22 @@
         "\n",
         "This notebook shows how to use the components defined in [`google_cloud_pipeline_components`](https://github.com/kubeflow/pipelines/tree/master/components/google-cloud) to build an AutoML tabular classification workflow on [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines).\n",
         "\n",
-        "You build a pipeline in this notebook that looks like this:\n",
+        "You'll build a pipeline that looks like this:\n",
         "\n",
         "<a href=\"https://storage.googleapis.com/amy-jo/images/mp/beans.png\" target=\"_blank\"><img src=\"https://storage.googleapis.com/amy-jo/images/mp/beans.png\" width=\"95%\"/></a>"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dataset:beans,lcn"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the UCI Machine Learning ['Dry beans dataset'](https://archive.ics.uci.edu/ml/datasets/Dry+Bean+Dataset), from: KOKLU, M. and OZKAN, I.A., (2020), \"Multiclass Classification of Dry Beans Using Computer Vision and Machine Learning Techniques.\"In Computers and Electronics in Agriculture, 174, 105507. [DOI](https://doi.org/10.1016/j.compag.2020.105507)."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -85,8 +96,8 @@
         "- `Vertex AI Pipelines`\n",
         "- `Google Cloud Pipeline Components`\n",
         "- `Vertex AutoML`\n",
-        "- `Vertex AI Model`\n",
-        "- `Vertex AI Endpoint`\n",
+        "- `Vertex AI Model` resource\n",
+        "- `Vertex AI Endpoint` resource\n",
         "\n",
         "The steps performed include:\n",
         "\n",
@@ -101,17 +112,6 @@
         "The components are [documented here](https://google-cloud-pipeline-components.readthedocs.io/en/latest/google_cloud_pipeline_components.aiplatform.html#module-google_cloud_pipeline_components.aiplatform)."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "dataset:beans,lcn"
-      },
-      "source": [
-        "### Dataset\n",
-        "\n",
-        "The dataset used for this tutorial is the UCI Machine Learning ['Dry beans dataset'](https://archive.ics.uci.edu/ml/datasets/Dry+Bean+Dataset), from: KOKLU, M. and OZKAN, I.A., (2020), \"Multiclass Classification of Dry Beans Using Computer Vision and Machine Learning Techniques.\"In Computers and Electronics in Agriculture, 174, 105507. [DOI](https://doi.org/10.1016/j.compag.2020.105507)."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -140,7 +140,7 @@
       "source": [
         "### Set up your local development environment\n",
         "\n",
-        "If you are using Colab or Vertex AI Workbench Notebook, your environment already meets all the requirements to run this notebook.\n",
+        "If you are using Colab or Vertex AI Workbench Notebook, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
         "\n",
         "Otherwise, make sure your environment meets this notebook's requirements. You need the following:\n",
         "\n",
@@ -197,10 +197,9 @@
         "if IS_WORKBENCH_NOTEBOOK:\n",
         "    USER_FLAG = \"--user\"\n",
         "\n",
-        "! pip3 install --upgrade {USER_FLAG} google-cloud-aiplatform \\\n",
-        "                                    google-cloud-storage \\\n",
-        "                                    kfp \\\n",
-        "                                    google-cloud-pipeline-components -q"
+        "! pip3 install --upgrade google-cloud-aiplatform {USER_FLAG} -q\n",
+        "! pip3 install -U google-cloud-storage $USER_FLAG -q\n",
+        "! pip3 install $USER kfp google-cloud-pipeline-components --upgrade -q"
       ]
     },
     {
@@ -238,8 +237,6 @@
         "id": "check_versions"
       },
       "source": [
-        "### Check the package versions\n",
-        "\n",
         "Check the versions of the packages you installed.  The KFP SDK version should be >=1.8."
       ]
     },
@@ -285,17 +282,6 @@
         "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$`."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "af794e75b7e3"
-      },
-      "source": [
-        "#### Set your project ID\n",
-        "\n",
-        "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -304,7 +290,7 @@
       },
       "outputs": [],
       "source": [
-        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
+        "PROJECT_ID = \"python-docs-samples-tests\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -315,7 +301,7 @@
       },
       "outputs": [],
       "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
+        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"python-docs-samples-tests\":\n",
         "    # Get your GCP project id from gcloud\n",
         "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
         "    PROJECT_ID = shell_output[0]\n",
@@ -361,10 +347,7 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
       ]
     },
     {
@@ -373,9 +356,9 @@
         "id": "timestamp"
       },
       "source": [
-        "#### UUID\n",
+        "#### Timestamp\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
       ]
     },
     {
@@ -386,16 +369,9 @@
       },
       "outputs": [],
       "source": [
-        "import random\n",
-        "import string\n",
+        "from datetime import datetime\n",
         "\n",
-        "\n",
-        "# Generate a uuid of a specifed length(default=8)\n",
-        "def generate_uuid(length: int = 8) -> str:\n",
-        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
-        "\n",
-        "\n",
-        "UUID = generate_uuid()"
+        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
     {
@@ -406,7 +382,7 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Vertex AI Workbench Notebook**, your environment is already authenticated.\n",
+        "**If you are using Vertex AI Workbench Notebook**, your environment is already authenticated. Skip this step.\n",
         "\n",
         "**If you are using Colab**, run the cell below and follow the instructions\n",
         "when prompted to authenticate your account via oAuth.\n",
@@ -501,8 +477,8 @@
       "outputs": [],
       "source": [
         "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
-        "    BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
+        "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
       ]
     },
     {
@@ -623,6 +599,9 @@
         "id": "setup_vars"
       },
       "source": [
+        "### Set up variables\n",
+        "\n",
+        "Next, set up some variables used throughout the tutorial.\n",
         "### Import libraries and define constants"
       ]
     },
@@ -634,13 +613,7 @@
       },
       "outputs": [],
       "source": [
-        "from typing import NamedTuple\n",
-        "\n",
-        "import kfp\n",
-        "from google.cloud import aiplatform\n",
-        "from kfp.v2 import dsl\n",
-        "from kfp.v2.dsl import (Artifact, ClassificationMetrics, Input, Metrics,\n",
-        "                        Output, component)"
+        "import google.cloud.aiplatform as aip"
       ]
     },
     {
@@ -651,9 +624,9 @@
       "source": [
         "#### Vertex AI constants\n",
         "\n",
-        "Setup up the following constants for Vertex AI Pipeline:\n",
-        "- `PIPELINE_NAME`: Set name for the Pipeline.\n",
-        "- `PIPELINE_ROOT`: Cloud Storage bucket path to store pipeline artifacts."
+        "Setup up the following constants for Vertex AI:\n",
+        "\n",
+        "- `API_ENDPOINT`: The Vertex AI API service endpoint for `Dataset`, `Model`, `Job`, `Pipeline` and `Endpoint` services."
       ]
     },
     {
@@ -664,18 +637,65 @@
       },
       "outputs": [],
       "source": [
-        "# set path for storing the pipeline artifacts\n",
-        "PIPELINE_NAME = \"automl-tabular-beans-training\"\n",
+        "# API service endpoint\n",
+        "API_ENDPOINT = \"{}-aiplatform.googleapis.com\".format(REGION)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "pipeline_constants"
+      },
+      "source": [
+        "#### Vertex AI Pipelines constants\n",
+        "\n",
+        "Setup up the following constants for Vertex AI Pipelines:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "pipeline_constants"
+      },
+      "outputs": [],
+      "source": [
         "PIPELINE_ROOT = \"{}/pipeline_root/beans\".format(BUCKET_URI)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "additional_imports"
+      },
+      "source": [
+        "Additional imports."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "import_pipelines"
+      },
+      "outputs": [],
+      "source": [
+        "from typing import NamedTuple\n",
+        "\n",
+        "import kfp\n",
+        "from google_cloud_pipeline_components import aiplatform as gcc_aip\n",
+        "from kfp.v2 import dsl\n",
+        "from kfp.v2.dsl import (Artifact, ClassificationMetrics, Input, Metrics,\n",
+        "                        Output, component)"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "init_aip:mbsdk"
       },
       "source": [
-        "### Initialize Vertex AI SDK for Python\n",
+        "## Initialize Vertex AI SDK for Python\n",
         "\n",
         "Initialize the Vertex AI SDK for Python for your project and corresponding bucket."
       ]
@@ -688,7 +708,7 @@
       },
       "outputs": [],
       "source": [
-        "aiplatform.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
+        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
       ]
     },
     {
@@ -731,7 +751,8 @@
         ")\n",
         "def classification_model_eval_metrics(\n",
         "    project: str,\n",
-        "    location: str,\n",
+        "    location: str,  # \"us-central1\",\n",
+        "    api_endpoint: str,  # \"us-central1-aiplatform.googleapis.com\",\n",
         "    thresholds_dict_str: str,\n",
         "    model: Input[Artifact],\n",
         "    metrics: Output[Metrics],\n",
@@ -741,21 +762,20 @@
         "    import json\n",
         "    import logging\n",
         "\n",
-        "    from google.cloud import aiplatform\n",
-        "\n",
-        "    aiplatform.init(project=project)\n",
+        "    from google.cloud import aiplatform as aip\n",
         "\n",
         "    # Fetch model eval info\n",
-        "    def get_eval_info(model):\n",
-        "        response = model.list_model_evaluations()\n",
+        "    def get_eval_info(client, model_name):\n",
+        "        from google.protobuf.json_format import MessageToDict\n",
+        "\n",
+        "        response = client.list_model_evaluations(parent=model_name)\n",
         "        metrics_list = []\n",
         "        metrics_string_list = []\n",
         "        for evaluation in response:\n",
-        "            evaluation = evaluation.to_dict()\n",
         "            print(\"model_evaluation\")\n",
-        "            print(\" name:\", evaluation[\"name\"])\n",
-        "            print(\" metrics_schema_uri:\", evaluation[\"metricsSchemaUri\"])\n",
-        "            metrics = evaluation[\"metrics\"]\n",
+        "            print(\" name:\", evaluation.name)\n",
+        "            print(\" metrics_schema_uri:\", evaluation.metrics_schema_uri)\n",
+        "            metrics = MessageToDict(evaluation._pb.metrics)\n",
         "            for metric in metrics.keys():\n",
         "                logging.info(\"metric: %s, value: %s\", metric, metrics[metric])\n",
         "            metrics_str = json.dumps(metrics)\n",
@@ -763,7 +783,7 @@
         "            metrics_string_list.append(metrics_str)\n",
         "\n",
         "        return (\n",
-        "            evaluation[\"name\"],\n",
+        "            evaluation.name,\n",
         "            metrics_list,\n",
         "            metrics_string_list,\n",
         "        )\n",
@@ -812,18 +832,20 @@
         "            if metric != \"confidenceMetrics\":\n",
         "                val_string = json.dumps(metrics_list[0][metric])\n",
         "                metrics.log_metric(metric, val_string)\n",
+        "        # metrics.metadata[\"model_type\"] = \"AutoML Tabular classification\"\n",
         "\n",
         "    logging.getLogger().setLevel(logging.INFO)\n",
-        "\n",
+        "    aip.init(project=project)\n",
         "    # extract the model resource name from the input Model Artifact\n",
         "    model_resource_path = model.metadata[\"resourceName\"]\n",
         "    logging.info(\"model path: %s\", model_resource_path)\n",
         "\n",
-        "    # Get the trained model resource\n",
-        "    model = aiplatform.Model(model_resource_path)\n",
-        "\n",
-        "    # Get model evaluation metrics from the the trained model\n",
-        "    eval_name, metrics_list, metrics_str_list = get_eval_info(model)\n",
+        "    client_options = {\"api_endpoint\": api_endpoint}\n",
+        "    # Initialize client that will be used to create and send requests.\n",
+        "    client = aip.gapic.ModelServiceClient(client_options=client_options)\n",
+        "    eval_name, metrics_list, metrics_str_list = get_eval_info(\n",
+        "        client, model_resource_path\n",
+        "    )\n",
         "    logging.info(\"got evaluation name: %s\", eval_name)\n",
         "    logging.info(\"got metrics list: %s\", metrics_list)\n",
         "    log_metrics(metrics_list, metricsc)\n",
@@ -845,9 +867,7 @@
         "id": "define_pipeline:gcpc,beans,lcn"
       },
       "source": [
-        "## Define pipeline \n",
-        "\n",
-        "Define the pipeline for AutoML tabular classification using the components from `google_cloud_pipeline_components`."
+        "## Define an AutoML tabular classification pipeline that uses components from `google_cloud_pipeline_components`"
       ]
     },
     {
@@ -858,34 +878,30 @@
       },
       "outputs": [],
       "source": [
+        "DISPLAY_NAME = \"automl-beans{}\".format(TIMESTAMP)\n",
+        "PIPELINE_NAME = \"automl-tabular-beans-training-v2\"\n",
+        "MACHINE_TYPE = \"n1-standard-4\"\n",
+        "\n",
+        "\n",
         "@kfp.dsl.pipeline(name=PIPELINE_NAME, pipeline_root=PIPELINE_ROOT)\n",
         "def pipeline(\n",
-        "    bq_source: str,\n",
-        "    DATASET_DISPLAY_NAME: str,\n",
-        "    TRAINING_DISPLAY_NAME: str,\n",
-        "    MODEL_DISPLAY_NAME: str,\n",
-        "    ENDPOINT_DISPLAY_NAME: str,\n",
-        "    MACHINE_TYPE: str,\n",
-        "    project: str,\n",
-        "    gcp_region: str,\n",
-        "    thresholds_dict_str: str,\n",
+        "    bq_source: str = \"bq://aju-dev-demos.beans.beans1\",\n",
+        "    display_name: str = DISPLAY_NAME,\n",
+        "    project: str = PROJECT_ID,\n",
+        "    gcp_region: str = REGION,\n",
+        "    api_endpoint: str = API_ENDPOINT,\n",
+        "    thresholds_dict_str: str = '{\"auRoc\": 0.95}',\n",
         "):\n",
-        "\n",
-        "    from google_cloud_pipeline_components.aiplatform import (\n",
-        "        AutoMLTabularTrainingJobRunOp, EndpointCreateOp, ModelDeployOp,\n",
-        "        TabularDatasetCreateOp)\n",
-        "\n",
-        "    dataset_create_op = TabularDatasetCreateOp(\n",
-        "        project=project, display_name=DATASET_DISPLAY_NAME, bq_source=bq_source\n",
+        "    dataset_create_op = gcc_aip.TabularDatasetCreateOp(\n",
+        "        project=project, display_name=display_name, bq_source=bq_source\n",
         "    )\n",
         "\n",
-        "    training_op = AutoMLTabularTrainingJobRunOp(\n",
+        "    training_op = gcc_aip.AutoMLTabularTrainingJobRunOp(\n",
         "        project=project,\n",
-        "        display_name=TRAINING_DISPLAY_NAME,\n",
+        "        display_name=display_name,\n",
         "        optimization_prediction_type=\"classification\",\n",
         "        optimization_objective=\"minimize-log-loss\",\n",
         "        budget_milli_node_hours=1000,\n",
-        "        model_display_name=MODEL_DISPLAY_NAME,\n",
         "        column_specs={\n",
         "            \"Area\": \"numeric\",\n",
         "            \"Perimeter\": \"numeric\",\n",
@@ -908,10 +924,10 @@
         "        dataset=dataset_create_op.outputs[\"dataset\"],\n",
         "        target_column=\"Class\",\n",
         "    )\n",
-        "\n",
         "    model_eval_task = classification_model_eval_metrics(\n",
         "        project,\n",
         "        gcp_region,\n",
+        "        api_endpoint,\n",
         "        thresholds_dict_str,\n",
         "        training_op.outputs[\"model\"],\n",
         "    )\n",
@@ -921,13 +937,13 @@
         "        name=\"deploy_decision\",\n",
         "    ):\n",
         "\n",
-        "        endpoint_op = EndpointCreateOp(\n",
+        "        endpoint_op = gcc_aip.EndpointCreateOp(\n",
         "            project=project,\n",
         "            location=gcp_region,\n",
-        "            display_name=ENDPOINT_DISPLAY_NAME,\n",
+        "            display_name=\"train-automl-beans\",\n",
         "        )\n",
         "\n",
-        "        ModelDeployOp(\n",
+        "        gcc_aip.ModelDeployOp(\n",
         "            model=training_op.outputs[\"model\"],\n",
         "            endpoint=endpoint_op.outputs[\"endpoint\"],\n",
         "            dedicated_resources_min_replica_count=1,\n",
@@ -944,7 +960,7 @@
       "source": [
         "## Compile the pipeline\n",
         "\n",
-        "Next, compile the pipeline to the specified json file."
+        "Next, compile the pipeline."
       ]
     },
     {
@@ -955,11 +971,11 @@
       },
       "outputs": [],
       "source": [
-        "from kfp.v2 import compiler\n",
+        "from kfp.v2 import compiler  # noqa: F811\n",
         "\n",
         "compiler.Compiler().compile(\n",
         "    pipeline_func=pipeline,\n",
-        "    package_path=\"tabular_classification_pipeline.json\",\n",
+        "    package_path=\"tabular classification_pipeline.json\".replace(\" \", \"_\"),\n",
         ")"
       ]
     },
@@ -971,53 +987,7 @@
       "source": [
         "## Run the pipeline\n",
         "\n",
-        "Next, pass the input parameters required for the pipeline and run it. The defined pipeline takes the following parameters:\n",
-        "\n",
-        "- `bq_source`: BigQuery source for the tabular dataset.\n",
-        "- `DATASET_DISPLAY_NAME`: Display name for the Vertex AI managed dataset.\n",
-        "- `TRAINIG_DISPLAY_NAME`: Display name for the AutoML Training job.\n",
-        "- `MODEL_DISPLAY_NAME`: Display name for the Vertex AI Model generated as a result of the training job.\n",
-        "- `ENDPOINT_DISPLAY_NAME`: Display name for the Vertex AI Endpoint where the model is deployed.\n",
-        "- `MACHINE_TYPE`: Machine type for the serving container.\n",
-        "- `project`: Project-id where the pipeline is run.\n",
-        "- `gcp_region`: Region for setting the pipeline location.\n",
-        "- `thresholds_dict_str`: dictionary of thresholds based on which the model deployment is conditioned.\n",
-        "- `pipeline_root`:  To override the pipeline root path specified in the pipeline job's definition, specify a path that your pipeline job can access, such as a Cloud Storage bucket URI."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "1adf9b056954"
-      },
-      "outputs": [],
-      "source": [
-        "# Set the display-names for Vertex AI resources\n",
-        "PIPELINE_DISPLAY_NAME = \"[your-pipeline-display-name]\"  # @param {type:\"string\"}\n",
-        "DATASET_DISPLAY_NAME = \"[your-dataset-display-name]\"  # @param {type:\"string\"}\n",
-        "MODEL_DISPLAY_NAME = \"[your-model-display-name]\"  # @param {type:\"string\"}\n",
-        "TRAINING_DISPLAY_NAME = \"[your-training-job-display-name]\"  # @param {type:\"string\"}\n",
-        "ENDPOINT_DISPLAY_NAME = \"[your-endpoint-display-name]\"  # @param {type:\"string\"}\n",
-        "\n",
-        "# Otherwise, use the default display-names\n",
-        "if PIPELINE_DISPLAY_NAME == \"[your-pipeline-display-name]\":\n",
-        "    PIPELINE_DISPLAY_NAME = f\"pipeline_beans_{UUID}\"\n",
-        "\n",
-        "if DATASET_DISPLAY_NAME == \"[your-dataset-display-name]\":\n",
-        "    DATASET_DISPLAY_NAME = f\"dataset_beans_{UUID}\"\n",
-        "\n",
-        "if MODEL_DISPLAY_NAME == \"[your-model-display-name]\":\n",
-        "    MODEL_DISPLAY_NAME = f\"model_beans_{UUID}\"\n",
-        "\n",
-        "if TRAINING_DISPLAY_NAME == \"[your-training-job-display-name]\":\n",
-        "    TRAINING_DISPLAY_NAME = f\"automl_training_beans_{UUID}\"\n",
-        "\n",
-        "if ENDPOINT_DISPLAY_NAME == \"[your-endpoint-display-name]\":\n",
-        "    ENDPOINT_DISPLAY_NAME = f\"endpoint_beans_{UUID}\"\n",
-        "\n",
-        "# Set machine type\n",
-        "MACHINE_TYPE = \"n1-standard-4\""
+        "Next, run the pipeline."
       ]
     },
     {
@@ -1028,24 +998,19 @@
       },
       "outputs": [],
       "source": [
-        "# Configure the pipeline\n",
-        "job = aiplatform.PipelineJob(\n",
-        "    display_name=PIPELINE_DISPLAY_NAME,\n",
-        "    template_path=\"tabular_classification_pipeline.json\",\n",
+        "DISPLAY_NAME = \"beans_\" + TIMESTAMP\n",
+        "\n",
+        "job = aip.PipelineJob(\n",
+        "    display_name=DISPLAY_NAME,\n",
+        "    template_path=\"tabular classification_pipeline.json\".replace(\" \", \"_\"),\n",
         "    pipeline_root=PIPELINE_ROOT,\n",
-        "    parameter_values={\n",
-        "        \"project\": PROJECT_ID,\n",
-        "        \"gcp_region\": REGION,\n",
-        "        \"bq_source\": \"bq://aju-dev-demos.beans.beans1\",\n",
-        "        \"thresholds_dict_str\": '{\"auRoc\": 0.95}',\n",
-        "        \"DATASET_DISPLAY_NAME\": DATASET_DISPLAY_NAME,\n",
-        "        \"TRAINING_DISPLAY_NAME\": TRAINING_DISPLAY_NAME,\n",
-        "        \"MODEL_DISPLAY_NAME\": MODEL_DISPLAY_NAME,\n",
-        "        \"ENDPOINT_DISPLAY_NAME\": ENDPOINT_DISPLAY_NAME,\n",
-        "        \"MACHINE_TYPE\": MACHINE_TYPE,\n",
-        "    },\n",
+        "    parameter_values={\"project\": PROJECT_ID, \"display_name\": DISPLAY_NAME},\n",
         "    enable_caching=False,\n",
-        ")"
+        ")\n",
+        "\n",
+        "job.submit()\n",
+        "\n",
+        "! rm tabular_classification_pipeline.json"
       ]
     },
     {
@@ -1054,19 +1019,11 @@
         "id": "view_pipeline_run:model"
       },
       "source": [
-        "Run the pipeline job. Click on the generated link to see your run in the Cloud Console."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "114ab8ff24ac"
-      },
-      "outputs": [],
-      "source": [
-        "# Run the job\n",
-        "job.run()"
+        "Click on the generated link to see your run in the Cloud Console.\n",
+        "\n",
+        "<!-- It should look something like this as it is running:\n",
+        "\n",
+        "<a href=\"https://storage.googleapis.com/amy-jo/images/mp/automl_tabular_classif.png\" target=\"_blank\"><img src=\"https://storage.googleapis.com/amy-jo/images/mp/automl_tabular_classif.png\" width=\"40%\"/></a> -->"
       ]
     },
     {
@@ -1088,7 +1045,7 @@
       },
       "outputs": [],
       "source": [
-        "pipeline_df = aiplatform.get_pipeline_df(pipeline=PIPELINE_NAME)\n",
+        "pipeline_df = aip.get_pipeline_df(pipeline=PIPELINE_NAME)\n",
         "print(pipeline_df.head(2))"
       ]
     },
@@ -1098,14 +1055,21 @@
         "id": "cleanup:pipelines"
       },
       "source": [
-        "## Cleaning up\n",
+        "# Cleaning up\n",
         "\n",
         "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
         "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
         "\n",
-        "Otherwise, you can delete the individual resources you created in this tutorial:\n",
+        "Otherwise, you can delete the individual resources you created in this tutorial -- *Note:* this is auto-generated and not all resources may be applicable for this tutorial:\n",
         "\n",
-        "(Set `delete_bucket` to **True** to delete the Cloud Storage bucket.)"
+        "- Dataset\n",
+        "- Pipeline\n",
+        "- Model\n",
+        "- Endpoint\n",
+        "- Batch Job\n",
+        "- Custom Job\n",
+        "- Hyperparameter Tuning Job\n",
+        "- Cloud Storage Bucket"
       ]
     },
     {
@@ -1116,37 +1080,94 @@
       },
       "outputs": [],
       "source": [
-        "delete_bucket = False\n",
+        "delete_dataset = True\n",
+        "delete_pipeline = True\n",
+        "delete_model = True\n",
+        "delete_endpoint = True\n",
+        "delete_batchjob = True\n",
+        "delete_customjob = True\n",
+        "delete_hptjob = True\n",
+        "delete_bucket = True\n",
         "\n",
-        "# Delete the Vertex AI Pipeline Job\n",
-        "job.delete()\n",
+        "try:\n",
+        "    if delete_model and \"DISPLAY_NAME\" in globals():\n",
+        "        models = aip.Model.list(\n",
+        "            filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
+        "        )\n",
+        "        model = models[0]\n",
+        "        aip.Model.delete(model)\n",
+        "        print(\"Deleted model:\", model)\n",
+        "except Exception as e:\n",
+        "    print(e)\n",
         "\n",
-        "# Delete the Vertex AI Endpoint\n",
-        "endpoints = aiplatform.Endpoint.list(\n",
-        "    filter=f\"display_name={ENDPOINT_DISPLAY_NAME}\", order_by=\"create_time\"\n",
-        ")\n",
+        "try:\n",
+        "    if delete_endpoint and \"DISPLAY_NAME\" in globals():\n",
+        "        endpoints = aip.Endpoint.list(\n",
+        "            filter=f\"display_name={DISPLAY_NAME}_endpoint\", order_by=\"create_time\"\n",
+        "        )\n",
+        "        endpoint = endpoints[0]\n",
+        "        endpoint.undeploy_all()\n",
+        "        aip.Endpoint.delete(endpoint.resource_name)\n",
+        "        print(\"Deleted endpoint:\", endpoint)\n",
+        "except Exception as e:\n",
+        "    print(e)\n",
         "\n",
-        "if len(endpoints) > 0:\n",
-        "    endpoint = endpoints[0]\n",
-        "    endpoint.delete(force=True)\n",
+        "if delete_dataset and \"DISPLAY_NAME\" in globals():\n",
+        "    if \"tabular\" == \"tabular\":\n",
+        "        try:\n",
+        "            datasets = aip.TabularDataset.list(\n",
+        "                filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
+        "            )\n",
+        "            dataset = datasets[0]\n",
+        "            aip.TabularDataset.delete(dataset.resource_name)\n",
+        "            print(\"Deleted dataset:\", dataset)\n",
+        "        except Exception as e:\n",
+        "            print(e)\n",
         "\n",
-        "# Delete the Vertex AI model\n",
-        "models = aiplatform.Model.list(\n",
-        "    filter=f\"display_name={MODEL_DISPLAY_NAME}\", order_by=\"create_time\"\n",
-        ")\n",
-        "if len(models) > 0:\n",
-        "    model = models[0]\n",
-        "    model.delete()\n",
+        "    if \"tabular\" == \"image\":\n",
+        "        try:\n",
+        "            datasets = aip.ImageDataset.list(\n",
+        "                filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
+        "            )\n",
+        "            dataset = datasets[0]\n",
+        "            aip.ImageDataset.delete(dataset.resource_name)\n",
+        "            print(\"Deleted dataset:\", dataset)\n",
+        "        except Exception as e:\n",
+        "            print(e)\n",
         "\n",
-        "# Delete the Vertex AI Dataset\n",
-        "datasets = aiplatform.TabularDataset.list(\n",
-        "    filter=f\"display_name={DATASET_DISPLAY_NAME}\", order_by=\"create_time\"\n",
-        ")\n",
-        "if len(datasets) > 0:\n",
-        "    dataset = datasets[0]\n",
-        "    dataset.delete()\n",
+        "    if \"tabular\" == \"text\":\n",
+        "        try:\n",
+        "            datasets = aip.TextDataset.list(\n",
+        "                filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
+        "            )\n",
+        "            dataset = datasets[0]\n",
+        "            aip.TextDataset.delete(dataset.resource_name)\n",
+        "            print(\"Deleted dataset:\", dataset)\n",
+        "        except Exception as e:\n",
+        "            print(e)\n",
+        "\n",
+        "    if \"tabular\" == \"video\":\n",
+        "        try:\n",
+        "            datasets = aip.VideoDataset.list(\n",
+        "                filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
+        "            )\n",
+        "            dataset = datasets[0]\n",
+        "            aip.VideoDataset.delete(dataset.resource_name)\n",
+        "            print(\"Deleted dataset:\", dataset)\n",
+        "        except Exception as e:\n",
+        "            print(e)\n",
+        "\n",
+        "try:\n",
+        "    if delete_pipeline and \"DISPLAY_NAME\" in globals():\n",
+        "        pipelines = aip.PipelineJob.list(\n",
+        "            filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
+        "        )\n",
+        "        pipeline = pipelines[0]\n",
+        "        aip.PipelineJob.delete(pipeline.resource_name)\n",
+        "        print(\"Deleted pipeline:\", pipeline)\n",
+        "except Exception as e:\n",
+        "    print(e)\n",
         "\n",
-        "# Delete the Cloud Storage bucket\n",
         "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
         "    ! gsutil rm -r $BUCKET_URI"
       ]

notebooks/official/pipelines/google_cloud_pipeline_components_automl_tabular.ipynb

@@ -64,6 +64,19 @@
         "This notebook shows how to use the components defined in [`google_cloud_pipeline_components`](https://github.com/kubeflow/pipelines/tree/master/components/google-cloud) to build an AutoML tabular regression workflow on [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines)."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dataset:cal_housing,lrg"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the [California Housing dataset from the 1990 Census](https://developers.google.com/machine-learning/crash-course/california-housing-data-description)\n",
+        "\n",
+        "The dataset predicts the median house price."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -96,19 +109,6 @@
         "The components are [documented here](https://google-cloud-pipeline-components.readthedocs.io/en/latest/google_cloud_pipeline_components.aiplatform.html#module-google_cloud_pipeline_components.aiplatform)."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "77de5c53ac91"
-      },
-      "source": [
-        "### Dataset\n",
-        "\n",
-        "The dataset used for this tutorial is the [California Housing dataset from the 1990 Census](https://developers.google.com/machine-learning/crash-course/california-housing-data-description)\n",
-        "\n",
-        "The dataset predicts the median house price."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -234,8 +234,6 @@
         "id": "check_versions"
       },
       "source": [
-        "### Check installed package versions\n",
-        "\n",
         "Check the versions of the packages you installed.  The KFP SDK version should be >=1.6."
       ]
     },
@@ -346,10 +344,7 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
       ]
     },
     {
@@ -358,9 +353,9 @@
         "id": "timestamp"
       },
       "source": [
-        "#### UUID\n",
+        "#### Timestamp\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append the uuid onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
       ]
     },
     {
@@ -371,16 +366,9 @@
       },
       "outputs": [],
       "source": [
-        "import random\n",
-        "import string\n",
+        "from datetime import datetime\n",
         "\n",
-        "\n",
-        "# Generate a uuid of length 8\n",
-        "def generate_uuid():\n",
-        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=8))\n",
-        "\n",
-        "\n",
-        "UUID = generate_uuid()"
+        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
     {
@@ -486,7 +474,7 @@
       "outputs": [],
       "source": [
         "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
         "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
       ]
     },
@@ -730,10 +718,10 @@
         "    endpoint_op = EndpointCreateOp(\n",
         "        project=project,\n",
         "        location=region,\n",
-        "        display_name=\"train-automl-cal_housing_endpoint\",\n",
+        "        display_name=\"train-automl-flowers\",\n",
         "    )\n",
         "\n",
-        "    _ = ModelDeployOp(\n",
+        "    ModelDeployOp(\n",
         "        model=training_op.outputs[\"model\"],\n",
         "        endpoint=endpoint_op.outputs[\"endpoint\"],\n",
         "        dedicated_resources_machine_type=\"n1-standard-4\",\n",
@@ -765,7 +753,7 @@
         "\n",
         "compiler.Compiler().compile(\n",
         "    pipeline_func=pipeline,\n",
-        "    package_path=\"tabular_regression_pipeline.json\",\n",
+        "    package_path=\"tabular regression_pipeline.json\".replace(\" \", \"_\"),\n",
         ")"
       ]
     },
@@ -788,11 +776,11 @@
       },
       "outputs": [],
       "source": [
-        "DISPLAY_NAME = \"cal_housing_\" + UUID\n",
+        "DISPLAY_NAME = \"cal_housing_\" + TIMESTAMP\n",
         "\n",
         "job = aip.PipelineJob(\n",
         "    display_name=DISPLAY_NAME,\n",
-        "    template_path=\"tabular_regression_pipeline.json\",\n",
+        "    template_path=\"tabular regression_pipeline.json\".replace(\" \", \"_\"),\n",
         "    pipeline_root=PIPELINE_ROOT,\n",
         "    enable_caching=False,\n",
         ")\n",
@@ -830,7 +818,16 @@
         "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
         "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
         "\n",
-        "Otherwise, you can delete the individual resources you created in this tutorial "
+        "Otherwise, you can delete the individual resources you created in this tutorial -- *Note:* this is auto-generated and not all resources may be applicable for this tutorial:\n",
+        "\n",
+        "- Dataset\n",
+        "- Pipeline\n",
+        "- Model\n",
+        "- Endpoint\n",
+        "- Batch Job\n",
+        "- Custom Job\n",
+        "- Hyperparameter Tuning Job\n",
+        "- Cloud Storage Bucket"
       ]
     },
     {
@@ -845,73 +842,89 @@
         "delete_pipeline = True\n",
         "delete_model = True\n",
         "delete_endpoint = True\n",
+        "delete_batchjob = True\n",
+        "delete_customjob = True\n",
+        "delete_hptjob = True\n",
         "delete_bucket = True\n",
         "\n",
-        "dataset_display_name = \"housing\"\n",
-        "pipeline_display_name = \"automl-tab-training-v2\"\n",
-        "model_display_name = \"train-automl-cal_housing\"\n",
-        "endpoint_display_name = \"train-automl-cal_housing_endpoint\"\n",
+        "try:\n",
+        "    if delete_model and \"DISPLAY_NAME\" in globals():\n",
+        "        models = aip.Model.list(\n",
+        "            filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
+        "        )\n",
+        "        model = models[0]\n",
+        "        aip.Model.delete(model)\n",
+        "        print(\"Deleted model:\", model)\n",
+        "except Exception as e:\n",
+        "    print(e)\n",
         "\n",
-        "\n",
-        "if delete_endpoint:\n",
-        "    endpoints = aip.Endpoint.list(\n",
-        "        filter=f\"display_name={endpoint_display_name}\", order_by=\"create_time\"\n",
-        "    )\n",
-        "    if endpoints:\n",
+        "try:\n",
+        "    if delete_endpoint and \"DISPLAY_NAME\" in globals():\n",
+        "        endpoints = aip.Endpoint.list(\n",
+        "            filter=f\"display_name={DISPLAY_NAME}_endpoint\", order_by=\"create_time\"\n",
+        "        )\n",
         "        endpoint = endpoints[0]\n",
         "        endpoint.undeploy_all()\n",
-        "        endpoint.delete()\n",
+        "        aip.Endpoint.delete(endpoint.resource_name)\n",
         "        print(\"Deleted endpoint:\", endpoint)\n",
+        "except Exception as e:\n",
+        "    print(e)\n",
         "\n",
-        "if delete_model:\n",
-        "    models = aip.Model.list(\n",
-        "        filter=f\"display_name={model_display_name}\", order_by=\"create_time\"\n",
-        "    )\n",
-        "    if models:\n",
-        "        model = models[0]\n",
-        "        model.delete()\n",
-        "        print(\"Deleted model:\", model)\n",
-        "\n",
-        "if delete_dataset:\n",
+        "if delete_dataset and \"DISPLAY_NAME\" in globals():\n",
         "    if \"tabular\" == \"tabular\":\n",
-        "        datasets = aip.TabularDataset.list(\n",
-        "            filter=f\"display_name={dataset_display_name}\", order_by=\"create_time\"\n",
-        "        )\n",
-        "        if datasets:\n",
+        "        try:\n",
+        "            datasets = aip.TabularDataset.list(\n",
+        "                filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
+        "            )\n",
         "            dataset = datasets[0]\n",
-        "            dataset.delete()\n",
+        "            aip.TabularDataset.delete(dataset.resource_name)\n",
         "            print(\"Deleted dataset:\", dataset)\n",
+        "        except Exception as e:\n",
+        "            print(e)\n",
         "\n",
         "    if \"tabular\" == \"image\":\n",
-        "        datasets = aip.ImageDataset.list(\n",
-        "            filter=f\"display_name={dataset_display_name}\", order_by=\"create_time\"\n",
-        "        )\n",
-        "        if datasets:\n",
+        "        try:\n",
+        "            datasets = aip.ImageDataset.list(\n",
+        "                filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
+        "            )\n",
         "            dataset = datasets[0]\n",
-        "            dataset.delete()\n",
+        "            aip.ImageDataset.delete(dataset.resource_name)\n",
         "            print(\"Deleted dataset:\", dataset)\n",
+        "        except Exception as e:\n",
+        "            print(e)\n",
         "\n",
         "    if \"tabular\" == \"text\":\n",
-        "        datasets = aip.TextDataset.list(\n",
-        "            filter=f\"display_name={dataset_display_name}\", order_by=\"create_time\"\n",
-        "        )\n",
-        "        if datasets:\n",
+        "        try:\n",
+        "            datasets = aip.TextDataset.list(\n",
+        "                filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
+        "            )\n",
         "            dataset = datasets[0]\n",
-        "            dataset.delete()\n",
+        "            aip.TextDataset.delete(dataset.resource_name)\n",
         "            print(\"Deleted dataset:\", dataset)\n",
+        "        except Exception as e:\n",
+        "            print(e)\n",
         "\n",
         "    if \"tabular\" == \"video\":\n",
-        "        datasets = aip.VideoDataset.list(\n",
-        "            filter=f\"display_name={dataset_display_name}\", order_by=\"create_time\"\n",
-        "        )\n",
-        "        if datasets:\n",
+        "        try:\n",
+        "            datasets = aip.VideoDataset.list(\n",
+        "                filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
+        "            )\n",
         "            dataset = datasets[0]\n",
-        "            dataset.delete()\n",
+        "            aip.VideoDataset.delete(dataset.resource_name)\n",
         "            print(\"Deleted dataset:\", dataset)\n",
+        "        except Exception as e:\n",
+        "            print(e)\n",
         "\n",
-        "if delete_pipeline:\n",
-        "    job.delete()\n",
-        "\n",
+        "try:\n",
+        "    if delete_pipeline and \"DISPLAY_NAME\" in globals():\n",
+        "        pipelines = aip.PipelineJob.list(\n",
+        "            filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
+        "        )\n",
+        "        pipeline = pipelines[0]\n",
+        "        aip.PipelineJob.delete(pipeline.resource_name)\n",
+        "        print(\"Deleted pipeline:\", pipeline)\n",
+        "except Exception as e:\n",
+        "    print(e)\n",
         "\n",
         "if delete_bucket and os.getenv(\"IS_TESTING\"):\n",
         "    ! gsutil rm -r $BUCKET_URI"

notebooks/official/pipelines/google_cloud_pipeline_components_dataproc_tabular.ipynb

@@ -29,7 +29,7 @@
         "id": "JAPoU8Sm5E6e"
       },
       "source": [
-        "# Vertex AI Pipelines: Loan eligibility prediction using `google-cloud-pipeline-components` and Spark ML\n",
+        "# Vertex AI Pipelines: Loan eligibility prediction using google-cloud-pipeline-components and Spark ML\n",
         "\n",
         "<table align=\"left\">\n",
         "\n",
@@ -206,7 +206,7 @@
         "    \n",
         "!pip3 install {USER_FLAG} --upgrade google-cloud-aiplatform==1.11.0 \\\n",
         "                                    kfp==1.8.11 \\\n",
-        "                                    google-cloud-pipeline-components==1.0.18 --quiet --no-warn-conflicts"
+        "                                    google-cloud-pipeline-components==1.0.1 --quiet --no-warn-conflicts"
       ]
     },
     {
@@ -425,19 +425,9 @@
         "3. In the **Service account name** field, enter a name, and\n",
         "   click **Create**.\n",
         "\n",
-        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type the following role and select them\n",
-        "\n",
-        "      - Artifact Registry Administrator\n",
-        "      - Artifact Registry Repository Administrator\n",
-        "      - Cloud Build Editor\n",
-        "      - Compute Network Admin\n",
-        "      - Dataproc Administrator\n",
-        "      - Dataproc Worker\n",
-        "      - Service Account User\n",
-        "      - Storage Admin\n",
-        "      - Storage Object Admin\n",
-        "      - Vertex AI Administrator\n",
-        "\n",
+        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
+        "into the filter box, and select\n",
+        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
         "\n",
         "5. Click *Create*. A JSON file that contains your key downloads to your\n",
         "local environment.\n",
@@ -649,10 +639,15 @@
       },
       "outputs": [],
       "source": [
+        "from pathlib import Path as path\n",
+        "\n",
+        "DATA_PATH = path(\"content/path/\")\n",
         "PUBLIC_DATA_URI = \"gs://cloud-samples-data/vertex-ai/dataset-management/datasets/loan_eligibilty/data.csv\"\n",
         "FEATURES_TRAIN_URI = f\"{BUCKET_URI}/data/features/snapshots/{UUID}\"\n",
         "\n",
-        "!gsutil cp -r $PUBLIC_DATA_URI $FEATURES_TRAIN_URI"
+        "!mkdir -m 777 -p $DATA_PATH\n",
+        "!gsutil cp -r $PUBLIC_DATA_URI $DATA_PATH\n",
+        "!gsutil cp -r $DATA_PATH $FEATURES_TRAIN_URI"
       ]
     },
     {
@@ -733,10 +728,14 @@
         "from pathlib import Path as path\n",
         "from typing import NamedTuple\n",
         "\n",
+        "# Part 1 - ML Training\n",
         "from google.cloud import aiplatform as vertex_ai\n",
+        "from google_cloud_pipeline_components import aiplatform as vertex_ai_components\n",
         "from kfp.v2 import compiler, dsl\n",
         "from kfp.v2.dsl import (Artifact, ClassificationMetrics, Condition, Input,\n",
-        "                        Metrics, Output, component)"
+        "                        Metrics, Output, component)\n",
+        "\n",
+        "ID = random.randint(1, 10000)"
       ]
     },
     {
@@ -761,14 +760,14 @@
         "PIPELINE_ROOT = f\"{BUCKET_URI}/pipelines\"\n",
         "PIPELINE_PACKAGE_PATH = str(BUILD_PATH / f\"pipeline_{UUID}.json\")\n",
         "RUNTIME_CONTAINER_IMAGE = f\"gcr.io/{PROJECT_ID}/{RUNTIME_IMAGE}:{IMAGE_TAG}\"\n",
-        "ML_APPLICATION = \"loan-eligibility\"\n",
-        "TASK = \"sparkml\"\n",
+        "ML_APPLICATION = \"spark\"\n",
+        "TASK = \"classifier\"\n",
         "MODEL_TYPE = \"rfor\"\n",
         "VERSION = \"1.0.0\"\n",
         "MODEL_NAME = f\"{ML_APPLICATION}-{TASK}-{MODEL_TYPE}-{VERSION}\"\n",
-        "ARTIFACT_URI = f\"{BUCKET_URI}/deliverables/bundle/{UUID}\"\n",
         "\n",
         "# Preprocessing\n",
+        "PREPROCESSING_BATCH_ID = f\"data-preprocessing-{ID}\"\n",
         "PREPROCESSING_PYTHON_FILE_URI = f\"{BUCKET_URI}/src/data_preprocessing.py\"\n",
         "PROCESSED_DATA_URI = f\"{BUCKET_URI}/data/processed\"\n",
         "PREPROCESSING_ARGS = [\n",
@@ -779,10 +778,11 @@
         "]\n",
         "\n",
         "# Dataset\n",
-        "DATASET_NAME = f\"preprocessed-dataset-{UUID}\"\n",
+        "DATASET_NAME = f\"preprocessed-dataset-{ID}\"\n",
         "GCS_PREPROCESSED_URI = f\"{PROCESSED_DATA_URI}/*/?.csv\"\n",
         "\n",
         "# Training\n",
+        "TRAINING_BATCH_ID = f\"model-training-{ID}\"\n",
         "TRAINING_PYTHON_FILE_URI = f\"{BUCKET_URI}/src/model_training.py\"\n",
         "MODEL_URI = f\"{BUCKET_URI}/deliverables/model/rfor/{UUID}/train_model\"\n",
         "METRICS_URI = f\"{BUCKET_URI}/deliverables/metrics/rfor/{UUID}/train_metrics.json\"\n",
@@ -797,9 +797,10 @@
         "\n",
         "# Condition\n",
         "AUPR_THRESHOLD = 0.5\n",
-        "AUPR_HYPERTUNE_CONDITION = \"hypertune\"\n",
+        "AUPR_HYPERTUNE_CONDITION = \"[AUPR_HYPERTUNE]\"\n",
         "\n",
         "# Hypertuning\n",
+        "HPT_TRAINING_BATCH_ID = f\"hyper-tuning-{ID}\"\n",
         "HPT_PYTHON_FILE_URI = f\"{BUCKET_URI}/src/hp_tuning.py\"\n",
         "HPT_MODEL_URI = f\"{BUCKET_URI}/deliverables/model/rfor/{UUID}/model\"\n",
         "HPT_METRICS_URI = f\"{BUCKET_URI}/deliverables/metrics/rfor/{UUID}/metrics.json\"\n",
@@ -810,24 +811,7 @@
         "    HPT_MODEL_URI,\n",
         "    \"--metrics-path\",\n",
         "    HPT_METRICS_URI,\n",
-        "]\n",
-        "HPT_BUNDLE_URI = f\"{ARTIFACT_URI}/model.zip\"\n",
-        "HPT_ARGS = [\n",
-        "    \"--train-path\",\n",
-        "    PROCESSED_DATA_URI,\n",
-        "    \"--model-path\",\n",
-        "    HPT_MODEL_URI,\n",
-        "    \"--metrics-path\",\n",
-        "    HPT_METRICS_URI,\n",
-        "    \"--bundle-path\",\n",
-        "    HPT_BUNDLE_URI,\n",
-        "]\n",
-        "HPT_RUNTIME_PROPERTIES = {\n",
-        "    \"spark.jars.packages\": \"ml.combust.mleap:mleap-spark-base_2.12:0.20.0,ml.combust.mleap:mleap-spark_2.12:0.20.0\"\n",
-        "}\n",
-        "\n",
-        "# Deploy\n",
-        "SERVING_IMAGE_URI = f\"{REGION}-docker.pkg.dev/{PROJECT_ID}/{REPO_NAME}/spark-ml-serving\""
+        "]"
       ]
     },
     {
@@ -856,7 +840,7 @@
         "id": "LB2aM7VyRyZG"
       },
       "source": [
-        "## Build the Vertex Pipeline to train and deploy a Spark model\n",
+        "## PART I - Build the Vertex Pipeline to train and deploy a Spark model\n",
         "\n",
         "In this case, the ML pipeline includes the following steps:\n",
         "\n",
@@ -864,16 +848,10 @@
         "2.   Train an `RandomForestClassifier` with `DataprocPySparkBatchOp`\n",
         "3.   Run a custom component in order to evaluate the model\n",
         "\n",
-        "If the model respects the performance condition, then:\n",
+        "If the model respects the performance condition, then\n",
         "\n",
         "4.   Hypertune the `RandomForestClassifier` with `DataprocPySparkBatchOp`\n",
-        "5.   Serializes the model to MLeap format to use the model outside of Spark.\n",
-        "\n",
-        "If the `deploy_model` pipeline parameter is set to `True`:\n",
-        "\n",
-        "6.   Upload the model to Vertex AI Model Registry.\n",
-        "7.   Creates a Vertex AI endpoint.\n",
-        "8.   Deploys the model to the Vertex AI endpoint for serving online prediction requests.\n"
+        "5.   Register the model in the Vertex AI Model Registry\n"
       ]
     },
     {
@@ -931,7 +909,6 @@
         "\n",
         "\"\"\"\n",
         "data_preprocessing.py is the module for\n",
-        "\n",
         "  - ingest data\n",
         "  - do simple preprocessing tasks\n",
         "  - upload processed data to gcs\n",
@@ -1044,7 +1021,8 @@
         "\n",
         "    spark = (SparkSession.builder\n",
         "             .master(\"local[*]\")\n",
-        "             .appName(\"loan eligibility\")\n",
+        "             .appName(\"spark go live\")\n",
+        "             .config('spark.ui.port', '4050')\n",
         "             .getOrCreate())\n",
         "    try:\n",
         "        logger.info(f'spark version: {spark.sparkContext.version}')\n",
@@ -1063,7 +1041,12 @@
         "        training_data_raw_df.show(truncate=False)\n",
         "\n",
         "        logger.info(f'load prepared data to {output_data_path}.')\n",
-        "        training_data_raw_df.write.mode('overwrite').csv(str(output_data_path), header=True)\n",
+        "        if output_data_path.startswith('gs://'):\n",
+        "            training_data_raw_df.write.mode('overwrite').csv(str(output_data_path), header=True)\n",
+        "        else:\n",
+        "            output_file_path = Path(output_data_path)\n",
+        "            output_file_path.mkdir(parents=True, exist_ok=True)\n",
+        "            training_data_raw_df.write.mode('overwrite').csv(str(output_file_path), header=True)\n",
         "    except RuntimeError as main_error:\n",
         "        logger.error(main_error)\n",
         "    else:\n",
@@ -1393,7 +1376,8 @@
         "        logger.info('start spark session.')\n",
         "        spark = (SparkSession.builder\n",
         "                 .master(\"local[*]\")\n",
-        "                 .appName(\"loan eligibility\")\n",
+        "                 .appName(\"spark go live\")\n",
+        "                 .config('spark.ui.port', '4050')\n",
         "                 .getOrCreate())\n",
         "        logger.info(f'spark version: {spark.sparkContext.version}')\n",
         "        logger.info('start bulding pipeline.')\n",
@@ -1418,12 +1402,23 @@
         "\n",
         "        logger.info(f'load model pipeline in {model_path}.')\n",
         "        pipeline.write().overwrite().save(model_path)\n",
+        "        if model_path.startswith('gs://'):\n",
+        "            pipeline.write().overwrite().save(model_path)\n",
+        "        else:\n",
+        "            path(model_path).mkdir(parents=True, exist_ok=True)\n",
+        "            pipeline.write().overwrite().save(model_path)\n",
         "\n",
-        "        logger.info(f'Upload metrics under {metrics_path}.')     \n",
-        "        bucket = urlparse(model_path).netloc\n",
-        "        metrics_file_path = urlparse(metrics_path).path.strip('/')\n",
-        "        write_metrics(bucket, metrics, metrics_file_path)\n",
-        "        \n",
+        "        logger.info(f'Upload metrics under {metrics_path}.')\n",
+        "        if metrics_path.startswith('gs://'):\n",
+        "            bucket = urlparse(model_path).netloc\n",
+        "            metrics_file_path = urlparse(metrics_path).path.strip('/')\n",
+        "            write_metrics(bucket, metrics, metrics_file_path)\n",
+        "        else:\n",
+        "            metrics_version_path = path(metrics_path).parents[0]\n",
+        "            metrics_version_path.mkdir(parents=True, exist_ok=True)\n",
+        "            with open(metrics_path, 'w') as json_file:\n",
+        "                json.dump(metrics, json_file)\n",
+        "            json_file.close()\n",
         "    except RuntimeError as main_error:\n",
         "        logger.error(main_error)\n",
         "    else:\n",
@@ -1449,9 +1444,7 @@
         "\n",
         "- `--train-path`: The GCS path of the training sample.\n",
         "- `--model-path`: The GCS path to store the trained model.\n",
-        "- `--metrics-path`: The GCS path to store the metrics of model.\n",
-        "\n",
-        "The hyperparameter tuning job will also serialize the best performing model to an MLeap bundle, which can be imported to Vertex AI as a model for serving predictions - see the *Serve your model in Vertex AI* section further below."
+        "- `--metrics-path`: The GCS path to store the metrics of model."
       ]
     },
     {
@@ -1488,9 +1481,6 @@
         "except ImportError as e:\n",
         "    print('WARN: Something wrong with pyspark library. Please check configuration settings!')\n",
         "    print(e)\n",
-        "    \n",
-        "import mleap.pyspark\n",
-        "from mleap.pyspark.spark_support import SimpleSparkSerializer\n",
         "\n",
         "from pyspark.sql.types import StructType, DoubleType, StringType\n",
         "from pyspark.sql.functions import col, udf\n",
@@ -1596,16 +1586,6 @@
         "        ''',\n",
         "        type=str,\n",
         "        required=True)\n",
-        "    args_parser.add_argument(\n",
-        "        '--bundle-path',\n",
-        "        help='''\n",
-        "        The GCS path to store the exported MLeap bundle. \n",
-        "        Format: \n",
-        "        - locally: /path/to/dir\n",
-        "        - cloud: gs://bucket/path\n",
-        "        ''',\n",
-        "        type=str,\n",
-        "        required=True)\n",
         "    return args_parser.parse_args()\n",
         "\n",
         "\n",
@@ -1762,14 +1742,18 @@
         "    train_path = args.train_path\n",
         "    model_path = args.model_path\n",
         "    metrics_path = args.metrics_path\n",
-        "    bundle_path = args.bundle_path\n",
         "\n",
         "    try:\n",
         "        logger.info('initializing pipeline training.')\n",
         "        logger.info('start spark session.')\n",
         "        spark = (SparkSession.builder\n",
         "                 .master(\"local[*]\")\n",
-        "                 .appName(\"loan eligibility\")\n",
+        "                 .appName(\"spark go live\")\n",
+        "                 .config('spark.ui.port', '4050')\n",
+        "                 .config('spark.jars.packages', 'ml.combust.mleap:mleap-runtime_2.12:0.19.0')\n",
+        "                 .config('spark.jars.packages', 'ml.combust.mleap:mleap-base_2.12:0.19.0')\n",
+        "                 .config('spark.jars.packages', 'ml.combust.mleap:mleap-spark_2.12:0.19.0')\n",
+        "                 .config('spark.jars.packages', 'ml.combust.mleap:mleap-spark-extension_2.12:0.19.0')\n",
         "                 .getOrCreate())\n",
         "        logger.info(f'spark version: {spark.sparkContext.version}')\n",
         "        logger.info('start building pipeline.')\n",
@@ -1779,7 +1763,12 @@
         "        pipeline_cross_validator = build_hp_pipeline(preprocessing_stages, feature_engineering_stages,\n",
         "                                                     model_training_stage)\n",
         "        logger.info(f'load train data from {train_path}.')\n",
-        "        raw_data = (spark.read.format('csv')\n",
+        "        if train_path.startswith('bq://'):\n",
+        "            raw_data = spark.read.format('bigquery') \\\n",
+        "                .option('table', train_path.replace('bq://', '')) \\\n",
+        "                .load()\n",
+        "        else:\n",
+        "            raw_data = (spark.read.format('csv')\n",
         "                        .option(\"header\", \"true\")\n",
         "                        .schema(DATA_SCHEMA)\n",
         "                        .load(train_path))\n",
@@ -1792,22 +1781,23 @@
         "            print(f'{m}: {v}')\n",
         "\n",
         "        logger.info(f'load model pipeline in {model_path}.')\n",
-        "        pipeline_model.write().overwrite().save(model_path)\n",
+        "        if model_path.startswith('gs://'):\n",
+        "            pipeline_model.write().overwrite().save(model_path)\n",
+        "        else:\n",
+        "            path(model_path).mkdir(parents=True, exist_ok=True)\n",
+        "            pipeline_model.write().overwrite().save(model_path)\n",
         "\n",
-        "        logger.info(f'upload metrics under {metrics_path}.')\n",
-        "        bucket = urlparse(model_path).netloc\n",
-        "        metrics_file_path = urlparse(metrics_path).path.strip('/')\n",
-        "        write_metrics(bucket, metrics, metrics_file_path)\n",
-        "        \n",
-        "        logger.info('export MLeap bundle to temporary location')\n",
-        "        pipeline_model.bestModel.serializeToBundle(f'jar:file:/tmp/bundle.zip', predictions)\n",
-        "        \n",
-        "        logger.info(f'upload MLeap bundle to {bundle_path}')\n",
-        "        bundle_file_path = urlparse(bundle_path).path.strip('/')\n",
-        "        bucket = urlparse(bundle_path).netloc\n",
-        "        logger.info(f'Copying /tmp/bundle.zip to bucket {bucket} using object name {bundle_file_path} ...')\n",
-        "        upload_file(bucket, '/tmp/bundle.zip', bundle_file_path)\n",
-        "        \n",
+        "        logger.info(f'Upload metrics under {metrics_path}.')\n",
+        "        if metrics_path.startswith('gs://'):\n",
+        "            bucket = urlparse(model_path).netloc\n",
+        "            metrics_file_path = urlparse(metrics_path).path.strip('/')\n",
+        "            write_metrics(bucket, metrics, metrics_file_path)\n",
+        "        else:\n",
+        "            metrics_version_path = path(metrics_path).parents[0]\n",
+        "            metrics_version_path.mkdir(parents=True, exist_ok=True)\n",
+        "            with open(metrics_path, 'w') as json_file:\n",
+        "                json.dump(metrics, json_file)\n",
+        "            json_file.close()\n",
         "    except RuntimeError as main_error:\n",
         "        logger.error(main_error)\n",
         "    else:\n",
@@ -1852,11 +1842,11 @@
         "id": "68nYBB5GS9TB"
       },
       "source": [
-        "### Build a custom Dataproc Serverless container image\n",
+        "### Build a custom dataproc serverless image\n",
         "\n",
-        "Dataproc Serverless provides [default runtime images](https://cloud.google.com/dataproc-serverless/docs/concepts/versions/spark-runtime-versions). You can also use custom container images for your Dataproc Serverless workloads. \n",
+        "The `DataprocPySparkBatchOp` allows you to pass custom image that you use when the [provided Dataproc Serverless runtime versions](https://cloud.google.com/dataproc-serverless/docs/concepts/versions/spark-runtime-versions) does not respect your requirements. \n",
         "\n",
-        "The steps in this section builds a custom container image that includes additional dependencies. The custom container image can be specified when using the `DataprocPySparkBatchOp` component to launch the workload within a pipeline."
+        "**Note:** This step is optional and is included here for general awareness."
       ]
     },
     {
@@ -1865,7 +1855,7 @@
         "id": "GF9_5IGYqLAX"
       },
       "source": [
-        "#### Define the Dataproc Serverless custom runtime image"
+        "#### Define the Dataproc serverless custom runtime image"
       ]
     },
     {
@@ -1937,7 +1927,7 @@
         "      scikit-image \\\n",
         "      scikit-learn \\\n",
         "      scipy \\\n",
-        "      mleap\n",
+        "      mleap \n",
         "\n",
         "# (Required) Create the 'spark' group/user.\n",
         "# The GID and UID must be 1099. Home directory is required.\n",
@@ -1973,9 +1963,7 @@
         "id": "ZXzI2xInqb3V"
       },
       "source": [
-        "#### Build the Dataproc Serverless custom runtime using Google Cloud Build\n",
-        "\n",
-        "**Note:** this step may take approximately 5 to 10 minutes to complete."
+        "#### Build the Dataproc serverless custom runtime using Google Cloud Build"
       ]
     },
     {
@@ -2152,12 +2140,12 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "28f3d22dd97f"
+        "id": "1-Ccx4uLDz4N"
       },
       "source": [
-        "#### Create component for passing args to hyperparameter tuning component\n",
+        "#### Model registration custom component\n",
         "\n",
-        "The following component passes the args `--train-path`, `--model-path` and `--metrics-path`, and `--bundle-path` in the required format for the hyperparamter tuning function defined earlier."
+        "Define a component to create a model resource for the trained model on Vertex AI Model registry."
       ]
     },
     {
@@ -2168,230 +2156,22 @@
       },
       "outputs": [],
       "source": [
+        "# TODO: Build a custom compiler using Spark docker image to compile the Mleap bundle\n",
+        "\n",
+        "\n",
         "@component(base_image=\"python:3.8-slim\")\n",
-        "def build_hpt_args(\n",
-        "    dataset_uri: Input[Artifact],\n",
-        "    train_path: str,\n",
-        "    model_path: str,\n",
-        "    metrics_path: str,\n",
-        "    bundle_path: str,\n",
-        ") -> list:\n",
-        "    return [\n",
-        "        \"--train-path\",\n",
-        "        train_path,\n",
-        "        \"--model-path\",\n",
-        "        model_path,\n",
-        "        \"--metrics-path\",\n",
-        "        metrics_path,\n",
-        "        \"--bundle-path\",\n",
-        "        bundle_path,\n",
-        "    ]"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "0db73fff95b3"
-      },
-      "source": [
-        "### (Optional) Serve your model using Vertex AI\n",
+        "def register_model(\n",
+        "    artifact_uri: str,\n",
+        "    model: Output[Artifact],\n",
+        ") -> NamedTuple(\"Outputs\", [(\"uri\", str)]):\n",
         "\n",
-        "The hyperparameter tuning task exports the best performing model as an MLeap bundle. The MLeap bundle can be imported into the Vertex AI Model Registry and used for prediction serving. See [Serving Spark ML model using Vertex AI](https://cloud.google.com/architecture/spark-ml-model-with-vertexai) for more information.\n",
-        "\n",
-        "Enable import of the MLeap bundle into the Vertex AI Model Registry and online prediction serving."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "2d7e7c8fc21b"
-      },
-      "outputs": [],
-      "source": [
-        "# Set DEPLOY_MODEL to True\n",
-        "DEPLOY_MODEL = False"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "0bb792622d9f"
-      },
-      "source": [
-        "### Build the model serving container image\n",
-        "\n",
-        "A *serving container image* is required to import your model into the Model Registry. The serving container image provides the model serving implementation for the model. The following replicates the instructions from [Serving Spark ML model using Vertex AI](https://cloud.google.com/architecture/spark-ml-model-with-vertexai) to build the serving container image.\n",
-        "\n",
-        "**Note:** this step may take approximately 5 to 10 minutes to complete."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "4703a0f969a3"
-      },
-      "outputs": [],
-      "source": [
-        "DEPLOY_MODEL_CONDITION = 'deploy'\n",
-        "\n",
-        "if DEPLOY_MODEL:\n",
-        "\n",
-        "    import os\n",
-        "    \n",
-        "    CWD = os.getcwd()\n",
-        "\n",
-        "    # Clone and build the scala-sbt cloud builder\n",
-        "    ! git clone https://github.com/GoogleCloudPlatform/cloud-builders-community.git\n",
-        "    ! cd ${CWD}/cloud-builders-community/scala-sbt && \\\n",
-        "        gcloud builds submit .\n",
-        "\n",
-        "    # Clone and build the serving container code\n",
-        "    ! cd {CWD} && git clone https://github.com/GoogleCloudPlatform/vertex-ai-spark-ml-serving.git\n",
-        "    ! cd {CWD}/vertex-ai-spark-ml-serving && \\\n",
-        "        gcloud builds submit --config=cloudbuild.yaml \\\n",
-        "            --substitutions=\"_LOCATION={REGION},_REPOSITORY={REPO_NAME},_IMAGE=spark-ml-serving\" ."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "947b51adc087"
-      },
-      "source": [
-        "### Create component for importing a model artifact into a pipeline\n",
-        "\n",
-        "The pipeline uses the `ModelImportOp` component to import (upload) a model to Vertex AI Model Registry.\n",
-        "\n",
-        "The `import_model_artifact` python component creates a model artifact that can be passed to the `ModelImportOp` component."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "2ed96e7ad046"
-      },
-      "outputs": [],
-      "source": [
-        "@dsl.component(\n",
-        "    base_image=\"python:3.8-slim\",\n",
-        "    packages_to_install=[\"google-cloud-aiplatform\"],\n",
-        ")\n",
-        "def import_model_artifact(\n",
-        "    model: dsl.Output[dsl.Artifact], artifact_uri: str, serving_image_uri: str\n",
-        "):\n",
-        "    model.metadata[\"containerSpec\"] = {\n",
-        "        \"imageUri\": serving_image_uri,\n",
-        "        \"healthRoute\": \"/health\",\n",
-        "        \"predictRoute\": \"/predict\",\n",
-        "    }\n",
-        "    model.uri = artifact_uri"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "0d83d9e80923"
-      },
-      "source": [
-        "The serving container requires the model schema in JSON format, which is read during container startup. See [Provide the model schema](https://cloud.google.com/architecture/spark-ml-model-with-vertexai#provide_the_model_schema) for more information.\n",
-        "\n",
-        "Write the model schema file:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "521d2f4d7992"
-      },
-      "outputs": [],
-      "source": [
-        "%%writefile $SRC/schema.json\n",
-        "{\n",
-        "  \"input\": [\n",
-        "    {\n",
-        "      \"name\": \"loan_amount\",\n",
-        "      \"type\": \"DOUBLE\"\n",
-        "    },\n",
-        "    {\n",
-        "      \"name\": \"loan_term\",\n",
-        "      \"type\": \"STRING\"\n",
-        "    },\n",
-        "    {\n",
-        "      \"name\": \"property_area\",\n",
-        "      \"type\": \"STRING\"\n",
-        "    },\n",
-        "    {\n",
-        "      \"name\": \"feature_7\",\n",
-        "      \"type\": \"DOUBLE\"\n",
-        "    },\n",
-        "    {\n",
-        "      \"name\": \"feature_3\",\n",
-        "      \"type\": \"DOUBLE\"\n",
-        "    },\n",
-        "    {\n",
-        "      \"name\": \"feature_1\",\n",
-        "      \"type\": \"DOUBLE\"\n",
-        "    },\n",
-        "    {\n",
-        "      \"name\": \"feature_9\",\n",
-        "      \"type\": \"DOUBLE\"\n",
-        "    },\n",
-        "    {\n",
-        "      \"name\": \"feature_5\",\n",
-        "      \"type\": \"DOUBLE\"\n",
-        "    },\n",
-        "    {\n",
-        "      \"name\": \"feature_0\",\n",
-        "      \"type\": \"DOUBLE\"\n",
-        "    },\n",
-        "    {\n",
-        "      \"name\": \"feature_8\",\n",
-        "      \"type\": \"DOUBLE\"\n",
-        "    },\n",
-        "    {\n",
-        "      \"name\": \"feature_4\",\n",
-        "      \"type\": \"DOUBLE\"\n",
-        "    },\n",
-        "    {\n",
-        "      \"name\": \"feature_2\",\n",
-        "      \"type\": \"DOUBLE\"\n",
-        "    },\n",
-        "    {\n",
-        "      \"name\": \"feature_6\",\n",
-        "      \"type\": \"DOUBLE\"\n",
-        "    }\n",
-        "  ],\n",
-        "  \"output\": [\n",
-        "    {\n",
-        "      \"name\": \"prediction\",\n",
-        "      \"type\": \"DOUBLE\"\n",
-        "    }\n",
-        "  ]\n",
-        "}"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "d0b88e26570a"
-      },
-      "source": [
-        "Copy the model schema configuration file to GCS. The serving container reads the model schema file location from the `AIP_STORAGE_URI` environment at startup. See [Import the model into Vertex AI](https://cloud.google.com/architecture/spark-ml-model-with-vertexai#import-the-model-into-vertex-ai) for more information."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "b7763bb558f3"
-      },
-      "outputs": [],
-      "source": [
-        "! gsutil cp $SRC/schema.json $ARTIFACT_URI/schema.json"
+        "    component_outputs = NamedTuple(\n",
+        "        \"Outputs\",\n",
+        "        [\n",
+        "            (\"uri\", str),\n",
+        "        ],\n",
+        "    )\n",
+        "    return component_outputs(artifact_uri)"
       ]
     },
     {
@@ -2415,35 +2195,30 @@
       "source": [
         "@dsl.pipeline(name=PIPELINE_NAME, description=\"A pipeline to train a PySpark model.\")\n",
         "def pipeline(\n",
+        "    preprocessing_batch_id: str = PREPROCESSING_BATCH_ID,\n",
         "    preprocessing_main_python_file_uri: str = PREPROCESSING_PYTHON_FILE_URI,\n",
         "    train_data_path: str = FEATURES_TRAIN_URI,\n",
         "    preprocessed_data_path: str = PROCESSED_DATA_URI,\n",
         "    dataset_name: str = DATASET_NAME,\n",
         "    dataset_uri: str = GCS_PREPROCESSED_URI,\n",
+        "    training_batch_id: str = TRAINING_BATCH_ID,\n",
         "    training_main_python_file_uri: str = TRAINING_PYTHON_FILE_URI,\n",
         "    train_path: str = PROCESSED_DATA_URI,\n",
         "    model_path: str = MODEL_URI,\n",
         "    metrics_path: str = METRICS_URI,\n",
         "    threshold: float = AUPR_THRESHOLD,\n",
+        "    hpt_batch_id: str = HPT_TRAINING_BATCH_ID,\n",
         "    hpt_main_python_file_uri: str = HPT_PYTHON_FILE_URI,\n",
         "    hpt_model_path: str = HPT_MODEL_URI,\n",
         "    hpt_metrics_path: str = HPT_METRICS_URI,\n",
-        "    hpt_bundle_path: str = HPT_BUNDLE_URI,\n",
         "    custom_container_image: str = RUNTIME_CONTAINER_IMAGE,\n",
         "    model_name: str = MODEL_NAME,\n",
         "    project_id: str = PROJECT_ID,\n",
         "    location: str = REGION,\n",
-        "    deploy_model: bool = DEPLOY_MODEL,\n",
-        "    artifact_uri: str = ARTIFACT_URI,\n",
-        "    serving_image_uri: str = SERVING_IMAGE_URI,\n",
         "):\n",
-        "    from google_cloud_pipeline_components.v1.dataproc import \\\n",
+        "\n",
+        "    from google_cloud_pipeline_components.experimental.dataproc import \\\n",
         "        DataprocPySparkBatchOp\n",
-        "    from google_cloud_pipeline_components.v1.dataset import \\\n",
-        "        TabularDatasetCreateOp\n",
-        "    from google_cloud_pipeline_components.v1.endpoint import (EndpointCreateOp,\n",
-        "                                                              ModelDeployOp)\n",
-        "    from google_cloud_pipeline_components.v1.model import ModelUploadOp\n",
         "\n",
         "    # build preprocessed data args\n",
         "    build_preprocessing_args_op = build_preprocessing_args(\n",
@@ -2455,12 +2230,13 @@
         "        project=project_id,\n",
         "        location=location,\n",
         "        container_image=custom_container_image,\n",
+        "        batch_id=preprocessing_batch_id,\n",
         "        main_python_file_uri=preprocessing_main_python_file_uri,\n",
         "        args=build_preprocessing_args_op.output,\n",
         "    ).after(build_preprocessing_args_op)\n",
         "\n",
         "    # create dataset\n",
-        "    create_dataset_op = TabularDatasetCreateOp(\n",
+        "    create_dataset_op = vertex_ai_components.TabularDatasetCreateOp(\n",
         "        display_name=dataset_name,\n",
         "        gcs_source=dataset_uri,\n",
         "        project=project_id,\n",
@@ -2480,6 +2256,7 @@
         "        project=project_id,\n",
         "        location=location,\n",
         "        container_image=custom_container_image,\n",
+        "        batch_id=training_batch_id,\n",
         "        main_python_file_uri=training_main_python_file_uri,\n",
         "        args=build_training_args_op.output,\n",
         "    ).after(build_training_args_op)\n",
@@ -2492,12 +2269,11 @@
         "        name=AUPR_HYPERTUNE_CONDITION,\n",
         "    ):\n",
         "\n",
-        "        build_hpt_args_op = build_hpt_args(\n",
+        "        build_hpt_args_op = build_training_args(\n",
         "            dataset_uri=create_dataset_op.output,\n",
         "            train_path=train_path,\n",
         "            model_path=hpt_model_path,\n",
         "            metrics_path=hpt_metrics_path,\n",
-        "            bundle_path=hpt_bundle_path,\n",
         "        ).after(evaluate_model_op)\n",
         "\n",
         "        # hyperparameter tuning\n",
@@ -2505,46 +2281,13 @@
         "            project=project_id,\n",
         "            location=location,\n",
         "            container_image=custom_container_image,\n",
+        "            batch_id=hpt_batch_id,\n",
         "            main_python_file_uri=hpt_main_python_file_uri,\n",
         "            args=build_hpt_args_op.output,\n",
-        "            runtime_config_properties=HPT_RUNTIME_PROPERTIES,\n",
         "        ).after(model_traning_op)\n",
         "\n",
-        "        # evaluate condition to upload and deploy model to Vertex AI\n",
-        "        with Condition(\n",
-        "            # kfp casts `bool` parameter to `str`\n",
-        "            deploy_model == \"True\",\n",
-        "            name=DEPLOY_MODEL_CONDITION,\n",
-        "        ):\n",
-        "            # import the model into the pipeline as a kfp model artifact\n",
-        "            import_model_artifact_op = import_model_artifact(\n",
-        "                artifact_uri=artifact_uri,\n",
-        "                serving_image_uri=serving_image_uri,\n",
-        "            )\n",
-        "\n",
-        "            # upload model to Vertex AI\n",
-        "            model_upload_op = ModelUploadOp(\n",
-        "                project=project_id,\n",
-        "                location=location,\n",
-        "                display_name=model_name,\n",
-        "                unmanaged_container_model=import_model_artifact_op.outputs[\"model\"],\n",
-        "            ).after(hyperparameter_tuning_op)\n",
-        "\n",
-        "            # create a serving endpoint\n",
-        "            endpoint_op = EndpointCreateOp(\n",
-        "                project=project_id,\n",
-        "                location=location,\n",
-        "                display_name=model_name,\n",
-        "            ).after(model_upload_op)\n",
-        "\n",
-        "            # deploy  model to the serving endpoint\n",
-        "            _ = ModelDeployOp(\n",
-        "                model=model_upload_op.outputs[\"model\"],\n",
-        "                endpoint=endpoint_op.outputs[\"endpoint\"],\n",
-        "                dedicated_resources_machine_type=\"n1-standard-2\",\n",
-        "                dedicated_resources_min_replica_count=1,\n",
-        "                dedicated_resources_max_replica_count=1,\n",
-        "            ).after(endpoint_op)"
+        "        # upload model\n",
+        "        register_model(artifact_uri=hpt_model_path).after(hyperparameter_tuning_op)"
       ]
     },
     {
@@ -2620,65 +2363,6 @@
         "pipeline.wait()"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "b584afa5a1b1"
-      },
-      "source": [
-        "### (Optional) Get online predictions from the deployed model\n",
-        "\n",
-        "You can request online predictions if the model was deployed to a Vertex AI endpoint. Use the `google-cloud-aiplatform` client library to request predictions, or use `curl` as per below:\n",
-        "\n",
-        "Create the prediction request payload with the instances that you want to predict:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "12d068e1877c"
-      },
-      "outputs": [],
-      "source": [
-        "%%writefile instances.json\n",
-        "{\n",
-        "    \"instances\": [\n",
-        "        [214.0, \"360\", \"Rural\", 2.13, 2.21, 0.0, 0.0, 2.31, 2.01, 0.0, 0.0, 0.0, 0.0],\n",
-        "        [213.0, \"360\", \"Semiurban\", 2.03, 2.11, 0.0, 0.0, 2.13, 2.02, 0.0, 0.0, 0.0, 0.0]\n",
-        "    ]\n",
-        "}"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "b7cbfec4537d"
-      },
-      "source": [
-        "Use `curl` to send the prediction request to the Vertex AI endpoint. The response contains the predicted label (`0 == not eligible`, `1 == eligible`) for each instance sent in the request payload."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "b1617d6e8a3d"
-      },
-      "outputs": [],
-      "source": [
-        "ENDPOINT_ID=!(gcloud ai endpoints list \\\n",
-        "              --region={REGION} \\\n",
-        "              --filter=display_name={MODEL_NAME} \\\n",
-        "              --format='value(name)')\n",
-        "\n",
-        "!curl -X POST \\\n",
-        "   -H \"Authorization: Bearer $(gcloud auth print-access-token)\" \\\n",
-        "   -H \"Content-Type: application/json\" \\\n",
-        "   https://{REGION}-aiplatform.googleapis.com/v1/projects/{PROJECT_ID}/locations/us-central1/endpoints/{ENDPOINT_ID[-1]}:predict \\\n",
-        "   -d \"@instances.json\""
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -2704,14 +2388,8 @@
         "# Delete pipeline\n",
         "pipeline.delete()\n",
         "\n",
-        "# Delete endpoints\n",
-        "endpoint_list = vertex_ai.Endpoint.list(filter=f'display_name=\"{MODEL_NAME}\"')\n",
-        "for endpoint in endpoint_list:\n",
-        "    endpoint.undeploy_all()\n",
-        "    endpoint.delete()\n",
-        "\n",
         "# Delete model\n",
-        "model_list = vertex_ai.Model.list(filter=f'display_name=\"{MODEL_NAME}\"')\n",
+        "model_list = vertex_ai.TabularDataset.list(filter=f'display_name=\"{MODEL_NAME}\"')\n",
         "for model in model_list:\n",
         "    model.delete()\n",
         "\n",
@@ -2755,17 +2433,6 @@
         "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
         "    ! gsutil -m rm -r $BUCKET_URI"
       ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "bc7f1247f0ec"
-      },
-      "outputs": [],
-      "source": [
-        "!rm -rf $SRC $BUILD_PATH"
-      ]
     }
   ],
   "metadata": {

notebooks/official/pipelines/metrics_viz_run_compare_kfp.ipynb

@@ -64,30 +64,6 @@
         "This notebook shows how to use [the Kubeflow Pipelines (KFP) SDK](https://www.kubeflow.org/docs/components/pipelines/) to build [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines) that generate model metrics and metrics visualizations, and comparing pipeline runs."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "objective:pipelines,metrics"
-      },
-      "source": [
-        "### Objective\n",
-        "\n",
-        "In this tutorial, you use the KFP SDK to build pipelines that generate evaluation metrics.\n",
-        "\n",
-        "This tutorial uses the following Google Cloud ML services:\n",
-        "\n",
-        "- `Vertex AI Pipelines`\n",
-        "\n",
-        "The steps performed include:\n",
-        "\n",
-        "- Create KFP components:\n",
-        "    - Generate ROC curve and confusion matrix visualizations for classification results\n",
-        "    - Write metrics\n",
-        "- Create KFP pipelines.\n",
-        "- Execute KFP pipelines\n",
-        "- Compare metrics across pipeline runs"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -112,6 +88,30 @@
         "The dataset predicts the type of Iris flower species from a class of three species: setosa, virginica, or versicolor."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "objective:pipelines,metrics"
+      },
+      "source": [
+        "### Objective\n",
+        "\n",
+        "In this tutorial, you use the KFP SDK to build pipelines that generate evaluation metrics.\n",
+        "\n",
+        "This tutorial uses the following Google Cloud ML services:\n",
+        "\n",
+        "- `Vertex AI Pipelines`\n",
+        "\n",
+        "The steps performed include:\n",
+        "\n",
+        "- Create KFP components:\n",
+        "    - Generate ROC curve and confusion matrix visualizations for classification results\n",
+        "    - Write metrics\n",
+        "- Create KFP pipelines.\n",
+        "- Execute KFP pipelines\n",
+        "- Compare metrics across pipeline runs"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -202,7 +202,7 @@
         "! pip3 install {USER_FLAG} kfp google-cloud-pipeline-components --upgrade -q\n",
         "\n",
         "if os.getenv(\"IS_TESTING\"):\n",
-        "    ! pip3 install --upgrade matplotlib $USER_FLAG -q"
+        "    ! pip3 install --upgrade matplotlib $USER_FLAG"
       ]
     },
     {
@@ -240,8 +240,6 @@
         "id": "check_versions"
       },
       "source": [
-        "### KFP SDK version\n",
-        "\n",
         "Check the versions of the packages you installed.  The KFP SDK version should be >=1.6."
       ]
     },
@@ -351,10 +349,7 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
       ]
     },
     {
@@ -363,9 +358,9 @@
         "id": "timestamp"
       },
       "source": [
-        "#### UUID\n",
+        "#### Timestamp\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
       ]
     },
     {
@@ -376,16 +371,9 @@
       },
       "outputs": [],
       "source": [
-        "import random\n",
-        "import string\n",
+        "from datetime import datetime\n",
         "\n",
-        "\n",
-        "# Generate a uuid of a specifed length(default=8)\n",
-        "def generate_uuid(length: int = 8) -> str:\n",
-        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
-        "\n",
-        "\n",
-        "UUID = generate_uuid()"
+        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
     {
@@ -396,7 +384,7 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Vertex AI Workbench Notebook**, your environment is already authenticated.\n",
+        "**If you are using Vertex AI Workbench Notebook**, your environment is already authenticated. Skip this step.\n",
         "\n",
         "**If you are using Colab**, run the cell below and follow the instructions\n",
         "when prompted to authenticate your account via oAuth.\n",
@@ -491,7 +479,7 @@
       "outputs": [],
       "source": [
         "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
         "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
       ]
     },
@@ -565,10 +553,6 @@
       },
       "outputs": [],
       "source": [
-        "import sys\n",
-        "\n",
-        "IS_COLAB = \"google.colab\" in sys.modules\n",
-        "\n",
         "if (\n",
         "    SERVICE_ACCOUNT == \"\"\n",
         "    or SERVICE_ACCOUNT is None\n",
@@ -928,12 +912,12 @@
       },
       "outputs": [],
       "source": [
-        "DISPLAY_NAME = \"iris_\" + UUID\n",
+        "DISPLAY_NAME = \"iris_\" + TIMESTAMP\n",
         "\n",
         "job = aip.PipelineJob(\n",
         "    display_name=DISPLAY_NAME,\n",
         "    template_path=\"tabular classification_pipeline.json\".replace(\" \", \"_\"),\n",
-        "    job_id=f\"tabular classification-v2{UUID}-1\".replace(\" \", \"\"),\n",
+        "    job_id=f\"tabular classification-v2{TIMESTAMP}-1\".replace(\" \", \"\"),\n",
         "    pipeline_root=PIPELINE_ROOT,\n",
         "    parameter_values={\"seed\": 7, \"splits\": 10},\n",
         ")\n",
@@ -966,18 +950,7 @@
         "\n",
         "Next, generate another pipeline run that uses a different `seed` and `split` for the `iris_logregression` step.\n",
         "\n",
-        "Submit the new pipeline run:\n",
-        "\n",
-        "\n",
-        "**pipeline_root :**   Specify a Cloud Storage URI that your pipelines service account can access. The artifacts of your pipeline runs are stored within the pipeline root. \n",
-        "\n",
-        "**display_name :**  The name of the pipeline, this will show up in the Google Cloud console. \n",
-        "\n",
-        "**parameter_values :**  The pipeline parameters to pass to this run. For example, create a dict() with the parameter names as the dictionary keys and the parameter values as the dictionary values. \n",
-        "\n",
-        "**job_id :**  A unique identifier for this pipeline run. If the job ID is not specified, Vertex AI Pipelines creates a job ID for you using the pipeline name and the timestamp of when the pipeline run was started. \n",
-        "\n",
-        "**template_path :**  complete pipeline path"
+        "Submit the new pipeline run:"
       ]
     },
     {
@@ -989,9 +962,9 @@
       "outputs": [],
       "source": [
         "job = aip.PipelineJob(\n",
-        "    display_name=\"iris_\" + UUID,\n",
+        "    display_name=\"iris_\" + TIMESTAMP,\n",
         "    template_path=\"tabular classification_pipeline.json\".replace(\" \", \"_\"),\n",
-        "    job_id=f\"tabular classification-pipeline-v2{UUID}-2\".replace(\" \", \"\"),\n",
+        "    job_id=f\"tabular classification-pipeline-v2{TIMESTAMP}-2\".replace(\" \", \"\"),\n",
         "    pipeline_root=PIPELINE_ROOT,\n",
         "    parameter_values={\"seed\": 5, \"splits\": 7},\n",
         ")\n",
@@ -1108,7 +1081,16 @@
         "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
         "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
         "\n",
-        "Otherwise, you can delete the individual resources you created in this tutorial -- *Note:* this is auto-generated and not all resources may be applicable for this tutorial:"
+        "Otherwise, you can delete the individual resources you created in this tutorial -- *Note:* this is auto-generated and not all resources may be applicable for this tutorial:\n",
+        "\n",
+        "- Dataset\n",
+        "- Pipeline\n",
+        "- Model\n",
+        "- Endpoint\n",
+        "- Batch Job\n",
+        "- Custom Job\n",
+        "- Hyperparameter Tuning Job\n",
+        "- Cloud Storage Bucket"
       ]
     },
     {
@@ -1119,9 +1101,94 @@
       },
       "outputs": [],
       "source": [
-        "delete_bucket = False\n",
+        "delete_dataset = True\n",
+        "delete_pipeline = True\n",
+        "delete_model = True\n",
+        "delete_endpoint = True\n",
+        "delete_batchjob = True\n",
+        "delete_customjob = True\n",
+        "delete_hptjob = True\n",
+        "delete_bucket = True\n",
+        "\n",
+        "try:\n",
+        "    if delete_model and \"DISPLAY_NAME\" in globals():\n",
+        "        models = aip.Model.list(\n",
+        "            filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
+        "        )\n",
+        "        model = models[0]\n",
+        "        aip.Model.delete(model)\n",
+        "        print(\"Deleted model:\", model)\n",
+        "except Exception as e:\n",
+        "    print(e)\n",
+        "\n",
+        "try:\n",
+        "    if delete_endpoint and \"DISPLAY_NAME\" in globals():\n",
+        "        endpoints = aip.Endpoint.list(\n",
+        "            filter=f\"display_name={DISPLAY_NAME}_endpoint\", order_by=\"create_time\"\n",
+        "        )\n",
+        "        endpoint = endpoints[0]\n",
+        "        endpoint.undeploy_all()\n",
+        "        aip.Endpoint.delete(endpoint.resource_name)\n",
+        "        print(\"Deleted endpoint:\", endpoint)\n",
+        "except Exception as e:\n",
+        "    print(e)\n",
+        "\n",
+        "if delete_dataset and \"DISPLAY_NAME\" in globals():\n",
+        "    if \"tabular\" == \"tabular\":\n",
+        "        try:\n",
+        "            datasets = aip.TabularDataset.list(\n",
+        "                filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
+        "            )\n",
+        "            dataset = datasets[0]\n",
+        "            aip.TabularDataset.delete(dataset.resource_name)\n",
+        "            print(\"Deleted dataset:\", dataset)\n",
+        "        except Exception as e:\n",
+        "            print(e)\n",
+        "\n",
+        "    if \"tabular\" == \"image\":\n",
+        "        try:\n",
+        "            datasets = aip.ImageDataset.list(\n",
+        "                filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
+        "            )\n",
+        "            dataset = datasets[0]\n",
+        "            aip.ImageDataset.delete(dataset.resource_name)\n",
+        "            print(\"Deleted dataset:\", dataset)\n",
+        "        except Exception as e:\n",
+        "            print(e)\n",
+        "\n",
+        "    if \"tabular\" == \"text\":\n",
+        "        try:\n",
+        "            datasets = aip.TextDataset.list(\n",
+        "                filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
+        "            )\n",
+        "            dataset = datasets[0]\n",
+        "            aip.TextDataset.delete(dataset.resource_name)\n",
+        "            print(\"Deleted dataset:\", dataset)\n",
+        "        except Exception as e:\n",
+        "            print(e)\n",
+        "\n",
+        "    if \"tabular\" == \"video\":\n",
+        "        try:\n",
+        "            datasets = aip.VideoDataset.list(\n",
+        "                filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
+        "            )\n",
+        "            dataset = datasets[0]\n",
+        "            aip.VideoDataset.delete(dataset.resource_name)\n",
+        "            print(\"Deleted dataset:\", dataset)\n",
+        "        except Exception as e:\n",
+        "            print(e)\n",
+        "\n",
+        "try:\n",
+        "    if delete_pipeline and \"DISPLAY_NAME\" in globals():\n",
+        "        pipelines = aip.PipelineJob.list(\n",
+        "            filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
+        "        )\n",
+        "        pipeline = pipelines[0]\n",
+        "        aip.PipelineJob.delete(pipeline.resource_name)\n",
+        "        print(\"Deleted pipeline:\", pipeline)\n",
+        "except Exception as e:\n",
+        "    print(e)\n",
         "\n",
-        "job.delete()\n",
         "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
         "    ! gsutil rm -r $BUCKET_URI"
       ]

notebooks/official/workbench/fraud_detection/fraud-detection-model.ipynb

@@ -1,5 +1,14 @@
 {
   "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "c4b363e1330b"
+      },
+      "source": [
+        "# Build a fraud detection model on Vertex AI"
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -29,8 +38,6 @@
         "id": "05c670d35496"
       },
       "source": [
-        "# Build a fraud detection model on Vertex AI\n",
-        "\n",
         "<table align=\"left\">\n",
         "\n",
         "  <td>\n",
@@ -53,6 +60,28 @@
         "</table>"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "4c5fb7f2090f"
+      },
+      "source": [
+        "## Table of contents\n",
+        "\n",
+        "* [Overview](#section-1)\n",
+        "* [Dataset](#section-2)\n",
+        "* [Objective](#section-3)\n",
+        "* [Costs](#section-4)\n",
+        "* [Analyze the dataset](#section-5)\n",
+        "* [Fit a random forest model](#section-6)\n",
+        "* [Analyzing results](#section-7)\n",
+        "* [Save the model to a Cloud Storagae path](#section-8)\n",
+        "* [Create a model in Vertex AI](#section-9)\n",
+        "* [Create an Endpoint](#section-10)  \n",
+        "* [What-If Tool ](#section-11)\n",
+        "* [Clean up](#section-12)"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -65,6 +94,19 @@
         "This tutorial shows you how to build, deploy, and analyze predictions from a simple [random forest](https://en.wikipedia.org/wiki/Random_forest) model using tools like scikit-learn, Vertex AI, and the [What-IF Tool (WIT)](https://cloud.google.com/ai-platform/prediction/docs/using-what-if-tool) on a synthetic fraud transaction dataset to solve a financial fraud detection problem.\n"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9625185ccee9"
+      },
+      "source": [
+        "## Dataset\n",
+        "<a name=\"section-2\"></a>\n",
+        "\n",
+        "\n",
+        "The dataset used in this tutorial is publicly available at Kaggle. See [Synthetic Financial Datasets For Fraud Detection](https://www.kaggle.com/ealaxi/paysim1)."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -76,13 +118,6 @@
         "\n",
         "This tutorial demonstrates data analysis and model-building using a synthetic financial dataset. The model is trained on identifying fraudulent cases among the transactions. Then, the trained model is deployed on a Vertex AI Endpoint and analyzed using the What-If Tool. The steps taken in this tutorial are as follows: \n",
         "\n",
-        "This tutorial uses the following Google Cloud ML services and resources:\n",
-        "\n",
-        "- Vertex AI Model\n",
-        "- Vertex AI Endpoint\n",
-        "\n",
-        "The steps performed include:\n",
-        "\n",
         "- Installation of required libraries\n",
         "- Reading the dataset from a Cloud Storage bucket\n",
         "- Performing exploratory analysis on the dataset\n",
@@ -94,19 +129,6 @@
         "- Un-deploying the model and cleaning up the model resources"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "3037523e7523"
-      },
-      "source": [
-        "## Dataset\n",
-        "<a name=\"section-2\"></a>\n",
-        "\n",
-        "\n",
-        "The dataset used in this tutorial is publicly available at Kaggle. See [Synthetic Financial Datasets For Fraud Detection](https://www.kaggle.com/ealaxi/paysim1)."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -132,15 +154,21 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "cd1bc75a1cb2"
+        "id": "1ba37fa1511f"
       },
       "source": [
         "### Set up your local development environment\n",
         "\n",
         "**If you are using Colab or Google Cloud Notebooks**, your environment already meets\n",
-        "all the requirements to run this notebook. You can skip this step.\n",
-        "\n",
-        "\n",
+        "all the requirements to run this notebook. You can skip this step."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "cd1bc75a1cb2"
+      },
+      "source": [
         "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
         "You need the following:\n",
         "\n",
@@ -183,44 +211,142 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 1,
+      "execution_count": null,
       "metadata": {
         "id": "172533a994ad"
       },
-      "outputs": [
-        {
-          "name": "stdout",
-          "output_type": "stream",
-          "text": [
-            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
-            "flake8 4.0.1 requires importlib-metadata<4.3; python_version < \"3.8\", but you have importlib-metadata 4.12.0 which is incompatible.\u001b[0m\u001b[31m\n",
-            "\u001b[0m"
-          ]
-        }
-      ],
+      "outputs": [],
       "source": [
         "import os\n",
         "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
+        "import google.auth\n",
         "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
         "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
+        "# Google Cloud Notebook requires dependencies to be installed with '--user'\n",
+        "if \"default\" in dir(google.auth):\n",
+        "    USER_FLAG = \"--user\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "a465cf9367de"
+      },
+      "source": [
+        "Install the latest version of the Vertex AI client library.\n",
         "\n",
-        "# Install the latest version of the Vertex AI client library.\n",
-        "! pip3 install --upgrade google-cloud-aiplatform {USER_FLAG} -q\n",
-        "\n",
-        "# Install additional libraries\n",
-        "! pip3 install {USER_FLAG} witwidget -q\n",
-        "! pip3 install {USER_FLAG} joblib -q\n",
-        "! pip3 install {USER_FLAG} scikit-learn -q\n",
-        "! pip3 install {USER_FLAG} fsspec -q\n",
-        "! pip3 install {USER_FLAG} gcsfs -q"
+        "Run the following command in your notebook environment to install the Vertex SDK for Python:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6380f7ee5f54"
+      },
+      "outputs": [],
+      "source": [
+        "! pip install {USER_FLAG} --upgrade google-cloud-aiplatform"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "1969a1cc46cf"
+      },
+      "source": [
+        "Run the following command in your notebook environment to install witwidget:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "8b10e59b0911"
+      },
+      "outputs": [],
+      "source": [
+        "! pip install {USER_FLAG} witwidget"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "4099ce79705a"
+      },
+      "source": [
+        "Run the following command in your notebook environment to install joblib:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "1e56d524753a"
+      },
+      "outputs": [],
+      "source": [
+        "! pip install {USER_FLAG} joblib"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "b87ee3041f7d"
+      },
+      "source": [
+        "Run the following command in your notebook environment to install scikit-learn:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "c3ebecd9bd72"
+      },
+      "outputs": [],
+      "source": [
+        "! pip install {USER_FLAG} scikit-learn"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "b624b5163531"
+      },
+      "source": [
+        "Run the following command in your notebook environment to install fsspec:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "79c7a64b04de"
+      },
+      "outputs": [],
+      "source": [
+        "! pip install {USER_FLAG} fsspec"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5593090dcf0a"
+      },
+      "source": [
+        "Run the following command in your notebook environment to install gcsfs:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7bf981bc5bf6"
+      },
+      "outputs": [],
+      "source": [
+        "! pip install {USER_FLAG} gcsfs"
       ]
     },
     {
@@ -252,14 +378,21 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2d9b3731b3e0"
+      },
+      "source": [
+        "## Before you begin"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "7a5cb1df1ef7"
       },
       "source": [
-        "## Before you begin\n",
-        "\n",
         "### Set up your Google Cloud project\n",
         "\n",
         "**The following steps are required, regardless of your notebook environment.**\n",
@@ -293,26 +426,19 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "dcdfccf50581"
+        "id": "b27f37ed1ccf"
       },
       "outputs": [],
       "source": [
-        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "5bf9979b96ff"
-      },
-      "outputs": [],
-      "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
-        "    # Get your GCP project id from gcloud\n",
-        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "import os\n",
+        "\n",
+        "PROJECT_ID = \"\"\n",
+        "\n",
+        "# Get your Google Cloud project ID from gcloud\n",
+        "if not os.getenv(\"IS_TESTING\"):\n",
+        "    shell_output = !gcloud config list --format 'value(core.project)' 2>/dev/null\n",
         "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID:\", PROJECT_ID)"
+        "    print(\"Project ID: \", PROJECT_ID)"
       ]
     },
     {
@@ -324,6 +450,18 @@
         "Otherwise, set your project ID here."
       ]
     },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3dbdf6a5c539"
+      },
+      "outputs": [],
+      "source": [
+        "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
+        "    PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -335,49 +473,15 @@
         "! gcloud config set project $PROJECT_ID"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "264543a144ad"
-      },
-      "source": [
-        "#### Region\n",
-        "\n",
-        "You can also change the `REGION` variable, which is used for operations\n",
-        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. It is recommended that you choose the region closest to you.\n",
-        "\n",
-        "- Americas: `us-central1`\n",
-        "- Europe: `europe-west4`\n",
-        "- Asia Pacific: `asia-east1`\n",
-        "\n",
-        "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
-        "\n",
-        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "3281bedf6d3c"
-      },
-      "outputs": [],
-      "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "e663bd062c6f"
       },
       "source": [
-        "#### UUID\n",
+        "#### Timestamp\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append it onto the name of resources you create in this tutorial."
       ]
     },
     {
@@ -388,16 +492,21 @@
       },
       "outputs": [],
       "source": [
-        "import random\n",
-        "import string\n",
+        "from datetime import datetime\n",
         "\n",
+        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0c7f603fcdcf"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
         "\n",
-        "# Generate a uuid of length 8\n",
-        "def generate_uuid():\n",
-        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=8))\n",
-        "\n",
-        "\n",
-        "UUID = generate_uuid()"
+        "**If you are using Google Cloud Notebooks**, your environment is already\n",
+        "authenticated. Skip this step."
       ]
     },
     {
@@ -406,11 +515,6 @@
         "id": "72bf8f7c9ab3"
       },
       "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
-        "authenticated. Skip this step.\n",
-        "\n",
         "**If you are using Colab**, run the cell below and follow the instructions\n",
         "when prompted to authenticate your account via oAuth.\n",
         "\n",
@@ -443,19 +547,19 @@
       },
       "outputs": [],
       "source": [
+        "import os\n",
+        "import sys\n",
+        "\n",
         "# If you are running this notebook in Colab, run this cell and follow the\n",
         "# instructions to authenticate your GCP account. This provides access to your\n",
         "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
         "# requests.\n",
         "\n",
-        "import os\n",
-        "import sys\n",
+        "# The Google Cloud Notebook product has specific requirements\n",
+        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
         "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = \"google.colab\" in sys.modules\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
+        "# If on Google Cloud Notebooks, then don't execute this code\n",
+        "if not IS_GOOGLE_CLOUD_NOTEBOOK:\n",
         "    if \"google.colab\" in sys.modules:\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
@@ -497,25 +601,27 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "5e9a782f5608"
+        "id": "f56c52ba662c"
       },
       "outputs": [],
       "source": [
         "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+        "REGION = \"[your-region]\"  # @param {type:\"string\"}"
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "6d0729c4ae94"
+        "id": "68d1f4908641"
       },
       "outputs": [],
       "source": [
         "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
-        "    BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+        "    BUCKET_NAME = PROJECT_ID + \"-vertex-ai-\" + TIMESTAMP\n",
+        "BUCKET_URI = f\"gs://{BUCKET_NAME}\"\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -591,7 +697,6 @@
         "import numpy as np\n",
         "import pandas as pd\n",
         "from google.cloud import aiplatform, storage\n",
-        "from IPython.display import display\n",
         "from sklearn.ensemble import RandomForestClassifier\n",
         "from sklearn.metrics import (average_precision_score, classification_report,\n",
         "                             confusion_matrix, f1_score)\n",
@@ -601,15 +706,6 @@
         "warnings.filterwarnings(\"ignore\")"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "fdcb614c716f"
-      },
-      "source": [
-        "## Load dataset"
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -618,6 +714,7 @@
       },
       "outputs": [],
       "source": [
+        "# Load dataset\n",
         "df = pd.read_csv(\n",
         "    \"gs://cloud-samples-data/vertex-ai/managed_notebooks/fraud_detection/fraud_detection_data.csv\"\n",
         ")"
@@ -934,8 +1031,6 @@
         "\n",
         "# Upload the saved model file to Cloud Storage\n",
         "BLOB_PATH = \"[your-blob-path]\"\n",
-        "if BLOB_PATH == \"[your-blob-path]\":\n",
-        "    BLOB_PATH = \"fraud-detection-model-path\"\n",
         "BLOB_NAME = os.path.join(BLOB_PATH, FILE_NAME)\n",
         "\n",
         "bucket = storage.Client(PROJECT_ID).bucket(BUCKET_NAME)\n",
@@ -962,8 +1057,6 @@
       "outputs": [],
       "source": [
         "MODEL_DISPLAY_NAME = \"[your-model-display-name]\"\n",
-        "if MODEL_DISPLAY_NAME == \"[your-model-display-name]\":\n",
-        "    MODEL_DISPLAY_NAME = \"fraud-detection-model-display-name\"\n",
         "ARTIFACT_GCS_PATH = f\"{BUCKET_URI}/{BLOB_PATH}\"\n",
         "SERVING_CONTAINER_IMAGE_URI = (\n",
         "    \"us-docker.pkg.dev/vertex-ai/prediction/sklearn-cpu.1-0:latest\"\n",
@@ -1012,9 +1105,7 @@
       },
       "outputs": [],
       "source": [
-        "ENDPOINT_DISPLAY_NAME = \"[your-endpoint-display-name]\"\n",
-        "if ENDPOINT_DISPLAY_NAME == \"[your-endpoint-display-name]\":\n",
-        "    ENDPOINT_DISPLAY_NAME = \"fraud-detection-endpoint\""
+        "ENDPOINT_DISPLAY_NAME = \"[your-endpoint-display-name]\""
       ]
     },
     {
@@ -1052,8 +1143,6 @@
       "outputs": [],
       "source": [
         "DEPLOYED_MODEL_NAME = \"[your-deployed-model-name]\"\n",
-        "if DEPLOYED_MODEL_NAME == \"[your-deployed-model-name]\":\n",
-        "    DEPLOYED_MODEL_NAME = \"fraud-detection-deployed-model\"\n",
         "MACHINE_TYPE = \"n1-standard-2\""
       ]
     },
@@ -1135,33 +1224,34 @@
       },
       "outputs": [],
       "source": [
-        "if not IS_COLAB:\n",
-        "    # define target and labels\n",
-        "    TARGET_FEATURE = \"isFraud\"\n",
-        "    LABEL_VOCAB = [\"not-fraud\", \"fraud\"]\n",
+        "# define target and labels\n",
+        "TARGET_FEATURE = \"isFraud\"\n",
+        "LABEL_VOCAB = [\"not-fraud\", \"fraud\"]\n",
         "\n",
-        "    # define the function to adjust the predictions\n",
+        "# define the function to adjust the predictions\n",
         "\n",
-        "    def adjust_prediction(pred):\n",
-        "        return [1 - pred, pred]\n",
         "\n",
-        "    # Combine the features and labels into one array for the What-If Tool\n",
-        "    test_examples = np.hstack(\n",
-        "        (test_samples_X.to_numpy(), test_samples_y.to_numpy().reshape(-1, 1))\n",
+        "def adjust_prediction(pred):\n",
+        "    return [1 - pred, pred]\n",
+        "\n",
+        "\n",
+        "# Combine the features and labels into one array for the What-If Tool\n",
+        "test_examples = np.hstack(\n",
+        "    (test_samples_X.to_numpy(), test_samples_y.to_numpy().reshape(-1, 1))\n",
+        ")\n",
+        "\n",
+        "# Configure the WIT to run on the locally trained model\n",
+        "config_builder = (\n",
+        "    WitConfigBuilder(\n",
+        "        test_examples.tolist(), test_samples_X.columns.tolist() + [\"isFraud\"]\n",
         "    )\n",
+        "    .set_custom_predict_fn(forest.predict_proba)\n",
+        "    .set_target_feature(TARGET_FEATURE)\n",
+        "    .set_label_vocab(LABEL_VOCAB)\n",
+        ")\n",
         "\n",
-        "    # Configure the WIT to run on the locally trained model\n",
-        "    config_builder = (\n",
-        "        WitConfigBuilder(\n",
-        "            test_examples.tolist(), test_samples_X.columns.tolist() + [\"isFraud\"]\n",
-        "        )\n",
-        "        .set_custom_predict_fn(forest.predict_proba)\n",
-        "        .set_target_feature(TARGET_FEATURE)\n",
-        "        .set_label_vocab(LABEL_VOCAB)\n",
-        "    )\n",
-        "\n",
-        "    # display the WIT widget\n",
-        "    display(WitWidget(config_builder, height=600))"
+        "# display the WIT widget\n",
+        "WitWidget(config_builder, height=600)"
       ]
     },
     {
@@ -1181,35 +1271,36 @@
       },
       "outputs": [],
       "source": [
-        "if not IS_COLAB:\n",
-        "    # configure the target and class-labels\n",
-        "    TARGET_FEATURE = \"isFraud\"\n",
-        "    LABEL_VOCAB = [\"not-fraud\", \"fraud\"]\n",
+        "# configure the target and class-labels\n",
+        "TARGET_FEATURE = \"isFraud\"\n",
+        "LABEL_VOCAB = [\"not-fraud\", \"fraud\"]\n",
         "\n",
-        "    # function to return predictions from the deployed Model\n",
+        "# function to return predictions from the deployed Model\n",
         "\n",
-        "    def endpoint_predict_sample(instances: list):\n",
-        "        prediction = endpoint.predict(instances=instances)\n",
-        "        preds = [[1 - i, i] for i in prediction.predictions]\n",
-        "        return preds\n",
         "\n",
-        "    # Combine the features and labels into one array for the What-If Tool\n",
-        "    test_examples = np.hstack(\n",
-        "        (test_samples_X.to_numpy(), test_samples_y.to_numpy().reshape(-1, 1))\n",
+        "def endpoint_predict_sample(instances: list):\n",
+        "    prediction = endpoint.predict(instances=instances)\n",
+        "    preds = [[1 - i, i] for i in prediction.predictions]\n",
+        "    return preds\n",
+        "\n",
+        "\n",
+        "# Combine the features and labels into one array for the What-If Tool\n",
+        "test_examples = np.hstack(\n",
+        "    (test_samples_X.to_numpy(), test_samples_y.to_numpy().reshape(-1, 1))\n",
+        ")\n",
+        "\n",
+        "# Configure the WIT with the prediction function\n",
+        "config_builder = (\n",
+        "    WitConfigBuilder(\n",
+        "        test_examples.tolist(), test_samples_X.columns.tolist() + [\"isFraud\"]\n",
         "    )\n",
+        "    .set_custom_predict_fn(endpoint_predict_sample)\n",
+        "    .set_target_feature(TARGET_FEATURE)\n",
+        "    .set_label_vocab(LABEL_VOCAB)\n",
+        ")\n",
         "\n",
-        "    # Configure the WIT with the prediction function\n",
-        "    config_builder = (\n",
-        "        WitConfigBuilder(\n",
-        "            test_examples.tolist(), test_samples_X.columns.tolist() + [\"isFraud\"]\n",
-        "        )\n",
-        "        .set_custom_predict_fn(endpoint_predict_sample)\n",
-        "        .set_target_feature(TARGET_FEATURE)\n",
-        "        .set_label_vocab(LABEL_VOCAB)\n",
-        "    )\n",
-        "\n",
-        "    # run the WIT-widget\n",
-        "    display(WitWidget(config_builder, height=400))"
+        "# run the WIT-widget\n",
+        "WitWidget(config_builder, height=400)"
       ]
     },
     {