Merge branch 'main' into batch_bq

* review dataproc serverless notebook

* linter passed

Co-authored-by: Andrew Ferlitsch <aferlitsch@google.com>

.cloud-build/execute_changed_notebooks_cli.py

@@ -68,6 +68,12 @@ parser.add_argument(
     help="A service account. This is used to inject a variable value into the notebook before running. This is not the account that will run the notebook.",
     required=True,
 )
+parser.add_argument(
+    "--variable_vpc_network",
+    type=str,
+    help="The full VPC network name. See https://cloud.google.com/compute/docs/networks-and-firewalls#networks. Format is projects/{project}/global/networks/{network}, where {project} is a project number, as in '12345', and {network} is network name. See <https://cloud.google.com/compute/docs/reference/rest/v1/networks/insert> for details. This is used to inject a variable value into the notebook before running.",
+    required=False,
+)
 parser.add_argument(
     "--staging_bucket",
     type=str,
@@ -114,10 +120,11 @@ execute_changed_notebooks_helper.process_and_execute_notebooks(
     container_uri=args.container_uri,
     staging_bucket=args.staging_bucket,
     artifacts_bucket=args.artifacts_bucket,
+    should_parallelize=args.should_parallelize,
+    timeout=args.timeout,
     variable_project_id=args.variable_project_id,
     variable_region=args.variable_region,
     variable_service_account=args.variable_service_account,
+    variable_vpc_network=args.variable_vpc_network,
     private_pool_id=args.private_pool_id,
-    should_parallelize=args.should_parallelize,
-    timeout=args.timeout,
 )

.cloud-build/execute_changed_notebooks_helper.py

@@ -67,7 +67,7 @@ class NotebookExecutionResult:
     output_uri: str
     build_id: str
     error_message: Optional[str]
-    
+
     @property
     def output_uri_web(self) -> Optional[str]:
         if self.output_uri.startswith("gs://"):
@@ -81,6 +81,7 @@ def _process_notebook(
     variable_project_id: str,
     variable_region: str,
     variable_service_account: str,
+    variable_vpc_network: Optional[str],
 ):
     # Read notebook
     with open(notebook_path) as f:
@@ -93,6 +94,7 @@ def _process_notebook(
             "PROJECT_ID": variable_project_id,
             "REGION": variable_region,
             "SERVICE_ACCOUNT": variable_service_account,
+            "VPC_NETWORK": variable_vpc_network,
         },
     )
 
@@ -128,8 +130,9 @@ def process_and_execute_notebook(
     variable_project_id: str,
     variable_region: str,
     variable_service_account: str,
+    variable_vpc_network: Optional[str],
     private_pool_id: Optional[str],
-    deadline: datetime,
+    deadline: datetime.datetime,
     notebook: str,
     should_get_tail_logs: bool = False,
 ) -> NotebookExecutionResult:
@@ -137,6 +140,13 @@ def process_and_execute_notebook(
 
     print(f"Running notebook: {notebook}")
 
+    # Handle empty strings
+    if not variable_vpc_network:
+        variable_vpc_network = None
+
+    if not private_pool_id:
+        private_pool_id = None
+
     # Create paths
     notebook_output_uri = "/".join([artifacts_bucket, pathlib.Path(notebook).name])
 
@@ -163,6 +173,7 @@ def process_and_execute_notebook(
             variable_project_id=variable_project_id,
             variable_region=variable_region,
             variable_service_account=variable_service_account,
+            variable_vpc_network=variable_vpc_network,
         )
 
         # Upload the pre-processed code to a GCS bucket
@@ -266,8 +277,8 @@ def get_changed_notebooks(
             notebooks = []
     else:
         print(f"Looking for all notebooks.")
-        notebooks = subprocess.check_output(["git", "ls-files"] + test_paths)
-        notebooks = notebooks.decode("utf-8").split("\n")
+        notebooks_str = subprocess.check_output(["git", "ls-files"] + test_paths)
+        notebooks = notebooks_str.decode("utf-8").split("\n")
 
     notebooks = [notebook for notebook in notebooks if notebook.endswith(".ipynb")]
     notebooks = [notebook for notebook in notebooks if len(notebook) > 0]
@@ -286,12 +297,13 @@ def process_and_execute_notebooks(
     container_uri: str,
     staging_bucket: str,
     artifacts_bucket: str,
+    should_parallelize: bool,
+    timeout: int,
     variable_project_id: str,
     variable_region: str,
     variable_service_account: str,
-    private_pool_id: Optional[str],
-    should_parallelize: bool,
-    timeout: int,
+    variable_vpc_network: Optional[str] = None,
+    private_pool_id: Optional[str] = None,
 ):
     """
     Run the notebooks that exist under the folders defined in the test_paths_file.
@@ -349,6 +361,7 @@ def process_and_execute_notebooks(
                             variable_project_id,
                             variable_region,
                             variable_service_account,
+                            variable_vpc_network,
                             private_pool_id,
                             deadline,
                         ),
@@ -364,6 +377,7 @@ def process_and_execute_notebooks(
                     variable_project_id=variable_project_id,
                     variable_region=variable_region,
                     variable_service_account=variable_service_account,
+                    variable_vpc_network=variable_vpc_network,
                     private_pool_id=private_pool_id,
                     deadline=deadline,
                     notebook=notebook,
@@ -389,11 +403,18 @@ def process_and_execute_notebooks(
                         format_timedelta(result.duration),
                         result.log_url,
                         result.output_uri,
-                        result.output_uri_web
+                        result.output_uri_web,
                     ]
                     for result in results_sorted
                 ],
-                headers=["build_tag", "status", "duration", "log_url", "output_uri", "output_uri_web"],
+                headers=[
+                    "build_tag",
+                    "status",
+                    "duration",
+                    "log_url",
+                    "output_uri",
+                    "output_uri_web",
+                ],
             )
         )
 
@@ -422,6 +443,7 @@ def process_and_execute_notebooks(
             variable_project_id=variable_project_id,
             variable_region=variable_region,
             variable_service_account=variable_service_account,
+            variable_vpc_network=variable_vpc_network,
         )
 
         execute_notebook_helper.execute_notebook(

.cloud-build/execute_notebook_helper.py

@@ -26,6 +26,9 @@ from utils import util
 
 # This script is used to execute a notebook and write out the output notebook.
 
+# This is used to force papermill to use this kernel to run the notebook instead of any defined inside the notebook itself
+DEFAULT_KERNEL_NAME = "python3"
+
 
 def execute_notebook(
     notebook_source: str,
@@ -50,14 +53,11 @@ def execute_notebook(
 
     execution_exception = None
 
-
     print("\n=== DOWNLOAD EXECUTED NOTEBOOK ===\n")
-    print(
-                f"Please debug the executed notebook by downloading the executed notebook:"
-            )
+    print(f"Please debug the executed notebook by downloading the executed notebook:")
 
     print("Option 1. Using gsutil. Run the following command in your terminal.")
-    print(f"\tgsutil cp \"{output_file_or_uri}\" .")
+    print(f'\tgsutil cp "{output_file_or_uri}" .')
 
     print("Option 2. Using this link.")
     print(f"\thttps://storage.googleapis.com/{output_file_or_uri[5:]}")
@@ -72,6 +72,7 @@ def execute_notebook(
             output_path=notebook_source,
             progress_bar=should_log_output,
             request_save_on_cell_execute=should_log_output,
+            kernel_name=DEFAULT_KERNEL_NAME,
             log_output=should_log_output,
             stdout_file=sys.stdout if should_log_output else None,
             stderr_file=sys.stderr if should_log_output else None,

.cloud-build/notebook-execution-test-cloudbuild.yaml

@@ -36,7 +36,7 @@ steps:
     - -c
     - |
       . workspace/env/bin/activate &&
-      python3 .cloud-build/execute_changed_notebooks_cli.py --test_paths_file "${_TEST_PATHS_FILE}" --base_branch "${_FORCED_BASE_BRANCH}" --container_uri ${_PYTHON_IMAGE} --staging_bucket ${_GCS_STAGING_BUCKET}  --artifacts_bucket ${_GCS_STAGING_BUCKET}/executed_notebooks/PR_${_PR_NUMBER}/BUILD_${BUILD_ID} --variable_project_id ${PROJECT_ID} --variable_region ${_GCP_REGION} --variable_service_account ${_GCP_SERVICE_ACCOUNT} `if [ ! -z "${_PRIVATE_POOL_NAME}" ]; then echo "--private_pool_id ${_PRIVATE_POOL_NAME}"; fi`
+      python3 .cloud-build/execute_changed_notebooks_cli.py --test_paths_file "${_TEST_PATHS_FILE}" --base_branch "${_FORCED_BASE_BRANCH}" --container_uri ${_PYTHON_IMAGE} --staging_bucket ${_GCS_STAGING_BUCKET}  --artifacts_bucket ${_GCS_STAGING_BUCKET}/executed_notebooks/PR_${_PR_NUMBER}/BUILD_${BUILD_ID} --variable_project_id ${PROJECT_ID} --variable_region ${_GCP_REGION} --variable_service_account ${_GCP_SERVICE_ACCOUNT} --variable_vpc_network "${_GPC_VPC_NETWORK_NAME}" `if [ ! -z "${_PRIVATE_POOL_NAME}" ]; then echo "--private_pool_id ${_PRIVATE_POOL_NAME}"; fi`
     env:
     - 'IS_TESTING=1'
 timeout: 86400s

.cloud-build/test_notebook_vm.txt

@@ -2,3 +2,4 @@ notebooks/official/vizier/gapic-vizier-multi-objective-optimization.ipynb
 notebooks/official/pipelines/lightweight_functions_component_io_kfp.ipynb
 notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-locally-trained-models.ipynb
 notebooks/official/pipelines/metrics_viz_run_compare_kfp.ipynb
+notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,4 +1,11 @@
-If you are opening a PR for `Official Notebooks` under the [notebooks/official](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/notebooks/official) folder, follow this mandatory checklist:
+**REQUIRED:** Add a summary of your PR here, typically including why the change is needed and what was changed. Include any design alternatives for discussion purposes.
+
+<br>
+--- YOUR PR SUMMARY GOES HERE ---
+<br><br><br>
+
+**REQUIRED:** Fill out the below checklists or remove if irrelevant
+1. If you are opening a PR for `Official Notebooks` under the [notebooks/official](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/notebooks/official) folder, follow this mandatory checklist:
 - [ ] Use the [notebook template](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/notebook_template.ipynb) as a starting point.
 - [ ] Follow the style and grammar rules outlined in the above notebook template.
 - [ ] Verify the notebook runs successfully in Colab since the automated tests cannot guarantee this even when it passes.
@@ -7,12 +14,15 @@ If you are opening a PR for `Official Notebooks` under the [notebooks/official](
 - [ ] This notebook has been added to the [CODEOWNERS](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/CODEOWNERS) file under the `Official Notebooks` section, pointing to the author or the author's team.
 - [ ] The Jupyter notebook cleans up any artifacts it has created (datasets, ML models, endpoints, etc) so as not to eat up unnecessary resources.
 
+<br>
 
-If you are opening a PR for `Community Notebooks` under the [notebooks/community](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/notebooks/community) folder:
+2. If you are opening a PR for `Community Notebooks` under the [notebooks/community](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/notebooks/community) folder:
 - [ ] This notebook has been added to the [CODEOWNERS](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/CODEOWNERS) file under the `Community Notebooks` section, pointing to the author or the author's team.
 - [ ] Passes all the required formatting and linting checks. You can locally test with these [instructions](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/CONTRIBUTING.md#code-quality-checks).
 
-If you are opening a PR for `Community Content` under the [community-content](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/community-content) folder:
+<br>
+
+3. If you are opening a PR for `Community Content` under the [community-content](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/community-content) folder:
 - [ ] Make sure your main `Content Directory Name` is descriptive, informative, and includes some of the key products and attributes of your content, so that it is differentiable from other content
 - [ ] The main content directory has been added to the [CODEOWNERS](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/community-content/CODEOWNERS) file under the `Community Content` section, pointing to the author or the author's team.
 - [ ] Passes all the required formatting and linting checks. You can locally test with these [instructions](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/CONTRIBUTING.md#code-quality-checks).

community-content/pytorch_image_classification_distributed_data_parallel_training_with_vertex_sdk/multi_node_ddp_gloo_vertex_training_with_custom_container.ipynb

@@ -1,474 +0,0 @@
-{
-  "cells": [
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "a6b56b1c7b76"
-      },
-      "outputs": [],
-      "source": [
-        "# Copyright 2021 Google LLC\n",
-        "#\n",
-        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
-        "# you may not use this file except in compliance with the License.\n",
-        "# You may obtain a copy of the License at\n",
-        "#\n",
-        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
-        "#\n",
-        "# Unless required by applicable law or agreed to in writing, software\n",
-        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
-        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
-        "# See the License for the specific language governing permissions and\n",
-        "# limitations under the License."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "c414a395a19b"
-      },
-      "source": [
-        "# PyTorch Image Classification Multi-Node Distributed Data Parallel Training on CPU using Vertex Training with Custom Container"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "b98238e32cf7"
-      },
-      "source": [
-        "<table align=\"left\">\n",
-        "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/community-content/pytorch_image_classification_distributed_data_parallel_training_with_vertex_sdk/multi_node_ddp_gloo_vertex_training_with_custom_container.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
-        "      View on GitHub\n",
-        "    </a>\n",
-        "  </td>\n",
-        "</table>"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "03d216c7f7b1"
-      },
-      "source": [
-        "## Setup"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "c5ac73516218"
-      },
-      "outputs": [],
-      "source": [
-        "PROJECT_ID = \"YOUR PROJECT ID\"\n",
-        "BUCKET_NAME = \"gs://YOUR BUCKET NAME\"\n",
-        "REGION = \"YOUR REGION\"\n",
-        "SERVICE_ACCOUNT = \"YOUR SERVICE ACCOUNT\""
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "0b5ae674177e"
-      },
-      "outputs": [],
-      "source": [
-        "! gsutil ls -al $BUCKET_NAME"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "19a9b3bdd553"
-      },
-      "outputs": [],
-      "source": [
-        "content_name = \"pt-img-cls-multi-node-ddp-cust-cont\""
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "57bf6f8b4361"
-      },
-      "source": [
-        "## Local Training"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "e5d8a3443da0"
-      },
-      "outputs": [],
-      "source": [
-        "! ls trainer"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "07f79309472d"
-      },
-      "outputs": [],
-      "source": [
-        "! cat trainer/requirements.txt"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "e16cd8bb7483"
-      },
-      "outputs": [],
-      "source": [
-        "! pip install -r trainer/requirements.txt"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "0b8a210718c4"
-      },
-      "outputs": [],
-      "source": [
-        "! cat trainer/task.py"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "c0c6e7dfb3c6"
-      },
-      "outputs": [],
-      "source": [
-        "%run trainer/task.py --epochs 5 --no-cuda --local-mode"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "31dfdeede587"
-      },
-      "outputs": [],
-      "source": [
-        "! ls ./tmp"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "48d56ec621cc"
-      },
-      "outputs": [],
-      "source": [
-        "! rm -rf ./tmp"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "8f3ea1210749"
-      },
-      "source": [
-        "## Vertex Training using Vertex SDK and Custom Container"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "93002a20a2a6"
-      },
-      "source": [
-        "### Build Custom Container"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "4130ce43fd08"
-      },
-      "outputs": [],
-      "source": [
-        "hostname = \"gcr.io\"\n",
-        "image_name = content_name\n",
-        "tag = \"latest\"\n",
-        "\n",
-        "custom_container_image_uri = f\"{hostname}/{PROJECT_ID}/{image_name}:{tag}\""
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "2f1fc5b05240"
-      },
-      "outputs": [],
-      "source": [
-        "! cd trainer && docker build -t $custom_container_image_uri -f Dockerfile ."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "b4f274f499ac"
-      },
-      "outputs": [],
-      "source": [
-        "! docker run --rm $custom_container_image_uri --epochs 5 --no-cuda --local-mode"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "ee1a0a06d0b4"
-      },
-      "outputs": [],
-      "source": [
-        "! docker push $custom_container_image_uri"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "cb763be12fc9"
-      },
-      "outputs": [],
-      "source": [
-        "! gcloud container images list --repository $hostname/$PROJECT_ID"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "10c8cc6b3334"
-      },
-      "source": [
-        "### Initialize Vertex SDK"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "1a12348169fa"
-      },
-      "outputs": [],
-      "source": [
-        "! pip install -r requirements.txt"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "42e981cefe41"
-      },
-      "outputs": [],
-      "source": [
-        "from google.cloud import aiplatform\n",
-        "\n",
-        "aiplatform.init(\n",
-        "    project=PROJECT_ID,\n",
-        "    staging_bucket=BUCKET_NAME,\n",
-        "    location=REGION,\n",
-        ")"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "73c92c9298e9"
-      },
-      "source": [
-        "### Create a Vertex Tensorboard Instance"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "bde509558cd5"
-      },
-      "outputs": [],
-      "source": [
-        "content_name = content_name + \"-cpu\""
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "6d7908c0083c"
-      },
-      "outputs": [],
-      "source": [
-        "tensorboard = aiplatform.Tensorboard.create(\n",
-        "    display_name=content_name,\n",
-        ")"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "a1f0a4f54037"
-      },
-      "source": [
-        "#### Option: Use a Previously Created Vertex Tensorboard Instance\n",
-        "\n",
-        "```\n",
-        "tensorboard_name = \"Your Tensorboard Resource Name or Tensorboard ID\"\n",
-        "tensorboard = aiplatform.Tensorboard(tensorboard_name=tensorboard_name)\n",
-        "```"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "a4cac84e04ac"
-      },
-      "source": [
-        "### Run a Vertex SDK CustomContainerTrainingJob"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "f92e8fdd44ee"
-      },
-      "outputs": [],
-      "source": [
-        "display_name = content_name\n",
-        "gcs_output_uri_prefix = f\"{BUCKET_NAME}/{display_name}\"\n",
-        "\n",
-        "replica_count = 4\n",
-        "machine_type = \"n1-standard-4\"\n",
-        "\n",
-        "args = [\n",
-        "    \"--backend\",\n",
-        "    \"gloo\",\n",
-        "    \"--no-cuda\",\n",
-        "    \"--batch-size\",\n",
-        "    \"128\",\n",
-        "    \"--epochs\",\n",
-        "    \"25\",\n",
-        "]"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "ae4c57df7e07"
-      },
-      "outputs": [],
-      "source": [
-        "custom_container_training_job = aiplatform.CustomContainerTrainingJob(\n",
-        "    display_name=display_name,\n",
-        "    container_uri=custom_container_image_uri,\n",
-        ")"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "35cf3ecdf0df"
-      },
-      "outputs": [],
-      "source": [
-        "custom_container_training_job.run(\n",
-        "    args=args,\n",
-        "    base_output_dir=gcs_output_uri_prefix,\n",
-        "    replica_count=replica_count,\n",
-        "    machine_type=machine_type,\n",
-        "    tensorboard=tensorboard.resource_name,\n",
-        "    service_account=SERVICE_ACCOUNT,\n",
-        ")"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "49d10dded73b"
-      },
-      "outputs": [],
-      "source": [
-        "print(f\"Custom Training Job Name: {custom_container_training_job.resource_name}\")\n",
-        "print(f\"GCS Output URI Prefix: {gcs_output_uri_prefix}\")"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "78398f52807b"
-      },
-      "source": [
-        "### Training Output Artifact"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "fc74422de1d1"
-      },
-      "outputs": [],
-      "source": [
-        "! gsutil ls $gcs_output_uri_prefix"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "5e99a6a05b10"
-      },
-      "source": [
-        "## Clean Up Artifact"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "b0c1b3f7466b"
-      },
-      "outputs": [],
-      "source": [
-        "! gsutil rm -rf $gcs_output_uri_prefix"
-      ]
-    }
-  ],
-  "metadata": {
-    "colab": {
-      "name": "multi_node_ddp_gloo_vertex_training_with_custom_container.ipynb",
-      "toc_visible": true
-    },
-    "kernelspec": {
-      "display_name": "Python 3",
-      "name": "python3"
-    }
-  },
-  "nbformat": 4,
-  "nbformat_minor": 0
-}

community-content/pytorch_image_classification_distributed_data_parallel_training_with_vertex_sdk/multi_node_ddp_nccl_vertex_training_with_custom_container.ipynb

@@ -1,347 +0,0 @@
-{
-  "cells": [
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "a6b56b1c7b76"
-      },
-      "outputs": [],
-      "source": [
-        "# Copyright 2021 Google LLC\n",
-        "#\n",
-        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
-        "# you may not use this file except in compliance with the License.\n",
-        "# You may obtain a copy of the License at\n",
-        "#\n",
-        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
-        "#\n",
-        "# Unless required by applicable law or agreed to in writing, software\n",
-        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
-        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
-        "# See the License for the specific language governing permissions and\n",
-        "# limitations under the License."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "20a5ea0081d0"
-      },
-      "source": [
-        "# PyTorch Image Classification Multi-Node Distributed Data Parallel Training on GPU using Vertex Training with Custom Container"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "8752d4a255fb"
-      },
-      "source": [
-        "<table align=\"left\">\n",
-        "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/community-content/pytorch_image_classification_distributed_data_parallel_training_with_vertex_sdk/multi_node_ddp_nccl_vertex_training_with_custom_container.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
-        "      View on GitHub\n",
-        "    </a>\n",
-        "  </td>\n",
-        "</table>"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "03d216c7f7b1"
-      },
-      "source": [
-        "## Setup"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "c5ac73516218"
-      },
-      "outputs": [],
-      "source": [
-        "PROJECT_ID = \"YOUR PROJECT ID\"\n",
-        "BUCKET_NAME = \"gs://YOUR BUCKET NAME\"\n",
-        "REGION = \"YOUR REGION\"\n",
-        "SERVICE_ACCOUNT = \"YOUR SERVICE ACCOUNT\""
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "0b5ae674177e"
-      },
-      "outputs": [],
-      "source": [
-        "! gsutil ls -al $BUCKET_NAME"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "19a9b3bdd553"
-      },
-      "outputs": [],
-      "source": [
-        "content_name = \"pt-img-cls-multi-node-ddp-cust-cont\""
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "5307fe28b633"
-      },
-      "source": [
-        "## Vertex Training using Vertex SDK and Custom Container"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "46cb58c7fbf9"
-      },
-      "source": [
-        "### Built Custom Container"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "97e66e9f9bab"
-      },
-      "outputs": [],
-      "source": [
-        "hostname = \"gcr.io\"\n",
-        "image_name = content_name\n",
-        "tag = \"latest\"\n",
-        "\n",
-        "custom_container_image_uri = f\"{hostname}/{PROJECT_ID}/{image_name}:{tag}\""
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "ae9b29c4773f"
-      },
-      "source": [
-        "### Initialize Vertex SDK"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "dc1e84d5dec2"
-      },
-      "outputs": [],
-      "source": [
-        "! pip install -r requirements.txt"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "6964be27b98e"
-      },
-      "outputs": [],
-      "source": [
-        "from google.cloud import aiplatform\n",
-        "\n",
-        "aiplatform.init(\n",
-        "    project=PROJECT_ID,\n",
-        "    staging_bucket=BUCKET_NAME,\n",
-        "    location=REGION,\n",
-        ")"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "594a91f438f2"
-      },
-      "source": [
-        "### Create a Vertex Tensorboard Instance"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "93134273261e"
-      },
-      "outputs": [],
-      "source": [
-        "content_name = content_name + \"-gpu\""
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "c2bd82dbcd9b"
-      },
-      "outputs": [],
-      "source": [
-        "tensorboard = aiplatform.Tensorboard.create(\n",
-        "    display_name=content_name,\n",
-        ")"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "ebc593c6472e"
-      },
-      "source": [
-        "#### Option: Use a Previously Created Vertex Tensorboard Instance\n",
-        "\n",
-        "```\n",
-        "tensorboard_name = \"Your Tensorboard Resource Name or Tensorboard ID\"\n",
-        "tensorboard = aiplatform.Tensorboard(tensorboard_name=tensorboard_name)\n",
-        "```"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "0769e8e34c2f"
-      },
-      "source": [
-        "### Run a Vertex SDK CustomContainerTrainingJob"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "023f33ece826"
-      },
-      "outputs": [],
-      "source": [
-        "display_name = content_name\n",
-        "gcs_output_uri_prefix = f\"{BUCKET_NAME}/{display_name}\"\n",
-        "\n",
-        "replica_count = 1\n",
-        "machine_type = \"n1-standard-4\"\n",
-        "accelerator_count = 4\n",
-        "accelerator_type = \"NVIDIA_TESLA_K80\"\n",
-        "\n",
-        "args = [\n",
-        "    \"--backend\",\n",
-        "    \"nccl\",\n",
-        "    \"--batch-size\",\n",
-        "    \"128\",\n",
-        "    \"--epochs\",\n",
-        "    \"25\",\n",
-        "]"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "d4b599e726ef"
-      },
-      "outputs": [],
-      "source": [
-        "custom_container_training_job = aiplatform.CustomContainerTrainingJob(\n",
-        "    display_name=display_name,\n",
-        "    container_uri=custom_container_image_uri,\n",
-        ")"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "81321e3bdf7f"
-      },
-      "outputs": [],
-      "source": [
-        "custom_container_training_job.run(\n",
-        "    args=args,\n",
-        "    base_output_dir=gcs_output_uri_prefix,\n",
-        "    replica_count=replica_count,\n",
-        "    machine_type=machine_type,\n",
-        "    accelerator_count=accelerator_count,\n",
-        "    accelerator_type=accelerator_type,\n",
-        "    tensorboard=tensorboard.resource_name,\n",
-        "    service_account=SERVICE_ACCOUNT,\n",
-        ")"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "5100712c2c4c"
-      },
-      "outputs": [],
-      "source": [
-        "print(f\"Custom Training Job Name: {custom_container_training_job.resource_name}\")\n",
-        "print(f\"GCS Output URI Prefix: {gcs_output_uri_prefix}\")"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "f9b77676e5a6"
-      },
-      "source": [
-        "### Training Output Artifact"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "0e171ce95ace"
-      },
-      "outputs": [],
-      "source": [
-        "! gsutil ls $gcs_output_uri_prefix"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "cf1b74a12b87"
-      },
-      "source": [
-        "## Clean Up Artifact"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "a0b15089c341"
-      },
-      "outputs": [],
-      "source": [
-        "! gsutil rm -rf $gcs_output_uri_prefix"
-      ]
-    }
-  ],
-  "metadata": {
-    "colab": {
-      "name": "multi_node_ddp_nccl_vertex_training_with_custom_container.ipynb",
-      "toc_visible": true
-    },
-    "kernelspec": {
-      "display_name": "Python 3",
-      "name": "python3"
-    }
-  },
-  "nbformat": 4,
-  "nbformat_minor": 0
-}

notebooks/community/ml_ops/stage2/get_started_vertex_tensorboard.ipynb

@@ -149,8 +149,8 @@
       "source": [
         "### Set up your local development environment\n",
         "\n",
-        "**If you are using Colab or Google Cloud Notebooks**, your environment already meets\n",
-        "all the requirements to run this notebook. You can skip this step.\n",
+        "**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets\n",
+        "all the requirements to run this notebook.  \n",
         "\n",
         "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
         "You need the following:\n",

notebooks/community/ml_ops/stage2/get_started_vertex_training_lightgbm.ipynb

@@ -138,7 +138,7 @@
       "source": [
         "### Set up your local development environment\n",
         "\n",
-        "If you are using Colab or Google Cloud Notebooks, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
+        "If you are using Colab or Vertex AI Workbench Notebooks, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
         "\n",
         "Otherwise, make sure your environment meets this notebook's requirements. You need the following:\n",
         "\n",

notebooks/community/ml_ops/stage2/get_started_vertex_training_pytorch.ipynb

@@ -128,7 +128,7 @@
       "source": [
         "### Set up your local development environment\n",
         "\n",
-        "**If you are using Colab or Google Cloud Notebooks**, your environment already meets\n",
+        "**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets\n",
         "all the requirements to run this notebook. You can skip this step."
       ]
     },

notebooks/community/ml_ops/stage2/get_started_vertex_training_sklearn.ipynb

@@ -127,7 +127,7 @@
       "source": [
         "### Set up your local development environment\n",
         "\n",
-        "**If you are using Colab or Google Cloud Notebooks**, your environment already meets\n",
+        "**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets\n",
         "all the requirements to run this notebook. You can skip this step.\n",
         "\n",
         "**Otherwise**, make sure your environment meets this notebook's requirements.\n",

notebooks/community/ml_ops/stage2/get_started_vertex_vizier.ipynb

@@ -62,7 +62,7 @@
         "## Overview\n",
         "\n",
         "\n",
-        "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 2 : experimentation: get started with Vertex Vizier."
+        "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 2 : experimentation: get started with Vertex AI Vizier."
       ]
     },
     {

notebooks/community/ml_ops/stage2/get_started_with_visionapi_and_automl.ipynb

@@ -151,7 +151,7 @@
       "source": [
         "### Set up your local development environment\n",
         "\n",
-        "If you are using Colab or Google Cloud Notebooks, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
+        "If you are using Colab or Vertex AI Workbench Notebooks, your environment already meets all the requirements to run this notebook.  \n",
         "\n",
         "Otherwise, make sure your environment meets this notebook's requirements. You need the following:\n",
         "\n",

notebooks/community/ml_ops/stage3/get_started_with_rapid_prototyping_bqml_automl.ipynb

@@ -204,7 +204,7 @@
       "source": [
         "### Set up your local development environment\n",
         "\n",
-        "If you are using Colab or Google Cloud Notebooks, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
+        "If you are using Colab or Vertex AI Workbench Notebooks, your environment already meets all the requirements to run this notebook.  \n",
         "\n",
         "Otherwise, make sure your environment meets this notebook's requirements. You need the following:\n",
         "\n",

notebooks/community/ml_ops/stage6/get_started_with_cpr.ipynb

@@ -169,15 +169,6 @@
         "scikit-learn~=0.24"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "faf22f3af1ce"
-      },
-      "source": [
-        "**The model you deploy will have a different set of dependencies pre-installed than your notebook environment has. You should not assume that because things work in the notebook, they will work in the model. Instead, you will be very explicit about the dependencies for the model by listing them in requirements.txt and then use `pip install` to install the exact same dependencies in the notebook. Please note, of course, that there is a chance that a dependency is missed in requirements.txt that already exists in the notebook. If that's the case, things will run in the notebook, but not in the model. To guard against that, you will test the model locally before deploying to the cloud.**"
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,

notebooks/community/ml_ops/stage6/get_started_with_matching_engine_swivel.ipynb

@@ -143,7 +143,7 @@
       "source": [
         "### Set up your local development environment\n",
         "\n",
-        "**If you are using Colab or Google Cloud Notebooks**, your environment already meets\n",
+        "**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets\n",
         "all the requirements to run this notebook. You can skip this step."
       ]
     },
@@ -424,7 +424,7 @@
         "### Authenticate your Google Cloud account\n",
         "\n",
         "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
-        "authenticated. Skip this step."
+        "authenticated. "
       ]
     },
     {
@@ -1068,7 +1068,7 @@
         "model = aiplatform.Model.upload(\n",
         "    display_name=\"movies_\" + TIMESTAMP,\n",
         "    artifact_uri=SAVEDMODEL_DIR,\n",
-        "    serving_container_image_uri=DELOY_IMAGE,\n",
+        "    serving_container_image_uri=DEPLOY_IMAGE,\n",
         ")"
       ]
     },
@@ -1355,8 +1355,6 @@
       "source": [
         "QUERY_EMBEDDING_PATH = f\"{BUCKET_URI}/embeddings/train.jsonl\"\n",
         "\n",
-        "import tensorflow as tf\n",
-        "\n",
         "with tf.io.gfile.GFile(QUERY_EMBEDDING_PATH, \"w\") as f:\n",
         "    for i in range(1, 200001):\n",
         "        query = str(i)\n",
@@ -1417,7 +1415,7 @@
         "MAX_NODES = 4\n",
         "\n",
         "batch_predict_job = model.batch_predict(\n",
-        "    job_display_name=f\"batch_predict_swivel\",\n",
+        "    job_display_name=\"batch_predict_swivel\",\n",
         "    gcs_source=[QUERY_EMBEDDING_PATH],\n",
         "    gcs_destination_prefix=f\"{BUCKET_URI}/embeddings/output\",\n",
         "    machine_type=DEPLOY_COMPUTE,\n",

notebooks/community/ml_ops/stage6/get_started_with_matching_engine_twotowers.ipynb

@@ -142,7 +142,7 @@
       "source": [
         "### Set up your local development environment\n",
         "\n",
-        "**If you are using Colab or Google Cloud Notebooks**, your environment already meets\n",
+        "**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets\n",
         "all the requirements to run this notebook. You can skip this step."
       ]
     },
@@ -422,7 +422,7 @@
         "### Authenticate your Google Cloud account\n",
         "\n",
         "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
-        "authenticated. Skip this step."
+        "authenticated. "
       ]
     },
     {

notebooks/community/ml_ops/stage6/get_started_with_tf_serving.ipynb

@@ -1214,6 +1214,20 @@
         "     [1.0,3.0,\"cat1\"],\n",
         "     [2.0,4.0,\"cat2\"]\n",
         "    ]}\n",
+        "    \n",
+        "**BigQuery**\n",
+        "\n",
+        "Each row is converted to a JSON array. For example:\n",
+        "\n",
+        "    [1.0,3.0,\"cat1\"]\n",
+        "    [2.0,4.0,\"cat2\"]\n",
+        "    \n",
+        "The batch server generates the pivot data with the same format. The generated pivot data is then wrapped into a payload request:\n",
+        "\n",
+        "    {\"instances\": [\n",
+        "     [1.0,3.0,\"cat1\"],\n",
+        "     [2.0,4.0,\"cat2\"]\n",
+        "    ]}\n",
         "\n",
         "**TFRecords**\n",
         "\n",

notebooks/community/ml_ops/stage6/get_started_with_xai_and_custom_server.ipynb

@@ -134,7 +134,7 @@
       "source": [
         "### Set up your local development environment\n",
         "\n",
-        "**If you are using Colab or Google Cloud Notebooks**, your environment already meets\n",
+        "**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets\n",
         "all the requirements to run this notebook. You can skip this step."
       ]
     },
@@ -395,7 +395,7 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. \n",
         "\n",
         "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
         "\n",

notebooks/community/sdk/SDK_AutoML_Video_Classification.ipynb

@@ -1,496 +0,0 @@
-{
-  "cells": [
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "modular-concentration"
-      },
-      "outputs": [],
-      "source": [
-        "# Copyright 2021 Google LLC\n",
-        "#\n",
-        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
-        "# you may not use this file except in compliance with the License.\n",
-        "# You may obtain a copy of the License at\n",
-        "#\n",
-        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
-        "#\n",
-        "# Unless required by applicable law or agreed to in writing, software\n",
-        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
-        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
-        "# See the License for the specific language governing permissions and\n",
-        "# limitations under the License."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "insured-graduation"
-      },
-      "source": [
-        "# Feedback or issues?\n",
-        "\n",
-        "For any feedback or questions, please open an [issue](https://github.com/googleapis/python-aiplatform/issues)."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "pregnant-going"
-      },
-      "source": [
-        "# Vertex SDK for Python: AutoML Video Classification Example\n",
-        "To use this Jupyter notebook, copy the notebook to a Google Cloud Notebooks instance with Tensorflow installed and open it. You can run each step, or cell, and see its results. To run a cell, use Shift+Enter. Jupyter automatically displays the return value of the last line in each cell. For more information about running notebooks in Google Cloud Notebook, see the [Google Cloud Notebook guide](https://cloud.google.com/vertex-ai/docs/general/notebooks).\n",
-        "\n",
-        "\n",
-        "This notebook demonstrate how to create an AutoML Video Classification Model, with a Vertex AI video dataset, and how to serve the model for batch prediction. It will require you provide a bucket where the dataset will be stored.\n",
-        "\n",
-        "Note: you may incur charges for training, prediction, storage or usage of other GCP products in connection with testing this SDK."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "pending-chamber"
-      },
-      "source": [
-        "### Install Vertex SDK for Python\n",
-        "\n",
-        "\n",
-        "After the SDK installation the kernel will be automatically restarted."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "coated-remark"
-      },
-      "outputs": [],
-      "source": [
-        "!pip3 uninstall -y google-cloud-aiplatform\n",
-        "!pip3 install google-cloud-aiplatform\n",
-        "import IPython\n",
-        "\n",
-        "app = IPython.Application.instance()\n",
-        "app.kernel.do_shutdown(True)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "incorporated-edgar"
-      },
-      "source": [
-        "### Enter Your Project and GCS Bucket\n",
-        "\n",
-        "Enter your Project Id in the cell below. Then run the cell to make sure the Cloud SDK uses the right project for all the commands in this notebook."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "hispanic-macedonia"
-      },
-      "outputs": [],
-      "source": [
-        "MY_PROJECT = \"YOUR PROJECT\"\n",
-        "MY_STAGING_BUCKET = \"gs://YOUR BUCKET\"  # bucket should be in same region as ucaip"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "efovKMU5WW7u"
-      },
-      "outputs": [],
-      "source": [
-        "import sys\n",
-        "\n",
-        "if \"google.colab\" in sys.modules:\n",
-        "    import os\n",
-        "\n",
-        "    from google.colab import auth\n",
-        "\n",
-        "    auth.authenticate_user()\n",
-        "    os.environ[\"GOOGLE_CLOUD_PROJECT\"] = MY_PROJECT"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "historical-consciousness"
-      },
-      "source": [
-        "### Set Your Task Name, and GCS Prefix\n",
-        "\n",
-        "If you want to centeralize all input and output files under the gcs location."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "organizational-salad"
-      },
-      "outputs": [],
-      "source": [
-        "TASK_TYPE = \"mbsdk_automl-video-training\"\n",
-        "PREDICTION_TYPE = \"classification\"\n",
-        "MODEL_TYPE = \"CLOUD\"\n",
-        "\n",
-        "TASK_NAME = f\"{TASK_TYPE}_{PREDICTION_TYPE}\"\n",
-        "BUCKET_NAME = MY_STAGING_BUCKET.split(\"gs://\")[1]\n",
-        "GCS_PREFIX = TASK_NAME\n",
-        "\n",
-        "print(f\"Bucket Name:    {BUCKET_NAME}\")\n",
-        "print(f\"Task Name:      {TASK_NAME}\")"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "compact-engagement"
-      },
-      "source": [
-        "# HMDB: a large human motion database\n",
-        "We prepared some training data and prediction data for the demo using the [HMDB Dataset](https://serre-lab.clps.brown.edu/resource/hmdb-a-large-human-motion-database).\n",
-        "\n",
-        "The HMDB Dataset is licensed under the Creative Commons Attribution 4.0 International License. To view a copy of this license, visit https://creativecommons.org/licenses/by/4.0/\n",
-        "\n",
-        "For more information about this dataset please visit: https://serre-lab.clps.brown.edu/resource/hmdb-a-large-human-motion-database/"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "SPDHQoFRD-vM"
-      },
-      "outputs": [],
-      "source": [
-        "automl_video_demo_train_data = (\n",
-        "    \"gs://automl-video-demo-data/hmdb_split1_5classes_all.csv\"\n",
-        ")\n",
-        "automl_video_demo_batch_prediction_data = (\n",
-        "    \"gs://automl-video-demo-data/hmdb_split1_predict.jsonl\"\n",
-        ")"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "professional-bulletin"
-      },
-      "source": [
-        "### Copy AutoML Video Demo Train Data for Creating Managed Dataset"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "accurate-producer"
-      },
-      "outputs": [],
-      "source": [
-        "gcs_source_train = f\"gs://{BUCKET_NAME}/{TASK_NAME}/data/video_classification.csv\""
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "sticky-casino"
-      },
-      "outputs": [],
-      "source": [
-        "!gsutil cp $automl_video_demo_train_data $gcs_source_train"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "rough-alert"
-      },
-      "source": [
-        "# Run AutoML Video Training with Managed Video Dataset"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "adaptive-slovakia"
-      },
-      "source": [
-        "## Initialize Vertex SDK for Python\n",
-        "\n",
-        "Initialize the *client* for Vertex AI."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "figured-fellow"
-      },
-      "outputs": [],
-      "source": [
-        "from google.cloud import aiplatform\n",
-        "\n",
-        "aiplatform.init(project=MY_PROJECT, staging_bucket=MY_STAGING_BUCKET)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "pleasant-holmes"
-      },
-      "source": [
-        "## Create a Dataset on Vertex AI\n",
-        "We will now create a Vertex AI video dataset using the previously prepared csv files. Choose one of the options below. "
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "Ln-8NdHjTfbH"
-      },
-      "source": [
-        "Option 1: Using MBSDK VideoDataset class"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "uVBfL-0TTjNS"
-      },
-      "outputs": [],
-      "source": [
-        "dataset = aiplatform.VideoDataset.create(\n",
-        "    display_name=f\"temp-{TASK_NAME}\",\n",
-        "    gcs_source=gcs_source_train,\n",
-        "    import_schema_uri=aiplatform.schema.dataset.ioformat.video.classification,\n",
-        "    sync=False,\n",
-        ")"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "lXCA_nvHTp_I"
-      },
-      "source": [
-        "Option 2: Using MBSDK Dataset class\n",
-        "```\n",
-        "dataset = aiplatform.Dataset.create(\n",
-        "    display_name=f'temp-{TASK_NAME}',\n",
-        "    metadata_schema_uri=aiplatform.schema.dataset.metadata.video,\n",
-        "    gcs_source=gcs_source_train, \n",
-        "    import_schema_uri=aiplatform.schema.dataset.ioformat.video.classification,\n",
-        "    sync=False\n",
-        ")\n",
-        "```"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "3x4xuyIbVR_N"
-      },
-      "outputs": [],
-      "source": [
-        "dataset.wait()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "mexican-spending"
-      },
-      "source": [
-        "## Launch a Training Job and Create a Model on Vertex AI"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "dynamic-piece"
-      },
-      "source": [
-        "### Config a Training Job"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "continuous-circular"
-      },
-      "outputs": [],
-      "source": [
-        "job = aiplatform.AutoMLVideoTrainingJob(\n",
-        "    display_name=f\"temp-{TASK_NAME}\",\n",
-        "    prediction_type=PREDICTION_TYPE,\n",
-        "    model_type=MODEL_TYPE,\n",
-        ")"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "juvenile-parameter"
-      },
-      "source": [
-        "### Run the Training Job"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "human-carrier"
-      },
-      "outputs": [],
-      "source": [
-        "model = job.run(\n",
-        "    dataset=dataset,\n",
-        "    training_fraction_split=0.8,\n",
-        "    test_fraction_split=0.2,\n",
-        "    model_display_name=f\"temp-{TASK_NAME}\",\n",
-        "    sync=False,\n",
-        ")"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "abstract-textbook"
-      },
-      "outputs": [],
-      "source": [
-        "model.wait()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "noted-usage"
-      },
-      "source": [
-        "# Batch Prediction Job on the Model"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "ruled-smith"
-      },
-      "source": [
-        "### Copy AutoML Video Demo Prediction Data for Creating Batch Prediction Job"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "polished-dispatch"
-      },
-      "outputs": [],
-      "source": [
-        "gcs_source_batch_prediction = (\n",
-        "    f\"gs://{BUCKET_NAME}/{TASK_NAME}/data/video_classification_batch_prediction.jsonl\"\n",
-        ")\n",
-        "gcs_destination_prefix_batch_prediction = (\n",
-        "    f\"gs://{BUCKET_NAME}/{TASK_NAME}/batch_prediction\"\n",
-        ")"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "objective-soldier"
-      },
-      "outputs": [],
-      "source": [
-        "!gsutil cp $automl_video_demo_batch_prediction_data $gcs_source_batch_prediction"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "piano-middle"
-      },
-      "outputs": [],
-      "source": [
-        "batch_predict_job = model.batch_predict(\n",
-        "    job_display_name=f\"temp-{TASK_NAME}\",\n",
-        "    gcs_source=gcs_source_batch_prediction,\n",
-        "    gcs_destination_prefix=gcs_destination_prefix_batch_prediction,\n",
-        "    sync=False,\n",
-        ")"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "visible-scientist"
-      },
-      "outputs": [],
-      "source": [
-        "batch_predict_job.wait()\n",
-        "bp_iter_outputs = batch_predict_job.iter_outputs()\n",
-        "\n",
-        "prediction_results = list()\n",
-        "for blob in bp_iter_outputs:\n",
-        "    if blob.name.split(\"/\")[-1].startswith(\"prediction\"):\n",
-        "        prediction_results.append(blob.name)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "moving-geneva"
-      },
-      "outputs": [],
-      "source": [
-        "import json\n",
-        "\n",
-        "import tensorflow as tf\n",
-        "\n",
-        "tags = list()\n",
-        "for prediction_result in prediction_results:\n",
-        "    gfile_name = f\"gs://{bp_iter_outputs.bucket.name}/{prediction_result}\"\n",
-        "    with tf.io.gfile.GFile(name=gfile_name, mode=\"r\") as gfile:\n",
-        "        for line in gfile.readlines():\n",
-        "            line = json.loads(line)\n",
-        "            break\n",
-        "\n",
-        "print(line)"
-      ]
-    }
-  ],
-  "metadata": {
-    "colab": {
-      "collapsed_sections": [],
-      "name": "AI_Platform_(Unified)_SDK_AutoML_Video_Classification.ipynb",
-      "toc_visible": true
-    },
-    "kernelspec": {
-      "display_name": "Python 3",
-      "name": "python3"
-    }
-  },
-  "nbformat": 4,
-  "nbformat_minor": 0
-}

notebooks/community/sdk/SDK_BigQuery_Custom_Container_Training.ipynb

@@ -23,6 +23,15 @@
         "# limitations under the License."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2d7a1a97d1ee"
+      },
+      "source": [
+        "# Vertex AI: SDK BigQuery Custom Container Training"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -58,18 +67,61 @@
       },
       "source": [
         "###  Overview \n",
-        "To use this Jupyter notebook, copy the notebook to a Google Cloud Notebooks instance and open it. You can run each step, or cell, and see its results. To run a cell, use Shift+Enter. Jupyter automatically displays the return value of the last line in each cell. For more information about running notebooks in Google Cloud Notebook, see the Google Cloud Notebook guide.. \n",
         "\n",
-        "### Objective \n",
+        "This Note book creates a custom container using bigquery dataset it will train container and crete ,train and Deploy the model to perform prediction. \n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "989999fbdab3"
+      },
+      "source": [
+        "### Objective\n",
         "\n",
-        "This notebook demonstrate how to create a Custom Model using Custom Container Training and a Big Query Dataset. It will require you provide a bucket where the dataset will be stored.\n",
+        "In this notebook, you will learn how to use Vertex AI Experiments to \n",
         "\n",
-        "Costs  \n",
-        "This tutorial uses billable components of Google Cloud: \n",
+        "*   Log Pipeline Job\n",
+        "*   Compare different Pipeline Jobs\n",
         "\n",
-        "Vertex AI\n",
-        "Cloud Storage\n",
-        "Learn about Vertex AI  pricing and Cloud Storage pricing, and use the Pricing Calculator to generate a cost estimate based on your projected usage."
+        "The steps covered include:\n",
+        "\n",
+        "* Formalize a training component\n",
+        "* Build a training a Model\n",
+        "* Run several Pipeline jobs and log their results\n",
+        "* Train the model for prediction\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3d29af7e49d8"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the [Iris dataset](https://www.tensorflow.org/datasets/catalog/iris) from [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/overview). This dataset does not require any feature engineering. The version of the dataset you use in this tutorial is stored in a public Cloud Storage bucket. The trained model predicts the type of Iris flower species from a class of three species: setosa, virginica, or versicolor."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e3e924989cce"
+      },
+      "source": [
+        "### Costs \n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI\n",
+        "pricing](https://cloud.google.com/vertex-ai/pricing)  [Bigquery\n",
+        "pricing](https://cloud.google.com/bigquery/pricing/) and [Cloud Storage\n",
+        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
+        "Calculator](https://cloud.google.com/products/calculator/)\n",
+        "to generate a cost estimate based on your projected usage."
       ]
     },
     {
@@ -139,10 +191,9 @@
         "id": "xOMNWzTbftDr"
       },
       "source": [
-        "# Install Vertex AI SDK for Python\n",
+        "### Install additional packages\n",
         "\n",
-        "\n",
-        "After the SDK installation the kernel will be automatically restarted."
+        "Install additional package dependencies not installed in your notebook environment, such as XGBoost, AdaNet, or TensorFlow Hub. Use the latest major GA version of each package."
       ]
     },
     {
@@ -161,6 +212,36 @@
         "app.kernel.do_shutdown(True)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d4f317591f55"
+      },
+      "source": [
+        "### Restart the kernel\n",
+        "\n",
+        "After you install the additional packages, you need to restart the notebook kernel so it can find the packages."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "f731803a16c0"
+      },
+      "outputs": [],
+      "source": [
+        "# Automatically restart kernel after installs\n",
+        "import os\n",
+        "\n",
+        "if not os.getenv(\"IS_TESTING\"):\n",
+        "    # Automatically restart kernel after installs\n",
+        "    import IPython\n",
+        "\n",
+        "    app = IPython.Application.instance()\n",
+        "    app.kernel.do_shutdown(True)"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -172,6 +253,17 @@
         "Enter your Project Id in the cell below. Then run the cell to make sure the Cloud SDK uses the right project for all the commands in this notebook."
       ]
     },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3c8049930470"
+      },
+      "outputs": [],
+      "source": [
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -183,14 +275,11 @@
         "import os\n",
         "\n",
         "PROJECT_ID = \"\"\n",
-        "\n",
         "# Get your Google Cloud project ID from gcloud\n",
         "if not os.getenv(\"IS_TESTING\"):\n",
         "    shell_output = !gcloud config list --format 'value(core.project)' 2>/dev/null\n",
         "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID: \", PROJECT_ID)\n",
-        "\n",
-        "MY_STAGING_BUCKET = \"gs://YOUR BUCKET\"  # bucket should be in same region as ucaip"
+        "    print(\"Project ID: \", PROJECT_ID)"
       ]
     },
     {
@@ -218,21 +307,14 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "6x6CSodKjMmg"
+        "id": "ZaQd5jNwjP_0"
       },
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Google Cloud Notebooks**, your environment is already\n",
-        "authenticated. Skip this step"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "ZaQd5jNwjP_0"
-      },
-      "source": [
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
+        "authenticated. Skip this step\n",
+        "\n",
         "**If you are using Colab**, run the cell below and follow the instructions\n",
         "when prompted to authenticate your account via oAuth.\n",
         "\n",
@@ -296,9 +378,9 @@
         "id": "r2lr6-MVpXLP"
       },
       "source": [
-        "#### Timestamp\n",
+        "#### UUID\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append it onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial.\n"
       ]
     },
     {
@@ -309,9 +391,16 @@
       },
       "outputs": [],
       "source": [
-        "from datetime import datetime\n",
+        "import random\n",
+        "import string\n",
         "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+        "\n",
+        "# Generate a uuid of a specifed length(default=8)\n",
+        "def generate_uuid(length: int = 8) -> str:\n",
+        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
+        "\n",
+        "\n",
+        "UUID = generate_uuid()"
       ]
     },
     {
@@ -343,15 +432,24 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "GF076Vmoioll"
+        "id": "2f6f0f6ec383"
       },
       "outputs": [],
       "source": [
         "BUCKET_URI = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "REGION = \"[your-region]\"  # @param {type:\"string\"}\n",
-        "\n",
+        "REGION = \"[your-region]\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "GF076Vmoioll"
+      },
+      "outputs": [],
+      "source": [
         "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP\n",
+        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + UUID\n",
         "\n",
         "if REGION == \"[your-region]\":\n",
         "    REGION = \"us-central1\""
@@ -403,8 +501,9 @@
         "id": "5T1d5uBoftDw"
       },
       "source": [
-        "# Copy Big Query Iris Dataset\n",
-        "We will make a Big Query dataset and copy Big Query's public iris table to that dataset. For more information about this dataset please visit: https://archive.ics.uci.edu/ml/datasets/iris "
+        "# Copy bigquery iris dataset\n",
+        "\n",
+        "You make a BigQuery dataset and copy BigQuery's public iris table to that dataset. For more information about this dataset please visit: https://archive.ics.uci.edu/ml/datasets/iris "
       ]
     },
     {
@@ -543,6 +642,15 @@
         "### Write the entrypoint script to invoke trainer"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "c5cdc477cd73"
+      },
+      "source": [
+        "The entrypoint script train adn validates the data and also compiles the model."
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -697,9 +805,9 @@
         "id": "736ddff8408b"
       },
       "source": [
-        "# Create a Managed Tabular Dataset from Big Query Dataset\n",
+        "# Create a managed tabular dataset from bigquery dataset\n",
         "\n",
-        "This section will create a managed Tabular dataset from the iris Big Query table we copied above."
+        "This section create a managed Tabular dataset from the iris BigQuery table we copied above.The param's used are BigQuery's public iris dataset."
       ]
     },
     {
@@ -723,7 +831,7 @@
       "source": [
         "# Launch The Training Job to Create a Model\n",
         "\n",
-        "We will train a model with the container we built above."
+        "We will train a model with the container we built above.To train the model you use the CustomeContanier TrainingJob method with Container Image and Container_uri as parametrs."
       ]
     },
     {
@@ -753,9 +861,9 @@
         "id": "a7fa9b59f919"
       },
       "source": [
-        "# Deploy The Model\n",
+        "# Deploy the model\n",
         "\n",
-        "Deploy your model, then wait until the model FINISHES deployment before proceeding to prediction."
+        "Deploy your model, then wait until the model Finishes deployment before proceeding to prediction.For prediction deploy method takes machine_type as parameter."
       ]
     },
     {
@@ -778,6 +886,15 @@
         "# Make a prediction\n"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e4b04d246ba9"
+      },
+      "source": [
+        "Endpoint predict method publish the prediction based on length and width feature parameters."
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -786,9 +903,11 @@
       },
       "outputs": [],
       "source": [
-        "endpoint.predict(\n",
+        "prediction = endpoint.predict(\n",
         "    [{\"sepal_length\": 5.1, \"sepal_width\": 2.5, \"petal_length\": 3.0, \"petal_width\": 1.1}]\n",
-        ")"
+        ")\n",
+        "\n",
+        "print(prediction)"
       ]
     },
     {
@@ -797,12 +916,16 @@
         "id": "MaoIczP8qu--"
       },
       "source": [
-        "## Cleaning up\n",
+        "# Cleaning up\n",
         "\n",
         "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
         "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
         "\n",
-        "Otherwise, you can delete the individual resources you created in this tutorial:"
+        "Otherwise, you can delete the individual resources you created in this tutorial:\n",
+        "\n",
+        "- Pipeline\n",
+        "- Endpoint\n",
+        "- Cloud Storage Bucket"
       ]
     },
     {
@@ -813,16 +936,28 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
+        "delete_pipeline = True\n",
+        "delete_endpoint = True\n",
         "\n",
-        "# Delete endpoint resource\n",
-        "! gcloud ai endpoints delete $ENDPOINT_NAME --quiet --region $REGION_NAME\n",
         "\n",
-        "# Delete Cloud Storage objects that were created\n",
-        "! gsutil -m rm -r $JOB_DIR\n",
+        "if delete_pipeline:\n",
+        "    job.delete()\n",
         "\n",
-        "if os.getenv(\"IS_TESTING\"):\n",
-        "! gsutil -m rm -r $BUCKET_URI  "
+        "    if delete_endpoint and \"DISPLAY_NAME\" in globals():\n",
+        "        endpoints = aip.Endpoint.list(\n",
+        "            filter=f\"display_name={DISPLAY_NAME}_endpoint\", order_by=\"create_time\"\n",
+        "        )\n",
+        "        if endpoints:\n",
+        "            endpoint = endpoints[0]\n",
+        "            endpoint.undeploy_all()\n",
+        "            aip.Endpoint.delete(endpoint.resource_name)\n",
+        "            print(\"Deleted endpoint:\", endpoint)\n",
+        "\n",
+        "\n",
+        "# Delete bucket\n",
+        "delete_bucket = False\n",
+        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "    ! gsutil rm -rf {BUCKET_URI}"
       ]
     }
   ],

notebooks/notebook_template_review.py

@@ -25,7 +25,11 @@ args = parser.parse_args()
 
 if args.errors_codes:
     args.errors_codes = args.errors_codes.split(',')
-    
+    args.errors = True
+
+if args.errors_csv:
+    args.errors = True
+
 def parse_dir(directory):
     entries = os.scandir(directory)
     for entry in entries:
@@ -38,7 +42,7 @@ def parse_dir(directory):
             parse_dir(entry.path)
         elif entry.name.endswith('.ipynb'):
             parse_notebook(entry.path)
-            
+
 def parse_notebook(path):
     with open(path, 'r') as f:
         try:
@@ -158,6 +162,11 @@ def parse_notebook(path):
             if cell['cell_type'] != 'code':
                 report_error(path, 22, "Installation code section not found")
             else:
+                if cell['source'][0].startswith('! mkdir'):
+                    cell, nth = get_cell(path, cells, nth)
+                if 'requirements.txt' in cell['source'][0]:
+                    cell, nth = get_cell(path, cells, nth)
+                    
                 text = ''
                 for line in cell['source']:
                     text += line
@@ -238,7 +247,7 @@ def parse_notebook(path):
         if cell['source'][0].startswith("### Region"): 
             report_error(path, 34, "Region section not found")
         '''
-                
+
 
 def get_cell(path, cells, nth):
     while empty_cell(path, cells, nth):
@@ -249,15 +258,38 @@ def get_cell(path, cells, nth):
         check_text_cell(path, cell)
     return cell, nth + 1
 
-                
+
 def empty_cell(path, cells, nth):
     if len(cells[nth]['source']) == 0:
         report_error(path, 10, f'empty cell: cell #{nth}')
         return True
     else:
         return False
-    
+
 def check_text_cell(path, cell):
+    
+    branding = {
+        'Vertex SDK': 'Vertex AI SDK',
+        'Vertex Training': 'Vertex AI Training',
+        'Vertex Prediction': 'Vertex AI Prediction',
+        'Vertex Batch Prediction': 'Vertex AI Batch Prediction',
+        'Vertex XAI': 'Vertex Explainable AI',
+        'Vertex Experiments': 'Vertex AI Experiments',
+        'Vertex TensorBoard': 'Vertex AI TensorBoard',
+        'Vertex Pipelines': 'Vertex AI Pipelines',
+        'Vertex Hyperparameter Tuning': 'Vertex AI Hyperparameter Tuning',
+        'Vertex Metadata': 'Vertex ML Metadata',
+        'Vertex AI Metadata': 'Vertex ML Metadata',
+        'Vertex Vizier': 'Vertex AI Vizier',
+        'Vertex Dataset': 'Vertex AI Dataset',
+        'Vertex Model': 'Vertex AI Model',
+        'Vertex Endpoint': 'Vertex AI Endpoint',
+        'Vertex Private Endpoint': 'Vertex AI Private Endpoint',
+        'Tensorflow': 'TensorFlow',
+        'Tensorboard': 'TensorBoard',
+        'Google Cloud Notebooks': 'Vertex AI Workbench Notebooks'
+    }
+    
     for line in cell['source']:
         if 'TODO' in line:
             report_error(path, 14, f'TODO in cell: {line}')
@@ -266,28 +298,9 @@ def check_text_cell(path, cell):
         if 'will' in line.lower() or 'would' in line.lower():
             report_error(path, 16, f'Do not use future tense (e.g., will), replace with present tense: {line}')
             
-        if 'Vertex SDK' in line:
-            report_error(path, 27, f"Branding: Vertex AI SDK: {line}")
-        if 'Vertex Training' in line:
-            report_error(path, 27, f"Branding: Vertex AI Training: {line}")
-        if 'Vertex Prediction' in line:
-            report_error(path, 27, f"Branding: Vertex AI Prediction: {line}")
-        if 'Vertex Batch Prediction' in line:
-            report_error(path, 27, f"Branding: Vertex AI Batch Prediction {line}")
-        if 'Vertex XAI' in line:
-            report_error(path, 27, f"Branding: Vertex Explainable AI: {line}")
-        if 'Vertex Experiments' in line:
-            report_error(path, 27, f"Branding: Vertex AI Experiments: {line}")
-        if 'Vertex TensorBoard' in line:
-            report_error(path, 27, f"Branding: Vertex AI TensorBoard: {line}")
-        if 'Vertex Pipelines' in line:
-            report_error(path, 27, f"Branding: Vertex AI Pipelines: {line}")
-        if 'Vertex Hyperparameter Tuning' in line:
-            report_error(path, 27, f"Branding: Vertex AI Hyperparameter Tuning: {line}")
-        if 'Tensorflow' in line:
-            report_error(path, 27, f"Branding: TensorFlow: {line}")
-        if 'Tensorboard' in line:
-            report_error(path, 27, f"Branding: TensorBoard: {line}")
+        for mistake, brand in branding.items():
+            if mistake in line:
+                report_error(path, 27, f"Branding {brand}: {line}")
 
 
 def check_sentence_case(path, heading):
@@ -298,12 +311,12 @@ def check_sentence_case(path, heading):
     for word in words[1:]:
         word = word.replace(':', '').replace('(', '').replace(')', '')
         if word in ['E2E', 'Vertex', 'AutoML', 'ML', 'AI', 'GCP', 'API', 'R', 'CMEK', 'TFX', 'TFDV', 'SDK',
-                    'VM', 'CPR', 'NVIDIA', 'ID']:
+                    'VM', 'CPR', 'NVIDIA', 'ID', 'DASK']:
             continue
         if word.isupper():
             report_error(path, 3, f"heading is not sentence case: {word}")
-            
-            
+
+
 def report_error(notebook, code, msg):
     if args.errors:
         if args.errors_codes:
@@ -314,7 +327,7 @@ def report_error(notebook, code, msg):
             print(notebook, ',', code)
         else:
             print(f"{notebook}: ERROR ({code}): {msg}")
-            
+
 def parse_objective(path, cell):
     desc = ''
     in_desc = True
@@ -392,7 +405,7 @@ def add_index(path, title, desc, uses, steps):
         
     if args.steps:
         print(steps)
-            
+
 
 if args.notebook_dir:
     if not os.path.isdir(args.notebook_dir):
@@ -406,4 +419,4 @@ elif args.notebook:
     parse_notebook(args.notebook)
 else:
         print("Error: must specify a directory or notebook")
-        exit(1)
+        exit(1)

notebooks/official/CODEOWNERS

@@ -5,7 +5,7 @@
 * @GoogleCloudPlatform/vertex-ai-samples-contributors  @GoogleCloudPlatform/caiis-tw
 
 # matching_engine folder
-/matching_engine @shenzhimo2
+/matching_engine @shenzhimo2 @ivanmkc
 
 /tabnet/tabnet_vertex_tutorial.ipynb @longtle
 
@@ -27,5 +27,5 @@
 /pipelines/google_cloud_pipelines_dataproc_tabular @inardini
 /automl/automl_forecasting_bqml_arima_plus_comparison.ipynb @TheMichaelHu
 /automl/automl_tabular_on_vertex_pipelines.ipynb @helinwang
-/custom/custom_training_tensorboard_profiler.ipynb @itseric
-/workbench/spark/spark_sample_notebook.ipynb @bmiro
+/custom/custom_training_tensorboard_profiler.ipynb @gericdong
+/workbench/spark/spark_sample_notebook.ipynb @bradmiro

notebooks/official/automl/automl-text-classification.ipynb

@@ -2,7 +2,7 @@
   "cells": [
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 54,
       "metadata": {
         "id": "ur8xi4C7S06n"
       },
@@ -17,12 +17,21 @@
         "#     https://www.apache.org/licenses/LICENSE-2.0\n",
         "#\n",
         "# Unless required by applicable law or agreed to in writing, software\n",
-        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# distributed under the Lice`nse is distributed on an \"AS IS\" BASIS,\n",
         "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
         "# See the License for the specific language governing permissions and\n",
         "# limitations under the License."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0d2298941703"
+      },
+      "source": [
+        "# Vertex AI: Create, train, and deploy an AutoML text classification model"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -43,8 +52,8 @@
         "    </a>\n",
         "  </td>\n",
         " <td>\n",
-        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/automl/automl-text-classification.ipynb\">\n",
-        "        <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/automl-text-classification.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
         "  </td>\n",
@@ -54,19 +63,20 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "0259a7ce8120"
+        "id": "1adb10a59bc3"
       },
       "source": [
-        "# Vertex AI: Create, train, and deploy an AutoML text classification model\n",
-        "\n",
         "## Overview\n",
         "\n",
-        "This notebook walks you through the major phases of building and using an AutoML text classification model on [Vertex AI](https://cloud.google.com/vertex-ai/docs/). \n",
-        "\n",
-        "### Dataset\n",
-        "\n",
-        "In this notebook, you use the \"Happy Moments\" sample dataset to train a model. The resulting model classifies happy moments into categores that reflect the causes of happiness. \n",
-        "\n",
+        "This notebook walks you through the major phases of building and using an AutoML text classification model on [Vertex AI](https://cloud.google.com/vertex-ai/docs/). \n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9b9824ae2c91"
+      },
+      "source": [
         "### Objective\n",
         "\n",
         "In this tutorial, you learn how to use `AutoML` to train a text classification model.\n",
@@ -84,8 +94,26 @@
         "* Create an `Endpoint` resource.\n",
         "* Deploy the `Model` resource to the `Endpoint` resource.\n",
         "* Make an online prediction\n",
-        "* Make a batch prediction\n",
+        "* Make a batch prediction"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "f67c62885df4"
+      },
+      "source": [
+        "### Dataset\n",
         "\n",
+        "In this notebook, you use the \"Happy Moments\" sample dataset to train a model. The resulting model classifies happy moments into categores that reflect the causes of happiness. "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0259a7ce8120"
+      },
+      "source": [
         "### Costs\n",
         "\n",
         "This tutorial uses billable components of Google Cloud:\n",
@@ -173,7 +201,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 55,
       "metadata": {
         "id": "b75757581291"
       },
@@ -188,6 +216,7 @@
         ")\n",
         "\n",
         "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "\n",
         "USER_FLAG = \"\"\n",
         "if IS_WORKBENCH_NOTEBOOK:\n",
         "    USER_FLAG = \"--user\"\n",
@@ -208,7 +237,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 56,
       "metadata": {
         "id": "0c0b2427998a"
       },
@@ -254,7 +283,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 1,
       "metadata": {
         "id": "be175254a715"
       },
@@ -311,7 +340,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 4,
       "metadata": {
         "id": "ae43d96c4b1b"
       },
@@ -336,7 +365,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 5,
       "metadata": {
         "id": "953fa6e5ddda"
       },
@@ -424,7 +453,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 6,
       "metadata": {
         "id": "d2de92accb67"
       },
@@ -436,7 +465,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 7,
       "metadata": {
         "id": "5ba09496accc"
       },
@@ -478,7 +507,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 9,
       "metadata": {
         "id": "96ad3d416327"
       },
@@ -498,7 +527,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 10,
       "metadata": {
         "id": "152013538e59"
       },
@@ -521,7 +550,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 11,
       "metadata": {
         "id": "740cd5c67c79"
       },
@@ -551,24 +580,15 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "6caf82e5e84e"
+        "id": "d35b8b6b94ae"
       },
       "outputs": [],
       "source": [
         "# Use a timestamp to ensure unique resources\n",
         "src_uris = \"gs://cloud-ml-data/NL-classification/happiness.csv\"\n",
-        "display_name = f\"e2e-text-dataset-{TIMESTAMP}\""
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "d35b8b6b94ae"
-      },
-      "outputs": [],
-      "source": [
-        "ds = aiplatform.TextDataset.create(\n",
+        "display_name = f\"e2e-text-dataset-{TIMESTAMP}\"\n",
+        "\n",
+        "text_dataset = aiplatform.TextDataset.create(\n",
         "    display_name=display_name,\n",
         "    gcs_source=src_uris,\n",
         "    import_schema_uri=aiplatform.schema.dataset.ioformat.text.single_label_classification,\n",
@@ -582,53 +602,7 @@
         "id": "5b3cc427353a"
       },
       "source": [
-        "## Train your text classification model\n",
-        "\n",
-        "Once your dataset has finished importing data, you are ready to train your model. To do this, you first need the full resource name of your dataset, where the full name has the format `projects/[YOUR_PROJECT]/locations/[YOUR_REGIO)N]/datasets/[YOUR_DATASET_ID]`. If you don't have the resource name handy, you can list all of the datasets in your project using `TextDataset.list()`. \n",
-        "\n",
-        "As shown in the following code block, you can pass in the display name of your dataset in the call to `list()` to filter the results.\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "52cf56f1c8a9"
-      },
-      "outputs": [],
-      "source": [
-        "datasets = aiplatform.TextDataset.list(filter=f'display_name=\"{display_name}\"')\n",
-        "print(datasets)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "58df3e02df82"
-      },
-      "source": [
-        "When you create a new model, you need a reference to the `TextDataset` object that corresponds to your dataset. You can use the `ds` variable you created previously when you created the dataset or you can also list all of your datasets to get a reference to your dataset. Each item returned from `TextDataset.list()` is an instance of `TextDataset`.\n",
-        "\n",
-        "The following code block shows how to instantiate a `TextDataset` object using a dataset ID. Note that this code is intentionally verbose for demonstration purposes."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "aa667203da03"
-      },
-      "outputs": [],
-      "source": [
-        "# Get the dataset ID if it's not available\n",
-        "dataset_id = \"[your-dataset-id]\"\n",
-        "\n",
-        "if dataset_id == \"[your-dataset-id]\":\n",
-        "    # Use the reference to the new dataset captured when we created it\n",
-        "    dataset_id = ds.resource_name.split(\"/\")[-1]\n",
-        "    print(f\"Dataset ID: {dataset_id}\")\n",
-        "\n",
-        "text_dataset = aiplatform.TextDataset(dataset_id)"
+        "## Train your text classification model\n"
       ]
     },
     {
@@ -653,7 +627,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 16,
       "metadata": {
         "id": "0aa0f01805ea"
       },
@@ -682,8 +656,8 @@
         "model = job.run(\n",
         "    dataset=text_dataset,\n",
         "    model_display_name=model_display_name,\n",
-        "    training_fraction_split=0.7,\n",
-        "    validation_fraction_split=0.2,\n",
+        "    training_fraction_split=0.1,\n",
+        "    validation_fraction_split=0.1,\n",
         "    test_fraction_split=0.1,\n",
         "    sync=True,\n",
         ")"
@@ -740,39 +714,11 @@
         "deployed_model_display_name = f\"e2e-deployed-text-classification-model-{TIMESTAMP}\"\n",
         "\n",
         "endpoint = model.deploy(\n",
-        "    deployed_model_display_name=deployed_model_display_name, sync=True\n",
+        "    deployed_model_display_name=deployed_model_display_name, \n",
+        "    sync=True\n",
         ")"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "531da446035b"
-      },
-      "source": [
-        "In case you didn't record the name of the new endpoint, you can get a list of all your endpoints as you did before with datasets and models. For each endpoint, you can list the models deployed to that endpoint. To get a reference to the model that you just deployed, you can check the `display_name` of each model deployed to the endpoint against the model you're looking for."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "f61fb44181b4"
-      },
-      "outputs": [],
-      "source": [
-        "endpoints = aiplatform.Endpoint.list()\n",
-        "\n",
-        "endpoint_with_deployed_model = []\n",
-        "\n",
-        "for endpoint_ in endpoints:\n",
-        "    for model in endpoint_.list_models():\n",
-        "        if model.display_name.find(deployed_model_display_name) == 0:\n",
-        "            endpoint_with_deployed_model.append(endpoint_)\n",
-        "\n",
-        "print(endpoint_with_deployed_model)"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -781,7 +727,7 @@
       "source": [
         "## Get online predictions from your model\n",
         "\n",
-        "Now that you have your endpoint's resource name, you can get online predictions from the text classification model. To get the online prediction, you send a prediction request to your endpoint."
+        "Now that you have your endpoint, you can get online predictions from the text classification model. To get the online prediction, you send a prediction request to your endpoint."
       ]
     },
     {
@@ -792,13 +738,6 @@
       },
       "outputs": [],
       "source": [
-        "endpoint_name = \"[your-endpoint-name]\"\n",
-        "if endpoint_name == \"[your-endpoint-name]\":\n",
-        "    endpoint_name = endpoint.resource_name\n",
-        "\n",
-        "print(f\"Endpoint name: {endpoint_name}\")\n",
-        "\n",
-        "endpoint = aiplatform.Endpoint(endpoint_name)\n",
         "content = \"I got a high score on my math final!\"\n",
         "\n",
         "response = endpoint.predict(instances=[{\"content\": content}])\n",
@@ -835,7 +774,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 23,
       "metadata": {
         "id": "e4b838cbcd99"
       },
@@ -858,7 +797,7 @@
         "\n",
         "+ All of your prediction instances as individual files on Google Cloud Storage, as TXT files for your instances\n",
         "+ A JSONL file that lists the URIs of all your prediction instances\n",
-        "+ A Google Cloud Storage bucket to hold the output from batch prediction\n",
+        "+ A Cloud Storage bucket to hold the output from batch prediction\n",
         "\n",
         "For this tutorial, the following cells create a new Storage bucket, upload individual prediction instances as text files to the bucket, and then create the JSONL file with the URIs of your prediction instances."
       ]
@@ -872,16 +811,15 @@
       "outputs": [],
       "source": [
         "# Instantiate the Storage client and create the new bucket\n",
-        "storage = storage.Client()\n",
-        "bucket = storage.bucket(BUCKET_URI)\n",
-        "\n",
+        "# from google.cloud import  storage\n",
+        "storage_client = storage.Client()\n",
+        "bucket = storage_client.bucket(BUCKET_NAME)\n",
         "# Iterate over the prediction instances, creating a new TXT file\n",
         "# for each.\n",
         "input_file_data = []\n",
         "for count, instance in enumerate(instances):\n",
         "    instance_name = f\"input_{count}.txt\"\n",
         "    instance_file_uri = f\"{BUCKET_URI}/{instance_name}\"\n",
-        "\n",
         "    # Add the data to store in the JSONL input file.\n",
         "    tmp_data = {\"content\": instance_file_uri, \"mimeType\": \"text/plain\"}\n",
         "    input_file_data.append(tmp_data)\n",
@@ -901,7 +839,7 @@
         "id": "31c262320610"
       },
       "source": [
-        "Now that you have the bucket with the prediction instances ready, you can send a batch prediction request to Vertex AI. When you send a request to the service, you must provide the URI of your JSONL file and your output bucket, including the `gs://` protocols.\n",
+        "Now that you have the bucket with the prediction instances ready, you can send a batch prediction rhttps://storage.googleapis.com/upload/storage/v1/b/gs://vertex-ai-devaip-20220728004429/o?uploadType=multipartequest to Vertex AI. When you send a request to the service, you must provide the URI of your JSONL file and your output bucket, including the `gs://` protocols.\n",
         "\n",
         "With the Python SDK, you can create a batch prediction job by calling `Model.batch_predict()`."
       ]
@@ -915,15 +853,13 @@
       "outputs": [],
       "source": [
         "job_display_name = \"e2e-text-classification-batch-prediction-job\"\n",
-        "model = aiplatform.Model(model_name=model_name)\n",
-        "\n",
+        "# model = aiplatform.Model(model_name=model.name)\n",
         "batch_prediction_job = model.batch_predict(\n",
         "    job_display_name=job_display_name,\n",
         "    gcs_source=f\"{BUCKET_URI}/{input_file_name}\",\n",
         "    gcs_destination_prefix=f\"{BUCKET_URI}/output\",\n",
         "    sync=True,\n",
         ")\n",
-        "\n",
         "batch_prediction_job_name = batch_prediction_job.resource_name"
       ]
     },
@@ -938,6 +874,15 @@
         "The following code snippet demonstrates how to create an instance of the `BatchPredictionJob` class to review its status. Note that you need the full resource name printed out from the Python SDK for this snippet.\n"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "cd014de40e2f"
+      },
+      "source": [
+        "## BatchPredictionJob"
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -999,6 +944,8 @@
       },
       "outputs": [],
       "source": [
+        "import os\n",
+        "\n",
         "RESULTS_DIRECTORY = \"prediction_results\"\n",
         "RESULTS_DIRECTORY_FULL = f\"{RESULTS_DIRECTORY}/output\"\n",
         "\n",
@@ -1020,6 +967,15 @@
         "print(f\"Local results folder: {latest_directory}\")"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e375109b7e40"
+      },
+      "source": [
+        "## JsonLines"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -1094,9 +1050,9 @@
         "    ! gsutil rm -r $BUCKET_URI\n",
         "\n",
         "batch_job.delete()\n",
-        "\n",
+        "endpoint.undeploy_all()\n",
         "# `force` parameter ensures that models are undeployed before deletion\n",
-        "endpoint.delete(force=True)\n",
+        "endpoint.delete()\n",
         "\n",
         "model.delete()\n",
         "\n",

notebooks/official/automl/sdk_automl_video_classification_batch.ipynb

@@ -3,6 +3,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "91417fdd",
       "metadata": {
         "id": "copyright"
       },
@@ -25,11 +26,12 @@
     },
     {
       "cell_type": "markdown",
+      "id": "f2902dac",
       "metadata": {
         "id": "title"
       },
       "source": [
-        "# Vertex SDK: AutoML training video classification model for batch prediction\n",
+        "# Vertex AI SDK: AutoML training video classification model for batch prediction\n",
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
@@ -55,6 +57,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "42cfbec0",
       "metadata": {
         "id": "overview:automl"
       },
@@ -62,29 +65,25 @@
         "## Overview\n",
         "\n",
         "\n",
-        "This tutorial demonstrates how to use the Vertex SDK to create video classification models and do batch prediction using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "dataset:hmdb,vcn"
-      },
-      "source": [
-        "### Dataset\n",
-        "\n",
-        "The dataset used for this tutorial is the golf swing recognition portion of the [Human Motion dataset from MIT](http://cbcl.mit.edu/publications/ps/Kuehne_etal_iccv11.pdf). The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket. The trained model will predict the start frame where a golf swing begins.\n"
+        "This tutorial demonstrates how to use the Vertex AI SDK to create video classification models and do batch prediction using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model."
       ]
     },
     {
       "cell_type": "markdown",
+      "id": "90b9b726",
       "metadata": {
         "id": "objective:automl,training,batch_prediction"
       },
       "source": [
         "### Objective\n",
         "\n",
-        "In this tutorial, you create an AutoML video classification model from a Python script, and then do a batch prediction using the Vertex SDK. You can alternatively create and deploy models using the `gcloud` command-line tool or online using the Cloud Console.\n",
+        "In this tutorial, you create an AutoML video classification model from a Python script, and then do a batch prediction using the Vertex AI SDK. You can alternatively create and deploy models using the `gcloud` command-line tool or online using the Cloud Console.\n",
+        "\n",
+        "This tutorial uses the following Google Cloud ML services and resources:\n",
+        "\n",
+        "- Vertex AI Training\n",
+        "- Vertex AI Models\n",
+        "- Vertex AI Batch Prediction\n",
         "\n",
         "The steps performed include:\n",
         "\n",
@@ -102,6 +101,19 @@
     },
     {
       "cell_type": "markdown",
+      "id": "44940826",
+      "metadata": {
+        "id": "dataset:hmdb,vcn"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the golf swing recognition portion of the [Human Motion dataset from MIT](http://cbcl.mit.edu/publications/ps/Kuehne_etal_iccv11.pdf). The version of the dataset you use in this tutorial is stored in a public Cloud Storage bucket. The trained model predicts the start frame where a golf swing begins.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "7183fc01",
       "metadata": {
         "id": "costs"
       },
@@ -122,89 +134,96 @@
     },
     {
       "cell_type": "markdown",
+      "id": "b88c255b-df72-4666-9403-0c96d7e657ca",
       "metadata": {
-        "id": "setup_local"
+        "id": "384b53dfdb54"
       },
       "source": [
         "### Set up your local development environment\n",
         "\n",
-        "If you are using Colab or Google Cloud Notebooks, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
-        "\n",
-        "Otherwise, make sure your environment meets this notebook's requirements. You need the following:\n",
-        "\n",
-        "- The Cloud Storage SDK\n",
-        "- Git\n",
-        "- Python 3\n",
-        "- virtualenv\n",
-        "- Jupyter notebook running in a virtual environment with Python 3\n",
-        "\n",
-        "The Cloud Storage guide to [Setting up a Python development environment](https://cloud.google.com/python/setup) and the [Jupyter installation guide](https://jupyter.org/install) provide detailed instructions for meeting these requirements. The following steps provide a condensed set of instructions:\n",
-        "\n",
-        "1. [Install and initialize the SDK](https://cloud.google.com/sdk/docs/).\n",
-        "\n",
-        "2. [Install Python 3](https://cloud.google.com/python/setup#installing_python).\n",
-        "\n",
-        "3. [Install virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv) and create a virtual environment that uses Python 3.  Activate the virtual environment.\n",
-        "\n",
-        "4. To install Jupyter, run `pip3 install jupyter` on the command-line in a terminal shell.\n",
-        "\n",
-        "5. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
-        "\n",
-        "6. Open this notebook in the Jupyter Notebook Dashboard.\n"
+        "**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets\n",
+        "all the requirements to run this notebook. You can skip this step."
       ]
     },
     {
       "cell_type": "markdown",
+      "id": "8c1be8fc",
+      "metadata": {
+        "id": "setup_local"
+      },
+      "source": [
+        "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
+        "You need the following:\n",
+        "\n",
+        "* The Google Cloud SDK\n",
+        "* Git\n",
+        "* Python 3\n",
+        "* virtualenv\n",
+        "* Jupyter notebook running in a virtual environment with Python 3\n",
+        "\n",
+        "The Google Cloud guide to [Setting up a Python development\n",
+        "environment](https://cloud.google.com/python/setup) and the [Jupyter\n",
+        "installation guide](https://jupyter.org/install) provide detailed instructions\n",
+        "for meeting these requirements. The following steps provide a condensed set of\n",
+        "instructions:\n",
+        "\n",
+        "1. [Install and initialize the Cloud SDK.](https://cloud.google.com/sdk/docs/)\n",
+        "\n",
+        "1. [Install Python 3.](https://cloud.google.com/python/setup#installing_python)\n",
+        "\n",
+        "1. [Install\n",
+        "   virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv)\n",
+        "   and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
+        "\n",
+        "1. To install Jupyter, run `pip3 install jupyter` on the\n",
+        "command-line in a terminal shell.\n",
+        "\n",
+        "1. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
+        "\n",
+        "1. Open this notebook in the Jupyter Notebook Dashboard.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "e131fbee",
       "metadata": {
         "id": "install_aip:mbsdk"
       },
       "source": [
         "## Installation\n",
         "\n",
-        "Install the latest version of Vertex SDK for Python."
+        "Install the latest versions of Vertex AI and Cloud Storage SDK for Python."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "484dcd52-ef9e-4928-b0f2-7940001bbc2e",
       "metadata": {
-        "id": "install_aip:mbsdk"
+        "id": "2abdd254e90f"
       },
       "outputs": [],
       "source": [
         "import os\n",
         "\n",
-        "# Google Cloud Notebook\n",
-        "if os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
-        "    USER_FLAG = \"--user\"\n",
-        "else:\n",
-        "    USER_FLAG = \"\"\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
         "\n",
-        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "install_storage"
-      },
-      "source": [
-        "Install the latest GA version of *google-cloud-storage* library as well."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "install_storage"
-      },
-      "outputs": [],
-      "source": [
-        "! pip3 install -U google-cloud-storage $USER_FLAG"
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "USER_FLAG = \"\"\n",
+        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\"\n",
+        "\n",
+        "! pip3 install --upgrade google-cloud-aiplatform {USER_FLAG} -q\n",
+        "! pip3 install -U google-cloud-storage {USER_FLAG} -q"
       ]
     },
     {
       "cell_type": "markdown",
+      "id": "aa8cefcd",
       "metadata": {
         "id": "restart"
       },
@@ -217,6 +236,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "4f079854",
       "metadata": {
         "id": "restart"
       },
@@ -234,6 +254,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "e96a43b8",
       "metadata": {
         "id": "before_you_begin:nogpu"
       },
@@ -254,19 +275,32 @@
         "\n",
         "3. [Enable the following APIs: Vertex AI APIs, Compute Engine APIs, and Cloud Storage.](https://console.cloud.google.com/flows/enableapi?apiid=ml.googleapis.com,compute_component,storage-component.googleapis.com)\n",
         "\n",
-        "4. If you are running this notebook locally, you will need to install the [Cloud SDK]((https://cloud.google.com/sdk)).\n",
+        "4. If you are running this notebook locally, you need to install the [Cloud SDK]((https://cloud.google.com/sdk)).\n",
         "\n",
         "5. Enter your project ID in the cell below. Then run the  cell to make sure the\n",
         "Cloud SDK uses the right project for all the commands in this notebook.\n",
         "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$`."
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these conmmands."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "305e7fa5-dcaf-477a-b20d-d9b69ecba381",
+      "metadata": {
+        "id": "1460fd744366"
+      },
+      "source": [
+        "#### Set your project ID\n",
+        "\n",
+        "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "ffd7caab-c2f8-41d3-a0e3-d2519f0bcf2c",
       "metadata": {
-        "id": "set_project_id"
+        "id": "cd85f5c794e5"
       },
       "outputs": [],
       "source": [
@@ -276,8 +310,9 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "ffb8077b",
       "metadata": {
-        "id": "autoset_project_id"
+        "id": "set_project_id"
       },
       "outputs": [],
       "source": [
@@ -291,6 +326,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "3c30f77a",
       "metadata": {
         "id": "set_gcloud_project_id"
       },
@@ -301,6 +337,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "61221789",
       "metadata": {
         "id": "region"
       },
@@ -322,68 +359,99 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "e05b6148",
       "metadata": {
         "id": "region"
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
       "cell_type": "markdown",
+      "id": "dab6b689",
       "metadata": {
         "id": "timestamp"
       },
       "source": [
-        "#### Timestamp\n",
+        "#### UUID\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "6dac7084",
       "metadata": {
         "id": "timestamp"
       },
       "outputs": [],
       "source": [
-        "from datetime import datetime\n",
+        "import random\n",
+        "import string\n",
         "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+        "\n",
+        "# Generate a uuid of a specifed length(default=8)\n",
+        "def generate_uuid(length: int = 8) -> str:\n",
+        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
+        "\n",
+        "\n",
+        "UUID = generate_uuid()"
       ]
     },
     {
       "cell_type": "markdown",
+      "id": "1bd3f05b-f17f-4341-be85-0bdcef3e6f13",
       "metadata": {
-        "id": "gcp_authenticate"
+        "id": "79055ac4078d"
       },
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Google Cloud Notebooks**, your environment is already authenticated. Skip this step.\n",
-        "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
+        "authenticated. Skip this step."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "c38fbff8",
+      "metadata": {
+        "id": "gcp_authenticate"
+      },
+      "source": [
+        "**If you are using Colab**, run the cell below and follow the instructions\n",
+        "when prompted to authenticate your account via oAuth.\n",
         "\n",
         "**Otherwise**, follow these steps:\n",
         "\n",
-        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
+        "1. In the Cloud Console, go to the [**Create service account key**\n",
+        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
         "\n",
-        "**Click Create service account**.\n",
+        "2. Click **Create service account**.\n",
         "\n",
-        "In the **Service account name** field, enter a name, and click **Create**.\n",
+        "3. In the **Service account name** field, enter a name, and\n",
+        "   click **Create**.\n",
         "\n",
-        "In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex\" into the filter box, and select **Vertex Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
+        "into the filter box, and select\n",
+        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
         "\n",
-        "Click Create. A JSON file that contains your key downloads to your local environment.\n",
+        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
+        "local environment.\n",
         "\n",
-        "Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
+        "6. Enter the path to your service account key as the\n",
+        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "8bae9ca0",
       "metadata": {
         "id": "gcp_authenticate"
       },
@@ -397,8 +465,11 @@
         "import os\n",
         "import sys\n",
         "\n",
-        "# If on Google Cloud Notebook, then don't execute this code\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = \"google.colab\" in sys.modules\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
         "    if \"google.colab\" in sys.modules:\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
@@ -413,6 +484,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "bbda1639",
       "metadata": {
         "id": "bucket:mbsdk"
       },
@@ -421,36 +493,42 @@
         "\n",
         "**The following steps are required, regardless of your notebook environment.**\n",
         "\n",
-        "When you initialize the Vertex SDK for Python, you specify a Cloud Storage staging bucket. The staging bucket is where all the data associated with your dataset and model resources are retained across sessions.\n",
+        "When you create a model in Vertex AI using the Cloud SDK, you give a Cloud Storage path where the trained model is saved. In this tutorial, you create a batch prediction job using the Vertex AI model. For this purpose, you need to save your test instances to a Cloud Storage bucket and give a destination Cloud Storage path to write the batch predictions.\n",
         "\n",
-        "Set the name of your Cloud Storage bucket below. Bucket names must be globally unique across all Google Cloud projects, including those outside of your organization."
+        "Set the name of your Cloud Storage bucket below. It must be unique across all\n",
+        "Cloud Storage buckets."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "be69ad8c",
       "metadata": {
         "id": "bucket"
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
+        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "2d0d674c",
       "metadata": {
         "id": "autoset_bucket"
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
+        "    BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
     {
       "cell_type": "markdown",
+      "id": "9307a615",
       "metadata": {
         "id": "create_bucket"
       },
@@ -461,16 +539,18 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "709e7b95",
       "metadata": {
         "id": "create_bucket"
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_NAME"
+        "! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
       ]
     },
     {
       "cell_type": "markdown",
+      "id": "b52bb2e6",
       "metadata": {
         "id": "validate_bucket"
       },
@@ -481,72 +561,76 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "e86c8b22",
       "metadata": {
         "id": "validate_bucket"
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_NAME"
+        "! gsutil ls -al $BUCKET_URI"
       ]
     },
     {
       "cell_type": "markdown",
+      "id": "cf0222d3",
       "metadata": {
         "id": "setup_vars"
       },
       "source": [
-        "### Set up variables\n",
-        "\n",
-        "Next, set up some variables used throughout the tutorial.\n",
-        "### Import libraries and define constants"
+        "### Import libraries"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 13,
+      "id": "7534d1a5",
       "metadata": {
         "id": "import_aip:mbsdk"
       },
       "outputs": [],
       "source": [
-        "import google.cloud.aiplatform as aiplatform"
+        "from google.cloud import aiplatform"
       ]
     },
     {
       "cell_type": "markdown",
+      "id": "15e5e61a",
       "metadata": {
         "id": "init_aip:mbsdk"
       },
       "source": [
-        "## Initialize Vertex SDK for Python\n",
+        "### Initialize Vertex AI SDK for Python\n",
         "\n",
-        "Initialize the Vertex SDK for Python for your project and corresponding bucket."
+        "Initialize the Vertex AI SDK for Python for your project and corresponding bucket."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "9df9b0b9",
       "metadata": {
         "id": "init_aip:mbsdk"
       },
       "outputs": [],
       "source": [
-        "aiplatform.init(project=PROJECT_ID, staging_bucket=BUCKET_NAME)"
+        "aiplatform.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
       ]
     },
     {
       "cell_type": "markdown",
+      "id": "866ae45f",
       "metadata": {
         "id": "tutorial_start:automl"
       },
       "source": [
-        "# Tutorial\n",
+        "## Tutorial\n",
         "\n",
         "Now you are ready to start creating your own AutoML video classification model."
       ]
     },
     {
       "cell_type": "markdown",
+      "id": "0adbd455",
       "metadata": {
         "id": "import_file:u_dataset,csv"
       },
@@ -559,6 +643,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "ab42c2d4",
       "metadata": {
         "id": "import_file:hmdb,csv,vcn"
       },
@@ -569,6 +654,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "2f7757ea",
       "metadata": {
         "id": "quick_peek:csv"
       },
@@ -583,6 +669,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "ea7bac53",
       "metadata": {
         "id": "quick_peek:csv"
       },
@@ -597,6 +684,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "aeadee6e",
       "metadata": {
         "id": "create_dataset:video,vcn"
       },
@@ -614,13 +702,14 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "9c55581d",
       "metadata": {
         "id": "create_dataset:video,vcn"
       },
       "outputs": [],
       "source": [
         "dataset = aiplatform.VideoDataset.create(\n",
-        "    display_name=\"MIT Human Motion\" + \"_\" + TIMESTAMP,\n",
+        "    display_name=\"MIT Human Motion\" + \"_\" + UUID,\n",
         "    gcs_source=[IMPORT_FILE],\n",
         "    import_schema_uri=aiplatform.schema.dataset.ioformat.video.classification,\n",
         ")\n",
@@ -630,6 +719,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "26f09f81",
       "metadata": {
         "id": "create_automl_pipeline:video,vcn"
       },
@@ -652,13 +742,14 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "9f35d88f",
       "metadata": {
         "id": "create_automl_pipeline:video,vcn"
       },
       "outputs": [],
       "source": [
         "job = aiplatform.AutoMLVideoTrainingJob(\n",
-        "    display_name=\"hmdb_\" + TIMESTAMP,\n",
+        "    display_name=\"hmdb_\" + UUID,\n",
         "    prediction_type=\"classification\",\n",
         ")\n",
         "\n",
@@ -667,6 +758,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "6bbaaf5f",
       "metadata": {
         "id": "run_automl_pipeline:video"
       },
@@ -688,6 +780,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "4b3f2c56",
       "metadata": {
         "id": "run_automl_pipeline:video"
       },
@@ -695,7 +788,7 @@
       "source": [
         "model = job.run(\n",
         "    dataset=dataset,\n",
-        "    model_display_name=\"hmdb_\" + TIMESTAMP,\n",
+        "    model_display_name=\"hmdb_\" + UUID,\n",
         "    training_fraction_split=0.8,\n",
         "    test_fraction_split=0.2,\n",
         ")"
@@ -703,6 +796,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "6d9e9f29",
       "metadata": {
         "id": "evaluate_the_model:mbsdk"
       },
@@ -710,35 +804,38 @@
         "## Review model evaluation scores\n",
         "After your model has finished training, you can review the evaluation scores for it.\n",
         "\n",
-        "First, you need to get a reference to the new model. As with datasets, you can either use the reference to the model variable you created when you deployed the model or you can list all of the models in your project."
+        "You can check the model's evaluation results using the `get_model_evaluation` method of the Vertex AI Model resource.\n",
+        "\n",
+        "Just like Vertex AI datasets, you can either use the reference to the model variable you created when you deployed the model or you can filter from the list of all of the models in your project using the model's display name as given below."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "59a76fa5",
       "metadata": {
         "id": "evaluate_the_model:mbsdk"
       },
       "outputs": [],
       "source": [
-        "# Get model resource ID\n",
-        "models = aiplatform.Model.list(filter=\"display_name=hmdb_\" + TIMESTAMP)\n",
+        "# Get model resource ID using the display_name\n",
+        "models = aiplatform.Model.list(filter=\"display_name=hmdb_\" + UUID)\n",
         "\n",
-        "# Get a reference to the Model Service client\n",
-        "client_options = {\"api_endpoint\": f\"{REGION}-aiplatform.googleapis.com\"}\n",
-        "model_service_client = aiplatform.gapic.ModelServiceClient(\n",
-        "    client_options=client_options\n",
-        ")\n",
+        "if len(models) != 0:\n",
         "\n",
-        "model_evaluations = model_service_client.list_model_evaluations(\n",
-        "    parent=models[0].resource_name\n",
-        ")\n",
-        "model_evaluation = list(model_evaluations)[0]\n",
-        "print(model_evaluation)"
+        "    # Get the model object\n",
+        "    model_rsc_name = models[0].resource_name\n",
+        "    print(\"Model resource name:\", model_rsc_name)\n",
+        "    model = aiplatform.Model(model_rsc_name)\n",
+        "\n",
+        "    # Print the model evaluation\n",
+        "    model_eval = model.get_model_evaluation()\n",
+        "    print(model_eval.to_dict())"
       ]
     },
     {
       "cell_type": "markdown",
+      "id": "060d3bae",
       "metadata": {
         "id": "make_prediction"
       },
@@ -750,18 +847,20 @@
     },
     {
       "cell_type": "markdown",
+      "id": "e614b9bf",
       "metadata": {
         "id": "get_test_items:batch_prediction"
       },
       "source": [
         "### Get test item(s)\n",
         "\n",
-        "Now do a batch prediction to your Vertex model. You will use arbitrary examples out of the dataset as a test items. Don't be concerned that the examples were likely used in training the model -- we just want to demonstrate how to make a prediction."
+        "Now do a batch prediction to your Vertex AI model. You use arbitrary examples from the dataset as a test items. Don't be concerned that the examples were likely used in training the model as this tutorial is just about how to make a batch prediction."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "bae97d10",
       "metadata": {
         "id": "get_test_items:automl,vcn,csv"
       },
@@ -783,13 +882,14 @@
     },
     {
       "cell_type": "markdown",
+      "id": "54138ea2",
       "metadata": {
         "id": "make_batch_file:automl,video"
       },
       "source": [
         "### Make a batch input file\n",
         "\n",
-        "Now make a batch input file, which you store in your local Cloud Storage bucket. The batch input file can be either CSV or JSONL. You will use JSONL in this tutorial. For JSONL file, you make one dictionary entry per line for each video. The dictionary contains the key/value pairs:\n",
+        "Now make a batch input file, which you store in your local Cloud Storage bucket. The batch input file can be either CSV or JSONL. You use JSONL in this tutorial. For JSONL file, you make one dictionary entry per line for each video. The dictionary contains the key/value pairs:\n",
         "\n",
         "- `content`: The Cloud Storage path to the video.\n",
         "- `mimeType`: The content type. In our example, it is a `avi` file.\n",
@@ -800,6 +900,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "ce7da5dd",
       "metadata": {
         "id": "make_batch_file:automl,video"
       },
@@ -810,7 +911,7 @@
         "from google.cloud import storage\n",
         "\n",
         "test_filename = \"test.jsonl\"\n",
-        "gcs_input_uri = BUCKET_NAME + \"/\" + test_filename\n",
+        "gcs_input_uri = BUCKET_URI + \"/\" + test_filename\n",
         "\n",
         "data_1 = {\n",
         "    \"content\": test_item_1,\n",
@@ -826,42 +927,66 @@
         "}\n",
         "\n",
         "\n",
-        "bucket = storage.Client(project=PROJECT_ID).bucket(BUCKET_NAME.replace(\"gs://\", \"\"))\n",
+        "bucket = storage.Client(project=PROJECT_ID).bucket(BUCKET_NAME)\n",
         "blob = bucket.blob(blob_name=test_filename)\n",
         "data = json.dumps(data_1) + \"\\n\" + json.dumps(data_2) + \"\\n\"\n",
         "blob.upload_from_string(data)\n",
-        "print(gcs_input_uri)\n",
+        "print(gcs_input_uri)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "5bbefe4a-e05f-4ed7-acf8-a0588757c376",
+      "metadata": {
+        "id": "d56366168ec5"
+      },
+      "source": [
+        "### Check input content\n",
+        "Check the contents of the `test.jsonl`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "a98d1c39-29f2-40c7-8267-91afebb8a440",
+      "metadata": {
+        "id": "378131e21a7e"
+      },
+      "outputs": [],
+      "source": [
         "! gsutil cat $gcs_input_uri"
       ]
     },
     {
       "cell_type": "markdown",
+      "id": "105f3bc5",
       "metadata": {
         "id": "batch_request:mbsdk"
       },
       "source": [
         "### Make the batch prediction request\n",
         "\n",
-        "Now that your Model resource is trained, you can make a batch prediction by invoking the batch_predict() method, with the following parameters:\n",
+        "Now that your Vertex AI Model resource is trained, you can make a batch prediction by invoking the batch_predict() method, with the following parameters:\n",
         "\n",
         "- `job_display_name`: The human readable name for the batch prediction job.\n",
         "- `gcs_source`: A list of one or more batch request input files.\n",
         "- `gcs_destination_prefix`: The Cloud Storage location for storing the batch prediction resuls.\n",
-        "- `sync`: If set to True, the call will block while waiting for the asynchronous batch job to complete."
+        "- `sync`: If set to True, the call blocks while waiting for the asynchronous batch job to complete."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "5657e704",
       "metadata": {
         "id": "batch_request:mbsdk"
       },
       "outputs": [],
       "source": [
         "batch_predict_job = model.batch_predict(\n",
-        "    job_display_name=\"hmdb_\" + TIMESTAMP,\n",
+        "    job_display_name=\"hmdb_\" + UUID,\n",
         "    gcs_source=gcs_input_uri,\n",
-        "    gcs_destination_prefix=BUCKET_NAME,\n",
+        "    gcs_destination_prefix=BUCKET_URI,\n",
         "    sync=False,\n",
         ")\n",
         "\n",
@@ -870,6 +995,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "c86ec9ec",
       "metadata": {
         "id": "batch_request_wait:mbsdk"
       },
@@ -882,6 +1008,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "2f108cc8",
       "metadata": {
         "id": "batch_request_wait:mbsdk"
       },
@@ -892,6 +1019,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "63e33110",
       "metadata": {
         "id": "get_batch_prediction:mbsdk,vcn"
       },
@@ -914,6 +1042,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "a76f3f2c",
       "metadata": {
         "id": "get_batch_prediction:mbsdk,vcn"
       },
@@ -928,7 +1057,7 @@
         "\n",
         "for prediction_result in prediction_results:\n",
         "    gfile_name = f\"gs://{bp_iter_outputs.bucket.name}/{prediction_result}\".replace(\n",
-        "        BUCKET_NAME + \"/\", \"\"\n",
+        "        BUCKET_URI + \"/\", \"\"\n",
         "    )\n",
         "    data = bucket.get_blob(gfile_name).download_as_string()\n",
         "    data = json.loads(data)\n",
@@ -937,11 +1066,12 @@
     },
     {
       "cell_type": "markdown",
+      "id": "000413e5",
       "metadata": {
         "id": "cleanup:mbsdk"
       },
       "source": [
-        "# Cleaning up\n",
+        "## Clean up\n",
         "\n",
         "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
         "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
@@ -958,6 +1088,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "7761ab4d",
       "metadata": {
         "id": "cleanup:mbsdk"
       },
@@ -979,8 +1110,8 @@
         "batch_predict_job.delete()\n",
         "\n",
         "# Delete the Cloud storage bucket\n",
-        "if delete_bucket is True:\n",
-        "    ! gsutil rm -r $BUCKET_NAME"
+        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "    ! gsutil rm -r $BUCKET_URI"
       ]
     }
   ],

notebooks/official/custom/custom-tabular-bq-managed-dataset.ipynb

@@ -33,12 +33,12 @@
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/official/custom/custom-tabular-bq-managed-dataset.ipynb\">\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/custom/custom-tabular-bq-managed-dataset.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/official/custom/custom-tabular-bq-managed-dataset.ipynb\">\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/custom/custom-tabular-bq-managed-dataset.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
@@ -64,17 +64,6 @@
         "This tutorial demonstrates how to use the Vertex AI SDK for Python to train and deploy a custom tabular classification model for online prediction."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "dataset:custom,cifar10,icn"
-      },
-      "source": [
-        "### Dataset\n",
-        "\n",
-        "The dataset used for this tutorial is the penguins dataset from [BigQuery public datasets](https://cloud.google.com/bigquery/public-data). In this version of the dataset, you will use only the fields `culmen_length_mm`, `culmen_depth_mm`, `flipper_length_mm`, `body_mass_g` to predict the penguins species (`species`)."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -83,7 +72,15 @@
       "source": [
         "### Objective\n",
         "\n",
-        "In this notebook, you create a custom-trained model from a Python script in a Docker container using the Vertex SDK for Python, and then get a prediction from the deployed model by sending data. Alternatively, you can create custom-trained models using `gcloud` command-line tool, or online using the Cloud Console.\n",
+        "In this notebook, you create a custom-trained model from a Python script in a Docker container using the Vertex AI SDK for Python, and then get a prediction from the deployed model by sending data. Alternatively, you can create custom-trained models using `gcloud` command-line tool, or online using the Cloud Console.\n",
+        "\n",
+        "This tutorial uses the following Google Cloud ML services and resources:\n",
+        "\n",
+        "- BigQuery\n",
+        "- Cloud Storage\n",
+        "- Vertex AI managed Datasets\n",
+        "- Vertex AI Training\n",
+        "- Vertex AI Endpoints\n",
         "\n",
         "The steps performed include:\n",
         "\n",
@@ -94,6 +91,17 @@
         "- Undeploy the `Model` resource."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dataset:custom,cifar10,icn"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the penguins dataset from [BigQuery public datasets](https://cloud.google.com/bigquery/public-data). For this tutorial, you use only the fields `culmen_length_mm`, `culmen_depth_mm`, `flipper_length_mm`, `body_mass_g` from the dataset to predict the penguins species (`species`)."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -106,14 +114,64 @@
         "\n",
         "* Vertex AI\n",
         "* Cloud Storage\n",
+        "* BigQuery\n",
         "\n",
         "Learn about [Vertex AI\n",
-        "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
-        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
+        "pricing](https://cloud.google.com/vertex-ai/pricing), [Cloud Storage\n",
+        "pricing](https://cloud.google.com/storage/pricing), [BigQuery pricing](https://cloud.google.com/bigquery/pricing) and use the [Pricing\n",
         "Calculator](https://cloud.google.com/products/calculator/)\n",
         "to generate a cost estimate based on your projected usage."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "384b53dfdb54"
+      },
+      "source": [
+        "### Set up your local development environment\n",
+        "\n",
+        "**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets\n",
+        "all the requirements to run this notebook. You can skip this step."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "7e689ee0bc3c"
+      },
+      "source": [
+        "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
+        "You need the following:\n",
+        "\n",
+        "* The Google Cloud SDK\n",
+        "* Git\n",
+        "* Python 3\n",
+        "* virtualenv\n",
+        "* Jupyter notebook running in a virtual environment with Python 3\n",
+        "\n",
+        "The Google Cloud guide to [Setting up a Python development\n",
+        "environment](https://cloud.google.com/python/setup) and the [Jupyter\n",
+        "installation guide](https://jupyter.org/install) provide detailed instructions\n",
+        "for meeting these requirements. The following steps provide a condensed set of\n",
+        "instructions:\n",
+        "\n",
+        "1. [Install and initialize the Cloud SDK.](https://cloud.google.com/sdk/docs/)\n",
+        "\n",
+        "1. [Install Python 3.](https://cloud.google.com/python/setup#installing_python)\n",
+        "\n",
+        "1. [Install\n",
+        "   virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv)\n",
+        "   and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
+        "\n",
+        "1. To install Jupyter, run `pip3 install jupyter` on the\n",
+        "command-line in a terminal shell.\n",
+        "\n",
+        "1. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
+        "\n",
+        "1. Open this notebook in the Jupyter Notebook Dashboard."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -122,7 +180,7 @@
       "source": [
         "## Installation\n",
         "\n",
-        "Install the latest version of Vertex AI SDK for Python."
+        "Install the latest version of Cloud Storage, Bigquery and Vertex AI SDKs for Python."
       ]
     },
     {
@@ -135,64 +193,22 @@
       "source": [
         "import os\n",
         "\n",
-        "# The Google Cloud Notebook product has specific requirements\n",
-        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
         "\n",
-        "# Google Cloud Notebook requires dependencies to be installed with '--user'\n",
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
         "USER_FLAG = \"\"\n",
-        "if IS_GOOGLE_CLOUD_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\""
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "YsxCgt1zlugo"
-      },
-      "outputs": [],
-      "source": [
-        "! pip install {USER_FLAG} --upgrade google-cloud-aiplatform"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "install_storage"
-      },
-      "source": [
-        "Install the latest version of *google-cloud-storage* library as well."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "qssss-KSlugo"
-      },
-      "outputs": [],
-      "source": [
-        "! pip install {USER_FLAG} -U google-cloud-storage"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "Z3hYaR6gLEK-"
-      },
-      "source": [
-        "Install the latest version of *google-cloud-bigquery* library as well."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "866ffc5cf763"
-      },
-      "outputs": [],
-      "source": [
-        "! pip install {USER_FLAG} -U \"google-cloud-bigquery[all]\""
+        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\"\n",
+        "\n",
+        "# Install the packages\n",
+        "! pip3 install {USER_FLAG} --upgrade google-cloud-aiplatform \\\n",
+        "                                    google-cloud-storage \\\n",
+        "                                    google-cloud-bigquery \\\n",
+        "                                    pyarrow -q"
       ]
     },
     {
@@ -224,18 +240,21 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "a47846030fef"
+      },
+      "source": [
+        "## Before you begin"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "before_you_begin"
       },
       "source": [
-        "## Before you begin\n",
-        "\n",
-        "### Select a GPU runtime\n",
-        "\n",
-        "Make sure you're running this notebook in a GPU runtime if you have that option. In Colab, select \"Runtime --> Change runtime type > GPU\"\n",
-        "\n",
         "### Set up your Google Cloud project\n",
         "\n",
         "**The following steps are required, regardless of your notebook environment.**\n",
@@ -246,7 +265,7 @@
         "\n",
         "3. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com).\n",
         "\n",
-        "4. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "4. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
         "\n",
         "5. Enter your project ID in the cell below. Then run the cell to make sure the\n",
         "Cloud SDK uses the right project for all the commands in this notebook.\n",
@@ -269,40 +288,71 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "autoset_project_id"
+        "id": "3c8049930470"
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
-        "PROJECT_ID = \"\"\n",
-        "\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    # Get your Google Cloud project ID from gcloud\n",
-        "    shell_output = !gcloud config list --format 'value(core.project)' 2>/dev/null\n",
-        "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID: \", PROJECT_ID)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "set_project_id"
-      },
-      "source": [
-        "Otherwise, set your project ID here."
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "USd_pUT0lugr"
+        "id": "a36c4b991a39"
       },
       "outputs": [],
       "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
-        "    PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
+        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
+        "    # Get your GCP project id from gcloud\n",
+        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "    PROJECT_ID = shell_output[0]\n",
+        "    print(\"Project ID:\", PROJECT_ID)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "f2e3c0f2cbfb"
+      },
+      "outputs": [],
+      "source": [
+        "! gcloud config set project $PROJECT_ID"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "a54f9d7c1876"
+      },
+      "source": [
+        "#### Region\n",
+        "\n",
+        "You can also change the `REGION` variable, which is used for operations\n",
+        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
+        "\n",
+        "- Americas: `us-central1`\n",
+        "- Europe: `europe-west4`\n",
+        "- Asia Pacific: `asia-east1`\n",
+        "\n",
+        "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
+        "\n",
+        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3aaadaaf9b30"
+      },
+      "outputs": [],
+      "source": [
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -311,9 +361,9 @@
         "id": "timestamp"
       },
       "source": [
-        "#### Timestamp\n",
+        "#### UUID\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, create a timestamp for each instance session, and append it onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
       ]
     },
     {
@@ -324,9 +374,28 @@
       },
       "outputs": [],
       "source": [
-        "from datetime import datetime\n",
+        "import random\n",
+        "import string\n",
         "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+        "\n",
+        "# Generate a uuid of length 8\n",
+        "def generate_uuid():\n",
+        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=8))\n",
+        "\n",
+        "\n",
+        "UUID = generate_uuid()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5c0404984792"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
+        "authenticated. Skip this step."
       ]
     },
     {
@@ -335,11 +404,6 @@
         "id": "gcp_authenticate"
       },
       "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Google Cloud Notebooks**, your environment is already\n",
-        "authenticated. Skip this step.\n",
-        "\n",
         "**If you are using Colab**, run the cell below and follow the instructions\n",
         "when prompted to authenticate your account via oAuth.\n",
         "\n",
@@ -372,19 +436,19 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "import sys\n",
-        "\n",
         "# If you are running this notebook in Colab, run this cell and follow the\n",
         "# instructions to authenticate your GCP account. This provides access to your\n",
         "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
         "# requests.\n",
         "\n",
-        "# The Google Cloud Notebook product has specific requirements\n",
-        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
+        "import os\n",
+        "import sys\n",
         "\n",
-        "# If on Google Cloud Notebooks, then don't execute this code\n",
-        "if not IS_GOOGLE_CLOUD_NOTEBOOK:\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = \"google.colab\" in sys.modules\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
         "    if \"google.colab\" in sys.modules:\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
@@ -415,12 +479,7 @@
         "online predictions.\n",
         "\n",
         "Set the name of your Cloud Storage bucket below. It must be unique across all\n",
-        "Cloud Storage buckets.\n",
-        "\n",
-        "You may also change the `REGION` variable, which is used for operations\n",
-        "throughout the rest of this notebook. Make sure to [choose a region where Vertex AI services are\n",
-        "available](https://cloud.google.com/vertex-ai/docs/general/locations#available_regions). You may\n",
-        "not use a Multi-Regional Storage bucket for training with Vertex AI."
+        "Cloud Storage buckets."
       ]
     },
     {
@@ -431,8 +490,8 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_URI = \"gs://[your-bucket-name]\"\n",
-        "REGION = \"[your-region]\"  # @param {type:\"string\"}"
+        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
     {
@@ -443,11 +502,9 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
+        "    BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
     {
@@ -467,7 +524,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_URI\n"
+        "! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
       ]
     },
     {
@@ -490,26 +547,13 @@
         "! gsutil ls -al $BUCKET_URI"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "setup_vars"
-      },
-      "source": [
-        "## Set up variables\n",
-        "\n",
-        "Next, set up some variables used throughout the tutorial."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "import_aip"
       },
       "source": [
-        "### Import Vertex AI SDK for Python\n",
-        "\n",
-        "Import the Vertex AI SDK for Python into your Python environment and initialize it."
+        "### Import libraries"
       ]
     },
     {
@@ -525,12 +569,55 @@
         "import sys\n",
         "\n",
         "import numpy as np\n",
-        "from google.cloud import aiplatform, bigquery\n",
-        "from google.cloud.aiplatform import gapic as aip\n",
+        "from google.cloud import aiplatform, bigquery"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "750d53e37094"
+      },
+      "source": [
+        "### Initialize Vertex AI SDK for Python\n",
         "\n",
+        "Initialize the Vertex AI SDK for Python for your project and corresponding bucket."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "c9d3ac73dfbc"
+      },
+      "outputs": [],
+      "source": [
+        "# Initialize the Vertex AI SDK\n",
         "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=BUCKET_URI)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "7c163842eabd"
+      },
+      "source": [
+        "### Initialize BigQuery Client\n",
+        "\n",
+        "Initialize the BigQuery Python client for your project."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "fad2ba1ad7c3"
+      },
+      "outputs": [],
+      "source": [
+        "# Set up BigQuery client\n",
+        "bqclient = bigquery.Client(project=PROJECT_ID)"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -560,9 +647,9 @@
       },
       "outputs": [],
       "source": [
-        "TRAIN_GPU, TRAIN_NGPU = (aip.AcceleratorType.NVIDIA_TESLA_K80, 1)\n",
+        "TRAIN_GPU, TRAIN_NGPU = (aiplatform.gapic.AcceleratorType.NVIDIA_TESLA_K80, 1)\n",
         "\n",
-        "DEPLOY_GPU, DEPLOY_NGPU = (aip.AcceleratorType.NVIDIA_TESLA_K80, 1)"
+        "DEPLOY_GPU, DEPLOY_NGPU = (aiplatform.gapic.AcceleratorType.NVIDIA_TESLA_K80, 1)"
       ]
     },
     {
@@ -657,17 +744,6 @@
         "Pass these summary statistics to the training script to normalize the data before training. Later, during prediction, use these summary statistics again to normalize the testing data."
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "890d562c6291"
-      },
-      "outputs": [],
-      "source": [
-        "BQ_SOURCE = \"bq://bigquery-public-data.ml_datasets.penguins\""
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -678,13 +754,15 @@
       "source": [
         "# Calculate mean and std across all rows\n",
         "\n",
+        "# Define the BigQuery source dataset\n",
+        "BQ_SOURCE = \"bq://bigquery-public-data.ml_datasets.penguins\"\n",
+        "\n",
+        "# Define NA values\n",
         "NA_VALUES = [\"NA\", \".\"]\n",
         "\n",
-        "# Set up BigQuery clients\n",
-        "bqclient = bigquery.Client(project=PROJECT_ID)\n",
-        "\n",
-        "\n",
         "# Download a table\n",
+        "\n",
+        "\n",
         "def download_table(bq_table_uri: str):\n",
         "    # Remove bq:// prefix if present\n",
         "    prefix = \"bq://\"\n",
@@ -740,9 +818,9 @@
         "id": "5c7732822757"
       },
       "source": [
-        "## Create a managed tabular dataset from BigQuery dataset\n",
+        "## Create a Vertex AI Tabular Dataset from BigQuery dataset\n",
         "\n",
-        "Your first step in training a model is to create a managed dataset instance."
+        "Your first step in training the model is to create a Vertex AI tabular dataset resource."
       ]
     },
     {
@@ -789,7 +867,7 @@
         "     - `\"single\"`: single device.\n",
         "     - `\"mirror\"`: all GPU devices on a single compute instance.\n",
         "     - `\"multi\"`: all GPU devices on all compute instances.\n",
-        "  - `\"--mean_and_std_json_file=\" + FILE_PATH`: The file on Google Cloud Storage with pre-calculated means and standard deviations."
+        "  - `\"--mean_and_std_json_file=\" + FILE_PATH`: The file on Cloud Storage with pre-calculated means and standard deviations."
       ]
     },
     {
@@ -800,7 +878,7 @@
       },
       "outputs": [],
       "source": [
-        "JOB_NAME = \"custom_job_\" + TIMESTAMP\n",
+        "JOB_NAME = \"custom_job_\" + UUID\n",
         "\n",
         "if not TRAIN_NGPU or TRAIN_NGPU < 2:\n",
         "    TRAIN_STRATEGY = \"single\"\n",
@@ -829,7 +907,7 @@
         "In the next cell, write the contents of the training script, `task.py`. In summary, the script does the following:\n",
         "\n",
         "- Loads the data from the BigQuery table using the BigQuery Python client library.\n",
-        "- Loads the pre-calculated mean and standard deviation from the Google Cloud Storage bucket.\n",
+        "- Loads the pre-calculated mean and standard deviation from the Cloud Storage bucket.\n",
         "- Builds a model using TF.Keras model API.\n",
         "- Compiles the model (`compile()`).\n",
         "- Sets a training distribution strategy according to the argument `args.distribute`.\n",
@@ -890,7 +968,7 @@
         "\n",
         "    # Construct a client side representation of a blob.\n",
         "    # Note `Bucket.blob` differs from `Bucket.get_blob` as it doesn't retrieve\n",
-        "    # any content from Google Cloud Storage. As we don't need additional data,\n",
+        "    # any content from Cloud Storage. As we don't need additional data,\n",
         "    # using `Bucket.blob` is preferred here.\n",
         "    blob = bucket.blob(source_blob_name)\n",
         "    blob.download_to_filename(destination_file_name)\n",
@@ -915,13 +993,13 @@
         "\n",
         "    Args:\n",
         "        gcs_path (str):\n",
-        "            Required. A full path to a Google Cloud Storage folder or resource.\n",
+        "            Required. A full path to a Cloud Storage folder or resource.\n",
         "            Can optionally include \"gs://\" prefix or end in a trailing slash \"/\".\n",
         "\n",
         "    Returns:\n",
         "        Tuple[str, Optional[str]]\n",
         "            A (bucket, prefix) pair from provided GCS path. If a prefix is not\n",
-        "            present, a None will be returned in its place.\n",
+        "            present, None is returned in its place.\n",
         "    \"\"\"\n",
         "    if gcs_path.startswith(\"gs://\"):\n",
         "        gcs_path = gcs_path[5:]\n",
@@ -999,7 +1077,6 @@
         "\n",
         "\n",
         "df_train = clean_dataframe(df_train)\n",
-        "#     df_validation = clean_dataframe(df_validation)\n",
         "df_validation = clean_dataframe(df_validation)\n",
         "\n",
         "_CATEGORICAL_TYPES = {\n",
@@ -1186,7 +1263,7 @@
         "    model_serving_container_image_uri=DEPLOY_IMAGE,\n",
         ")\n",
         "\n",
-        "MODEL_DISPLAY_NAME = \"penguins-\" + TIMESTAMP\n",
+        "MODEL_DISPLAY_NAME = \"penguins-\" + UUID\n",
         "\n",
         "# Start the training\n",
         "if TRAIN_GPU:\n",
@@ -1220,7 +1297,7 @@
       "source": [
         "### Deploy the model\n",
         "\n",
-        "Before you use your model to make predictions, you must deploy it to an `Endpoint`. You can do this by calling the `deploy` function on the `Model` resource. This will do two things:\n",
+        "Before you use your model to make predictions, you must deploy it to an `Endpoint`. You can do this by calling the `deploy` function on the `Model` resource. This does two things:\n",
         "\n",
         "1. Create an `Endpoint` resource for deploying the `Model` resource to.\n",
         "2. Deploy the `Model` resource to the `Endpoint` resource.\n",
@@ -1231,7 +1308,7 @@
         "- `deployed_model_display_name`: A human readable name for the deployed model.\n",
         "- `traffic_split`: Percent of traffic at the endpoint that goes to this model, which is specified as a dictionary of one or more key/value pairs.\n",
         "   - If only one model, then specify `{ \"0\": 100 }`, where \"0\" refers to this model being uploaded and 100 means 100% of the traffic.\n",
-        "   - If there are existing models on the endpoint, for which the traffic will be split, then use `model_id` to specify `{ \"0\": percent, model_id: percent, ... }`, where `model_id` is the ID of an existing `DeployedModel` on the endpoint. The percentages must add up to 100.\n",
+        "   - If there are existing models on the endpoint, for which the traffic is split, then use `model_id` to specify `{ \"0\": percent, model_id: percent, ... }`, where `model_id` is the ID of an existing `DeployedModel` on the endpoint. The percentages must add up to 100.\n",
         "- `machine_type`: The type of machine to use for training.\n",
         "- `accelerator_type`: The hardware accelerator type.\n",
         "- `accelerator_count`: The number of accelerators to attach to a worker replica.\n",
@@ -1252,7 +1329,7 @@
         "\n",
         "### Endpoint\n",
         "\n",
-        "The method will block until the model is deployed and eventually return an `Endpoint` object. If this is the first time a model is deployed to the endpoint, it may take a few additional minutes to complete provisioning of resources."
+        "The `deploy` method waits until the model is deployed and eventually returns an `Endpoint` object. If this is the first time a model is deployed to the endpoint, it may take a few additional minutes to complete provisioning of resources."
       ]
     },
     {
@@ -1263,7 +1340,7 @@
       },
       "outputs": [],
       "source": [
-        "DEPLOYED_NAME = \"penguins_deployed-\" + TIMESTAMP\n",
+        "DEPLOYED_NAME = \"penguins_deployed-\" + UUID\n",
         "\n",
         "TRAFFIC_SPLIT = {\"0\": 100}\n",
         "\n",
@@ -1427,7 +1504,7 @@
         "\n",
         "- `instances`: A list of penguin measurement instances. According to your custom model, each instance should be an array of numbers. You prepared this list in the previous step.\n",
         "\n",
-        "The `predict` function returns a list, where each element in the list corresponds to the an instance in the request. In the output for each prediction, you will see the following:\n",
+        "The `predict` function returns a list, where each element in the list corresponds to the an instance in the request. In the output for each prediction, you see the following:\n",
         "\n",
         "- Confidence level for the prediction (`predictions`), between 0 and 1, for each of the ten classes.\n",
         "\n",
@@ -1508,7 +1585,7 @@
       },
       "outputs": [],
       "source": [
-        "# Warning: Setting this to true will delete everything in your bucket\n",
+        "# Warning: Setting this to true deletes everything in your bucket\n",
         "delete_bucket = False\n",
         "\n",
         "# Delete the training job\n",

notebooks/official/experiments/build_model_experimentation_lineage_with_prebuild_code.ipynb

@@ -174,9 +174,8 @@
         "if IS_WORKBENCH_NOTEBOOK:\n",
         "    USER_FLAG = \"--user\"\n",
         "\n",
-        "! pip3 install {USER_FLAG} --upgrade fsspec gcsfs joblib -q\n",
-        "! pip3 install {USER_FLAG} --force-reinstall 'google-cloud-aiplatform>=1.15' -q\n",
-        "! pip3 install {USER_FLAG} --upgrade sklearn"
+        "! pip3 install {USER_FLAG} --upgrade joblib fsspec gcsfs scikit-learn -q\n",
+        "! pip3 install {USER_FLAG} --force-reinstall 'google-cloud-aiplatform>=1.15' -q"
       ]
     },
     {
@@ -232,7 +231,7 @@
         "\n",
         "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
         "\n",
-        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com).\n",
+        "1. [Enable APIs](https://console.cloud.google.com/flows/enableapi?apiid=cloudresourcemanager.googleapis.com,aiplatform.googleapis.com).\n",
         "\n",
         "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
         "\n",
@@ -393,9 +392,14 @@
         "3. In the **Service account name** field, enter a name, and\n",
         "   click **Create**.\n",
         "\n",
-        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
-        "into the filter box, and select\n",
-        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type and select\n",
+        "the following role into the filter box:\n",
+        "\n",
+        "    *   Storage Admin\n",
+        "    *   Storage Object Admin\n",
+        "    *   Service Account User\n",
+        "    *   Vertex AI Administrator\n",
+        "\n",
         "\n",
         "5. Click *Create*. A JSON file that contains your key downloads to your\n",
         "local environment.\n",
@@ -623,10 +627,13 @@
         "logger = logging.getLogger(\"logger\")\n",
         "logging.basicConfig(level=logging.INFO)\n",
         "\n",
+        "import collections\n",
         "import tempfile\n",
         "import time\n",
         "from json import dumps\n",
         "\n",
+        "collections.Iterable = collections.abc.Iterable\n",
+        "\n",
         "# Vertex AI\n",
         "from google.cloud import aiplatform as vertex_ai"
       ]
@@ -1240,6 +1247,17 @@
         "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
         "    ! gsutil -m rm -r $BUCKET_URI"
       ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3f00c455b930"
+      },
+      "outputs": [],
+      "source": [
+        "!rm -Rf {DATA_PATH}"
+      ]
     }
   ],
   "metadata": {

notebooks/official/experiments/comparing_local_trained_models.ipynb

@@ -56,26 +56,63 @@
       "metadata": {
         "id": "tvgnzT1CKxrO"
       },
+      "source": [
+        "# Compare trained and evaluated model experiments using Vertex AI Experiments"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3a0651225470"
+      },
       "source": [
         "## Overview\n",
         "\n",
-        "As a Data Scientist, you probably start running model experiments locally on your notebook. Depending on the framework you use, you would need to track parameters, training time series and evaluation metrics. In this way, you would be able to explain the modelling approach you would choose. \n",
-        "\n",
-        "### Dataset\n",
-        "\n",
-        "The dataset is the Tensorflow Dataset's Large Yelp Review Dataset. The Yelp reviews dataset consists of reviews from Yelp. For more information, please refer to this [link](http://www.yelp.com/dataset).\n",
-        "\n",
+        "As a Data Scientist, you probably start running model experiments locally on your notebook. Depending on the framework you use, you would need to track parameters, training time series and evaluation metrics. In this way, you would be able to explain the modelling approach you would choose. "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "f3021b2963a6"
+      },
+      "source": [
         "### Objective\n",
         "\n",
-        "In this notebook, you will learn how to \n",
+        "In this tutorial, you learn how to use Vertex AI Experiments to compare and evaluate model experiments.\n",
+        "\n",
+        "This tutorial uses the following Google Cloud ML services and resources:\n",
+        "\n",
+        "- Vertex AI Workbench\n",
+        "- Vertex AI Experiments\n",
+        "\n",
+        "The steps performed include:\n",
         "\n",
         "- log the model parameters\n",
         "- log the loss and metrics on every epoch to TensorBoard\n",
         "- log the evaluation metrics\n",
         "- compare two experiments\n",
         "\n",
-        "in Vertex AI Experiment of a recurrent neural network (RNN) for sentiment analysis.\n",
+        "in Vertex AI Experiment of a recurrent neural network (RNN) for sentiment analysis."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "20a5168cf05e"
+      },
+      "source": [
+        "### Dataset\n",
         "\n",
+        "The dataset is the Tensorflow Dataset's Large Yelp Review Dataset. The Yelp reviews dataset consists of reviews from Yelp. For more information, please refer to this [link](http://www.yelp.com/dataset).\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "de76bb18c85b"
+      },
+      "source": [
         "### Costs \n",
         "\n",
         "This tutorial uses billable components of Google Cloud:\n",
@@ -99,15 +136,8 @@
         "### Set up your local development environment\n",
         "\n",
         "**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets\n",
-        "all the requirements to run this notebook. You can skip this step."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "gCuSR8GkAgzl"
-      },
-      "source": [
+        "all the requirements to run this notebook. You can skip this step.\n",
+        "\n",
         "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
         "You need the following:\n",
         "\n",
@@ -169,19 +199,10 @@
         "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
         "USER_FLAG = \"\"\n",
         "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\""
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "wyy5Lbnzg5fi"
-      },
-      "outputs": [],
-      "source": [
+        "    USER_FLAG = \"--user\"\n",
+        "\n",
         "! pip3 install {USER_FLAG} --upgrade tensorflow==2.8.0 tensorflow_datasets==4.5.2 -q\n",
-        "! pip3 install --user --force-reinstall git+https://github.com/googleapis/python-aiplatform@main -q"
+        "! pip3 install --user --force-reinstall 'google-cloud-aiplatform>=1.15' -q"
       ]
     },
     {
@@ -376,15 +397,8 @@
         "### Authenticate your Google Cloud account\n",
         "\n",
         "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
-        "authenticated. Skip this step."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "sBCra4QMA2wR"
-      },
-      "source": [
+        "authenticated. Skip this step.\n",
+        "\n",
         "**If you are using Colab**, run the cell below and follow the instructions\n",
         "when prompted to authenticate your account via oAuth.\n",
         "\n",
@@ -398,9 +412,14 @@
         "3. In the **Service account name** field, enter a name, and\n",
         "   click **Create**.\n",
         "\n",
-        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
-        "into the filter box, and select\n",
-        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type and select\n",
+        "the following role into the filter box:\n",
+        "\n",
+        "    *   Storage Admin\n",
+        "    *   Storage Object Admin\n",
+        "    *   Service Account User\n",
+        "    *   Vertex AI Administrator\n",
+        "\n",
         "\n",
         "5. Click *Create*. A JSON file that contains your key downloads to your\n",
         "local environment.\n",
@@ -426,16 +445,11 @@
         "import sys\n",
         "\n",
         "# If on Vertex AI Workbench, then don't execute this code\n",
-        "\n",
-        "IS_COLAB = False\n",
-        "\n",
+        "IS_COLAB = \"google.colab\" in sys.modules\n",
         "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
         "    \"DL_ANACONDA_HOME\"\n",
         "):\n",
         "    if \"google.colab\" in sys.modules:\n",
-        "\n",
-        "        IS_COLAB = True\n",
-        "\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
         "        google_auth.authenticate_user()\n",
@@ -443,9 +457,7 @@
         "    # If you are running this notebook locally, replace the string below with the\n",
         "    # path to your service account key and run this cell to authenticate your GCP\n",
         "    # account.\n",
-        "\n",
         "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "\n",
         "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
       ]
     },
@@ -650,7 +662,15 @@
         "\n",
         "Vertex AI enables users to track the steps (for example, preprocessing, training) of an experiment run, and track inputs (for example, algorithm, parameters, datasets) and outputs (for example, models, checkpoints, metrics) of those steps. \n",
         "\n",
-        "Below you have some example of how track experiments to train recurrent neural network for sentiment analysis."
+        "Below you have some example of how track experiments to train recurrent neural network for sentiment analysis. \n",
+        "\n",
+        "To simplify the code, here you have helper function to cover the following steps:\n",
+        "\n",
+        "- Collect training data\n",
+        "- Create text encoder\n",
+        "- Build a RNN as baseline model\n",
+        "- Build a LSTM as challenger model\n",
+        "- Train the model"
       ]
     },
     {
@@ -721,16 +741,16 @@
         "    return encoder\n",
         "\n",
         "\n",
-        "def get_model(encoder, model_params, role):\n",
+        "def get_baseline_model(encoder, model_params):\n",
         "    \"\"\"\n",
-        "    Returns a tf.keras.Model object for the model\n",
+        "    Returns a tf.keras.Model object for the model as baseline\n",
         "    Args:\n",
         "        encoder: A TextVectorization object for the encoder\n",
         "        model_params: A dictionary with model parameters\n",
-        "        role: A variable to set the role of model\n",
         "    Returns:\n",
         "        tf.keras.Model: A tf.keras.Model object for the model\n",
         "    \"\"\"\n",
+        "\n",
         "    model = tf.keras.Sequential()\n",
         "    model.add(encoder)\n",
         "    model.add(\n",
@@ -738,20 +758,46 @@
         "            input_dim=len(encoder.get_vocabulary()), output_dim=64, mask_zero=True\n",
         "        )\n",
         "    )\n",
-        "    if role == \"baseline\":\n",
-        "        model.add(tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(64)))\n",
-        "        model.add(tf.keras.layers.Dense(64, activation=\"relu\"))\n",
-        "        model.add(tf.keras.layers.Dense(1))\n",
-        "    else:\n",
-        "        model.add(\n",
-        "            tf.keras.layers.Bidirectional(\n",
-        "                tf.keras.layers.LSTM(64, return_sequences=True)\n",
-        "            )\n",
-        "        )\n",
-        "        model.add(tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(32)))\n",
-        "        model.add(tf.keras.layers.Dropout(0.5))\n",
-        "        model.add(tf.keras.layers.Dense(1))\n",
+        "    model.add(tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(64)))\n",
+        "    model.add(tf.keras.layers.Dense(64, activation=\"relu\"))\n",
+        "    model.add(tf.keras.layers.Dense(1))\n",
+        "    model.compile(\n",
+        "        loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),\n",
+        "        optimizer=tf.keras.optimizers.Adam(\n",
+        "            learning_rate=model_params[\"learning_rate\"],\n",
+        "            beta_1=model_params[\"beta_1\"],\n",
+        "            beta_2=model_params[\"beta_2\"],\n",
+        "            epsilon=model_params[\"epsilon\"],\n",
+        "        ),\n",
+        "        metrics=[\"accuracy\"],\n",
+        "    )\n",
+        "    return model\n",
         "\n",
+        "\n",
+        "def get_stacked_model(encoder, model_params):\n",
+        "    \"\"\"\n",
+        "    Returns a tf.keras.Model object for the model as challenger\n",
+        "        Args:\n",
+        "            encoder: A TextVectorization object for the encoder\n",
+        "            model_params: A dictionary with model parameters\n",
+        "        Returns:\n",
+        "            tf.keras.Model: A tf.keras.Model object for the model\n",
+        "    \"\"\"\n",
+        "\n",
+        "    model = tf.keras.Sequential()\n",
+        "    model.add(encoder)\n",
+        "    model.add(\n",
+        "        tf.keras.layers.Embedding(\n",
+        "            input_dim=len(encoder.get_vocabulary()), output_dim=64, mask_zero=True\n",
+        "        )\n",
+        "    )\n",
+        "    model.add(\n",
+        "        tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(64, return_sequences=True))\n",
+        "    )\n",
+        "    model.add(tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(32)))\n",
+        "    model.add(tf.keras.layers.Dense(64, activation=\"relu\"))\n",
+        "    model.add(tf.keras.layers.Dropout(0.5))\n",
+        "    model.add(tf.keras.layers.Dense(1))\n",
         "    model.compile(\n",
         "        loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),\n",
         "        optimizer=tf.keras.optimizers.Adam(\n",
@@ -806,7 +852,15 @@
       "source": [
         "#### Run experiment and evaluate experiment runs using `with` statement\n",
         "\n",
-        "This step would takes **10 min** approx. to finish.\n"
+        "This step would takes **10 min** approx. to finish. And it covers the following steps:\n",
+        "\n",
+        "- Initialize an experiment run\n",
+        "- Log the parameters associated to training data\n",
+        "- Log the parameters of the encoder\n",
+        "- Log the parameters of the model\n",
+        "- Train the model\n",
+        "- Log the metrics for each epochs\n",
+        "- Log the overall training metrics"
       ]
     },
     {
@@ -818,7 +872,7 @@
       "outputs": [],
       "source": [
         "# Experiment Settings ----------------------------------------------------------\n",
-        "ID_1 = \"run-1\"\n",
+        "RUN_ID_1 = \"run-1\"\n",
         "BUFFER_SIZE = 10000\n",
         "BATCH_SIZE = 64\n",
         "VOCAB_SIZE = 1000\n",
@@ -832,7 +886,7 @@
         "\n",
         "# Initialize the experiment\n",
         "logging.info(\"Initialize the experiment.\")\n",
-        "with vertex_ai.start_run(ID_1) as run:\n",
+        "with vertex_ai.start_run(RUN_ID_1) as run:\n",
         "\n",
         "    # Get the training and testing datasets\n",
         "    logging.info(\"Get the training and testing datasets.\")\n",
@@ -856,7 +910,7 @@
         "    logging.info(\"Get the model.\")\n",
         "    run.log_params({\"role\": ROLE})\n",
         "    model_params = {\"learning_rate\": LR, \"beta_1\": B_1, \"beta_2\": B_2, \"epsilon\": EPS}\n",
-        "    model = get_model(encoder=encoder, model_params=model_params, role=ROLE)\n",
+        "    model = get_baseline_model(encoder=encoder, model_params=model_params)\n",
         "    run.log_params(model_params)\n",
         "\n",
         "    # Train the model\n",
@@ -907,7 +961,7 @@
         "# Get experiment\n",
         "logging.info(\"Get experiment status.\")\n",
         "experiment_df = vertex_ai.get_experiment_df()\n",
-        "experiment_df"
+        "experiment_df.T"
       ]
     },
     {
@@ -953,17 +1007,17 @@
       "outputs": [],
       "source": [
         "# Experiment Settings ----------------------------------------------------------\n",
-        "ID_2 = \"run-2\"\n",
+        "RUN_ID_2 = \"run-2\"\n",
         "ROLE = \"stacked\"\n",
         "\n",
         "# Initialize the experiment\n",
         "logger.info(\"Initialize the experiment.\")\n",
-        "vertex_ai.start_run(ID_2)\n",
+        "vertex_ai.start_run(RUN_ID_2)\n",
         "\n",
         "# Get the model\n",
         "logging.info(\"Get the model.\")\n",
         "run.log_params({\"role\": ROLE})\n",
-        "model = get_model(encoder=encoder, model_params=model_params, role=ROLE)\n",
+        "model = get_stacked_model(encoder=encoder, model_params=model_params)\n",
         "vertex_ai.log_params(model_params)\n",
         "\n",
         "# Train the model\n",
@@ -1014,7 +1068,7 @@
         "# Get experiment\n",
         "logging.info(\"Get experiment status.\")\n",
         "experiment_df = vertex_ai.get_experiment_df()\n",
-        "experiment_df"
+        "experiment_df.T"
       ]
     },
     {
@@ -1026,10 +1080,10 @@
       "outputs": [],
       "source": [
         "# Get time series metrics\n",
-        "exp_run = vertex_ai.ExperimentRun(ID_2, experiment=EXPERIMENT_NAME)\n",
+        "exp_run = vertex_ai.ExperimentRun(RUN_ID_2, experiment=EXPERIMENT_NAME)\n",
         "logging.info(\"Get time series metrics.\")\n",
         "ts_runs_df = exp_run.get_time_series_data_frame()\n",
-        "print(ts_runs_df)"
+        "ts_runs_df"
       ]
     },
     {
@@ -1056,34 +1110,29 @@
       "source": [
         "# Delete experiment\n",
         "exp = vertex_ai.Experiment(EXPERIMENT_NAME)\n",
-        "exp.delete(delete_backing_tensorboard_runs=True)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "dde8937123d4"
-      },
-      "outputs": [],
-      "source": [
+        "exp.delete(delete_backing_tensorboard_runs=True)\n",
+        "\n",
         "# Delete Tensorboard\n",
-        "vertex_ai_tb.delete()"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "sx_vKniMq9ZX"
-      },
-      "outputs": [],
-      "source": [
+        "vertex_ai_tb.delete()\n",
+        "\n",
         "# Delete Cloud Storage objects that were created\n",
-        "delete_bucket = False\n",
+        "delete_bucket = True\n",
         "\n",
         "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
-        "    ! gsutil rm -rf {BUCKET_URI}"
+        "    ! gsutil rm -rf {BUCKET_URI}\n",
+        "\n",
+        "!rm -Rf $DATA_DIR $LOG_DIR"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "15fbfe47e022"
+      },
+      "outputs": [],
+      "source": [
+        "!rm -Rf $DATA_DIR $LOG_DIR"
       ]
     }
   ],

notebooks/official/experiments/comparing_pipeline_runs.ipynb

@@ -3,6 +3,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "ed0fca3f",
       "metadata": {
         "id": "ur8xi4C7S06n"
       },
@@ -25,6 +26,17 @@
     },
     {
       "cell_type": "markdown",
+      "id": "6e92def7-b4f0-4100-b981-82972665d19d",
+      "metadata": {
+        "id": "847715f095b5"
+      },
+      "source": [
+        "# Vertex AI: Comparing Pipeline Runs"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "ffada1ce",
       "metadata": {
         "id": "JAPoU8Sm5E6e"
       },
@@ -43,7 +55,7 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/experiments/comparing_pipeline_runs.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/experiments/comparing_pipeline_runs.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
@@ -53,19 +65,25 @@
     },
     {
       "cell_type": "markdown",
+      "id": "d118d181",
       "metadata": {
         "id": "tvgnzT1CKxrO"
       },
       "source": [
+        "# Compare pipeline runs with Vertex AI Experiments\n",
+        "\n",
         "## Overview\n",
         "\n",
-        "Depending on the model life cycle of your data science team, you would like to experiment and track training Pipeline runs and its associated parameters. Then, you would to compare runs of these Pipelines to each others in order to figure out which is the best configuration generates the model you will register in the Vertex AI Model Registry.\n",
-        "\n",
-        "### Dataset\n",
-        "\n",
-        "The dataset used for this tutorial is the [Iris dataset](https://www.tensorflow.org/datasets/catalog/iris) from [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/overview). This dataset does not require any feature engineering. The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket. The trained model predicts the type of Iris flower species from a class of three species: setosa, virginica, or versicolor.\n",
-        "\n",
-        "\n",
+        "Depending on the model life cycle of your data science team, you would like to experiment and track training Pipeline runs and its associated parameters. Then, you would to compare runs of these Pipelines to each others in order to figure out which is the best configuration generates the model you will register in the Vertex AI Model Registry."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "b6201ad0-af42-48fd-b03d-403fd235e268",
+      "metadata": {
+        "id": "d220917f1302"
+      },
+      "source": [
         "### Objective\n",
         "\n",
         "In this notebook, you will learn how to use Vertex AI Experiments to \n",
@@ -78,9 +96,28 @@
         "* Formalize a training component\n",
         "* Build a training pipeline\n",
         "* Run several Pipeline jobs and log their results\n",
-        "* Compare different Pipeline jobs\n",
-        "\n",
+        "* Compare different Pipeline jobs"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "cffa7608-f550-4913-8f88-30cbcd525685",
+      "metadata": {
+        "id": "263933842022"
+      },
+      "source": [
+        "### Dataset\n",
         "\n",
+        "The dataset used for this tutorial is the [Iris dataset](https://www.tensorflow.org/datasets/catalog/iris) from [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/overview). This dataset does not require any feature engineering. The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket. The trained model predicts the type of Iris flower species from a class of three species: setosa, virginica, or versicolor.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "b46c0eb6-e65d-4b28-b17d-dc9bf9b08120",
+      "metadata": {
+        "id": "de76bb18c85b"
+      },
+      "source": [
         "### Costs \n",
         "\n",
         "This tutorial uses billable components of Google Cloud:\n",
@@ -97,22 +134,16 @@
     },
     {
       "cell_type": "markdown",
+      "id": "ee1e6851",
       "metadata": {
-        "id": "ze4-nDLfK4pw"
+        "id": "gCuSR8GkAgzl"
       },
       "source": [
         "### Set up your local development environment\n",
         "\n",
         "**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets\n",
-        "all the requirements to run this notebook. You can skip this step."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "gCuSR8GkAgzl"
-      },
-      "source": [
+        "all the requirements to run this notebook. You can skip this step.\n",
+        "\n",
         "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
         "You need the following:\n",
         "\n",
@@ -146,6 +177,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "97e5b386",
       "metadata": {
         "id": "i7EUnXsZhAGF"
       },
@@ -158,6 +190,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "5b01540f",
       "metadata": {
         "id": "2b4ef9b72d43"
       },
@@ -176,13 +209,13 @@
         "if IS_WORKBENCH_NOTEBOOK:\n",
         "    USER_FLAG = \"--user\"\n",
         "\n",
-        "!pip3 install {USER_FLAG} --force-reinstall git+https://github.com/sasha-gitg/python-aiplatform@main -q\n",
-        "!pip3 install {USER_FLAG} google-cloud-aiplatform[metadata] -q\n",
-        "!pip3 install {USER_FLAG} kfp -q"
+        "!pip3 install {USER_FLAG} --force-reinstall 'google-cloud-aiplatform>=1.15' -q --no-warn-conflicts\n",
+        "!pip3 install {USER_FLAG} kfp -q --no-warn-conflicts"
       ]
     },
     {
       "cell_type": "markdown",
+      "id": "c6806ee8",
       "metadata": {
         "id": "hhq5zEbGg0XX"
       },
@@ -195,6 +228,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "584f0887",
       "metadata": {
         "id": "EzrelQZ22IZj"
       },
@@ -213,6 +247,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "bb098c06",
       "metadata": {
         "id": "lWEdiXsJg0XY"
       },
@@ -222,6 +257,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "3706598b",
       "metadata": {
         "id": "BF1j6f9HApxa"
       },
@@ -234,7 +270,7 @@
         "\n",
         "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
         "\n",
-        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com).\n",
+        "1. [Enable APIs](https://console.cloud.google.com/flows/enableapi?apiid=cloudresourcemanager.googleapis.com,aiplatform.googleapis.com).\n",
         "\n",
         "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
         "\n",
@@ -246,6 +282,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "c7f6a1ea",
       "metadata": {
         "id": "WReHDGG5g0XY"
       },
@@ -258,6 +295,19 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "6984e874-ae15-4094-9c5b-8d12661645c9",
+      "metadata": {
+        "id": "3c8049930470"
+      },
+      "outputs": [],
+      "source": [
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "ebaef2f9",
       "metadata": {
         "id": "oM1iC_MfAts1"
       },
@@ -265,17 +315,28 @@
       "source": [
         "import os\n",
         "\n",
-        "PROJECT_ID = \"\"\n",
-        "\n",
-        "# Get your Google Cloud project ID from gcloud\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    shell_output = !gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
+        "    # Get your GCP project id from gcloud\n",
+        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
         "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID: \", PROJECT_ID)"
+        "    print(\"Project ID:\", PROJECT_ID)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "d9745b2b-cd37-4a1c-aae7-4cd75a3c126c",
+      "metadata": {
+        "id": "f2e3c0f2cbfb"
+      },
+      "outputs": [],
+      "source": [
+        "! gcloud config set project $PROJECT_ID"
       ]
     },
     {
       "cell_type": "markdown",
+      "id": "fff3c4af",
       "metadata": {
         "id": "qJYoRfYng0XZ"
       },
@@ -286,17 +347,19 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "89a05131",
       "metadata": {
         "id": "riG_qUokg0XZ"
       },
       "outputs": [],
       "source": [
         "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
-        "    PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
+        "    PROJECT_ID = \"python-docs-samples-tests\"  # @param {type:\"string\"}"
       ]
     },
     {
       "cell_type": "markdown",
+      "id": "9aa4ad5f",
       "metadata": {
         "id": "2aa333eca058"
       },
@@ -318,6 +381,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "244d416e",
       "metadata": {
         "id": "d8b34ef9a3d0"
       },
@@ -331,46 +395,49 @@
     },
     {
       "cell_type": "markdown",
+      "id": "eed6c3ba",
       "metadata": {
-        "id": "06571eb4063b"
+        "id": "126548a06aa1"
       },
       "source": [
-        "#### Timestamp\n",
+        "#### UUID\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append it onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial.\n"
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "6ef7c7b1",
       "metadata": {
-        "id": "697568e92bd6"
+        "id": "e660b8504e63"
       },
       "outputs": [],
       "source": [
-        "from datetime import datetime\n",
+        "import random\n",
+        "import string\n",
         "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+        "\n",
+        "# Generate a uuid of a specifed length(default=8)\n",
+        "def generate_uuid(length: int = 8) -> str:\n",
+        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
+        "\n",
+        "\n",
+        "UUID = generate_uuid()"
       ]
     },
     {
       "cell_type": "markdown",
+      "id": "5763da2c",
       "metadata": {
-        "id": "dr--iN2kAylZ"
+        "id": "sBCra4QMA2wR"
       },
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
         "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
-        "authenticated. Skip this step."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "sBCra4QMA2wR"
-      },
-      "source": [
+        "authenticated. Skip this step.\n",
+        "\n",
         "**If you are using Colab**, run the cell below and follow the instructions\n",
         "when prompted to authenticate your account via oAuth.\n",
         "\n",
@@ -384,9 +451,14 @@
         "3. In the **Service account name** field, enter a name, and\n",
         "   click **Create**.\n",
         "\n",
-        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
-        "into the filter box, and select\n",
-        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type and select\n",
+        "the following role into the filter box:\n",
+        "\n",
+        "    *   Storage Admin\n",
+        "    *   Storage Object Admin\n",
+        "    *   Service Account User\n",
+        "    *   Vertex AI Administrator\n",
+        "\n",
         "\n",
         "5. Click *Create*. A JSON file that contains your key downloads to your\n",
         "local environment.\n",
@@ -398,6 +470,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "85b826d2",
       "metadata": {
         "id": "PyQmSRbKA8r-"
       },
@@ -412,16 +485,11 @@
         "import sys\n",
         "\n",
         "# If on Vertex AI Workbench, then don't execute this code\n",
-        "\n",
-        "IS_COLAB = False\n",
-        "\n",
+        "IS_COLAB = \"google.colab\" in sys.modules\n",
         "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
         "    \"DL_ANACONDA_HOME\"\n",
         "):\n",
         "    if \"google.colab\" in sys.modules:\n",
-        "\n",
-        "        IS_COLAB = True\n",
-        "\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
         "        google_auth.authenticate_user()\n",
@@ -429,14 +497,13 @@
         "    # If you are running this notebook locally, replace the string below with the\n",
         "    # path to your service account key and run this cell to authenticate your GCP\n",
         "    # account.\n",
-        "\n",
         "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "\n",
         "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
       ]
     },
     {
       "cell_type": "markdown",
+      "id": "3d31b8b8",
       "metadata": {
         "id": "zgPO1eR3CYjk"
       },
@@ -453,6 +520,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "95a18950",
       "metadata": {
         "id": "MzGDU7TWdts_"
       },
@@ -465,18 +533,20 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "379d758a",
       "metadata": {
         "id": "cf221059d072"
       },
       "outputs": [],
       "source": [
         "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
         "    BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
     {
       "cell_type": "markdown",
+      "id": "50b86adc",
       "metadata": {
         "id": "-EcIXiGsCePi"
       },
@@ -487,6 +557,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "564ce38f",
       "metadata": {
         "id": "NIq7R4HZCfIc"
       },
@@ -497,6 +568,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "a4324d2e",
       "metadata": {
         "id": "ucvCsknMCims"
       },
@@ -507,6 +579,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "df030498-f6e7-4e45-96f4-0d36590865aa",
       "metadata": {
         "id": "vhOb7YnwClBb"
       },
@@ -517,6 +590,88 @@
     },
     {
       "cell_type": "markdown",
+      "id": "49d12f54",
+      "metadata": {
+        "id": "b7e24e522bee"
+      },
+      "source": [
+        "#### Service Account\n",
+        "\n",
+        "**If you don't know your service account**, try to get your service account using `gcloud` command by executing the second cell below."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "8da829a1",
+      "metadata": {
+        "id": "77b01a1fdbb4"
+      },
+      "outputs": [],
+      "source": [
+        "SERVICE_ACCOUNT = \"[your-service-account]\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "f2c8c41d",
+      "metadata": {
+        "id": "121d7ca29426"
+      },
+      "outputs": [],
+      "source": [
+        "import sys\n",
+        "\n",
+        "IS_COLAB = \"google.colab\" in sys.modules\n",
+        "\n",
+        "if (\n",
+        "    SERVICE_ACCOUNT == \"\"\n",
+        "    or SERVICE_ACCOUNT is None\n",
+        "    or SERVICE_ACCOUNT == \"[your-service-account]\"\n",
+        "):\n",
+        "    # Get your service account from gcloud\n",
+        "    if not IS_COLAB:\n",
+        "        shell_output = !gcloud auth list 2>/dev/null\n",
+        "        SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
+        "\n",
+        "    if IS_COLAB:\n",
+        "        shell_output = ! gcloud projects describe  $PROJECT_ID\n",
+        "        project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n",
+        "        SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n",
+        "\n",
+        "    print(\"Service Account:\", SERVICE_ACCOUNT)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "f0552e61",
+      "metadata": {
+        "id": "aa175e2960ac"
+      },
+      "source": [
+        "#### Set service account access for Vertex AI Pipelines\n",
+        "\n",
+        "Run the following commands to grant your service account access to read and write pipeline artifacts in the bucket that you created in the previous step -- you only need to run these once per service account."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "abbfcad3",
+      "metadata": {
+        "id": "f88cb0488c08"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
+        "\n",
+        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "2689f52b",
       "metadata": {
         "id": "fXUqOdIaLbjf"
       },
@@ -527,27 +682,20 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "754e5f7b",
       "metadata": {
         "id": "9fYX14c0LfmU"
       },
       "outputs": [],
       "source": [
-        "DATASET_URI = \"gs://cloud-samples-data/ai-platform/iris\""
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "z5WFzPetLl3Y"
-      },
-      "outputs": [],
-      "source": [
+        "DATASET_URI = \"gs://cloud-samples-data/ai-platform/iris\"\n",
+        "\n",
         "!gsutil cp -r $DATASET_URI $BUCKET_URI"
       ]
     },
     {
       "cell_type": "markdown",
+      "id": "231e5499",
       "metadata": {
         "id": "XoEqT2Y4DJmf"
       },
@@ -558,6 +706,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "884b2d31",
       "metadata": {
         "id": "pRUOFELefqf1"
       },
@@ -576,12 +725,14 @@
         "import kfp.v2.dsl as dsl\n",
         "# Vertex AI\n",
         "from google.cloud import aiplatform as vertex_ai\n",
+        "from google.cloud.aiplatform_v1.types.pipeline_state import PipelineState\n",
         "from kfp.v2.dsl import Metrics, Model, Output, component"
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "20e96b6b",
       "metadata": {
         "id": "OAY0QKZD8qNP"
       },
@@ -590,9 +741,10 @@
         "# Experiments\n",
         "TASK = \"classification\"\n",
         "MODEL_TYPE = \"xgboost\"\n",
-        "EXPERIMENT_NAME = f\"{PROJECT_ID}-{TASK}-{MODEL_TYPE}-{TIMESTAMP}\"\n",
+        "EXPERIMENT_NAME = f\"{PROJECT_ID}-{TASK}-{MODEL_TYPE}-{UUID}\"\n",
         "\n",
         "# Pipeline\n",
+        "PIPELINE_TEMPLATE_FILE = \"pipeline.json\"\n",
         "PIPELINE_URI = f\"{BUCKET_URI}/pipelines\"\n",
         "TRAIN_URI = f\"{BUCKET_URI}/iris/iris_data.csv\"\n",
         "LABEL_URI = f\"{BUCKET_URI}/iris/iris_target.csv\"\n",
@@ -601,6 +753,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "3fb7c387",
       "metadata": {
         "id": "inR70nh38PeK"
       },
@@ -613,6 +766,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "6eb614be",
       "metadata": {
         "id": "Nz0nasrh8T3c"
       },
@@ -623,6 +777,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "01448e2a",
       "metadata": {
         "id": "container:training,prediction,xgboost"
       },
@@ -641,6 +796,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "ece742dc",
       "metadata": {
         "id": "XujRA5ueox9U"
       },
@@ -653,6 +809,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "32a2397a",
       "metadata": {
         "id": "t1NLYz1R-KWv"
       },
@@ -662,18 +819,20 @@
     },
     {
       "cell_type": "markdown",
+      "id": "64570881",
       "metadata": {
         "id": "jnfKxpj0-Z0H"
       },
       "source": [
         "Before you start running your pipeline experiments, you have to formalize your training as pipeline component.\n",
         "\n",
-        "To do that, you will use the `kfp.v2.dsl.component` decorator to convert your training task into a pipeline component. "
+        "To do that, you build the pipeline by using the `kfp.v2.dsl.component` decorator to convert your training task into a pipeline component. "
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "f612dbf2",
       "metadata": {
         "id": "jv_-vU46_eFN"
       },
@@ -812,25 +971,20 @@
     },
     {
       "cell_type": "markdown",
+      "id": "bf048b2a",
       "metadata": {
         "id": "U1UiTZhkVoFM"
       },
       "source": [
-        "## Build a pipeline"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "7ABYbPz5UmJQ"
-      },
-      "source": [
-        "### Define your workflow using Kubeflow Pipelines DSL package"
+        "## Build a pipeline\n",
+        "\n",
+        "Below code will perform creating pipelineJob in associated project."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "7684850a",
       "metadata": {
         "id": "9Gfr6pNLU-dB"
       },
@@ -853,6 +1007,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "cb6cae0b",
       "metadata": {
         "id": "RkfZ7qVAVjBO"
       },
@@ -863,6 +1018,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "c6b9ec3f",
       "metadata": {
         "id": "oYlLBGUSVibG"
       },
@@ -873,6 +1029,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "cc940f17",
       "metadata": {
         "id": "95vG4-zPWc0B"
       },
@@ -882,6 +1039,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "bb2b2eb4",
       "metadata": {
         "id": "ZNb6kZ2l5t-O"
       },
@@ -894,6 +1052,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "77314a6f",
       "metadata": {
         "id": "XPy0Jc8xXgpa"
       },
@@ -911,6 +1070,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "aee97ebf",
       "metadata": {
         "id": "G0hm1no_WY8o"
       },
@@ -920,7 +1080,7 @@
         "\n",
         "    job = vertex_ai.PipelineJob(\n",
         "        display_name=f\"{EXPERIMENT_NAME}-pipeline-run-{i}\",\n",
-        "        template_path=\"pipeline.json\",\n",
+        "        template_path=PIPELINE_TEMPLATE_FILE,\n",
         "        pipeline_root=PIPELINE_URI,\n",
         "        parameter_values={\n",
         "            \"train_uri\": TRAIN_URI,\n",
@@ -934,11 +1094,12 @@
     },
     {
       "cell_type": "markdown",
+      "id": "0ca08588",
       "metadata": {
         "id": "O8TV4q535c2M"
       },
       "source": [
-        "### Check Pipeline run states\n",
+        "### Check Pipeline run states\n",
         "\n",
         "Vertex AI SDK provides you `get_experiment_df` method to monitor the status of pipeline runs. You can use it either to return parameters and metrics of the Pipeline Runs in the Vertex AI Experiment or in combination with `get` method of `PipelineJob` to return the pipeline job in Vertex AI Pipeline.\n"
       ]
@@ -946,6 +1107,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "a65f2574",
       "metadata": {
         "id": "dlCEJKfH5xR7"
       },
@@ -955,9 +1117,20 @@
         "vertex_ai.get_experiment_df(EXPERIMENT_NAME)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "id": "f605666a-9f2a-479f-9948-6f2f29e4e76b",
+      "metadata": {
+        "id": "98c022ca36b4"
+      },
+      "source": [
+        "The pipeline runs in the Vertex AI Experiment will be monitored based on pipeline run status."
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "dc6661c5",
       "metadata": {
         "id": "FA9W85vs7LLD"
       },
@@ -965,7 +1138,7 @@
       "source": [
         "while True:\n",
         "    pipeline_experiments_df = vertex_ai.get_experiment_df(EXPERIMENT_NAME)\n",
-        "    if all(\n",
+        "    if any(\n",
         "        pipeline_state != \"COMPLETE\" for pipeline_state in pipeline_experiments_df.state\n",
         "    ):\n",
         "        print(\"Pipeline runs are still running...\")\n",
@@ -984,6 +1157,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "ef041ba2",
       "metadata": {
         "id": "ISsK9Msi-Kqs"
       },
@@ -992,12 +1166,13 @@
         "# Get the PipelineJob resource using the experiment run name\n",
         "pipeline_experiments_df = vertex_ai.get_experiment_df(EXPERIMENT_NAME)\n",
         "job = vertex_ai.PipelineJob.get(pipeline_experiments_df.run_name[0])\n",
-        "print(job.resource_name)\n",
-        "print(job._dashboard_uri())"
+        "print(\"Pipeline job name: \", job.resource_name)\n",
+        "print(\"Pipeline Run UI link: \", job._dashboard_uri())"
       ]
     },
     {
       "cell_type": "markdown",
+      "id": "02a718ab",
       "metadata": {
         "id": "TpV-iwP9qw9c"
       },
@@ -1013,42 +1188,36 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "e90fb0b1",
       "metadata": {
         "id": "6xbYQn5t5Noe"
       },
       "outputs": [],
       "source": [
         "# Delete the pipeline\n",
-        "for i in range(0, len(runs)):\n",
-        "    pipeline_job = vertex_ai.PipelineJob.get(pipeline_experiments_df.run_name[i])\n",
-        "    pipeline_job.delete()"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "eRC5iZOh_I8B"
-      },
-      "outputs": [],
-      "source": [
+        "while True:\n",
+        "    for i in range(0, len(runs)):\n",
+        "        pipeline_job = vertex_ai.PipelineJob.get(pipeline_experiments_df.run_name[i])\n",
+        "        if pipeline_job.state != PipelineState.PIPELINE_STATE_SUCCEEDED:\n",
+        "            print(\"Pipeline job is still running...\")\n",
+        "            time.sleep(60)\n",
+        "        else:\n",
+        "            print(\"Pipeline job is complete.\")\n",
+        "            pipeline_job.delete()\n",
+        "    break\n",
+        "\n",
         "# Delete experiment\n",
         "exp = vertex_ai.Experiment(EXPERIMENT_NAME)\n",
-        "exp.delete()"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "sx_vKniMq9ZX"
-      },
-      "outputs": [],
-      "source": [
-        "delete_bucket = False\n",
+        "exp.delete()\n",
         "\n",
+        "# Delete bucket\n",
+        "delete_bucket = False\n",
         "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
-        "    ! gsutil rm -rf {BUCKET_URI}"
+        "    ! gsutil rm -rf {BUCKET_URI}\n",
+        "\n",
+        "# Remove local files\n",
+        "\n",
+        "!rm {PIPELINE_TEMPLATE_FILE}"
       ]
     }
   ],

notebooks/official/explainable_ai/sdk_custom_image_classification_batch_explain.ipynb

@@ -29,22 +29,22 @@
         "id": "title"
       },
       "source": [
-        "# Vertex SDK: Custom training image classification model for batch prediction with explainabilty\n",
+        "# Vertex AI SDK: Custom training image classification model for batch prediction with explainabilty\n",
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_custom_image_classification_batch_explain.ipynb\">\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/explainable_ai/sdk_custom_image_classification_batch_explain.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_custom_image_classification_batch_explain.ipynb\">\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/explainable_ai/sdk_custom_image_classification_batch_explain.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/notebooks/official/explainable_ai/sdk_custom_image_classification_batch_explain.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/explainable_ai/sdk_custom_image_classification_batch_explain.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
@@ -65,17 +65,6 @@
         "This tutorial demonstrates how to use the Vertex AI SDK to train and deploy a custom image classification model for batch prediction with explanation."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "dataset:custom,cifar10,icn"
-      },
-      "source": [
-        "### Dataset\n",
-        "\n",
-        "The dataset used for this tutorial is the [CIFAR10 dataset](https://www.tensorflow.org/datasets/catalog/cifar10) from [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/overview). The version of the dataset you will use is built into TensorFlow. The trained model predicts which type of class an image is from ten classes: airplane, automobile, bird, cat, deer, dog, frog, horse, ship, truck."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -91,7 +80,7 @@
         "- `Vertex AI Training`\n",
         "- `Vertex AI Batch Prediction`\n",
         "- `Vertex Explainable AI`\n",
-        "- `Vertex AI Model` resource\n",
+        "- `Vertex AI Models`\n",
         "\n",
         "The steps performed include:\n",
         "\n",
@@ -102,6 +91,17 @@
         "- Make a batch prediction with explanations."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dataset:custom,cifar10,icn"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the [CIFAR10 dataset](https://www.tensorflow.org/datasets/catalog/cifar10) from [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/overview). The version of the dataset you use in this notebook is built into TensorFlow. The trained model predicts the class of the provided input image from 10 classes namely airplane, automobile, bird, cat, deer, dog, frog, horse, ship and truck."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -190,11 +190,11 @@
         "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG -q\n",
         "! pip3 install {USER_FLAG} --upgrade google-cloud-storage -q\n",
         "! pip3 install --upgrade tensorflow $USER_FLAG -q\n",
+        "! pip3 install --upgrade opencv-python-headless $USER_FLAG -q\n",
         "\n",
         "if os.getenv(\"IS_TESTING\"):\n",
         "    ! apt-get update && apt-get install -y python3-opencv-headless\n",
-        "    ! apt-get install -y libgl1-mesa-dev\n",
-        "    ! pip3 install --upgrade opencv-python-headless $USER_FLAG"
+        "    ! apt-get install -y libgl1-mesa-dev"
       ]
     },
     {
@@ -256,6 +256,17 @@
         "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$`."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5aee4379e8e5"
+      },
+      "source": [
+        "#### Set your project ID\n",
+        "\n",
+        "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -321,7 +332,10 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -330,9 +344,9 @@
         "id": "timestamp"
       },
       "source": [
-        "#### Timestamp\n",
+        "#### UUID\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
       ]
     },
     {
@@ -343,9 +357,16 @@
       },
       "outputs": [],
       "source": [
-        "from datetime import datetime\n",
+        "import random\n",
+        "import string\n",
         "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+        "\n",
+        "# Generate a uuid of length 8\n",
+        "def generate_uuid():\n",
+        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=8))\n",
+        "\n",
+        "\n",
+        "UUID = generate_uuid()"
       ]
     },
     {
@@ -426,7 +447,7 @@
         "\n",
         "**The following steps are required, regardless of your notebook environment.**\n",
         "\n",
-        "When you initialize the Vertex SDK for Python, you specify a Cloud Storage staging bucket. The staging bucket is where all the data associated with your dataset and model resources are retained across sessions.\n",
+        "When you initialize the Vertex AI SDK for Python, you specify a Cloud Storage staging bucket. The staging bucket is where all the data associated with your dataset and model resources are retained across sessions.\n",
         "\n",
         "Set the name of your Cloud Storage bucket below. Bucket names must be globally unique across all Google Cloud projects, including those outside of your organization."
       ]
@@ -439,7 +460,7 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
         "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
@@ -452,7 +473,7 @@
       "outputs": [],
       "source": [
         "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
         "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
       ]
     },
@@ -516,7 +537,7 @@
       },
       "outputs": [],
       "source": [
-        "import google.cloud.aiplatform as aip"
+        "from google.cloud import aiplatform"
       ]
     },
     {
@@ -538,7 +559,7 @@
       },
       "outputs": [],
       "source": [
-        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
+        "aiplatform.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
       ]
     },
     {
@@ -560,7 +581,7 @@
         "\n",
         "Learn more [here](https://cloud.google.com/vertex-ai/docs/general/locations#accelerators) hardware accelerator support for your region\n",
         "\n",
-        "*Note*: TF releases before 2.3 for GPU support will fail to load the custom model in this tutorial. It is a known issue and fixed in TF 2.3 -- which is caused by static graph ops that are generated in the serving function. If you encounter this issue on your own custom models, use a container image for TF 2.3 with GPU support."
+        "*Note*: TF releases before 2.3 for GPU support fail to load the custom model in this tutorial. It is a known issue and is fixed in TF 2.3 -- which is caused by static graph ops that are generated in the serving function. If you encounter this issue on your own custom models, use a container image for TF 2.3 with GPU support."
       ]
     },
     {
@@ -573,7 +594,7 @@
       "source": [
         "if os.getenv(\"IS_TESTING_TRAIN_GPU\"):\n",
         "    TRAIN_GPU, TRAIN_NGPU = (\n",
-        "        aip.gapic.AcceleratorType.NVIDIA_TESLA_K80,\n",
+        "        aiplatform.gapic.AcceleratorType.NVIDIA_TESLA_K80,\n",
         "        int(os.getenv(\"IS_TESTING_TRAIN_GPU\")),\n",
         "    )\n",
         "else:\n",
@@ -581,7 +602,7 @@
         "\n",
         "if os.getenv(\"IS_TESTING_DEPLOY_GPU\"):\n",
         "    DEPLOY_GPU, DEPLOY_NGPU = (\n",
-        "        aip.gapic.AcceleratorType.NVIDIA_TESLA_K80,\n",
+        "        aiplatform.gapic.AcceleratorType.NVIDIA_TESLA_K80,\n",
         "        int(os.getenv(\"IS_TESTING_DEPLOY_GPU\")),\n",
         "    )\n",
         "else:\n",
@@ -659,7 +680,7 @@
         "\n",
         "Next, set the machine type to use for training and prediction.\n",
         "\n",
-        "- Set the variables `TRAIN_COMPUTE` and `DEPLOY_COMPUTE` to configure  the compute resources for the VMs you will use for for training and prediction.\n",
+        "- Set the variables `TRAIN_COMPUTE` and `DEPLOY_COMPUTE` to configure  the compute resources for the VMs you're going to use for for training and prediction.\n",
         " - `machine type`\n",
         "     - `n1-standard`: 3.75GB of memory per vCPU.\n",
         "     - `n1-highmem`: 6.5GB of memory per vCPU\n",
@@ -722,7 +743,7 @@
         "\n",
         "#### Package layout\n",
         "\n",
-        "Before you start the training, you will look at how a Python package is assembled for a custom training job. When unarchived, the package contains the following directory/file layout.\n",
+        "Before you start the training, you look at how a Python package is assembled for a custom training job. When unarchived, the package contains the following directory/file layout.\n",
         "\n",
         "- PKG-INFO\n",
         "- README.md\n",
@@ -738,7 +759,7 @@
         "\n",
         "#### Package Assembly\n",
         "\n",
-        "In the following cells, you will assemble the training package."
+        "In the following cells, you create the training package."
       ]
     },
     {
@@ -952,8 +973,8 @@
       },
       "outputs": [],
       "source": [
-        "job = aip.CustomTrainingJob(\n",
-        "    display_name=\"cifar10_\" + TIMESTAMP,\n",
+        "job = aiplatform.CustomTrainingJob(\n",
+        "    display_name=\"cifar10_\" + UUID,\n",
         "    script_path=\"custom/trainer/task.py\",\n",
         "    container_uri=TRAIN_IMAGE,\n",
         "    requirements=[\"gcsfs==0.7.1\", \"tensorflow-datasets==4.4\"],\n",
@@ -988,7 +1009,7 @@
       },
       "outputs": [],
       "source": [
-        "MODEL_DIR = \"{}/{}\".format(BUCKET_NAME, TIMESTAMP)\n",
+        "MODEL_DIR = \"{}/{}\".format(BUCKET_URI, UUID)\n",
         "\n",
         "EPOCHS = 20\n",
         "STEPS = 100\n",
@@ -1094,9 +1115,9 @@
         "\n",
         "### Load evaluation data\n",
         "\n",
-        "You will load the CIFAR10 test (holdout) data from `tf.keras.datasets`, using the method `load_data()`. This returns the dataset as a tuple of two elements. The first element is the training data and the second is the test data. Each element is also a tuple of two elements: the image data, and the corresponding labels.\n",
+        "Load the CIFAR10 test (holdout) data from `tf.keras.datasets`, using the method `load_data()`. This returns the dataset as a tuple of two elements. The first element is the training data and the second is the test data. Each element is also a tuple of two elements: the image data, and the corresponding labels.\n",
         "\n",
-        "You don't need the training data, and hence why we loaded it as `(_, _)`.\n",
+        "You don't need the training data, and hence why it was loaded into `(_, _)`.\n",
         "\n",
         "Before you can run the data through evaluation, you need to preprocess it:\n",
         "\n",
@@ -1104,7 +1125,7 @@
         "1. Normalize (rescale) the pixel data by dividing each pixel by 255. This replaces each single byte integer pixel with a 32-bit floating point number between 0 and 1.\n",
         "\n",
         "`y_test`:<br/>\n",
-        "2. The labels are currently scalar (sparse). If you look back at the `compile()` step in the `trainer/task.py` script, you will find that it was compiled for sparse labels. So we don't need to do anything more."
+        "2. The labels are currently scalar (sparse). At the `compile()` step in the `trainer/task.py` script, it can be noticed that it was compiled for sparse labels already."
       ]
     },
     {
@@ -1238,7 +1259,7 @@
         "\n",
         "You can get the signatures of your model's input and output layers by reloading the model into memory, and querying it for the signatures corresponding to each layer.\n",
         "\n",
-        "When making a prediction request, you need to route the request to the serving function instead of the model, so you need to know the input layer name of the serving function -- which you will use later when you make a prediction request.\n",
+        "When making a prediction request, you need to route the request to the serving function instead of the model, so you need to know the input layer name of the serving function -- which you use later when you make a prediction request.\n",
         "\n",
         "You also need to know the name of the serving function's input and output layer for constructing the explanation metadata -- which is discussed subsequently."
       ]
@@ -1295,7 +1316,7 @@
         "\n",
         "Parameters:\n",
         "\n",
-        "- `path_count`: This is the number of paths over the features that will be processed by the algorithm. An exact approximation of the Shapley values requires M! paths, where M is the number of features. For the CIFAR10 dataset, this would be 784 (28*28).\n",
+        "- `path_count`: The number of paths over the features that are processed by the algorithm. An exact approximation of the Shapley values requires M! paths, where M is the number of features. For the CIFAR10 dataset, this would be 784 (28*28).\n",
         "\n",
         "For any non-trival number of features, this is too compute expensive. You can reduce the number of paths over the features to M * `path_count`.\n",
         "\n",
@@ -1323,7 +1344,7 @@
         "\n",
         "- `step_count`: This is the number of steps to approximate the remaining sum. The more steps, the more accurate the integral approximation. The general rule of thumb is 50 steps, but as you increase so does the compute time.\n",
         "\n",
-        "In the next code cell, set the variable `XAI` to which explainabilty algorithm you will use on your custom model."
+        "In the next code cell, set the variable `XAI` to which explainabilty algorithm you want to use on your custom model."
       ]
     },
     {
@@ -1343,7 +1364,7 @@
         "elif XAI == \"xrai\":\n",
         "    PARAMETERS = {\"xrai_attribution\": {\"step_count\": 50}}\n",
         "\n",
-        "parameters = aip.explain.ExplanationParameters(PARAMETERS)"
+        "parameters = aiplatform.explain.ExplanationParameters(PARAMETERS)"
       ]
     },
     {
@@ -1422,10 +1443,10 @@
         "\n",
         "OUTPUT_METADATA = {\"output_tensor_name\": serving_output}\n",
         "\n",
-        "input_metadata = aip.explain.ExplanationMetadata.InputMetadata(INPUT_METADATA)\n",
-        "output_metadata = aip.explain.ExplanationMetadata.OutputMetadata(OUTPUT_METADATA)\n",
+        "input_metadata = aiplatform.explain.ExplanationMetadata.InputMetadata(INPUT_METADATA)\n",
+        "output_metadata = aiplatform.explain.ExplanationMetadata.OutputMetadata(OUTPUT_METADATA)\n",
         "\n",
-        "metadata = aip.explain.ExplanationMetadata(\n",
+        "metadata = aiplatform.explain.ExplanationMetadata(\n",
         "    inputs={\"image\": input_metadata}, outputs={\"class\": output_metadata}\n",
         ")"
       ]
@@ -1458,8 +1479,8 @@
       },
       "outputs": [],
       "source": [
-        "model = aip.Model.upload(\n",
-        "    display_name=\"cifar10_\" + TIMESTAMP,\n",
+        "model = aiplatform.Model.upload(\n",
+        "    display_name=\"cifar10_\" + UUID,\n",
         "    artifact_uri=MODEL_DIR,\n",
         "    serving_container_image_uri=DEPLOY_IMAGE,\n",
         "    explanation_parameters=parameters,\n",
@@ -1478,7 +1499,7 @@
       "source": [
         "### Get test items\n",
         "\n",
-        "You will use examples out of the test (holdout) portion of the dataset as a test items."
+        "Use examples from the test (holdout) portion of the dataset as a test items."
       ]
     },
     {
@@ -1532,7 +1553,7 @@
       "source": [
         "### Copy test item(s)\n",
         "\n",
-        "For the batch prediction, you will copy the test items over to your Cloud Storage bucket."
+        "For the batch prediction, copy the test items over to your Cloud Storage bucket."
       ]
     },
     {
@@ -1543,11 +1564,11 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil cp tmp1.jpg $BUCKET_NAME/tmp1.jpg\n",
-        "! gsutil cp tmp2.jpg $BUCKET_NAME/tmp2.jpg\n",
+        "! gsutil cp tmp1.jpg $BUCKET_URI/tmp1.jpg\n",
+        "! gsutil cp tmp2.jpg $BUCKET_URI/tmp2.jpg\n",
         "\n",
-        "test_item_1 = BUCKET_NAME + \"/tmp1.jpg\"\n",
-        "test_item_2 = BUCKET_NAME + \"/tmp2.jpg\""
+        "test_item_1 = BUCKET_URI + \"/tmp1.jpg\"\n",
+        "test_item_2 = BUCKET_URI + \"/tmp2.jpg\""
       ]
     },
     {
@@ -1558,7 +1579,7 @@
       "source": [
         "### Make the batch input file\n",
         "\n",
-        "Now make a batch input file, which you will store in your local Cloud Storage bucket. The batch input file can only be in JSONL format. For JSONL file, you make one dictionary entry per line for each data item (instance). The dictionary contains the key/value pairs:\n",
+        "Now make a batch input file, which is then stored to your Cloud Storage bucket. The batch input file can only be in JSONL format. For JSONL file, you make one dictionary entry per line for each data item (instance). The dictionary contains the key/value pairs:\n",
         "\n",
         "- `input_name`: the name of the input layer of the underlying model.\n",
         "- `'b64'`: A key that indicates the content is base64 encoded.\n",
@@ -1568,7 +1589,7 @@
         "\n",
         "                        {serving_input: {'b64': content}}\n",
         "\n",
-        "To pass the image data to the prediction service you encode the bytes into base64 -- which makes the content safe from modification when transmitting binary data over the network.\n",
+        "To pass the image data to the prediction service you encode the bytes into base64. It makes the content safe from modification when transmitting binary data over the network.\n",
         "\n",
         "- `tf.io.read_file`: Read the compressed JPG images into memory as raw bytes.\n",
         "- `base64.b64encode`: Encode the raw bytes into a base64 encoded string."
@@ -1585,7 +1606,7 @@
         "import base64\n",
         "import json\n",
         "\n",
-        "gcs_input_uri = BUCKET_NAME + \"/\" + \"test.jsonl\"\n",
+        "gcs_input_uri = BUCKET_URI + \"/\" + \"test.jsonl\"\n",
         "with tf.io.gfile.GFile(gcs_input_uri, \"w\") as f:\n",
         "    bytes = tf.io.read_file(test_item_1)\n",
         "    b64str = base64.b64encode(bytes.numpy()).decode(\"utf-8\")\n",
@@ -1613,7 +1634,7 @@
         "- `instances_format`: The format for the input instances, either 'csv' or 'jsonl'. Defaults to 'jsonl'.\n",
         "- `predictions_format`: The format for the output predictions, either 'csv' or 'jsonl'. Defaults to 'jsonl'.\n",
         "- `machine_type`: The type of machine to use for training.\n",
-        "- `sync`: If set to True, the call will block while waiting for the asynchronous batch job to complete."
+        "- `sync`: Whether to execute the job synchronously. If False, the job executes in concurrent Future and any downstream object gets immediately returned and synced when the Future has completed."
       ]
     },
     {
@@ -1628,9 +1649,9 @@
         "MAX_NODES = 1\n",
         "\n",
         "batch_predict_job = model.batch_predict(\n",
-        "    job_display_name=\"cifar10_\" + TIMESTAMP,\n",
+        "    job_display_name=\"cifar10_\" + UUID,\n",
         "    gcs_source=gcs_input_uri,\n",
-        "    gcs_destination_prefix=BUCKET_NAME,\n",
+        "    gcs_destination_prefix=BUCKET_URI,\n",
         "    instances_format=\"jsonl\",\n",
         "    model_parameters=None,\n",
         "    machine_type=DEPLOY_COMPUTE,\n",
@@ -1717,7 +1738,9 @@
         "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
         "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
         "\n",
-        "Otherwise, you can delete the individual resources you created in this tutorial."
+        "Otherwise, you can delete the individual resources you created in this tutorial. \n",
+        "\n",
+        "Set `delete_bucket` to **True** to delete the Cloud Storage bucket."
       ]
     },
     {
@@ -1735,7 +1758,7 @@
         "\n",
         "\n",
         "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
-        "    ! gsutil rm -r $BUCKET_NAME"
+        "    ! gsutil rm -r $BUCKET_URI"
       ]
     }
   ],

notebooks/official/matching_engine/intro-swivel.ipynb

@@ -214,7 +214,7 @@
         "if not os.getenv(\"IS_TESTING\"):\n",
         "    # Automatically restart kernel after installs\n",
         "    import IPython\n",
-        "    \n",
+        "\n",
         "    app = IPython.Application.instance()\n",
         "    app.kernel.do_shutdown(True)"
       ]
@@ -675,6 +675,18 @@
         "!./swivel_template_configuration.sh -pipeline_suffix {YOUR_PIPELINE_SUFFIX} -project_number {PROJECT_NUMBER} -project_id {PROJECT_ID} -machine_type {MACHINE_TYPE} -accelerator_count {ACCELERATOR_COUNT} -accelerator_type {ACCELERATOR_TYPE} -pipeline_root {BUCKET}"
       ]
     },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "1cacea95d68c"
+      },
+      "outputs": [],
+      "source": [
+        "! sed \"s:\\t:    :g\" swivel_pipeline_basic.json >tmp.json\n",
+        "! mv tmp.json swivel_pipeline_basic.json"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -765,19 +777,15 @@
         "# Instantiate PipelineJob object\n",
         "pl = aiplatform.PipelineJob(\n",
         "    display_name=YOUR_PIPELINE_SUFFIX,\n",
-        "\n",
         "    # Whether or not to enable caching\n",
         "    # True = always cache pipeline step result\n",
         "    # False = never cache pipeline step result\n",
         "    # None = defer to cache option for each pipeline component in the pipeline definition\n",
         "    enable_caching=False,\n",
-        "\n",
         "    # Local or GCS path to a compiled pipeline definition\n",
         "    template_path=\"swivel_pipeline_basic.json\",\n",
-        "\n",
         "    # Dictionary containing input parameters for your pipeline\n",
         "    parameter_values=PARAMETER_VALUES,\n",
-        "\n",
         "    # GCS path to act as the pipeline root\n",
         "    pipeline_root=PIPELINE_ROOT,\n",
         ")\n",

notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb

@@ -32,13 +32,13 @@
         "<table align=\"left\">\n",
         "\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/matching_engine/sdk_matching_engine_for_indexing.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Run in Vertex Workbench\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/matching_engine/sdk_matching_engine_for_indexing.ipynb\">\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
@@ -95,8 +95,182 @@
         "id": "S5zc4kbEiYCm"
       },
       "source": [
-        "## Before you begin\n",
+        "## Before you begin"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d1e95a984673"
+      },
+      "source": [
+        "### Set up your Google Cloud project\n",
         "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager).\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI API and Compute Engine API, and Service Networking API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component,servicenetworking.googleapis.com).\n",
+        "\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2b9daa35336a"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using a Vertex AI Workbench notebook**, your environment is already\n",
+        "authenticated. Skip this step."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "c6bed8c6a6b3"
+      },
+      "source": [
+        "**If you are using Colab**, run the cell below and follow the instructions\n",
+        "when prompted to authenticate your account via oAuth.\n",
+        "\n",
+        "**Otherwise**, follow these steps:\n",
+        "\n",
+        "1. In the Cloud Console, go to the [**Create service account key**\n",
+        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
+        "\n",
+        "2. Click **Create service account**.\n",
+        "\n",
+        "3. In the **Service account name** field, enter a name, and\n",
+        "   click **Create**.\n",
+        "\n",
+        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
+        "into the filter box, and select\n",
+        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "\n",
+        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
+        "local environment.\n",
+        "\n",
+        "6. Enter the path to your service account key as the\n",
+        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3e2b43c2d2bf"
+      },
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "Your browser has been opened to visit:\n",
+            "\n",
+            "    https://accounts.google.com/o/oauth2/auth?response_type=code&client_id=32555940559.apps.googleusercontent.com&redirect_uri=http%3A%2F%2Flocalhost%3A8085%2F&scope=openid+https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fuserinfo.email+https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fcloud-platform+https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fappengine.admin+https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fsqlservice.login+https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fcompute+https%3A%2F%2Fwww.googleapis.com%2Fauth%2Faccounts.reauth&state=UY9jjYfhoSedWWUOWXp5Pmicq0Ic04&access_type=offline&code_challenge=OQefcewSwkT7ZwfzzOVidtngvZspdY1NgN6rltw8x7A&code_challenge_method=S256\n",
+            "\n"
+          ]
+        }
+      ],
+      "source": [
+        "import os\n",
+        "import sys\n",
+        "\n",
+        "# If you are running this notebook in Colab, run this cell and follow the\n",
+        "# instructions to authenticate your GCP account. This provides access to your\n",
+        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
+        "# requests.\n",
+        "\n",
+        "# The Vertex AI Workbench notebook product has specific requirements\n",
+        "IS_VERTEX_AI_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
+        "\n",
+        "# If on a Vertex AI Workbench notebook, then don't execute this code\n",
+        "if not IS_VERTEX_AI_WORKBENCH_NOTEBOOK:\n",
+        "    if \"google.colab\" in sys.modules:\n",
+        "        from google.colab import auth as google_auth\n",
+        "\n",
+        "        google_auth.authenticate_user()\n",
+        "\n",
+        "    # If you are running this notebook locally, log in using gcloud\n",
+        "    elif not os.getenv(\"IS_TESTING\"):\n",
+        "        ! gcloud auth login"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WReHDGG5g0XY"
+      },
+      "source": [
+        "#### Set your project ID\n",
+        "\n",
+        "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 1,
+      "metadata": {
+        "id": "beb72f394541"
+      },
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "Project ID:  python-docs-samples-tests\n"
+          ]
+        }
+      ],
+      "source": [
+        "import os\n",
+        "\n",
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# Get your Google Cloud project ID from gcloud\n",
+        "if not os.getenv(\"IS_TESTING\"):\n",
+        "    shell_output = !gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "    PROJECT_ID = shell_output[0]\n",
+        "    print(\"Project ID: \", PROJECT_ID)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "f4c6d0a9e66c"
+      },
+      "source": [
+        "Otherwise, set your project ID here."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "1dc3fa9ac4f7"
+      },
+      "outputs": [],
+      "source": [
+        "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
+        "    PROJECT_ID = \"<your_project_id>\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "4962667eec8e"
+      },
+      "source": [
         "* **Prepare a VPC network**.  To reduce any network overhead that might lead to unnecessary increase in overhead latency, it is best to call the ANN endpoints from your VPC via a direct [VPC Peering](https://cloud.google.com/vertex-ai/docs/general/vpc-peering) connection. \n",
         "  * The following section describes how to setup a VPC Peering connection if you don't have one. \n",
         "  * This is a one-time initial setup task. You can also reuse existing VPC network and skip this section."
@@ -110,9 +284,7 @@
       },
       "outputs": [],
       "source": [
-        "PROJECT_ID = \"python-docs-samples-tests\"  # @param {type:\"string\"}\n",
-        "\n",
-        "NETWORK_NAME = \"ann-vpc-network\"  # @param {type:\"string\"}\n",
+        "VPC_NETWORK = \"[your-vpc-network-name]\"  # @param {type:\"string\"}\n",
         "\n",
         "PEERING_RANGE_NAME = \"ann-haystack-range\""
       ]
@@ -125,24 +297,28 @@
       },
       "outputs": [],
       "source": [
-        "# Create a VPC network\n",
-        "! gcloud compute networks create {NETWORK_NAME} --bgp-routing-mode=regional --subnet-mode=auto --project={PROJECT_ID}\n",
+        "import os\n",
         "\n",
-        "# Add necessary firewall rules\n",
-        "! gcloud compute firewall-rules create {NETWORK_NAME}-allow-icmp --network {NETWORK_NAME} --priority 65534 --project {PROJECT_ID} --allow icmp\n",
+        "# Remove the if condition to run the encapsulated code\n",
+        "if not os.getenv(\"IS_TESTING\"):\n",
+        "    # Create a VPC network\n",
+        "    ! gcloud compute networks create {VPC_NETWORK} --bgp-routing-mode=regional --subnet-mode=auto --project={PROJECT_ID}\n",
         "\n",
-        "! gcloud compute firewall-rules create {NETWORK_NAME}-allow-internal --network {NETWORK_NAME} --priority 65534 --project {PROJECT_ID} --allow all --source-ranges 10.128.0.0/9\n",
+        "    # Add necessary firewall rules\n",
+        "    ! gcloud compute firewall-rules create {VPC_NETWORK}-allow-icmp --network {VPC_NETWORK} --priority 65534 --project {PROJECT_ID} --allow icmp\n",
         "\n",
-        "! gcloud compute firewall-rules create {NETWORK_NAME}-allow-rdp --network {NETWORK_NAME} --priority 65534 --project {PROJECT_ID} --allow tcp:3389\n",
+        "    ! gcloud compute firewall-rules create {VPC_NETWORK}-allow-internal --network {VPC_NETWORK} --priority 65534 --project {PROJECT_ID} --allow all --source-ranges 10.128.0.0/9\n",
         "\n",
-        "! gcloud compute firewall-rules create {NETWORK_NAME}-allow-ssh --network {NETWORK_NAME} --priority 65534 --project {PROJECT_ID} --allow tcp:22\n",
+        "    ! gcloud compute firewall-rules create {VPC_NETWORK}-allow-rdp --network {VPC_NETWORK} --priority 65534 --project {PROJECT_ID} --allow tcp:3389\n",
         "\n",
-        "# Reserve IP range\n",
-        "! gcloud compute addresses create {PEERING_RANGE_NAME} --global --prefix-length=16 --network={NETWORK_NAME} --purpose=VPC_PEERING --project={PROJECT_ID} --description=\"peering range\"\n",
+        "    ! gcloud compute firewall-rules create {VPC_NETWORK}-allow-ssh --network {VPC_NETWORK} --priority 65534 --project {PROJECT_ID} --allow tcp:22\n",
         "\n",
-        "# Set up peering with service networking\n",
-        "# Your account must have the \"Compute Network Admin\" role to run the following.\n",
-        "! gcloud services vpc-peerings connect --service=servicenetworking.googleapis.com --network={NETWORK_NAME} --ranges={PEERING_RANGE_NAME} --project={PROJECT_ID}"
+        "    # Reserve IP range\n",
+        "    ! gcloud compute addresses create {PEERING_RANGE_NAME} --global --prefix-length=16 --network={VPC_NETWORK} --purpose=VPC_PEERING --project={PROJECT_ID} --description=\"peering range\"\n",
+        "\n",
+        "    # Set up peering with service networking\n",
+        "    # Your account must have the \"Compute Network Admin\" role to run the following.\n",
+        "    ! gcloud services vpc-peerings connect --service=servicenetworking.googleapis.com --network={VPC_NETWORK} --ranges={PEERING_RANGE_NAME} --project={PROJECT_ID}"
       ]
     },
     {
@@ -230,9 +406,6 @@
       },
       "outputs": [],
       "source": [
-        "# Automatically restart kernel after installs\n",
-        "import os\n",
-        "\n",
         "if not os.getenv(\"IS_TESTING\"):\n",
         "    # Automatically restart kernel after installs\n",
         "    import IPython\n",
@@ -241,88 +414,15 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "BF1j6f9HApxa"
-      },
-      "source": [
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager).\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI API and Compute Engine API, and Service Networking API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component,servicenetworking.googleapis.com).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "WReHDGG5g0XY"
-      },
-      "source": [
-        "#### Set your project ID\n",
-        "\n",
-        "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "oM1iC_MfAts1"
-      },
-      "outputs": [],
-      "source": [
-        "import os\n",
-        "\n",
-        "PROJECT_ID = \"python-docs-samples-tests\"\n",
-        "\n",
-        "# Get your Google Cloud project ID from gcloud\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    shell_output = !gcloud config list --format 'value(core.project)' 2>/dev/null\n",
-        "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID: \", PROJECT_ID)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "qJYoRfYng0XZ"
-      },
-      "source": [
-        "Otherwise, set your project ID here."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "riG_qUokg0XZ"
-      },
-      "outputs": [],
-      "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
-        "    PROJECT_ID = \"<your_project_id>\"  # @param {type:\"string\"}"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "q7tcBkCDI1_M"
       },
       "source": [
-        "#### Timestamp\n",
+        "### Random ID\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append it onto the name of resources you create in this tutorial."
+        "To avoid name collisions between users on resources created, create a random ID for each instance session, and append the id onto the name of resources you create in this tutorial."
       ]
     },
     {
@@ -333,84 +433,10 @@
       },
       "outputs": [],
       "source": [
-        "from datetime import datetime\n",
+        "import random\n",
+        "import string\n",
         "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "t6Ggbb4DI6by"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using a Vertex AI Workbench notebook**, your environment is already\n",
-        "authenticated. Skip this step."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "RpIzUmpOI9G7"
-      },
-      "source": [
-        "**If you are using Colab**, run the cell below and follow the instructions\n",
-        "when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "1. In the Cloud Console, go to the [**Create service account key**\n",
-        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
-        "\n",
-        "2. Click **Create service account**.\n",
-        "\n",
-        "3. In the **Service account name** field, enter a name, and\n",
-        "   click **Create**.\n",
-        "\n",
-        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
-        "into the filter box, and select\n",
-        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
-        "local environment.\n",
-        "\n",
-        "6. Enter the path to your service account key as the\n",
-        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "AW9vQHeoI-q_"
-      },
-      "outputs": [],
-      "source": [
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "# The Vertex AI Workbench notebook product has specific requirements\n",
-        "IS_VERTEX_AI_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# If on a Vertex AI Workbench notebook, then don't execute this code\n",
-        "if not IS_VERTEX_AI_WORKBENCH_NOTEBOOK:\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, log in using gcloud\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        ! gcloud auth login"
+        "RANDOM_ID = \"\".join(random.choices(string.ascii_lowercase + string.digits, k=8))"
       ]
     },
     {
@@ -453,7 +479,7 @@
       "outputs": [],
       "source": [
         "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP\n",
+        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + RANDOM_ID\n",
         "\n",
         "if REGION == \"[your-region]\":\n",
         "    REGION = \"us-central1\""
@@ -528,6 +554,15 @@
         "import h5py"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "76f7b9ffde0b"
+      },
+      "source": [
+        "Use gcloud to retrieve the project number."
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -748,12 +783,10 @@
       ]
     },
     {
-      "cell_type": "code",
-      "execution_count": null,
+      "cell_type": "markdown",
       "metadata": {
         "id": "0f1a9fbecabb"
       },
-      "outputs": [],
       "source": [
         "Using the resource name, you can retrieve an existing MatchingEngineIndex."
       ]
@@ -766,7 +799,7 @@
       },
       "outputs": [],
       "source": [
-        "tree_ah_index = aiplatform.MatchingEngineIndex(INDEX_RESOURCE_NAME)"
+        "tree_ah_index = aiplatform.MatchingEngineIndex(index_name=INDEX_RESOURCE_NAME)"
       ]
     },
     {
@@ -821,7 +854,7 @@
       "outputs": [],
       "source": [
         "brute_force_index = aiplatform.MatchingEngineIndex(\n",
-        "    \"projects/1012616486416/locations/us-central1/indexes/6738176690918260736\"\n",
+        "    index_name=INDEX_BRUTE_FORCE_RESOURCE_NAME\n",
         ")"
       ]
     },
@@ -932,8 +965,9 @@
       },
       "outputs": [],
       "source": [
-        "VPC_NETWORK_NAME = \"projects/{}/global/networks/{}\".format(PROJECT_NUMBER, NETWORK_NAME)\n",
-        "VPC_NETWORK_NAME"
+        "VPC_NETWORK = \"[your-network-name]\"\n",
+        "VPC_NETWORK_FULL = \"projects/{}/global/networks/{}\".format(PROJECT_NUMBER, VPC_NETWORK)\n",
+        "VPC_NETWORK_FULL"
       ]
     },
     {
@@ -947,7 +981,7 @@
         "my_index_endpoint = aiplatform.MatchingEngineIndexEndpoint.create(\n",
         "    display_name=\"index_endpoint_for_demo\",\n",
         "    description=\"index endpoint description\",\n",
-        "    network=VPC_NETWORK_NAME,\n",
+        "    network=VPC_NETWORK_FULL,\n",
         ")"
       ]
     },
@@ -989,7 +1023,7 @@
       },
       "outputs": [],
       "source": [
-        "DEPLOYED_INDEX_ID = f\"tree_ah_glove_deployed_{TIMESTAMP}\""
+        "DEPLOYED_INDEX_ID = f\"tree_ah_glove_deployed_{RANDOM_ID}\""
       ]
     },
     {
@@ -1024,7 +1058,7 @@
       },
       "outputs": [],
       "source": [
-        "DEPLOYED_BRUTE_FORCE_INDEX_ID = f\"glove_brute_force_deployed_{TIMESTAMP}\""
+        "DEPLOYED_BRUTE_FORCE_INDEX_ID = f\"glove_brute_force_deployed_{RANDOM_ID}\""
       ]
     },
     {
@@ -1171,8 +1205,8 @@
       "outputs": [],
       "source": [
         "# Delete indexes\n",
-        "tree_ah_index.delete(force=True)\n",
-        "brute_force_index.delete(force=True)"
+        "tree_ah_index.delete()\n",
+        "brute_force_index.delete()"
       ]
     }
   ],

notebooks/official/migration/UJ14 Vertex SDK AutoML Video Classification.ipynb

@@ -32,17 +32,24 @@
         "# Vertex AI: Vertex AI Migration: AutoML Video Classificaton\n",
         "\n",
         "<table align=\"left\">\n",
+        "\n",
         "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/ai-platform-samples/blob/master/vertex-ai-samples/tree/master/notebooks/official/migration/UJ14%20Vertex%20SDK%20AutoML%20Video%20Classification.ipynb\">\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ14 Vertex SDK AutoML Video Classification.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/ai-platform-samples/blob/master/vertex-ai-samples/tree/master/notebooks/official/migration/UJ14%20Vertex%20SDK%20AutoML%20Video%20Classification.ipynb\">\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ14 Vertex SDK AutoML Video Classification.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ14 Vertex SDK AutoML Video Classification.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "  </td>                                                                                               \n",
         "</table>\n",
         "<br/><br/><br/>"
       ]
@@ -150,17 +157,6 @@
         "Install the latest GA version of *google-cloud-storage* library as well."
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "install_storage"
-      },
-      "outputs": [],
-      "source": [
-        "! pip3 install -U google-cloud-storage $USER_FLAG"
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -170,7 +166,7 @@
       "outputs": [],
       "source": [
         "if os.getenv(\"IS_TESTING\"):\n",
-        "    ! pip3 install --upgrade tensorflow $USER_FLAG"
+        "    ! pip3 install -U google-cloud-storage --upgrade tensorflow $USER_FLAG"
       ]
     },
     {
@@ -297,7 +293,10 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -306,9 +305,9 @@
         "id": "timestamp"
       },
       "source": [
-        "#### Timestamp\n",
+        "#### UUID\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
       ]
     },
     {
@@ -319,9 +318,16 @@
       },
       "outputs": [],
       "source": [
-        "from datetime import datetime\n",
+        "import random\n",
+        "import string\n",
         "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+        "\n",
+        "# Generate a uuid of a specifed length(default=8)\n",
+        "def generate_uuid(length: int = 8) -> str:\n",
+        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
+        "\n",
+        "\n",
+        "UUID = generate_uuid()"
       ]
     },
     {
@@ -332,7 +338,7 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Google Cloud Notebooks**, your environment is already authenticated. Skip this step.\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
         "\n",
         "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
         "\n",
@@ -404,7 +410,8 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
+        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
     {
@@ -415,8 +422,9 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
+        "    BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
     {
@@ -436,7 +444,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_NAME"
+        "! gsutil mb -l $REGION $BUCKET_URI"
       ]
     },
     {
@@ -456,7 +464,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_NAME"
+        "! gsutil ls -al $BUCKET_URI"
       ]
     },
     {
@@ -501,7 +509,7 @@
       },
       "outputs": [],
       "source": [
-        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_NAME)"
+        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
       ]
     },
     {
@@ -602,7 +610,7 @@
       "outputs": [],
       "source": [
         "dataset = aip.VideoDataset.create(\n",
-        "    display_name=\"MIT Human Motion\" + \"_\" + TIMESTAMP,\n",
+        "    display_name=\"MIT Human Motion\" + \"_\" + UUID,\n",
         "    gcs_source=[IMPORT_FILE],\n",
         "    import_schema_uri=aip.schema.dataset.ioformat.video.classification,\n",
         ")\n",
@@ -677,7 +685,7 @@
       "outputs": [],
       "source": [
         "dag = aip.AutoMLVideoTrainingJob(\n",
-        "    display_name=\"hmdb_\" + TIMESTAMP,\n",
+        "    display_name=\"hmdb_\" + UUID,\n",
         "    prediction_type=\"classification\",\n",
         ")\n",
         "\n",
@@ -725,7 +733,7 @@
       "source": [
         "model = dag.run(\n",
         "    dataset=dataset,\n",
-        "    model_display_name=\"hmdb_\" + TIMESTAMP,\n",
+        "    model_display_name=\"hmdb_\" + UUID,\n",
         "    training_fraction_split=0.8,\n",
         "    test_fraction_split=0.2,\n",
         ")"
@@ -800,7 +808,7 @@
       "outputs": [],
       "source": [
         "# Get model resource ID\n",
-        "models = aip.Model.list(filter=\"display_name=hmdb_\" + TIMESTAMP)\n",
+        "models = aip.Model.list(filter=\"display_name=hmdb_\" + UUID)\n",
         "\n",
         "# Get a reference to the Model Service client\n",
         "client_options = {\"api_endpoint\": f\"{REGION}-aiplatform.googleapis.com\"}\n",
@@ -932,7 +940,7 @@
         "\n",
         "import tensorflow as tf\n",
         "\n",
-        "gcs_input_uri = BUCKET_NAME + \"/test.jsonl\"\n",
+        "gcs_input_uri = BUCKET_URI + \"/test.jsonl\"\n",
         "with tf.io.gfile.GFile(gcs_input_uri, \"w\") as f:\n",
         "    data = {\n",
         "        \"content\": test_item_1,\n",
@@ -978,9 +986,9 @@
       "outputs": [],
       "source": [
         "batch_predict_job = model.batch_predict(\n",
-        "    job_display_name=\"hmdb_\" + TIMESTAMP,\n",
+        "    job_display_name=\"hmdb_\" + UUID,\n",
         "    gcs_source=gcs_input_uri,\n",
-        "    gcs_destination_prefix=BUCKET_NAME,\n",
+        "    gcs_destination_prefix=BUCKET_URI,\n",
         "    sync=False,\n",
         ")\n",
         "\n",
@@ -1208,8 +1216,8 @@
         "    except Exception as e:\n",
         "        print(e)\n",
         "\n",
-        "    if \"BUCKET_NAME\" in globals():\n",
-        "        ! gsutil rm -r $BUCKET_NAME"
+        "    if \"BUCKET_URI\" in globals():\n",
+        "        ! gsutil rm -r $BUCKET_URI"
       ]
     }
   ],

notebooks/official/migration/UJ2,12 Vertex SDK Custom Image Classification with pre-built training container.ipynb

@@ -2,7 +2,7 @@
   "cells": [
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 1,
       "metadata": {
         "id": "copyright"
       },
@@ -32,17 +32,24 @@
         "# Vertex AI: Vertex AI Migration: Custom Image Classification w/pre-built training container\n",
         "\n",
         "<table align=\"left\">\n",
+        "\n",
         "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/ai-platform-samples/blob/master/vertex-ai-samples/tree/master/notebooks/official/migration/UJ2,12%20Vertex%20SDK%20Custom%20Image%20Classification%20with%20pre-built%20training%20container.ipynb\">\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ2,12 Vertex SDK Custom Image Classification with pre-built training container.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/ai-platform-samples/blob/master/vertex-ai-samples/tree/master/notebooks/official/migration/UJ2,12%20Vertex%20SDK%20Custom%20Image%20Classification%20with%20pre-built%20training%20container.ipynb\">\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ2,12 Vertex SDK Custom Image Classification with pre-built training container.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ2,12 Vertex SDK Custom Image Classification with pre-built training container.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "  </td>                                                                                               \n",
         "</table>\n",
         "<br/><br/><br/>"
       ]
@@ -119,7 +126,7 @@
       "source": [
         "## Installation\n",
         "\n",
-        "Install the latest version of Vertex SDK for Python."
+        "Install the latest version of Vertex AI SDK for Python."
       ]
     },
     {
@@ -169,22 +176,10 @@
       },
       "outputs": [],
       "source": [
-        "if os.getenv(\"IS_TESTING\"):\n",
-        "    ! apt-get update && apt-get install -y python3-opencv-headless\n",
-        "    ! apt-get install -y libgl1-mesa-dev\n",
-        "    ! pip3 install --upgrade opencv-python-headless $USER_FLAG"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "install_tensorflow"
-      },
-      "outputs": [],
-      "source": [
-        "if os.getenv(\"IS_TESTING\"):\n",
-        "    ! pip3 install --upgrade tensorflow $USER_FLAG"
+        "! apt-get update && apt-get install -y python3-opencv-headless\n",
+        "! apt-get install -y libgl1-mesa-dev\n",
+        "! pip3 install --upgrade opencv-python-headless $USER_FLAG\n",
+        "! pip3 install --upgrade tensorflow $USER_FLAG"
       ]
     },
     {
@@ -200,7 +195,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 5,
       "metadata": {
         "id": "restart"
       },
@@ -248,7 +243,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 1,
       "metadata": {
         "id": "set_project_id"
       },
@@ -305,13 +300,16 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 4,
       "metadata": {
         "id": "region"
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -320,22 +318,29 @@
         "id": "timestamp"
       },
       "source": [
-        "#### Timestamp\n",
+        "#### UUID\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 5,
       "metadata": {
-        "id": "timestamp"
+        "id": "e87d5856317d"
       },
       "outputs": [],
       "source": [
-        "from datetime import datetime\n",
+        "import random\n",
+        "import string\n",
         "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+        "\n",
+        "# Generate a uuid of a specifed length(default=8)\n",
+        "def generate_uuid(length: int = 8) -> str:\n",
+        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
+        "\n",
+        "\n",
+        "UUID = generate_uuid()"
       ]
     },
     {
@@ -346,7 +351,7 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Google Cloud Notebooks**, your environment is already authenticated. Skip this step.\n",
+        "**If you are using Vertex AI Workbench Notebook**, your environment is already authenticated. Skip this step.\n",
         "\n",
         "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
         "\n",
@@ -381,8 +386,10 @@
         "import os\n",
         "import sys\n",
         "\n",
-        "# If on Google Cloud Notebook, then don't execute this code\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
         "    if \"google.colab\" in sys.modules:\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
@@ -392,7 +399,7 @@
         "    # path to your service account key and run this cell to authenticate your GCP\n",
         "    # account.\n",
         "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''\n"
       ]
     },
     {
@@ -412,7 +419,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 6,
       "metadata": {
         "id": "bucket"
       },
@@ -423,14 +430,14 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 7,
       "metadata": {
         "id": "autoset_bucket"
       },
       "outputs": [],
       "source": [
         "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + UUID"
       ]
     },
     {
@@ -464,7 +471,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 10,
       "metadata": {
         "id": "validate_bucket"
       },
@@ -487,7 +494,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 11,
       "metadata": {
         "id": "import_aip:mbsdk"
       },
@@ -504,12 +511,12 @@
       "source": [
         "## Initialize Vertex SDK for Python\n",
         "\n",
-        "Initialize the Vertex SDK for Python for your project and corresponding bucket."
+        "Initialize the Vertex AI SDK for Python for your project and corresponding bucket."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 12,
       "metadata": {
         "id": "init_aip:mbsdk"
       },
@@ -542,12 +549,14 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 13,
       "metadata": {
         "id": "accelerators:training,cpu,prediction,cpu,mbsdk"
       },
       "outputs": [],
       "source": [
+        "import os\n",
+        "\n",
         "if os.getenv(\"IS_TESTING_TRAIN_GPU\"):\n",
         "    TRAIN_GPU, TRAIN_NGPU = (\n",
         "        aip.gapic.AcceleratorType.NVIDIA_TESLA_K80,\n",
@@ -699,12 +708,12 @@
         "\n",
         "#### Package Assembly\n",
         "\n",
-        "In the following cells, you will assemble the training package."
+        "In the following cells, you assemble the training package."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 16,
       "metadata": {
         "id": "examine_training_package"
       },
@@ -932,7 +941,7 @@
       "outputs": [],
       "source": [
         "job = aip.CustomTrainingJob(\n",
-        "    display_name=\"cifar10_\" + TIMESTAMP,\n",
+        "    display_name=\"cifar10_\" + UUID,\n",
         "    script_path=\"custom/trainer/task.py\",\n",
         "    container_uri=TRAIN_IMAGE,\n",
         "    requirements=[\"gcsfs==0.7.1\", \"tensorflow-datasets==4.4\"],\n",
@@ -979,7 +988,7 @@
       },
       "outputs": [],
       "source": [
-        "MODEL_DIR = \"{}/{}\".format(BUCKET_NAME, TIMESTAMP)\n",
+        "MODEL_DIR = \"{}/{}\".format(BUCKET_NAME, UUID)\n",
         "\n",
         "EPOCHS = 20\n",
         "STEPS = 100\n",
@@ -1262,7 +1271,7 @@
       "outputs": [],
       "source": [
         "model = aip.Model.upload(\n",
-        "    display_name=\"cifar10_\" + TIMESTAMP,\n",
+        "    display_name=\"cifar10_\" + UUID,\n",
         "    artifact_uri=MODEL_DIR,\n",
         "    serving_container_image_uri=DEPLOY_IMAGE,\n",
         "    sync=False,\n",
@@ -1346,11 +1355,22 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 29,
       "metadata": {
         "id": "prepare_test_items:test,image"
       },
-      "outputs": [],
+      "outputs": [
+        {
+          "data": {
+            "text/plain": [
+              "True"
+            ]
+          },
+          "execution_count": 29,
+          "metadata": {},
+          "output_type": "execute_result"
+        }
+      ],
       "source": [
         "import cv2\n",
         "\n",
@@ -1410,7 +1430,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 31,
       "metadata": {
         "id": "make_batch_file:custom,image"
       },
@@ -1464,7 +1484,7 @@
         "MAX_NODES = 1\n",
         "\n",
         "batch_predict_job = model.batch_predict(\n",
-        "    job_display_name=\"cifar10_\" + TIMESTAMP,\n",
+        "    job_display_name=\"cifar10_\" + UUID,\n",
         "    gcs_source=gcs_input_uri,\n",
         "    gcs_destination_prefix=BUCKET_NAME,\n",
         "    instances_format=\"jsonl\",\n",
@@ -1657,7 +1677,7 @@
       },
       "outputs": [],
       "source": [
-        "DEPLOYED_NAME = \"cifar10-\" + TIMESTAMP\n",
+        "DEPLOYED_NAME = \"cifar10-\" + UUID\n",
         "\n",
         "TRAFFIC_SPLIT = {\"0\": 100}\n",
         "\n",
@@ -1755,7 +1775,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 40,
       "metadata": {
         "id": "prepare_test_item:test,image"
       },

notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb

@@ -29,6 +29,8 @@
         "id": "JAPoU8Sm5E6e"
       },
       "source": [
+        "# Vertex AI: Track parameters and metrics for custom training jobs\n",
+        "\n",
         "<table align=\"left\">\n",
         "\n",
         "  <td>\n",
@@ -51,15 +53,6 @@
         "</table>"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "j9gUDU_3vV9d"
-      },
-      "source": [
-        "# Vertex AI: Track parameters and metrics for custom training jobs"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -68,11 +61,15 @@
       "source": [
         "## Overview\n",
         "\n",
-        "This notebook demonstrates how to track metrics and parameters for `Vertex AI` custom training jobs, and how to perform detailed analysis using this data.\n",
-        "\n",
-        "### Dataset\n",
-        "\n",
-        "This example uses the Abalone Dataset. For more information about this dataset please visit: https://archive.ics.uci.edu/ml/datasets/abalone\n",
+        "This notebook demonstrates how to track metrics and parameters for `Vertex AI` custom training jobs, and how to perform detailed analysis using this data."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "37147bd9c3c4"
+      },
+      "source": [
         "### Objective\n",
         "\n",
         "In this notebook, you learn how to use `Vertex ML Metadata` to track training parameters and evaluation metrics.\n",
@@ -85,8 +82,26 @@
         "The steps performed include:\n",
         "\n",
         "- Track parameters and metrics for a `Vertex AI` custom trained model.\n",
-        "- Extract and perform analysis for all parameters and metrics within an Experiment.\n",
+        "- Extract and perform analysis for all parameters and metrics within an Experiment."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "96cb18467417"
+      },
+      "source": [
+        "### Dataset\n",
         "\n",
+        "This example uses the Abalone Dataset. For more information about this dataset please visit: https://archive.ics.uci.edu/ml/datasets/abalone"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "c831245dc1d5"
+      },
+      "source": [
         "### Costs \n",
         "\n",
         "\n",
@@ -285,8 +300,7 @@
       },
       "outputs": [],
       "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
-        "    PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}\n",
+        "if PROJECT_ID == \"[your-project-id]\" or PROJECT_ID == \"\" or PROJECT_ID is None:\n",
         "    # Get your GCP project id from gcloud\n",
         "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
         "    PROJECT_ID = shell_output[0]\n",
@@ -332,7 +346,10 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -368,7 +385,7 @@
         "### Authenticate your Google Cloud account\n",
         "\n",
         "**If you are using Vertex AI Workbench**, your environment is already\n",
-        "authenticated. Skip this step."
+        "authenticated. "
       ]
     },
     {
@@ -664,9 +681,9 @@
         "id": "35QVNhACqcTJ"
       },
       "source": [
-        "### Create a managed tabular dataset from a CSV\n",
+        "### Create a Vertex AI Dataset from a CSV\n",
         "\n",
-        "A Managed dataset can be used to create an AutoML model or a custom model. "
+        "A Vertex AI Dataset can be used to create an AutoML model or a custom model. "
       ]
     },
     {
@@ -679,7 +696,7 @@
       "source": [
         "ds = aiplatform.TabularDataset.create(display_name=\"abalone\", gcs_source=[gcs_csv_path])\n",
         "\n",
-        "ds.resource_name"
+        "print(ds.resource_name)"
       ]
     },
     {
@@ -780,7 +797,11 @@
         "id": "k_QorXXztzPH"
       },
       "source": [
-        "Start a new experiment run to track training parameters and start the training job. Note that this operation will take around 10 mins."
+        "Start a new experiment run to track training parameters and start the training job. \n",
+        "\n",
+        "Prior to executing the training job, you call the `start_run()` method to initialize the start of the experiment, and then use the `log_params()` to log the parameters used in the experiment.\n",
+        "\n",
+        "*Note:* This operation will take around 10 mins."
       ]
     },
     {
@@ -818,7 +839,7 @@
         "id": "O-uCOL3Naap4"
       },
       "source": [
-        "Deploy model to Google Cloud. This operation will take 10-20 mins."
+        "Deploy model to Google Cloud. This operation may take a few minutes."
       ]
     },
     {
@@ -899,7 +920,7 @@
         "id": "_HphZ38obJeB"
       },
       "source": [
-        "Perform online prediction."
+        "### Perform online prediction"
       ]
     },
     {
@@ -911,7 +932,7 @@
       "outputs": [],
       "source": [
         "prediction = endpoint.predict(test_dataset.tolist())\n",
-        "prediction"
+        "print(prediction)"
       ]
     },
     {
@@ -920,7 +941,11 @@
         "id": "TDKiv_O7bNwE"
       },
       "source": [
-        "Calculate and track prediction evaluation metrics."
+        "### Calculate and track prediction evaluation metrics.\n",
+        "\n",
+        "Next, log the evaluation metrics for your experiment.\n",
+        "\n",
+        "Once the experiment is completed, you call the `end_run()` method to indicate the end of tracking for the experiment."
       ]
     },
     {
@@ -934,7 +959,9 @@
         "mse = mean_squared_error(test_labels, prediction.predictions)\n",
         "mae = mean_absolute_error(test_labels, prediction.predictions)\n",
         "\n",
-        "aiplatform.log_metrics({\"mse\": mse, \"mae\": mae})"
+        "aiplatform.log_metrics({\"mse\": mse, \"mae\": mae})\n",
+        "\n",
+        "aiplatform.end_run()"
       ]
     },
     {

notebooks/official/model_monitoring/model_monitoring.ipynb

@@ -59,15 +59,8 @@
         "id": "lA32H1oKGgpf"
       },
       "source": [
-        "## Overview"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "t6Cd51FkG09E"
-      },
-      "source": [
+        "## Overview\n",
+        "\n",
         "### What is Vertex AI Model Monitoring?\n",
         "\n",
         "Modern applications rely on a well established set of capabilities to monitor the health of their services. Examples include:\n",
@@ -94,7 +87,7 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "yG7FcXWKHOhC"
+        "id": "t6Cd51FkG09E"
       },
       "source": [
         "### Objective\n",
@@ -115,18 +108,38 @@
         "- Deploy the `Model` resource to the `Endpoint` resource.\n",
         "- Configure the `Endpoint` resource for model monitoring.\n",
         "- Generate synthetic prediction requests.\n",
-        "- Understand how to interpret the statistics, visualizations, other data reported by the model monitoring feature.\n",
+        "- Understand how to interpret the statistics, visualizations, other data reported by the model monitoring feature."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "edba71dc9840"
+      },
+      "source": [
+        "### Model\n",
         "\n",
+        "This tutorial uses a pre-trained model, where the model artifacts are stored in a public Cloud Storage bucket. The model predicts for an online gaming site, the probability that a player will churn, i.e. stop being an active player."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5abcd585354f"
+      },
+      "source": [
         "### Costs \n",
         "\n",
         "This tutorial uses billable components of Google Cloud:\n",
         "\n",
-        "* Vertext AI\n",
+        "* Vertex AI\n",
         "* BigQuery\n",
+        "* Cloud Storage\n",
         "\n",
-        "Learn about [Vertext AI\n",
-        "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
-        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
+        "Learn about [Vertext AI pricing](https://cloud.google.com/vertex-ai/pricing), \n",
+        "[Cloud Storage pricing](https://cloud.google.com/storage/pricing), \n",
+        "and [BigQuery pricing](https://cloud.google.com/bigquery/pricing)\n",
+        "and use the [Pricing\n",
         "Calculator](https://cloud.google.com/products/calculator/)\n",
         "to generate a cost estimate based on your projected usage."
       ]
@@ -137,6 +150,8 @@
         "id": "8yVpQt-JHKPF"
       },
       "source": [
+        "## Before you begin\n",
+        "\n",
         "### Set up your local development environment\n",
         "\n",
         "**If you are using Colab or Vertex AI Workbench notebooks**, your environment already meets\n",
@@ -173,15 +188,6 @@
         "1. Open this notebook in the Jupyter Notebook dashboard."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "ze4-nDLfK4pw"
-      },
-      "source": [
-        "## Before you begin"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -309,8 +315,7 @@
       },
       "outputs": [],
       "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
-        "    PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}\n",
+        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
         "    # Get your GCP project id from gcloud\n",
         "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
         "    PROJECT_ID = shell_output[0]\n",
@@ -336,13 +341,14 @@
       "source": [
         "#### Region\n",
         "\n",
-        "You can also change the `REGION` variable, which is used for operations\n",
-        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
+        "You can also change the `REGION` variable, which is used for operations throughout the rest of this notebook.  Below are regions supported for Vertex AI.\n",
         "\n",
         "- Americas: `us-central1`\n",
         "- Europe: `europe-west4`\n",
         "- Asia Pacific: `asia-east1`\n",
         "\n",
+        "**For this notebook, we recommend that you leave the region set to the default value us-central1**.\n",
+        "\n",
         "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
         "\n",
         "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)"
@@ -356,7 +362,41 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "06571eb4063b"
+      },
+      "source": [
+        "#### UUID\n",
+        "\n",
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "4e166d927e36"
+      },
+      "outputs": [],
+      "source": [
+        "import random\n",
+        "import string\n",
+        "\n",
+        "\n",
+        "# Generate a uuid of a specifed length(default=8)\n",
+        "def generate_uuid(length: int = 8) -> str:\n",
+        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
+        "\n",
+        "\n",
+        "UUID = generate_uuid()"
       ]
     },
     {
@@ -368,7 +408,7 @@
         "### Authenticate your Google Cloud account\n",
         "\n",
         "**If you are using Vertex AI Workbench notebooks**, your environment is already\n",
-        "authenticated. Skip this step.\n",
+        "authenticated.\n",
         "\n",
         "**If you are using Colab**, run the cell below and follow the instructions\n",
         "when prompted to authenticate your account via oAuth.\n",
@@ -481,12 +521,14 @@
       "source": [
         "# Import required packages.\n",
         "import os\n",
-        "import random\n",
+        "import pprint as pp\n",
         "import sys\n",
-        "import time\n",
         "\n",
+        "import google.cloud.aiplatform as aiplatform\n",
         "import matplotlib.pyplot as plt\n",
-        "import numpy as np"
+        "from google.cloud.aiplatform import model_monitoring\n",
+        "from google.cloud.aiplatform.explain.metadata.tf.v2 import \\\n",
+        "    saved_model_metadata_builder"
       ]
     },
     {
@@ -497,16 +539,36 @@
       },
       "outputs": [],
       "source": [
-        "SUFFIX = \"aiplatform.googleapis.com\"\n",
-        "API_ENDPOINT = f\"{REGION}-{SUFFIX}\"\n",
-        "PREDICT_API_ENDPOINT = f\"{REGION}-prediction-{SUFFIX}\"\n",
         "if os.getenv(\"IS_TESTING\"):\n",
-        "    !gcloud --quiet components install beta\n",
-        "    !gcloud --quiet components update\n",
-        "!gcloud config set ai/region $REGION\n",
+        "    ! gcloud --quiet components install beta\n",
+        "    ! gcloud --quiet components update\n",
+        "\n",
+        "! gcloud config set ai/region $REGION\n",
         "os.environ[\"GOOGLE_CLOUD_PROJECT\"] = PROJECT_ID"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "init_aip:mbsdk,region"
+      },
+      "source": [
+        "### Initialize Vertex AI SDK for Python\n",
+        "\n",
+        "Initialize the Vertex AI SDK for Python for your project and corresponding bucket."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "init_aip:mbsdk,region"
+      },
+      "outputs": [],
+      "source": [
+        "aiplatform.init(project=PROJECT_ID, location=REGION)"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -515,14 +577,14 @@
       "source": [
         "### The example model\n",
         "\n",
-        "The model you'll use in this notebook is based on [this blog post](https://cloud.google.com/blog/topics/developers-practitioners/churn-prediction-game-developers-using-google-analytics-4-ga4-and-bigquery-ml). The idea behind this model is that your company has extensive log data describing how your game users have interacted with the site. The raw data contains the following categories of information:\n",
+        "The model you use in this notebook is based on [this blog post](https://cloud.google.com/blog/topics/developers-practitioners/churn-prediction-game-developers-using-google-analytics-4-ga4-and-bigquery-ml). The idea behind this model is that your company has extensive log data describing how your game users have interacted with the site. The raw data contains the following categories of information:\n",
         "\n",
         "- identity - unique player identitity numbers\n",
         "- demographic features - information about the player, such as the geographic region in which a player is located\n",
         "- behavioral features - counts of the number of times a  player has triggered certain game events, such as reaching a new level\n",
         "- churn propensity - this is the label or target feature, it provides an estimated probability that this player will churn, i.e. stop being an active player.\n",
         "\n",
-        "The blog article referenced above explains how to use BigQuery to store the raw data, pre-process it for use in machine learning, and train a model. Because this notebook focuses on model monitoring, rather than training models, you're going to reuse a pre-trained version of this model, which has been exported to Cloud Storage. In the next section, you will setup your environment and import this model into your own project."
+        "The blog article referenced above explains how to use BigQuery to store the raw data, pre-process the data for machine learning, and train the corresponding model. Because this notebook focuses on model monitoring, rather than training models, you're going to reuse a pre-trained version of this model, which has been exported to Cloud Storage. In the next section, you will setup your environment and import this model into your own project."
       ]
     },
     {
@@ -531,170 +593,9 @@
         "id": "btZeLzqQ7pXc"
       },
       "source": [
-        "### Define some helper functions and data structures\n",
+        "### Define some helper data structures\n",
         "\n",
-        "Run the following cells to define some utility functions and data structures used throughout this notebook. Some highlights:\n",
-        "\n",
-        "* create_monitoring_job - convenience function for requesting a model monitoring job\n",
-        "* send_predict_request - convenience function for sending a prediction request and receiving the response\n",
-        "\n",
-        "Although these functions and data strctures are not critical to understand the main concepts, feel free to expand the  cell if you're curious or want to dive deeper into how some of your API requests are made."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "cellView": "form",
-        "id": "yhDFSB2YDvfT"
-      },
-      "outputs": [],
-      "source": [
-        "# @title Utility functions\n",
-        "import copy\n",
-        "import os\n",
-        "\n",
-        "from google.cloud.aiplatform_v1.services.endpoint_service import \\\n",
-        "    EndpointServiceClient\n",
-        "from google.cloud.aiplatform_v1.services.job_service import JobServiceClient\n",
-        "from google.cloud.aiplatform_v1.services.prediction_service import \\\n",
-        "    PredictionServiceClient\n",
-        "from google.cloud.aiplatform_v1.types.io import BigQuerySource\n",
-        "from google.cloud.aiplatform_v1.types.model_deployment_monitoring_job import (\n",
-        "    ModelDeploymentMonitoringJob, ModelDeploymentMonitoringObjectiveConfig,\n",
-        "    ModelDeploymentMonitoringScheduleConfig)\n",
-        "from google.cloud.aiplatform_v1.types.model_monitoring import (\n",
-        "    ModelMonitoringAlertConfig, ModelMonitoringObjectiveConfig,\n",
-        "    SamplingStrategy, ThresholdConfig)\n",
-        "from google.cloud.aiplatform_v1.types.prediction_service import (\n",
-        "    ExplainRequest, PredictRequest)\n",
-        "from google.protobuf import json_format\n",
-        "from google.protobuf.duration_pb2 import Duration\n",
-        "from google.protobuf.struct_pb2 import Value\n",
-        "\n",
-        "DEFAULT_THRESHOLD_VALUE = 0.001\n",
-        "\n",
-        "\n",
-        "def create_monitoring_job(objective_configs):\n",
-        "    # Create sampling configuration.\n",
-        "    random_sampling = SamplingStrategy.RandomSampleConfig(sample_rate=LOG_SAMPLE_RATE)\n",
-        "    sampling_config = SamplingStrategy(random_sample_config=random_sampling)\n",
-        "\n",
-        "    # Create schedule configuration.\n",
-        "    duration = Duration(seconds=MONITOR_INTERVAL)\n",
-        "    schedule_config = ModelDeploymentMonitoringScheduleConfig(monitor_interval=duration)\n",
-        "\n",
-        "    # Create alerting configuration.\n",
-        "    emails = [USER_EMAIL]\n",
-        "    email_config = ModelMonitoringAlertConfig.EmailAlertConfig(user_emails=emails)\n",
-        "    alerting_config = ModelMonitoringAlertConfig(email_alert_config=email_config)\n",
-        "\n",
-        "    # Create the monitoring job.\n",
-        "    endpoint = f\"projects/{PROJECT_ID}/locations/{REGION}/endpoints/{ENDPOINT_ID}\"\n",
-        "    predict_schema = \"\"\n",
-        "    analysis_schema = \"\"\n",
-        "    job = ModelDeploymentMonitoringJob(\n",
-        "        display_name=JOB_NAME,\n",
-        "        endpoint=endpoint,\n",
-        "        model_deployment_monitoring_objective_configs=objective_configs,\n",
-        "        logging_sampling_strategy=sampling_config,\n",
-        "        model_deployment_monitoring_schedule_config=schedule_config,\n",
-        "        model_monitoring_alert_config=alerting_config,\n",
-        "        predict_instance_schema_uri=predict_schema,\n",
-        "        analysis_instance_schema_uri=analysis_schema,\n",
-        "    )\n",
-        "    options = dict(api_endpoint=API_ENDPOINT)\n",
-        "    client = JobServiceClient(client_options=options)\n",
-        "    parent = f\"projects/{PROJECT_ID}/locations/{REGION}\"\n",
-        "    response = client.create_model_deployment_monitoring_job(\n",
-        "        parent=parent, model_deployment_monitoring_job=job\n",
-        "    )\n",
-        "    print(\"Created monitoring job:\")\n",
-        "    print(response)\n",
-        "    return response\n",
-        "\n",
-        "\n",
-        "def get_thresholds(default_thresholds, custom_thresholds):\n",
-        "    thresholds = {}\n",
-        "    default_threshold = ThresholdConfig(value=DEFAULT_THRESHOLD_VALUE)\n",
-        "    for feature in default_thresholds.split(\",\"):\n",
-        "        feature = feature.strip()\n",
-        "        thresholds[feature] = default_threshold\n",
-        "    for custom_threshold in custom_thresholds.split(\",\"):\n",
-        "        pair = custom_threshold.split(\":\")\n",
-        "        if len(pair) != 2:\n",
-        "            print(f\"Invalid custom skew threshold: {custom_threshold}\")\n",
-        "            return\n",
-        "        feature, value = pair\n",
-        "        thresholds[feature] = ThresholdConfig(value=float(value))\n",
-        "    return thresholds\n",
-        "\n",
-        "\n",
-        "def get_deployed_model_ids(endpoint_id):\n",
-        "    client_options = dict(api_endpoint=API_ENDPOINT)\n",
-        "    client = EndpointServiceClient(client_options=client_options)\n",
-        "    parent = f\"projects/{PROJECT_ID}/locations/{REGION}\"\n",
-        "    response = client.get_endpoint(name=f\"{parent}/endpoints/{endpoint_id}\")\n",
-        "    model_ids = []\n",
-        "    for model in response.deployed_models:\n",
-        "        model_ids.append(model.id)\n",
-        "    return model_ids\n",
-        "\n",
-        "\n",
-        "def set_objectives(model_ids, objective_template):\n",
-        "    # Use the same objective config for all models.\n",
-        "    objective_configs = []\n",
-        "    for model_id in model_ids:\n",
-        "        objective_config = copy.deepcopy(objective_template)\n",
-        "        objective_config.deployed_model_id = model_id\n",
-        "        objective_configs.append(objective_config)\n",
-        "    return objective_configs\n",
-        "\n",
-        "\n",
-        "def send_predict_request(endpoint, input, type=\"predict\"):\n",
-        "    client_options = {\"api_endpoint\": PREDICT_API_ENDPOINT}\n",
-        "    client = PredictionServiceClient(client_options=client_options)\n",
-        "    if type == \"predict\":\n",
-        "        obj = PredictRequest\n",
-        "        method = client.predict\n",
-        "    elif type == \"explain\":\n",
-        "        obj = ExplainRequest\n",
-        "        method = client.explain\n",
-        "    else:\n",
-        "        raise Exception(\"unsupported request type:\" + type)\n",
-        "    params = {}\n",
-        "    params = json_format.ParseDict(params, Value())\n",
-        "    request = obj(endpoint=endpoint, parameters=params)\n",
-        "    inputs = [json_format.ParseDict(input, Value())]\n",
-        "    request.instances.extend(inputs)\n",
-        "    response = None\n",
-        "    try:\n",
-        "        response = method(request)\n",
-        "    except Exception as ex:\n",
-        "        print(ex)\n",
-        "    return response\n",
-        "\n",
-        "\n",
-        "def list_monitoring_jobs():\n",
-        "    client_options = dict(api_endpoint=API_ENDPOINT)\n",
-        "    parent = f\"projects/{PROJECT_ID}/locations/us-central1\"\n",
-        "    client = JobServiceClient(client_options=client_options)\n",
-        "    response = client.list_model_deployment_monitoring_jobs(parent=parent)\n",
-        "    print(response)\n",
-        "\n",
-        "\n",
-        "def pause_monitoring_job(job):\n",
-        "    client_options = dict(api_endpoint=API_ENDPOINT)\n",
-        "    client = JobServiceClient(client_options=client_options)\n",
-        "    response = client.pause_model_deployment_monitoring_job(name=job)\n",
-        "    print(response)\n",
-        "\n",
-        "\n",
-        "def delete_monitoring_job(job):\n",
-        "    client_options = dict(api_endpoint=API_ENDPOINT)\n",
-        "    client = JobServiceClient(client_options=client_options)\n",
-        "    response = client.delete_model_deployment_monitoring_job(name=job)\n",
-        "    print(response)"
+        "Run the following cell to define some data structures used throughout this notebook."
       ]
     },
     {
@@ -710,6 +611,7 @@
         "\n",
         "# Sampling distributions for categorical features...\n",
         "DAYOFWEEK = {1: 1040, 2: 1223, 3: 1352, 4: 1217, 5: 1078, 6: 1011, 7: 1110}\n",
+        "\n",
         "LANGUAGE = {\n",
         "    \"en-us\": 4807,\n",
         "    \"en-gb\": 678,\n",
@@ -732,8 +634,11 @@
         "    \"en-nz\": 29,\n",
         "    \"es-es\": 25,\n",
         "}\n",
+        "\n",
         "OS = {\"IOS\": 3980, \"ANDROID\": 3798, \"null\": 253}\n",
+        "\n",
         "MONTH = {6: 3125, 7: 1838, 8: 1276, 9: 1718, 10: 74}\n",
+        "\n",
         "COUNTRY = {\n",
         "    \"United States\": 4395,\n",
         "    \"India\": 486,\n",
@@ -802,7 +707,7 @@
       "source": [
         "### Generate model metadata for Vertex Explainable AI\n",
         "\n",
-        "Run the following cell to extract metadata from the exported model, which is needed for generating the prediction explanations."
+        "Run the following cell to extract metadata from the exported model, which is needed for generating the explanations for a prediction request."
       ]
     },
     {
@@ -813,15 +718,13 @@
       },
       "outputs": [],
       "source": [
-        "import google.cloud.aiplatform as aiplatform\n",
-        "from google.cloud.aiplatform.explain.metadata.tf.v2 import \\\n",
-        "    saved_model_metadata_builder\n",
-        "\n",
         "MODEL_PATH = \"gs://mco-mm/churn\"\n",
+        "\n",
         "params = {\"sampled_shapley_attribution\": {\"path_count\": 10}}\n",
         "EXPLAIN_PARAMS = aiplatform.explain.ExplanationParameters(params)\n",
+        "\n",
         "builder = saved_model_metadata_builder.SavedModelMetadataBuilder(\n",
-        "    MODEL_PATH, outputs_to_explain=[\"churned_probs\"]\n",
+        "    model_path=MODEL_PATH, outputs_to_explain=[\"churned_probs\"]\n",
         ")\n",
         "EXPLAIN_META = builder.get_metadata_protobuf()"
       ]
@@ -834,7 +737,7 @@
       "source": [
         "## Upload your model\n",
         "\n",
-        "The churn propensity model you'll be using in this notebook has been trained in BigQuery ML and exported to a Cloud Storage bucket. This illustrates how you can easily export a trained model and move a model from one cloud service to another. \n",
+        "The churn propensity model you use in this notebook has been trained in BigQuery ML and exported to a Cloud Storage bucket. This illustrates how you can easily export a trained model and move a model from one cloud service to another. \n",
         "\n",
         "Run the next cell to import this model into your project. **If you've already imported your model, you can skip this step.**"
       ]
@@ -850,17 +753,16 @@
         "MODEL_NAME = \"churn\"\n",
         "IMAGE = \"us-docker.pkg.dev/cloud-aiplatform/prediction/tf2-cpu.2-5:latest\"\n",
         "\n",
-        "aiplatform.init(project=PROJECT_ID, location=REGION)\n",
         "model = aiplatform.Model.upload(\n",
         "    display_name=MODEL_NAME,\n",
         "    artifact_uri=MODEL_PATH,\n",
         "    serving_container_image_uri=IMAGE,\n",
         "    explanation_parameters=EXPLAIN_PARAMS,\n",
         "    explanation_metadata=EXPLAIN_META,\n",
+        "    sync=True,\n",
         ")\n",
-        "model.wait()\n",
-        "print(f\"model display name: {model.display_name}\")\n",
-        "print(f\"model resource name: {model.resource_name}\")"
+        "\n",
+        "MODEL_ID = model.resource_name.split(\"/\")[-1]"
       ]
     },
     {
@@ -869,7 +771,7 @@
         "id": "e2030b028cef"
       },
       "source": [
-        "Once the above cell completes, you should see a new model on the Vertex AI Model Inventory page on the Cloud Console."
+        "Once the above cell completes, you should see a new model on the Vertex AI Model Registry page on the Cloud Console."
       ]
     },
     {
@@ -878,9 +780,9 @@
         "id": "d7cbb0fb73cc"
       },
       "source": [
-        "## Deploy your endpoint\n",
+        "## Deploy your Model resource to an Endpoint resource\n",
         "\n",
-        "Now that you've imported your model into your project, you need to create an endpoint to serve your model. An endpoint can be thought of as a channel through which your model provides prediction services. Once established, you'll be able to make prediction requests on your model via the public internet.  Your endpoint is also serverless, in the sense that Google Cloud ensures high availability by reducing single points of failure, and scalability by dynamically allocating resources to meet the demand for your service. In this way, you are able to focus on your model quality, and freed from adminstrative and infrastructure concerns.\n",
+        "Now that you've imported your model into your project, you need to create an endpoint to serve your model. An endpoint can be thought of as a channel through which your model provides prediction services. Once established, you can make online prediction requests on your model via the public internet.  Your endpoint is also serverless, in the sense that Google Cloud ensures high availability by reducing single points of failure, and scalability by dynamically allocating resources to meet the demand for your service. In this way, you are able to focus on your model quality, and freed from adminstrative and infrastructure concerns.\n",
         "\n",
         "Run the next cell to deploy your model to an endpoint. **This will take about ten minutes to complete.**"
       ]
@@ -932,13 +834,14 @@
       "outputs": [],
       "source": [
         "try:\n",
-        "    resp = send_predict_request(ENDPOINT, DEFAULT_INPUT)\n",
+        "    resp = endpoint.predict([DEFAULT_INPUT])\n",
         "    for i in resp.predictions:\n",
         "        vals = i[\"churned_values\"]\n",
         "        probs = i[\"churned_probs\"]\n",
         "    for i in range(len(vals)):\n",
         "        print(vals[i], probs[i])\n",
         "    plt.pie(probs, labels=vals)\n",
+        "    pp.pprint(resp)\n",
         "except Exception as ex:\n",
         "    print(\"prediction request failed\", ex)"
       ]
@@ -949,7 +852,9 @@
         "id": "a1eb4131bb5e"
       },
       "source": [
-        "Taking a closer look at the results, we see the following elements:\n",
+        "### Test results\n",
+        "\n",
+        "Taking a look at the results, you see the following elements:\n",
         "\n",
         "- **churned_values** - a set of possible values (0 and 1) for the target field\n",
         "- **churned_probs** - a corresponding set of probabilities for each possible target field value (5x10^-40 and 1.0, respectively)\n",
@@ -966,7 +871,7 @@
       "source": [
         "## Run an explanation test\n",
         "\n",
-        "We can also run a test of explainable AI on this endpoint. Run the next cell to send a test explanation request. If everything works as expected, you should receive a response encoding the feature importance of this prediction in a text representation called JSON, along with a bar chart summarizing the results.\n",
+        "You can run a test of Explainable AI on this endpoint. Run the next cell to send a test explanation request. The response you receive encodes the feature importance of this prediction in a text representation called JSON, along with a bar chart summarizing the results.\n",
         "\n",
         "**Try this now by running the next cell.**"
       ]
@@ -982,8 +887,7 @@
         "try:\n",
         "    features = []\n",
         "    scores = []\n",
-        "    resp = send_predict_request(ENDPOINT, DEFAULT_INPUT, type=\"explain\")\n",
-        "    # pp.pprint(resp)\n",
+        "    resp = endpoint.explain([DEFAULT_INPUT])\n",
         "    for i in resp.explanations:\n",
         "        for j in i.attributions:\n",
         "            for k in j.feature_attributions:\n",
@@ -1009,7 +913,7 @@
         "\n",
         "Now that you've created an endpoint to serve prediction requests on your model, you're ready to start a monitoring job to keep an eye on model quality and to alert you if and when input begins to deviate in way that may impact your model's prediction quality.\n",
         "\n",
-        "In this section, you will configure and create a model monitoring job based on the churn propensity model you imported from BigQuery ML."
+        "In this section, you configure and create a model monitoring job based on the churn propensity model you imported from BigQuery ML."
       ]
     },
     {
@@ -1051,17 +955,35 @@
         "# Prediction target column name in training dataset.\n",
         "TARGET = \"churned\"\n",
         "\n",
-        "# Skew and drift thresholds.\n",
-        "SKEW_DEFAULT_THRESHOLDS = \"country,cnt_user_engagement\"  # @param {type:\"string\"}\n",
-        "SKEW_CUSTOM_THRESHOLDS = \"cnt_level_start_quickplay:.01\"  # @param {type:\"string\"}\n",
-        "DRIFT_DEFAULT_THRESHOLDS = \"country,cnt_user_engagement\"  # @param {type:\"string\"}\n",
-        "DRIFT_CUSTOM_THRESHOLDS = \"cnt_level_start_quickplay:.01\"  # @param {type:\"string\"}\n",
-        "ATTRIB_SKEW_DEFAULT_THRESHOLDS = \"country,cnt_user_engagement\"  # @param {type:\"string\"}\n",
-        "# fmt: off\n",
-        "ATTRIB_SKEW_CUSTOM_THRESHOLDS = \"cnt_level_start_quickplay:.01\"  # @param {type:\"string\"}\n",
-        "ATTRIB_DRIFT_DEFAULT_THRESHOLDS = \"country,cnt_user_engagement\"  # @param {type:\"string\"}\n",
-        "ATTRIB_DRIFT_CUSTOM_THRESHOLDS = \"cnt_level_start_quickplay:.01\"  # @param {type:\"string\"}\n",
-        "# fmt: on"
+        "# # Skew and drift thresholds.\n",
+        "\n",
+        "DEFAULT_THRESHOLD_VALUE = 0.001\n",
+        "\n",
+        "SKEW_THRESHOLDS = {\n",
+        "    \"country\": DEFAULT_THRESHOLD_VALUE,\n",
+        "    \"cnt_user_engagement\": DEFAULT_THRESHOLD_VALUE,\n",
+        "}\n",
+        "DRIFT_THRESHOLDS = {\n",
+        "    \"country\": DEFAULT_THRESHOLD_VALUE,\n",
+        "    \"cnt_user_engagement\": DEFAULT_THRESHOLD_VALUE,\n",
+        "}\n",
+        "ATTRIB_SKEW_THRESHOLDS = {\n",
+        "    \"country\": DEFAULT_THRESHOLD_VALUE,\n",
+        "    \"cnt_user_engagement\": DEFAULT_THRESHOLD_VALUE,\n",
+        "}\n",
+        "ATTRIB_DRIFT_THRESHOLDS = {\n",
+        "    \"country\": DEFAULT_THRESHOLD_VALUE,\n",
+        "    \"cnt_user_engagement\": DEFAULT_THRESHOLD_VALUE,\n",
+        "}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e10f3d0fa538"
+      },
+      "source": [
+        "You can change the threshold values and the configuration settings, so that you can monitor other features in the model as well."
       ]
     },
     {
@@ -1083,42 +1005,74 @@
       },
       "outputs": [],
       "source": [
-        "skew_thresholds = get_thresholds(SKEW_DEFAULT_THRESHOLDS, SKEW_CUSTOM_THRESHOLDS)\n",
-        "drift_thresholds = get_thresholds(DRIFT_DEFAULT_THRESHOLDS, DRIFT_CUSTOM_THRESHOLDS)\n",
-        "attrib_skew_thresholds = get_thresholds(\n",
-        "    ATTRIB_SKEW_DEFAULT_THRESHOLDS, ATTRIB_SKEW_CUSTOM_THRESHOLDS\n",
-        ")\n",
-        "attrib_drift_thresholds = get_thresholds(\n",
-        "    ATTRIB_DRIFT_DEFAULT_THRESHOLDS, ATTRIB_DRIFT_CUSTOM_THRESHOLDS\n",
+        "skew_config = model_monitoring.SkewDetectionConfig(\n",
+        "    data_source=DATASET_BQ_URI,\n",
+        "    skew_thresholds=SKEW_THRESHOLDS,\n",
+        "    attribute_skew_thresholds=ATTRIB_SKEW_THRESHOLDS,\n",
+        "    target_field=TARGET,\n",
         ")\n",
         "\n",
-        "skew_config = ModelMonitoringObjectiveConfig.TrainingPredictionSkewDetectionConfig(\n",
-        "    skew_thresholds=skew_thresholds,\n",
-        "    attribution_score_skew_thresholds=attrib_skew_thresholds,\n",
-        ")\n",
-        "drift_config = ModelMonitoringObjectiveConfig.PredictionDriftDetectionConfig(\n",
-        "    drift_thresholds=drift_thresholds,\n",
-        "    attribution_score_drift_thresholds=attrib_drift_thresholds,\n",
-        ")\n",
-        "explanation_config = ModelMonitoringObjectiveConfig.ExplanationConfig(\n",
-        "    enable_feature_attributes=True\n",
+        "drift_config = model_monitoring.DriftDetectionConfig(\n",
+        "    drift_thresholds=DRIFT_THRESHOLDS,\n",
+        "    attribute_drift_thresholds=ATTRIB_DRIFT_THRESHOLDS,\n",
         ")\n",
         "\n",
-        "training_dataset = ModelMonitoringObjectiveConfig.TrainingDataset(target_field=TARGET)\n",
-        "training_dataset.bigquery_source = BigQuerySource(input_uri=DATASET_BQ_URI)\n",
-        "objective_config = ModelMonitoringObjectiveConfig(\n",
-        "    training_dataset=training_dataset,\n",
-        "    training_prediction_skew_detection_config=skew_config,\n",
-        "    prediction_drift_detection_config=drift_config,\n",
-        "    explanation_config=explanation_config,\n",
+        "explanation_config = model_monitoring.ExplanationConfig()\n",
+        "objective_config = model_monitoring.ObjectiveConfig(\n",
+        "    skew_config, drift_config, explanation_config\n",
         ")\n",
-        "model_ids = get_deployed_model_ids(ENDPOINT_ID)\n",
-        "objective_template = ModelDeploymentMonitoringObjectiveConfig(\n",
-        "    objective_config=objective_config\n",
-        ")\n",
-        "objective_configs = set_objectives(model_ids, objective_template)\n",
         "\n",
-        "monitoring_job = create_monitoring_job(objective_configs)"
+        "# Create sampling configuration\n",
+        "random_sampling = model_monitoring.RandomSampleConfig(sample_rate=LOG_SAMPLE_RATE)\n",
+        "\n",
+        "# Create schedule configuration\n",
+        "schedule_config = model_monitoring.ScheduleConfig(monitor_interval=MONITOR_INTERVAL)\n",
+        "\n",
+        "# Create alerting configuration.\n",
+        "emails = [USER_EMAIL]\n",
+        "alerting_config = model_monitoring.EmailAlertConfig(\n",
+        "    user_emails=emails, enable_logging=True\n",
+        ")\n",
+        "\n",
+        "# Create the monitoring job.\n",
+        "job = aiplatform.ModelDeploymentMonitoringJob.create(\n",
+        "    display_name=JOB_NAME,\n",
+        "    logging_sampling_strategy=random_sampling,\n",
+        "    schedule_config=schedule_config,\n",
+        "    alert_config=alerting_config,\n",
+        "    objective_configs=objective_config,\n",
+        "    project=PROJECT_ID,\n",
+        "    location=REGION,\n",
+        "    endpoint=endpoint,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "SaXYVFFslRru"
+      },
+      "source": [
+        "### Receiving email alert\n",
+        "After a minute or two, you should receive email at the address you configured above for USER_EMAIL. This email confirms successful deployment of your monitoring job. Here's a sample of what this email might look like:\n",
+        "<br>\n",
+        "<br>\n",
+        "<img src=\"https://storage.googleapis.com/mco-general/img/mm6.png\" />\n",
+        "<br>\n",
+        "As your monitoring job collects data, measurements are stored in Cloud Storage and you are free to examine your data at any time. The \"Statistics and Anomalies Root Path\" specifies the location of your measurements in Cloud Storage. Run the following cell to see an example of the layout of these measurements in Cloud Storage. If you substitute the Cloud Storage URL in your job creation email, you can view the structure and content of the data files for your own monitoring job."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6f38e8423bce"
+      },
+      "source": [
+        "### Create the sampling distribution\n",
+        "\n",
+        "Next, you send a first test prediction request. The model monitoring service will analyze the distribution of features and automatically create a baseline to monitor deviations from the baseline.\n",
+        "\n",
+        "*Note:* You need to wait for the email notification before making the first prediction request."
       ]
     },
     {
@@ -1131,24 +1085,10 @@
       "source": [
         "# Run a prediction request to generate schema, if necessary.\n",
         "try:\n",
-        "    _ = send_predict_request(ENDPOINT, DEFAULT_INPUT)\n",
+        "    _ = endpoint.predict([DEFAULT_INPUT])\n",
         "    print(\"prediction succeeded\")\n",
-        "except Exception:\n",
-        "    print(\"prediction failed\")"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "SaXYVFFslRru"
-      },
-      "source": [
-        "After a minute or two, you should receive email at the address you configured above for USER_EMAIL. This email confirms successful deployment of your monitoring job. Here's a sample of what this email might look like:\n",
-        "<br>\n",
-        "<br>\n",
-        "<img src=\"https://storage.googleapis.com/mco-general/img/mm6.png\" />\n",
-        "<br>\n",
-        "As your monitoring job collects data, measurements are stored in Cloud Storage and you are free to examine your data at any time. The \"Statistics and Anomalies Root Path\" specifies the location of your measurements in Cloud Storage. Run the following cell to see an example of the layout of these measurements in Cloud Storage. If you substitute the Cloud Storage URL in your job creation email, you can view the structure and content of the data files for your own monitoring job."
+        "except Exception as e:\n",
+        "    print(f\"prediction failed: {e}\")"
       ]
     },
     {
@@ -1168,7 +1108,8 @@
         "id": "XgUwU0sDpUUD"
       },
       "source": [
-        "You will notice the following components in these Cloud Storage paths:\n",
+        "### Cloud storage layout\n",
+        "Notice the following components in these Cloud Storage paths:\n",
         "\n",
         "- **cloud-ai-platform-..** - This is a bucket created for you and assigned to capture your service's prediction data. Each monitoring job you create will trigger creation of a new folder in this bucket.\n",
         "- **[model_monitoring|instance_schemas]/job-..** - This is your unique monitoring job number, which you can see above in both the response to your job creation requesst and the email notification. \n",
@@ -1187,94 +1128,7 @@
       "source": [
         "### You can create monitoring jobs with other user interfaces\n",
         "\n",
-        "In the previous cells, you created a monitoring job using the Python client library. You can also use the *gcloud* command line tool or the Cloud Console to create a model monitoring job. \n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "Q106INuFCXKX"
-      },
-      "source": [
-        "## Generate test data to trigger alerting\n",
-        "\n",
-        "It takes some time (in the case of this particular model, up to two hours) for the model monitoring system to pre-process your training data and before your model monitoring job is ready to detect and report on anomalies. \n",
-        "\n",
-        "**Read through the rest of this notebook to understand more about this feature and then come back here after two hours to run the test in this cell**.\n",
-        "\n",
-        "Now you are ready to test the monitoring function. Run the following cell, which will generate fabricated test predictions designed to exceed the thresholds you specified above.\n",
-        "\n",
-        "The first test sends 600 fabricated requests (ten per second for 60 seconds). It does this three times, each repetition  perturbing two features of interest (*cnt_level_start_quickplay* and *country*) by successive powers of two (2, 4, 8). By perturbing data in two experiments, we're able to trigger both skew and drift alerts.\n",
-        "\n",
-        "After running this test, it takes up to an hour to assess and report skew and drift alerts. The following cells give examples of the resulting reports."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "obZYLLAuKmG8"
-      },
-      "outputs": [],
-      "source": [
-        "from IPython.display import clear_output\n",
-        "\n",
-        "\n",
-        "def random_uid():\n",
-        "    digits = [str(i) for i in range(10)] + [\"A\", \"B\", \"C\", \"D\", \"E\", \"F\"]\n",
-        "    return \"\".join(random.choices(digits, k=32))\n",
-        "\n",
-        "\n",
-        "def monitoring_test(test, count, sleep, perturb_num={}, perturb_cat={}):\n",
-        "    # Use random sampling and mean/sd with gaussian distribution to model\n",
-        "    # training data. Then modify sampling distros for two categorical features\n",
-        "    # and mean/sd for two numerical features.\n",
-        "    mean_sd = MEAN_SD.copy()\n",
-        "    country = COUNTRY.copy()\n",
-        "    for k, (mean_fn, sd_fn) in perturb_num.items():\n",
-        "        orig_mean, orig_sd = MEAN_SD[k]\n",
-        "        mean_sd[k] = (mean_fn(orig_mean), sd_fn(orig_sd))\n",
-        "    for k, v in perturb_cat.items():\n",
-        "        country[k] = v\n",
-        "    for i in range(0, count):\n",
-        "        input = DEFAULT_INPUT.copy()\n",
-        "        input[\"user_pseudo_id\"] = str(random_uid())\n",
-        "        input[\"country\"] = random.choices([*country], list(country.values()))[0]\n",
-        "        input[\"dayofweek\"] = random.choices([*DAYOFWEEK], list(DAYOFWEEK.values()))[0]\n",
-        "        input[\"language\"] = str(random.choices([*LANGUAGE], list(LANGUAGE.values()))[0])\n",
-        "        input[\"operating_system\"] = str(random.choices([*OS], list(OS.values()))[0])\n",
-        "        input[\"month\"] = random.choices([*MONTH], list(MONTH.values()))[0]\n",
-        "        for key, (mean, sd) in mean_sd.items():\n",
-        "            sample_val = round(float(np.random.normal(mean, sd, 1)))\n",
-        "            val = max(sample_val, 0)\n",
-        "            input[key] = val\n",
-        "        clear_output(wait=True)\n",
-        "        print(f\"Sending prediction (round: {test:2}, iteration: {i:3})\", end=\"\")\n",
-        "        try:\n",
-        "            send_predict_request(ENDPOINT, input)\n",
-        "        except Exception:\n",
-        "            print(\"prediction request failed\")\n",
-        "        time.sleep(sleep)\n",
-        "    print(\"\\nTest Completed.\")\n",
-        "\n",
-        "\n",
-        "tests_per_sec = 10\n",
-        "tests = 3\n",
-        "multiplier = 1\n",
-        "for test in range(tests):\n",
-        "    multiplier *= 2\n",
-        "    test_time = 60\n",
-        "    tests_per_sec = 10\n",
-        "    sleep_time = 1 / tests_per_sec\n",
-        "    iterations = test_time * tests_per_sec\n",
-        "    perturb_num = {\n",
-        "        \"cnt_level_start_quickplay\": (\n",
-        "            lambda x: x * multiplier,\n",
-        "            lambda x: x / multiplier,\n",
-        "        )\n",
-        "    }\n",
-        "    perturb_cat = {\"Japan\": max(COUNTRY.values()) * multiplier}\n",
-        "    monitoring_test(test, iterations, sleep_time, perturb_num, perturb_cat)"
+        "In the previous cells, you created a monitoring job using the Python client library. Alternatively, you can use the *gcloud* command line tool or the Cloud Console to create a model monitoring job. \n"
       ]
     },
     {
@@ -1285,9 +1139,9 @@
       "source": [
         "## Interpret your results\n",
         "\n",
-        "Model Monitoring detects an anomaly when the threshold set for a feature is exceeded. While waiting to conduct your test or receive your results, read ahead to get a sense of the alerting experience.\n",
+        "Vertex AI Model Monitoring detects an anomaly when the threshold set for a feature is exceeded. The following cells give you a sense of the alerting and reporting experience after model monitoring anomalies have been detected.\n",
         "\n",
-        "Model Monitoring automatically notifies you of detected anomalies through email, but you can also [set up alerts through Cloud Logging](https://cloud.google.com/vertex-ai/docs/model-monitoring/using-model-monitoring#monitor-job)."
+        "Vertex AI Model Monitoring automatically notifies you of detected anomalies through email, but you can also [set up alerts through Cloud Logging](https://cloud.google.com/vertex-ai/docs/model-monitoring/using-model-monitoring#monitor-job)."
       ]
     },
     {
@@ -1368,15 +1222,21 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "TPP_ImwJDFJf"
+        "id": "d6cc924aa1fb"
       },
       "outputs": [],
       "source": [
-        "# Delete endpoint resource\n",
-        "!gcloud ai endpoints delete $ENDPOINT_NAME --quiet\n",
+        "# Undeploy the model and delete the endpoint\n",
+        "endpoint.undeploy_all()\n",
+        "endpoint.delete()\n",
         "\n",
-        "# Delete model resource\n",
-        "!gcloud ai models delete $MODEL_NAME --quiet"
+        "model.delete()\n",
+        "\n",
+        "# Delete BQ table and dataset\n",
+        "rmtable = f\"bq rm -f model_deployment_monitoring_{ENDPOINT_ID}.serving_predict\"\n",
+        "! $rmtable\n",
+        "rmdataset = f\"bq rm -f model_deployment_monitoring_{ENDPOINT_ID}\"\n",
+        "! $rmdataset"
       ]
     },
     {

notebooks/official/pipelines/custom_model_training_and_batch_prediction.ipynb

@@ -2,7 +2,7 @@
   "cells": [
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 1,
       "metadata": {
         "id": "b0b4f2bf"
       },
@@ -32,6 +32,7 @@
         "# Vertex AI Pipelines: Custom training with pre-built Google Cloud Pipeline Components\n",
         "\n",
         "<table align=\"left\">\n",
+        "\n",
         "  <td>\n",
         "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/custom_model_training_and_batch_prediction.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
@@ -44,11 +45,11 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/custom_model_training_and_batch_prediction.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooksofficial/pipelines/custom_model_training_and_batch_prediction.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
-        "     </a>\n",
-        "  </td>\n",
+        "    </a>\n",
+        "  </td>                                                                                               \n",
         "</table>\n",
         "<br/><br/><br/>"
       ]
@@ -65,17 +66,6 @@
         "This tutorial demonstrates how to use Vertex AI Pipelines with pre-built Google Cloud Pipeline Components for custom training."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "57139e75264f"
-      },
-      "source": [
-        "### Dataset\n",
-        "\n",
-        "The dataset used for this tutorial is the [CIFAR10 dataset](https://www.tensorflow.org/datasets/catalog/cifar10) from [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/overview). The version of the dataset you will use is built into TensorFlow. The trained model predicts which type of class an image is from ten classes: airplane, automobile, bird, cat, deer, dog, frog, horse, ship, or truck."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -107,6 +97,17 @@
         "Learn more about [Google Cloud Pipeline Components](https://cloud.google.com/vertex-ai/docs/pipelines/build-pipeline)."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "57139e75264f"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the [CIFAR10 dataset](https://www.tensorflow.org/datasets/catalog/cifar10) from [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/overview). The version of the dataset you will use is built into TensorFlow. The trained model predicts which type of class an image is from ten classes: airplane, automobile, bird, cat, deer, dog, frog, horse, ship, or truck."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -196,8 +197,8 @@
         "! pip3 install -U google-cloud-storage {USER_FLAG} -q\n",
         "! pip3 install {USER_FLAG} kfp google-cloud-pipeline-components --upgrade -q\n",
         "\n",
-        "if os.getenv(\"IS_TESTING\"):\n",
-        "    ! pip3 install --upgrade --force-reinstall $USER_FLAG tensorflow==2.5 kfp google-cloud-aiplatform google-cloud-storage google-cloud-pipeline-components"
+        "\n",
+        "! pip3 install --upgrade --force-reinstall $USER_FLAG tensorflow kfp google-cloud-aiplatform google-cloud-storage google-cloud-pipeline-components -q"
       ]
     },
     {
@@ -213,7 +214,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 3,
       "metadata": {
         "id": "restart"
       },
@@ -261,7 +262,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 1,
       "metadata": {
         "id": "set_project_id"
       },
@@ -318,13 +319,16 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 1,
       "metadata": {
         "id": "region"
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -333,22 +337,29 @@
         "id": "timestamp"
       },
       "source": [
-        "#### Timestamp\n",
+        "#### UUID\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 6,
       "metadata": {
         "id": "wJjft8z1IA81"
       },
       "outputs": [],
       "source": [
-        "from datetime import datetime\n",
+        "import random\n",
+        "import string\n",
         "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+        "\n",
+        "# Generate a uuid of a specifed length(default=8)\n",
+        "def generate_uuid(length: int = 8) -> str:\n",
+        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
+        "\n",
+        "\n",
+        "UUID = generate_uuid()"
       ]
     },
     {
@@ -435,7 +446,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 7,
       "metadata": {
         "id": "bucket"
       },
@@ -447,14 +458,14 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 8,
       "metadata": {
         "id": "autoset_bucket"
       },
       "outputs": [],
       "source": [
         "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
         "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
       ]
     },
@@ -489,7 +500,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 10,
       "metadata": {
         "id": "validate_bucket"
       },
@@ -511,7 +522,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 11,
       "metadata": {
         "id": "set_service_account"
       },
@@ -559,7 +570,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 15,
       "metadata": {
         "id": "set_service_account:pipelines"
       },
@@ -591,7 +602,6 @@
       "outputs": [],
       "source": [
         "import google.cloud.aiplatform as aip\n",
-        "import tensorflow as tf\n",
         "from google_cloud_pipeline_components.experimental.custom_job import utils\n",
         "from kfp.v2 import compiler, dsl\n",
         "from kfp.v2.dsl import component"
@@ -610,7 +620,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 17,
       "metadata": {
         "id": "pipeline_constants"
       },
@@ -632,7 +642,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 18,
       "metadata": {
         "id": "yX-aimhGRRRl"
       },
@@ -665,7 +675,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 19,
       "metadata": {
         "id": "ipG9uBUDRRRm"
       },
@@ -835,12 +845,12 @@
         "\n",
         "#### Package Assembly\n",
         "\n",
-        "In the following cells, you will assemble the training package."
+        "In the following cells, you assemble the training package."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 22,
       "metadata": {
         "id": "YpA6MFcLRRRn"
       },
@@ -880,7 +890,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 23,
       "metadata": {
         "id": "2etUCVVMRRRo"
       },
@@ -1096,12 +1106,14 @@
       "source": [
         "### Convert the component to a Vertex AI Custom Job\n",
         "\n",
-        "Next, use the `create_custom_training_job_op_from_component` method to convert the custom component into a Vertex AI Custom Job pre-built component."
+        "Next, use the `create_custom_training_job_op_from_component` method to convert the custom component into a Vertex AI Custom Job pre-built component.\n",
+        "\n",
+        "**replica_count :**  The  number of machine replicas the batch operation may be scaled to. Only used if machine_type is set. Default is 10."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 24,
       "metadata": {
         "id": "deb6c1cc"
       },
@@ -1129,7 +1141,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 25,
       "metadata": {
         "id": "5R23d2J_HJr-"
       },
@@ -1219,7 +1231,7 @@
         "    pipeline_func=pipeline, package_path=\"custom_model_training_spec.json\"\n",
         ")\n",
         "\n",
-        "DISPLAY_NAME = \"cifar10_\" + TIMESTAMP\n",
+        "DISPLAY_NAME = \"cifar10_\" + UUID\n",
         "\n",
         "job = aip.PipelineJob(\n",
         "    display_name=DISPLAY_NAME,\n",
@@ -1240,7 +1252,7 @@
       "source": [
         "### View custom training pipeline results\n",
         "\n",
-        "Finally, you will view the artifact outputs of each task in the pipeline."
+        "Finally, you view the artifact outputs of each task in the pipeline."
       ]
     },
     {
@@ -1253,6 +1265,8 @@
       "source": [
         "import json\n",
         "\n",
+        "import tensorflow as tf\n",
+        "\n",
         "PROJECT_NUMBER = job.gca_resource.name.split(\"/\")[1]\n",
         "print(PROJECT_NUMBER)\n",
         "\n",
@@ -1358,6 +1372,7 @@
         "batch_job = aip.BatchPredictionJob(batch_job_id)\n",
         "batch_job.delete()\n",
         "\n",
+        "\n",
         "delete_bucket = False\n",
         "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
         "    ! gsutil rm -rf {BUCKET_URI}"

notebooks/official/pipelines/google_cloud_pipeline_components_bqml_text.ipynb

@@ -29,21 +29,24 @@
         "id": "JAPoU8Sm5E6e"
       },
       "source": [
+        "# Training an acquisition-prediction model using Swivel, BigQuery ML and Vertex AI Pipelines\n",
+        "\n",
         "<table align=\"left\">\n",
         "\n",
         "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/official/pipelines/google_cloud_pipeline_components_bqml_text.ipynb\"\">\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/google_cloud_pipeline_components_bqml_text.ipynb\"\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/notebooks/official/pipelines/google_cloud_pipeline_components_bqml_text.ipynb\">\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/google_cloud_pipeline_components_bqml_text.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/notebooks/official/pipelines/google_cloud_pipeline_components_bqml_text.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/google_cloud_pipeline_components_bqml_text.ipynb\">\n",
+        "    <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
         "  </td>\n",
@@ -55,33 +58,67 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "tvgnzT1CKxrO"
+        "id": "991ab00f3d75"
       },
       "source": [
         "## Overview\n",
         "\n",
-        "This notebooks shows the DataflowPythonJobOp and the main BQML components in a Text Categorization Vertex AI Pipeline. \n",
+        "This notebook demonstrates the usage of `DataflowPythonJobOp` and BigQuery ML components through buidling a Text Categorization model and running it on Vertex AI Pipelines. \n",
         "\n",
-        "The pipeline will \n",
+        "The steps performed include:\n",
         "\n",
-        "1. Read raw text (HTML) documents stored in Google Cloud Storage\n",
-        "2. Extract title, content and topic of (HTML) documents using Dataflow and ingest into BigQuery\n",
-        "3. Apply the Swivel model to generate embeddings of our document’s content\n",
+        "1. Read raw text (HTML) documents stored in Google Cloud Storage.\n",
+        "2. Extract title, content and topic of (HTML) documents using Dataflow and ingest into BigQuery.\n",
+        "3. Apply the Swivel model to generate embeddings of your document’s content.\n",
         "4. Train a Logistic regression model to classify if an article is about corporate acquisitions (`acq` category). \n",
-        "5. Evaluate the model \n",
-        "6. Apply the model to a dataset in order to generate predictions\n",
-        "\n",
-        "### Dataset\n",
-        "\n",
-        "The dataset is [Reuters-21578 Text Categorization Collection Data Set](https://archive.ics.uci.edu/ml/datasets/reuters-21578+text+categorization+collection).\n",
-        "\n",
-        "The dataset is a collection of publicly available news articles appeared on the Reuters newswire in 1987. They were assembled and indexed with categories by personnel from Reuters Ltd. and Carnegie Group, Inc. in 1987.\n",
-        "\n",
+        "5. Evaluate the model.\n",
+        "6. Apply the model to a dataset in order to generate predictions."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "acc98a3361cc"
+      },
+      "source": [
         "### Objective\n",
         "\n",
-        "In this notebook, you will learn how to build a simple BigQuery ML pipeline on Vertex AI pipeline in order to calculate text embeddings of articles' content and classify them\n",
-        "depending the *corporate acquisitions* category.\n",
+        "In this notebook, you learn how to build a simple BigQuery ML pipeline using Vertex AI pipelines in order to calculate text embeddings of content from articles and classify them\n",
+        "into the *corporate acquisitions* category.\n",
         "\n",
+        "\n",
+        "This tutorial uses the following Google Cloud ML services and resources:\n",
+        "\n",
+        "- Vertex AI Pipelines\n",
+        "- BigQuery ML\n",
+        "\n",
+        "\n",
+        "The steps performed include:\n",
+        "\n",
+        "- Creating a component for Dataflow job that ingests data to BigQuery.\n",
+        "- Creating a component for preprocessing steps to run on the data in BigQuery.\n",
+        "- Creating a component for training a logistic regression model using BigQuery ML.\n",
+        "- Building and configuring a Kubeflow DSL pipeline with all the created components.\n",
+        "- Compiling and running the pipeline in Vertex AI Pipelines."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9bc6d52899ba"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used in this notebook is the [Reuters-21578 Text Categorization Collection Data Set](https://archive.ics.uci.edu/ml/datasets/reuters-21578+text+categorization+collection). This dataset is a collection of publicly available news articles appeared on the Reuters newswire in 1987. They were assembled and indexed with categories by personnel from Reuters Ltd. and Carnegie Group, Inc. in 1987."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tvgnzT1CKxrO"
+      },
+      "source": [
         "### Costs \n",
         "\n",
         "This tutorial uses billable components of Google Cloud:\n",
@@ -89,19 +126,15 @@
         "* Vertex AI\n",
         "* Cloud Storage\n",
         "* BigQuery\n",
-        "* Dataflow"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "ze4-nDLfK4pw"
-      },
-      "source": [
-        "### Set up your local development environment\n",
+        "* Dataflow\n",
         "\n",
-        "**If you are using Colab or Google Cloud Notebooks**, your environment already meets\n",
-        "all the requirements to run this notebook. You can skip this step."
+        "Learn about [Vertex AI\n",
+        "pricing](https://cloud.google.com/vertex-ai/pricing), [Cloud Storage\n",
+        "pricing](https://cloud.google.com/storage/pricing), [BigQuery\n",
+        "pricing](https://cloud.google.com/bigquery/pricing), [Dataflow\n",
+        "pricing](https://cloud.google.com/dataflow/pricing) and use the [Pricing\n",
+        "Calculator](https://cloud.google.com/products/calculator/)\n",
+        "to generate a cost estimate based on your projected usage."
       ]
     },
     {
@@ -110,6 +143,11 @@
         "id": "gCuSR8GkAgzl"
       },
       "source": [
+        "### Set up your local development environment\n",
+        "\n",
+        "**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets\n",
+        "all the requirements to run this notebook. You can skip this step.\n",
+        "\n",
         "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
         "You need the following:\n",
         "\n",
@@ -147,7 +185,7 @@
         "id": "i7EUnXsZhAGF"
       },
       "source": [
-        "### Install additional packages\n",
+        "## Install additional packages\n",
         "\n",
         "Install additional package dependencies not installed in your notebook environment, such as Vertex AI SDK. Use the latest major GA version of each package."
       ]
@@ -162,43 +200,26 @@
       "source": [
         "import os\n",
         "\n",
-        "# The Google Cloud Notebook product has specific requirements\n",
-        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
         "\n",
-        "# Google Cloud Notebook requires dependencies to be installed with '--user'\n",
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
         "USER_FLAG = \"\"\n",
-        "if IS_GOOGLE_CLOUD_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\""
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "e6d33c55a3c5"
-      },
-      "outputs": [],
-      "source": [
-        "if os.getenv(\"IS_TESTING\"):\n",
-        "    ! touch /builder/home/.local/lib/python3.9/site-packages/google_api_core-2.7.1.dist-info/METADATA"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "wyy5Lbnzg5fi"
-      },
-      "outputs": [],
-      "source": [
-        "! pip3 install {USER_FLAG} --upgrade \"apache-beam[gcp]==2.36.0\"\n",
-        "! pip3 install {USER_FLAG} --upgrade \"bs4==0.0.1\"\n",
-        "! pip3 install {USER_FLAG} --upgrade \"nltk==3.7\"\n",
-        "! pip3 install {USER_FLAG} --upgrade \"tensorflow<2.8.0\"\n",
-        "! pip3 install {USER_FLAG} --upgrade \"tensorflow-hub==0.12.0\"\n",
-        "! pip3 install {USER_FLAG} --upgrade \"kfp==1.8.2\"\n",
-        "! pip3 install {USER_FLAG} --upgrade \"google-cloud-aiplatform==1.10.0\"\n",
-        "! pip3 install {USER_FLAG} --upgrade \"google_cloud_pipeline_components==1.0.1\""
+        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\"\n",
+        "     \n",
+        "# Install dependencies\n",
+        "! pip3 install {USER_FLAG} --upgrade google-cloud-aiplatform==1.10.0 \\\n",
+        "                                    google_cloud_pipeline_components==1.0.1 \\\n",
+        "                                    google-api-core==2.8.2 \\\n",
+        "                                    google-auth==1.35.0 -q\n",
+        "\n",
+        "! pip3 install {USER_FLAG} --upgrade tensorflow==2.8.0 \\\n",
+        "                                    tensorflow-hub==0.12.0 \\\n",
+        "                                    kfp==1.8.9 -q"
       ]
     },
     {
@@ -254,9 +275,9 @@
         "\n",
         "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
         "\n",
-        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com)\n",
+        "1. [Enable the required APIs](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,bigquery.googleapis.com,dataflow.googleapis.com,storage-component.googleapis.com).\n",
         "\n",
-        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
         "\n",
         "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
         "Cloud SDK uses the right project for all the commands in this notebook.\n",
@@ -279,40 +300,26 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "oM1iC_MfAts1"
+        "id": "cd85f5c794e5"
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
-        "PROJECT_ID = \"\"\n",
-        "\n",
-        "# Get your Google Cloud project ID from gcloud\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    shell_output = !gcloud config list --format 'value(core.project)' 2>/dev/null\n",
-        "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID: \", PROJECT_ID)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "qJYoRfYng0XZ"
-      },
-      "source": [
-        "Otherwise, set your project ID here."
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "riG_qUokg0XZ"
+        "id": "oM1iC_MfAts1"
       },
       "outputs": [],
       "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
-        "    PROJECT_ID = \"\"  # @param {type:\"string\"}"
+        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
+        "    # Get your GCP project id from gcloud\n",
+        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "    PROJECT_ID = shell_output[0]\n",
+        "    print(\"Project ID:\", PROJECT_ID)"
       ]
     },
     {
@@ -323,7 +330,41 @@
       },
       "outputs": [],
       "source": [
-        "!gcloud config set project $PROJECT_ID"
+        "! gcloud config set project $PROJECT_ID"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "264543a144ad"
+      },
+      "source": [
+        "#### Region\n",
+        "\n",
+        "You can also change the `REGION` variable, which is used for operations\n",
+        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. It is recommended that you choose the region closest to you.\n",
+        "\n",
+        "- Americas: `us-central1`\n",
+        "- Europe: `europe-west4`\n",
+        "- Asia Pacific: `asia-east1`\n",
+        "\n",
+        "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
+        "\n",
+        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3281bedf6d3c"
+      },
+      "outputs": [],
+      "source": [
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -332,9 +373,9 @@
         "id": "06571eb4063b"
       },
       "source": [
-        "#### Timestamp\n",
+        "#### UUID\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append it onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
       ]
     },
     {
@@ -345,9 +386,16 @@
       },
       "outputs": [],
       "source": [
-        "from datetime import datetime\n",
+        "import random\n",
+        "import string\n",
         "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+        "\n",
+        "# Generate a uuid of length 8\n",
+        "def generate_uuid():\n",
+        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=8))\n",
+        "\n",
+        "\n",
+        "UUID = generate_uuid()"
       ]
     },
     {
@@ -358,16 +406,9 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Google Cloud Notebooks**, your environment is already\n",
-        "authenticated. Skip this step."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "sBCra4QMA2wR"
-      },
-      "source": [
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
+        "authenticated.\n",
+        "\n",
         "**If you are using Colab**, run the cell below and follow the instructions\n",
         "when prompted to authenticate your account via oAuth.\n",
         "\n",
@@ -400,19 +441,19 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "import sys\n",
-        "\n",
         "# If you are running this notebook in Colab, run this cell and follow the\n",
         "# instructions to authenticate your GCP account. This provides access to your\n",
         "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
         "# requests.\n",
         "\n",
-        "# The Google Cloud Notebook product has specific requirements\n",
-        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
+        "import os\n",
+        "import sys\n",
         "\n",
-        "# If on Google Cloud Notebooks, then don't execute this code\n",
-        "if not IS_GOOGLE_CLOUD_NOTEBOOK:\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = \"google.colab\" in sys.modules\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
         "    if \"google.colab\" in sys.modules:\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
@@ -435,12 +476,10 @@
         "\n",
         "**The following steps are required, regardless of your notebook environment.**\n",
         "\n",
-        "Set the name of your Cloud Storage bucket below. It must be unique across all\n",
-        "Cloud Storage buckets.\n",
+        "When you use services like Vertex AI and Dataflow, you need to specify Cloud Storage bucket paths as staging locations. Cloud Storage bucket is used to save the artifacts that are required or that are generated while using the services.\n",
         "\n",
-        "You may also change the `REGION` variable, which is used for operations\n",
-        "throughout the rest of this notebook. We suggest that you [choose a region where Vertex AI services are\n",
-        "available](https://cloud.google.com/vertex-ai/docs/general/locations#available_regions)."
+        "Set the name of your Cloud Storage bucket below. It must be unique across all\n",
+        "Cloud Storage buckets."
       ]
     },
     {
@@ -451,8 +490,8 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_URI = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "REGION = \"[your-region]\"  # @param {type:\"string\"}"
+        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
     {
@@ -463,11 +502,9 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"-aip-\" + TIMESTAMP\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
+        "    BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
     {
@@ -510,6 +547,78 @@
         "! gsutil ls -al $BUCKET_URI"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3708bd0b1855"
+      },
+      "source": [
+        "#### Service Account\n",
+        "\n",
+        "You use a service account to create Vertex AI Pipeline jobs. If you do not want to use your project's Compute Engine service account, set `SERVICE_ACCOUNT` to another service account ID."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "199a32a35466"
+      },
+      "outputs": [],
+      "source": [
+        "SERVICE_ACCOUNT = \"[your-service-account]\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "abb872bb98c1"
+      },
+      "outputs": [],
+      "source": [
+        "if (\n",
+        "    SERVICE_ACCOUNT == \"\"\n",
+        "    or SERVICE_ACCOUNT is None\n",
+        "    or SERVICE_ACCOUNT == \"[your-service-account]\"\n",
+        "):\n",
+        "    # Get your service account from gcloud\n",
+        "    if not IS_COLAB:\n",
+        "        shell_output = !gcloud auth list 2>/dev/null\n",
+        "        SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
+        "\n",
+        "    else:  # IS_COLAB:\n",
+        "        shell_output = ! gcloud projects describe  $PROJECT_ID\n",
+        "        project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n",
+        "        SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n",
+        "\n",
+        "    print(\"Service Account:\", SERVICE_ACCOUNT)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "4a63f2d1cd52"
+      },
+      "source": [
+        "#### Set service account access for Vertex AI Pipelines\n",
+        "\n",
+        "Run the following commands to grant your service account access to read and write pipeline artifacts in the bucket that you created in the previous step. You only need to run this step once per service account."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "749c598c5f5d"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
+        "\n",
+        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -552,11 +661,11 @@
       "source": [
         "### Prepare input data\n",
         "\n",
-        "In the following code, you will \n",
+        "In the following cell, you:\n",
         "\n",
-        "1) Get dataset from UCI archive.\n",
-        "2) Untar the dataset\n",
-        "3) Copy the dataset to a Cloud Storage location."
+        "1) Get the dataset from UCI archive.\n",
+        "2) Untar the dataset.\n",
+        "3) Copy the dataset to the Cloud Storage location."
       ]
     },
     {
@@ -601,18 +710,16 @@
       },
       "outputs": [],
       "source": [
-        "import random\n",
         "from pathlib import Path as path\n",
         "from urllib.parse import urlparse\n",
         "\n",
         "import tensorflow_hub as hub\n",
-        "\n",
-        "os.environ[\"TFHUB_MODEL_LOAD_FORMAT\"] = \"UNCOMPRESSED\"\n",
-        "\n",
-        "import google.cloud.aiplatform as vertex_ai\n",
+        "from google.cloud import aiplatform as vertex_ai\n",
         "from kfp import dsl\n",
         "from kfp.v2 import compiler\n",
-        "from kfp.v2.dsl import component"
+        "from kfp.v2.dsl import component\n",
+        "\n",
+        "os.environ[\"TFHUB_MODEL_LOAD_FORMAT\"] = \"UNCOMPRESSED\""
       ]
     },
     {
@@ -623,7 +730,7 @@
       "source": [
         "### Define constants\n",
         "\n",
-        "About the model we are going to use in preprocessing, we use the [Swivel](https://tfhub.dev/google/tf2-preview/gnews-swivel-20dim/1) embedding which was trained on English Google News 130GB corpus and has 20 dimensions."
+        "About the model you are going to use in preprocessing, you use the [Swivel](https://tfhub.dev/google/tf2-preview/gnews-swivel-20dim/1) embedding which was trained on English Google News 130GB corpus and has 20 dimensions."
       ]
     },
     {
@@ -634,7 +741,7 @@
       },
       "outputs": [],
       "source": [
-        "JOB_NAME = f\"reuters-ingest-{TIMESTAMP}\"\n",
+        "JOB_NAME = f\"reuters-ingest-{UUID}\"\n",
         "SETUP_FILE_URI = urlparse(BUCKET_URI)._replace(path=\"setup.py\").geturl()\n",
         "RUNNER = \"DataflowRunner\"\n",
         "STAGING_LOCATION_URI = urlparse(BUCKET_URI)._replace(path=\"staging\").geturl()\n",
@@ -643,13 +750,13 @@
         "BQ_DATASET = \"mlops_bqml_text_analyisis\"\n",
         "BQ_TABLE = \"reuters_ingested\"\n",
         "MODEL_NAME = \"swivel_text_embedding_model\"\n",
-        "EMBEDDINGS_TABLE = f\"reuters_text_embeddings_{TIMESTAMP}\"\n",
+        "EMBEDDINGS_TABLE = f\"reuters_text_embeddings_{UUID}\"\n",
         "MODEL_PATH = (\n",
         "    f'{hub.resolve(\"https://tfhub.dev/google/tf2-preview/gnews-swivel-20dim/1\")}/*'\n",
         ")\n",
-        "PREPROCESSED_TABLE = f\"reuters_text_preprocessed_{TIMESTAMP}\"\n",
+        "PREPROCESSED_TABLE = f\"reuters_text_preprocessed_{UUID}\"\n",
         "CLASSIFICATION_MODEL_NAME = \"logistic_reg\"\n",
-        "PREDICT_TABLE = f\"reuters_text_predict_{TIMESTAMP}\""
+        "PREDICT_TABLE = f\"reuters_text_predict_{UUID}\""
       ]
     },
     {
@@ -658,7 +765,7 @@
         "id": "NlyOjKrjCXsI"
       },
       "source": [
-        "### Initialize client"
+        "### Initialize Vertex AI SDK client"
       ]
     },
     {
@@ -678,7 +785,9 @@
         "id": "ZrgOD30o7HcL"
       },
       "source": [
-        "## Pipeline formalization"
+        "## Pipeline formalization\n",
+        "\n",
+        "In this step, you create various components for the pipeline and build the final pipeline."
       ]
     },
     {
@@ -757,7 +866,7 @@
         "from apache_beam.options.pipeline_options import SetupOptions\n",
         "\n",
         "\n",
-        "# Helpers -------------------------------------------------------- -------------\n",
+        "# Helpers ---------------------------------------------------------------------\n",
         "\n",
         "def get_args():\n",
         "    \"\"\"\n",
@@ -949,7 +1058,7 @@
         "apache-beam[gcp]==2.36.0\n",
         "bs4==0.0.1\n",
         "nltk==3.7\n",
-        "tensorflow<2.8.0"
+        "tensorflow==2.8.0"
       ]
     },
     {
@@ -993,7 +1102,7 @@
         "REQUIRED_PACKAGES = [\n",
         "    'bs4==0.0.1',\n",
         "    'nltk==3.7',\n",
-        "    'tensorflow<2.8.0']\n",
+        "    'tensorflow==2.8.0']\n",
         "\n",
         "setuptools.setup(\n",
         "    name='ingest',\n",
@@ -1010,7 +1119,7 @@
         "id": "5Nc7ByK1AEe8"
       },
       "source": [
-        "#### Copy the setup, the python module and requirements file to Cloud Storage\n",
+        "#### Copy the setup, module and requirements files to Cloud Storage\n",
         "\n",
         "Finally, copy the Python module, requirements and setup file to your Cloud Storage bucket."
       ]
@@ -1035,15 +1144,15 @@
         "id": "MI_wYYwdAZZs"
       },
       "source": [
-        "### BQML components\n",
+        "### BigQuery ML components\n",
         "\n",
-        "To build the next steps of our pipelines, we define a set of queries to:\n",
+        "For the next steps in building the pipeline, you define a set of queries to:\n",
         "\n",
         "1) Create the BigQuery dataset schema.\n",
-        "2) Preprocess our text data and generate the embeddings using Swevel model\n",
+        "2) Preprocess your text data and generate the embeddings using Swivel model.\n",
         "2) Train the BigQuery ML Logistic Regression model.\n",
         "3) Evaluate the model.\n",
-        "4) Run a batch prediction\n"
+        "4) Run a batch prediction.\n"
       ]
     },
     {
@@ -1066,9 +1175,9 @@
         "id": "8tYtZMRiepKe"
       },
       "source": [
-        "#### Create BQ Dataset query\n",
+        "#### Create BigQuery Dataset query\n",
         "\n",
-        "With this query, we create the Bigquery dataset schema we are going to use to train our model."
+        "With this query, you create the Bigquery dataset schema that you are going to use to train your model."
       ]
     },
     {
@@ -1096,9 +1205,9 @@
         "id": "T2_kuUSZUBDY"
       },
       "source": [
-        "#### Create BQ Preprocess query\n",
+        "#### Create BigQuery Preprocess query\n",
         "\n",
-        "The following query use the TFHub Swevel model to generate the embedding of our text data and split the dataset for training and serving purposes."
+        "The following query uses the TFHub Swivel model to generate embeddings for your text data and splits the dataset for training and serving purposes."
       ]
     },
     {
@@ -1187,9 +1296,9 @@
         "id": "yF9W5x4HgUQb"
       },
       "source": [
-        "#### Create BQ Model query\n",
+        "#### Create BigQuery Model query\n",
         "\n",
-        "Below you have a simple query to build a BigQuery ML Logistic Classifier model for topic's articles classification."
+        "Below, you have a simple query to build a BigQuery ML Logistic Classifier model for topic's articles classification."
       ]
     },
     {
@@ -1224,9 +1333,9 @@
         "id": "RlLTcuUhdzFU"
       },
       "source": [
-        "#### Create BQ Prediction query\n",
+        "#### Create BigQuery Prediction query\n",
         "\n",
-        "With the following query, we run a prediction job using the table with the preprocessing query."
+        "With the following query, you run a prediction job using the table with the preprocessing query."
       ]
     },
     {
@@ -1252,7 +1361,14 @@
         "id": "OxemUVCxAiSo"
       },
       "source": [
-        "### Build Pipeline"
+        "### Build the pipeline\n",
+        "\n",
+        "In this step, you build the pipeline using the individual components.\n",
+        "\n",
+        "Define the `JOB_NAME` and `JOB_CONFIG` below. `JOB_CONFIG` consists of the following parameters for the destination table:\n",
+        "- `PROJECT_ID`: Id of the project.\n",
+        "- `BQ_DATASET`: Id of the BigQuery dataset.\n",
+        "- `PREDICT_TABLE`: Id of the BigQuery table where predictions are stored."
       ]
     },
     {
@@ -1264,7 +1380,7 @@
       "outputs": [],
       "source": [
         "ID = random.randint(1, 10000)\n",
-        "JOB_NAME = f\"reuters-preprocess-{TIMESTAMP}-{ID}\"\n",
+        "JOB_NAME = f\"reuters-preprocess-{UUID}-{ID}\"\n",
         "JOB_CONFIG = {\n",
         "    \"destinationTable\": {\n",
         "        \"projectId\": PROJECT_ID,\n",
@@ -1280,7 +1396,9 @@
         "id": "mdO8st_gLKBZ"
       },
       "source": [
-        "#### Create a custom component to pass `DataflowPythonJobOp` arguments"
+        "#### Create a custom component for arguments\n",
+        "\n",
+        "Next, you create a component to pass arguments to the `DataflowPythonJobOp` component."
       ]
     },
     {
@@ -1323,7 +1441,26 @@
         "id": "pcSL1FHk69KT"
       },
       "source": [
-        "#### Create the pipeline"
+        "#### Create the pipeline\n",
+        "\n",
+        "Define the workflow of the pipeline and build the pipeline. The parameters passed to the pipeline include:\n",
+        "\n",
+        "- `create_bq_dataset_query`: SQL query to create the dataset in BigQuery.\n",
+        "- `job_name`: Name of the Cloud Dataflow job to be configured in `PipelineOptions`.\n",
+        "- `inputs_uri`: A directory location of input data.\n",
+        "-  `bq_dataset`: Dataset name used in BigQuery.\n",
+        "-  `bq_table`: Table name used in BigQuery for ingestion.\n",
+        "-  `requirements_file_path`: The GCS path to the pip requirements file.\n",
+        "-  `python_file_path`: The GCS path to the python file to run.\n",
+        "-  `setup_file_uri`: Path to a Python setup file containing package dependencies.\n",
+        "-  `temp_location`:  GCS path for Dataflow to stage temporary job files created during the execution of the pipeline.\n",
+        "-  `runner`: Pipeline runner used to execute the workflow.\n",
+        "-  `create_bq_preprocess_query`: SQL query to preprocess the data in BigQuery.\n",
+        "-  `create_bq_model_query`: SQL query to create the BigQuery ML model.\n",
+        "-  `create_bq_prediction_query`: SQL query for prediction.\n",
+        "-  `job_config`: A json formatted string describing the job configuration. For more information, vist this [page]( https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery).\n",
+        "-  `project`: Project ID.\n",
+        "-  `region`: Selected region to run the Dataflow job."
       ]
     },
     {
@@ -1371,7 +1508,7 @@
         "        project=project,\n",
         "        location=\"US\",\n",
         "    )\n",
-        "    # instanciate dataflow args\n",
+        "    # instantiate dataflow args\n",
         "    build_dataflow_args_op = build_dataflow_args(\n",
         "        job_name=job_name,\n",
         "        inputs_uri=inputs_uri,\n",
@@ -1431,7 +1568,9 @@
         "id": "nghLONQX7JNg"
       },
       "source": [
-        "## Compile and Run the pipeline"
+        "## Compile and Run the pipeline\n",
+        "\n",
+        "Pass the necessary constants and parameters to the pipeline and compile it to a json file."
       ]
     },
     {
@@ -1451,6 +1590,15 @@
         "compiler.Compiler().compile(pipeline_func=pipeline, package_path=PIPELINE_PACKAGE)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "b5c1b7b8b290"
+      },
+      "source": [
+        "Using the compiled json file, create Vertex AI Pipeline Job and run it by passing the `SERVICE_ACCOUNT` details configured earlier."
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -1460,7 +1608,7 @@
       "outputs": [],
       "source": [
         "pipeline = vertex_ai.PipelineJob(\n",
-        "    display_name=f\"data_preprocess_{TIMESTAMP}\",\n",
+        "    display_name=f\"data_preprocess_{UUID}\",\n",
         "    template_path=PIPELINE_PACKAGE,\n",
         "    pipeline_root=PIPELINE_ROOT,\n",
         "    parameter_values={\n",
@@ -1482,7 +1630,27 @@
         "    enable_caching=False,\n",
         ")\n",
         "\n",
-        "pipeline.run()"
+        "pipeline.run(service_account=SERVICE_ACCOUNT)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "bcb582e65740"
+      },
+      "source": [
+        "Once the pipeline job gets finished successfully, the trained model can be found created in the BigQuery dataset. Run the following cell to see the model listed in the output."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "c0d194c006ae"
+      },
+      "outputs": [],
+      "source": [
+        "! bq ls $PROJECT_ID:$BQ_DATASET"
       ]
     },
     {
@@ -1494,7 +1662,9 @@
         "## Cleaning up\n",
         "\n",
         "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
-        "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n"
+        "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
+        "\n",
+        "Otherwise, you can delete the individual resources you created in this tutorial in the following cell. Set `delete_bucket` and `delete_dataset` to **True** to delete the Cloud Storage bucket and the Bigquery dataset used in this notebook respectively."
       ]
     },
     {
@@ -1505,11 +1675,19 @@
       },
       "outputs": [],
       "source": [
+        "# delete the pipeline job\n",
+        "pipeline.delete()\n",
+        "\n",
+        "delete_bucket = False\n",
+        "delete_dataset = False\n",
+        "\n",
         "# delete bucket\n",
-        "! gsutil -m rm -r $BUCKET_URI\n",
+        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "    ! gsutil -m rm -r $BUCKET_URI\n",
         "\n",
         "# delete dataset\n",
-        "! bq rm -r -f -d $PROJECT_ID:$BQ_DATASET"
+        "if delete_dataset or os.getenv(\"IS_TESTING\"):\n",
+        "    ! bq rm -r -f -d $PROJECT_ID:$BQ_DATASET"
       ]
     }
   ],

notebooks/official/pipelines/google_cloud_pipelines_dataproc_tabular/src/data_preprocessing.py

@@ -1,169 +0,0 @@
-# Copyright 2022 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-
-"""
-data_preprocessing.py is the module for
-
-  - ingest data
-  - do simple preprocessing tasks
-  - upload processed data to gcs
-"""
-
-# Libraries --------------------------------------------------------------------------------
-import logging
-import argparse
-from pathlib import Path
-import sys
-
-try:
-    from pyspark import SparkContext, SparkConf
-    from pyspark.sql import SparkSession
-except ImportError as error:
-    print('WARN: Something wrong with pyspark library. Please check configuration settings!')
-    print(error)
-
-from pyspark.sql.types import StructType, DoubleType, StringType
-
-# Variables --------------------------------------------------------------------------------
-DATA_SCHEMA = (StructType()
-               .add("label", StringType(), True)
-               .add("loan_amount", StringType(), True)
-               .add("loan_term", StringType(), True)
-               .add("property_area", StringType(), True)
-               .add("timestamp", StringType(), True)
-               .add("entity_type_customer_id", StringType(), True)
-               .add("feature_7", DoubleType(), True)
-               .add("feature_3", DoubleType(), True)
-               .add("feature_1", DoubleType(), True)
-               .add("feature_9", DoubleType(), True)
-               .add("feature_5", DoubleType(), True)
-               .add("feature_0", DoubleType(), True)
-               .add("feature_8", DoubleType(), True)
-               .add("feature_4", DoubleType(), True)
-               .add("feature_2", DoubleType(), True)
-               .add("feature_6", DoubleType(), True)
-               )
-
-ENTITY_CUSTOMER_ID = 'entity_type_customer_id'
-FEATURE_STORE_IDS = ['timestamp', 'entity_type_customer_id']
-CATEGORICAL_VARIABLES = ['loan_term', 'property_area']
-IDX_CATEGORICAL_FEATURES = [f'{col}_idx' for col in CATEGORICAL_VARIABLES]
-TARGET = 'label'
-
-
-# Helpers ----------------------------------------------------------------------------------
-
-def set_logger():
-    """
-    Set logger for the module
-    Returns:
-        logger: logger object
-    """
-    fmt_pattern = "%(asctime)s — %(name)s — %(levelname)s —" "%(funcName)s:%(lineno)d — %(message)s"
-    main_logger = logging.getLogger(__name__)
-    main_logger.setLevel(logging.INFO)
-    main_logger.propagate = False
-    stream_handler = logging.StreamHandler(sys.stdout)
-    stream_handler.setLevel(logging.INFO)
-    formatter = logging.Formatter(fmt_pattern)
-    stream_handler.setFormatter(formatter)
-    main_logger.addHandler(stream_handler)
-    return main_logger
-
-
-def get_args():
-    """
-    Get arguments from command line
-    Returns:
-        args: arguments from command line
-    """
-    args_parser = argparse.ArgumentParser()
-    args_parser.add_argument(
-        '--train-data-path',
-        help='The GCS path of training sample',
-        type=str,
-        required=True)
-    args_parser.add_argument(
-        '--out-process-path',
-        help='''
-        The path to load processed data. 
-        Format: 
-        - locally: /path/to/dir
-        - cloud: gs://bucket/path
-        ''',
-        type=str,
-        required=True)
-    return args_parser.parse_args()
-
-
-# Main -------------------------------------------------------------------------------------
-
-def main(logger, args):
-    """
-    Main function
-    Args:
-        logger: logger object
-        args: arguments from command line
-    Returns:
-        None
-    """
-    # variables
-    train_data_path = args.train_data_path
-    output_data_path = args.out_process_path
-
-    logger.info('initializing data preprocessing.')
-    logger.info('start spark session.')
-
-    spark = (SparkSession.builder
-             .master("local[*]")
-             .appName("spark go live")
-             .config('spark.ui.port', '4050')
-             .getOrCreate())
-    try:
-        logger.info(f'spark version: {spark.sparkContext.version}')
-        logger.info('start ingesting data.')
-
-        training_data_raw_df = (spark.read.option("header", True)
-                                .option("delimiter", ',')
-                                .schema(DATA_SCHEMA)
-                                .csv(train_data_path)
-                                .drop(*FEATURE_STORE_IDS))
-
-        training_data_raw_df = training_data_raw_df.withColumn("label",
-                                                               training_data_raw_df.label.cast('double'))
-        training_data_raw_df = training_data_raw_df.withColumn("loan_amount",
-                                                               training_data_raw_df.loan_amount.cast('double'))
-        training_data_raw_df.show(truncate=False)
-
-        logger.info(f'load prepared data to {output_data_path}.')
-        if output_data_path.startswith('gs://'):
-            training_data_raw_df.write.mode('overwrite').csv(str(output_data_path), header=True)
-        else:
-            output_file_path = Path(output_data_path)
-            output_file_path.mkdir(parents=True, exist_ok=True)
-            training_data_raw_df.write.mode('overwrite').csv(str(output_file_path), header=True)
-    except RuntimeError as main_error:
-        logger.error(main_error)
-    else:
-        logger.info('data preprocessing successfully completed!')
-        return 0
-
-
-if __name__ == "__main__":
-    runtime_args = get_args()
-    runtime_logger = set_logger()
-    main(runtime_logger, runtime_args)

notebooks/official/pipelines/google_cloud_pipelines_dataproc_tabular/src/hp_tuning.py

@@ -1,366 +0,0 @@
-# Copyright 2022 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-
-"""
-hp_model_tuning.py is the module for hypertune the spark pipeline
-"""
-
-# Libraries --------------------------------------------------------------------------------
-import logging
-import sys
-import argparse
-from os import environ
-from datetime import datetime
-from pathlib import Path as path
-import tempfile
-from urllib.parse import urlparse, urljoin
-import json
-
-try:
-    from pyspark import SparkContext, SparkConf
-    from pyspark.sql import SparkSession
-except ImportError as e:
-    print('WARN: Something wrong with pyspark library. Please check configuration settings!')
-    print(e)
-
-from pyspark.sql.types import StructType, DoubleType, StringType
-from pyspark.sql.functions import col, udf
-from pyspark.sql.functions import round as spark_round
-from pyspark.ml.feature import StringIndexer, StandardScaler, VectorAssembler
-from pyspark.ml.classification import RandomForestClassifier
-from pyspark.ml.evaluation import BinaryClassificationEvaluator, MulticlassClassificationEvaluator
-from pyspark.ml.tuning import ParamGridBuilder, CrossValidator
-from pyspark.ml import Pipeline
-
-from google.cloud import storage
-
-# Variables --------------------------------------------------------------------------------
-
-# Data schema
-DATA_SCHEMA = (StructType()
-               .add("label", DoubleType(), True)
-               .add("loan_amount", DoubleType(), True)
-               .add("loan_term", StringType(), True)
-               .add("property_area", StringType(), True)
-               .add("feature_7", DoubleType(), True)
-               .add("feature_3", DoubleType(), True)
-               .add("feature_1", DoubleType(), True)
-               .add("feature_9", DoubleType(), True)
-               .add("feature_5", DoubleType(), True)
-               .add("feature_0", DoubleType(), True)
-               .add("feature_8", DoubleType(), True)
-               .add("feature_4", DoubleType(), True)
-               .add("feature_2", DoubleType(), True)
-               .add("feature_6", DoubleType(), True)
-               )
-
-# Training
-TARGET = 'label'
-CATEGORICAL_VARIABLES = ['loan_term', 'property_area']
-IDX_CATEGORICAL_FEATURES = [f'{col}_idx' for col in CATEGORICAL_VARIABLES]
-REAL_TIME_FEATURES_VECTOR = 'real_time_features_vector'
-REAL_TIME_FEATURES = 'real_time_features'
-FEATURES_SELECTED = ['feature_0', 'feature_1', 'feature_2', 'feature_3', 'feature_4', 'feature_5',
-                     'feature_6', 'feature_7', 'feature_8', 'feature_9', 'real_time_features']
-FEATURES = 'features'
-RANDOM_SEED = 8
-RANDOM_QUOTAS = [0.8, 0.2]
-MAX_DEPTH = [5, 10, 15]
-MAX_BINS = [24, 32, 40]
-N_TREES = [25, 30, 35]
-N_FOLDS = 5
-
-
-# Helpers ----------------------------------------------------------------------------------
-def set_logger():
-    """
-    Set logger for the module
-    Returns:
-        logger: logger object
-    """
-    fmt_pattern = "%(asctime)s — %(name)s — %(levelname)s —" "%(funcName)s:%(lineno)d — %(message)s"
-    main_logger = logging.getLogger(__name__)
-    main_logger.setLevel(logging.INFO)
-    main_logger.propagate = False
-    stream_handler = logging.StreamHandler(sys.stdout)
-    stream_handler.setLevel(logging.INFO)
-    formatter = logging.Formatter(fmt_pattern)
-    stream_handler.setFormatter(formatter)
-    main_logger.addHandler(stream_handler)
-    return main_logger
-
-
-def get_args():
-    """
-    Get arguments from command line
-    Returns:
-        args: arguments from command line
-    """
-    args_parser = argparse.ArgumentParser()
-    args_parser.add_argument(
-        '--train-path',
-        help='''
-        The GCS path of training data'
-        Format: 
-        - locally: /path/to/dir
-        - cloud: gs://bucket/path
-        ''',
-        type=str,
-        required=False)
-    args_parser.add_argument(
-        '--model-path',
-        help='''
-        The GCS path to store the trained model. 
-        Format: 
-        - locally: /path/to/dir
-        - cloud: gs://bucket/path
-        ''',
-        type=str,
-        required=False)
-    args_parser.add_argument(
-        '--metrics-path',
-        help='''
-        The GCS path to store the metrics of model. 
-        Format: 
-        - locally: /path/to/dir
-        - cloud: gs://bucket/path
-        ''',
-        type=str,
-        required=True)
-    return args_parser.parse_args()
-
-
-def build_preprocessing_components():
-    """
-    Build preprocessing components
-    Returns:
-        preprocessing_components: preprocessing components
-    """
-    loan_term_indexer = StringIndexer(inputCol=CATEGORICAL_VARIABLES[0], outputCol=IDX_CATEGORICAL_FEATURES[0],
-                                      stringOrderType='frequencyDesc', handleInvalid='keep')
-    property_area_indexer = StringIndexer(inputCol=CATEGORICAL_VARIABLES[1], outputCol=IDX_CATEGORICAL_FEATURES[1],
-                                          stringOrderType='frequencyDesc', handleInvalid='keep')
-    data_preprocessing_stages = [loan_term_indexer, property_area_indexer]
-    return data_preprocessing_stages
-
-
-def build_feature_engineering_components():
-    """
-    Build feature engineering components
-    Returns:
-        feature_engineering_components: feature engineering components
-    """
-    feature_engineering_stages = []
-    realtime_vector_assembler = VectorAssembler(inputCols=IDX_CATEGORICAL_FEATURES, outputCol=REAL_TIME_FEATURES_VECTOR)
-    realtime_scaler = StandardScaler(inputCol=REAL_TIME_FEATURES_VECTOR, outputCol=REAL_TIME_FEATURES)
-    features_vector_assembler = VectorAssembler(inputCols=FEATURES_SELECTED, outputCol=FEATURES)
-    feature_engineering_stages.extend((realtime_vector_assembler,
-                                       realtime_scaler,
-                                       features_vector_assembler))
-    return feature_engineering_stages
-
-
-def build_training_model_component():
-    """
-    Build training model component
-    Returns:
-        training_model_component: training model component
-    """
-    model_training_stage = []
-    rfor = RandomForestClassifier(featuresCol=FEATURES, labelCol=TARGET, seed=RANDOM_SEED)
-    model_training_stage.append(rfor)
-    return model_training_stage
-
-
-def build_hp_pipeline(data_preprocessing_stages, feature_engineering_stages, model_training_stage):
-    """
-    Build hyperparameter pipeline
-    Args:
-        data_preprocessing_stages: preprocessing components
-        feature_engineering_stages: feature engineering components
-        model_training_stage: training model component
-    Returns:
-        hp_pipeline: hyperparameter pipeline
-    """
-    pipeline = Pipeline(stages=data_preprocessing_stages + feature_engineering_stages + model_training_stage)
-    params_grid = (ParamGridBuilder()
-                   .addGrid(model_training_stage[0].maxDepth, MAX_DEPTH)
-                   .addGrid(model_training_stage[0].maxBins, MAX_BINS)
-                   .addGrid(model_training_stage[0].numTrees, N_TREES)
-                   .build())
-    evaluator = BinaryClassificationEvaluator(labelCol=TARGET)
-    cross_validator = CrossValidator(estimator=pipeline,
-                                     estimatorParamMaps=params_grid,
-                                     evaluator=evaluator,
-                                     numFolds=N_FOLDS)
-    return cross_validator
-
-
-def get_true_score_prediction(predictions, target):
-    """
-    Get true score and prediction
-    Args:
-        predictions: predictions
-        target: target column
-    Returns:
-        roc_dict: a dict of roc values for each class
-    """
-
-    split1_udf = udf(lambda value: value[1].item(), DoubleType())
-    roc_dataset = predictions.select(col(target).alias('true'),
-                                     spark_round(split1_udf('probability'), 5).alias('score'),
-                                     'prediction')
-    roc_df = roc_dataset.toPandas()
-    roc_dict = roc_df.to_dict(orient='list')
-    return roc_dict
-
-
-def get_metrics(predictions, target, mode):
-    """
-    Get metrics
-    Args:
-        predictions: predictions
-        target: target column
-        mode: train or test
-    Returns:
-        metrics: metrics
-    """
-    metric_labels = ['area_roc', 'area_prc', 'accuracy', 'f1', 'precision', 'recall']
-    metric_cols = ['true', 'score', 'prediction']
-    metric_keys = [f'{mode}_{ml}' for ml in metric_labels] + metric_cols
-
-    bc_evaluator = BinaryClassificationEvaluator(labelCol=target)
-    mc_evaluator = MulticlassClassificationEvaluator(labelCol=target)
-
-    # areas, acc, f1, prec, rec
-    metric_values = []
-    area_roc = round(bc_evaluator.evaluate(predictions, {bc_evaluator.metricName: 'areaUnderROC'}), 5)
-    area_prc = round(bc_evaluator.evaluate(predictions, {bc_evaluator.metricName: 'areaUnderPR'}), 5)
-    acc = round(mc_evaluator.evaluate(predictions, {mc_evaluator.metricName: "accuracy"}), 5)
-    f1 = round(mc_evaluator.evaluate(predictions, {mc_evaluator.metricName: "f1"}), 5)
-    prec = round(mc_evaluator.evaluate(predictions, {mc_evaluator.metricName: "weightedPrecision"}), 5)
-    rec = round(mc_evaluator.evaluate(predictions, {mc_evaluator.metricName: "weightedRecall"}), 5)
-
-    # true, score, prediction
-    roc_dict = get_true_score_prediction(predictions, target)
-    true = roc_dict['true']
-    score = roc_dict['score']
-    pred = roc_dict['prediction']
-
-    metric_values.extend((area_roc, area_prc, acc, f1, prec, rec, true, score, pred))
-    metrics = dict(zip(metric_keys, metric_values))
-
-    return metrics
-
-
-def upload_file(bucket_name, source_file_name, destination_blob_name):
-    storage_client = storage.Client()
-    bucket = storage_client.bucket(bucket_name)
-    blob = bucket.blob(destination_blob_name)
-    blob.upload_from_filename(source_file_name)
-
-
-def write_metrics(bucket_name, metrics, destination, dir='/tmp'):
-    temp_dir = tempfile.TemporaryDirectory(dir=dir)
-    temp_metrics_file_path = str(path(temp_dir.name) / path(destination).name)
-    with open(temp_metrics_file_path, 'w') as temp_file:
-        json.dump(metrics, temp_file)
-    upload_file(bucket_name, temp_metrics_file_path, destination)
-    temp_dir.cleanup()
-
-
-# Main -------------------------------------------------------------------------------------
-
-def main(logger, args):
-    """
-    Main function
-    Args:
-        logger: logger
-        args: args
-    Returns:
-        None
-    """
-    train_path = args.train_path
-    model_path = args.model_path
-    metrics_path = args.metrics_path
-
-    try:
-        logger.info('initializing pipeline training.')
-        logger.info('start spark session.')
-        spark = (SparkSession.builder
-                 .master("local[*]")
-                 .appName("spark go live")
-                 .config('spark.ui.port', '4050')
-                 .config('spark.jars.packages', 'ml.combust.mleap:mleap-runtime_2.12:0.19.0')
-                 .config('spark.jars.packages', 'ml.combust.mleap:mleap-base_2.12:0.19.0')
-                 .config('spark.jars.packages', 'ml.combust.mleap:mleap-spark_2.12:0.19.0')
-                 .config('spark.jars.packages', 'ml.combust.mleap:mleap-spark-extension_2.12:0.19.0')
-                 .getOrCreate())
-        logger.info(f'spark version: {spark.sparkContext.version}')
-        logger.info('start building pipeline.')
-        preprocessing_stages = build_preprocessing_components()
-        feature_engineering_stages = build_feature_engineering_components()
-        model_training_stage = build_training_model_component()
-        pipeline_cross_validator = build_hp_pipeline(preprocessing_stages, feature_engineering_stages,
-                                                     model_training_stage)
-        logger.info(f'load train data from {train_path}.')
-        if train_path.startswith('bq://'):
-            raw_data = spark.read.format('bigquery') \
-                .option('table', train_path.replace('bq://', '')) \
-                .load()
-        else:
-            raw_data = (spark.read.format('csv')
-                        .option("header", "true")
-                        .schema(DATA_SCHEMA)
-                        .load(train_path))
-        logger.info(f'fit model pipeline.')
-        train, test = raw_data.randomSplit(RANDOM_QUOTAS, seed=RANDOM_SEED)
-        pipeline_model = pipeline_cross_validator.fit(train)
-        predictions = pipeline_model.transform(test)
-        metrics = get_metrics(predictions, TARGET, 'test')
-        for m, v in metrics.items():
-            print(f'{m}: {v}')
-
-        logger.info(f'load model pipeline in {model_path}.')
-        if model_path.startswith('gs://'):
-            pipeline_model.write().overwrite().save(model_path)
-        else:
-            path(model_path).mkdir(parents=True, exist_ok=True)
-            pipeline_model.write().overwrite().save(model_path)
-
-        logger.info(f'Upload metrics under {metrics_path}.')
-        if metrics_path.startswith('gs://'):
-            bucket = urlparse(model_path).netloc
-            metrics_file_path = urlparse(metrics_path).path.strip('/')
-            write_metrics(bucket, metrics, metrics_file_path)
-        else:
-            metrics_version_path = path(metrics_path).parents[0]
-            metrics_version_path.mkdir(parents=True, exist_ok=True)
-            with open(metrics_path, 'w') as json_file:
-                json.dump(metrics, json_file)
-            json_file.close()
-    except RuntimeError as main_error:
-        logger.error(main_error)
-    else:
-        logger.info('model pipeline training successfully completed!')
-        return 0
-
-
-if __name__ == "__main__":
-    runtime_args = get_args()
-    runtime_logger = set_logger()
-    main(runtime_logger, runtime_args)

notebooks/official/pipelines/google_cloud_pipelines_dataproc_tabular/src/model_training.py

@@ -1,359 +0,0 @@
-# Copyright 2022 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-
-"""
-model_training.py is the module for training spark pipeline
-"""
-
-# Libraries --------------------------------------------------------------------------------
-import logging
-import sys
-import argparse
-from pathlib import Path as path
-import tempfile
-import json
-from urllib.parse import urlparse
-
-try:
-    from pyspark import SparkContext, SparkConf
-    from pyspark.sql import SparkSession
-except ImportError as e:
-    print('WARN: Something wrong with pyspark library. Please check configuration settings!')
-    print(e)
-
-from pyspark.sql.types import StructType, DoubleType, StringType
-from pyspark.sql.functions import col, udf
-from pyspark.sql.functions import round as spark_round
-from pyspark.ml.feature import StringIndexer, StandardScaler, VectorAssembler
-from pyspark.ml.classification import RandomForestClassifier
-from pyspark.ml.evaluation import BinaryClassificationEvaluator, MulticlassClassificationEvaluator
-from pyspark.ml import Pipeline
-
-from google.cloud import storage
-
-# Variables --------------------------------------------------------------------------------
-
-# Data schema
-DATA_SCHEMA = (StructType()
-               .add("label", DoubleType(), True)
-               .add("loan_amount", DoubleType(), True)
-               .add("loan_term", StringType(), True)
-               .add("property_area", StringType(), True)
-               .add("feature_7", DoubleType(), True)
-               .add("feature_3", DoubleType(), True)
-               .add("feature_1", DoubleType(), True)
-               .add("feature_9", DoubleType(), True)
-               .add("feature_5", DoubleType(), True)
-               .add("feature_0", DoubleType(), True)
-               .add("feature_8", DoubleType(), True)
-               .add("feature_4", DoubleType(), True)
-               .add("feature_2", DoubleType(), True)
-               .add("feature_6", DoubleType(), True)
-               )
-
-# Training
-TARGET = 'label'
-CATEGORICAL_VARIABLES = ['loan_term', 'property_area']
-IDX_CATEGORICAL_FEATURES = [f'{col}_idx' for col in CATEGORICAL_VARIABLES]
-REAL_TIME_FEATURES_VECTOR = 'real_time_features_vector'
-REAL_TIME_FEATURES = 'real_time_features'
-FEATURES_SELECTED = ['feature_0', 'feature_1', 'feature_2', 'feature_3', 'feature_4', 'feature_5',
-                     'feature_6', 'feature_7', 'feature_8', 'feature_9', 'real_time_features']
-FEATURES = 'features'
-RANDOM_SEED = 8
-RANDOM_QUOTAS = [0.8, 0.2]
-
-
-# Helpers ----------------------------------------------------------------------------------
-def set_logger():
-    """
-    Set logger
-    Returns:
-        logger: logger
-    """
-    fmt_pattern = "%(asctime)s — %(name)s — %(levelname)s —" "%(funcName)s:%(lineno)d — %(message)s"
-    main_logger = logging.getLogger(__name__)
-    main_logger.setLevel(logging.INFO)
-    main_logger.propagate = False
-    stream_handler = logging.StreamHandler(sys.stdout)
-    stream_handler.setLevel(logging.INFO)
-    formatter = logging.Formatter(fmt_pattern)
-    stream_handler.setFormatter(formatter)
-    main_logger.addHandler(stream_handler)
-    return main_logger
-
-
-def get_args():
-    """
-    Get arguments
-    Returns:
-        args: arguments
-    """
-    args_parser = argparse.ArgumentParser()
-    args_parser.add_argument(
-        '--train-path',
-        help='''
-        The GCS path of training data'
-        Format: 
-        - locally: /path/to/dir
-        - cloud: gs://bucket/path
-        ''',
-        type=str,
-        required=True)
-    args_parser.add_argument(
-        '--model-path',
-        help='''
-        The GCS path to store the trained model. 
-        Format: 
-        - locally: /path/to/dir
-        - cloud: gs://bucket/path
-        ''',
-        type=str,
-        required=True)
-    args_parser.add_argument(
-        '--metrics-path',
-        help='''
-        The GCS path to store the metrics of model. 
-        Format: 
-        - locally: /path/to/dir
-        - cloud: gs://bucket/path
-        ''',
-        type=str,
-        required=True)
-    return args_parser.parse_args()
-
-
-def build_preprocessing_components():
-    """
-    Build preprocessing components
-    Returns:
-        data_preprocessing_stages: data preprocessing stages
-    """
-    loan_term_indexer = StringIndexer(inputCol=CATEGORICAL_VARIABLES[0], outputCol=IDX_CATEGORICAL_FEATURES[0],
-                                      stringOrderType='frequencyDesc', handleInvalid='keep')
-    property_area_indexer = StringIndexer(inputCol=CATEGORICAL_VARIABLES[1], outputCol=IDX_CATEGORICAL_FEATURES[1],
-                                          stringOrderType='frequencyDesc', handleInvalid='keep')
-    data_preprocessing_stages = [loan_term_indexer, property_area_indexer]
-    return data_preprocessing_stages
-
-
-def build_feature_engineering_components():
-    """
-    Build feature engineering components
-    Returns:
-        feature_engineering_stages: feature engineering stages
-    """
-    feature_engineering_stages = []
-    realtime_vector_assembler = VectorAssembler(inputCols=IDX_CATEGORICAL_FEATURES, outputCol=REAL_TIME_FEATURES_VECTOR)
-    realtime_scaler = StandardScaler(inputCol=REAL_TIME_FEATURES_VECTOR, outputCol=REAL_TIME_FEATURES)
-    features_vector_assembler = VectorAssembler(inputCols=FEATURES_SELECTED, outputCol=FEATURES)
-    feature_engineering_stages.extend((realtime_vector_assembler,
-                                       realtime_scaler,
-                                       features_vector_assembler))
-    return feature_engineering_stages
-
-
-def build_training_model_component():
-    """
-    Build training model component
-    Returns:
-        model_training_stage: model_training_stage
-    """
-    model_training_stage = []
-    rfor = RandomForestClassifier(featuresCol=FEATURES, labelCol=TARGET, seed=RANDOM_SEED)
-    model_training_stage.append(rfor)
-    return model_training_stage
-
-
-def build_pipeline(data_preprocessing_stages, feature_engineering_stages, model_training_stage):
-    """
-    Build pipeline
-    Args:
-        data_preprocessing_stages:  data preprocessing stages
-        feature_engineering_stages: feature engineering stages
-        model_training_stage: model_training_stage
-    Returns:
-        pipeline: pipeline
-    """
-    pipeline = Pipeline(stages=data_preprocessing_stages + feature_engineering_stages + model_training_stage)
-    return pipeline
-
-
-def get_true_score_prediction(predictions, target):
-    """
-    Get true score prediction
-    Args:
-        predictions: predictions
-        target: target
-    Returns:
-        roc_dict: a dict of roc values for each class
-    """
-    split1_udf = udf(lambda value: value[1].item(), DoubleType())
-    roc_dataset = predictions.select(col(target).alias('true'),
-                                     spark_round(split1_udf('probability'), 5).alias('score'),
-                                     'prediction')
-    roc_df = roc_dataset.toPandas()
-    roc_dict = roc_df.to_dict(orient='list')
-    return roc_dict
-
-
-def get_metrics(predictions, target, mode):
-    """
-    Get metrics
-    Args:
-        predictions: predictions
-        target: target column name
-        mode: train or test
-    Returns:
-        metrics: metrics
-    """
-    metric_labels = ['area_roc', 'area_prc', 'accuracy', 'f1', 'precision', 'recall']
-    metric_cols = ['true', 'score', 'prediction']
-    metric_keys = [f'{mode}_{ml}' for ml in metric_labels] + metric_cols
-    bc_evaluator = BinaryClassificationEvaluator(labelCol=target)
-    mc_evaluator = MulticlassClassificationEvaluator(labelCol=target)
-
-    # areas, acc, f1, prec, rec
-    metric_values = []
-    area_roc = round(bc_evaluator.evaluate(predictions, {bc_evaluator.metricName: 'areaUnderROC'}), 5)
-    area_prc = round(bc_evaluator.evaluate(predictions, {bc_evaluator.metricName: 'areaUnderPR'}), 5)
-    acc = round(mc_evaluator.evaluate(predictions, {mc_evaluator.metricName: "accuracy"}), 5)
-    f1 = round(mc_evaluator.evaluate(predictions, {mc_evaluator.metricName: "f1"}), 5)
-    prec = round(mc_evaluator.evaluate(predictions, {mc_evaluator.metricName: "weightedPrecision"}), 5)
-    rec = round(mc_evaluator.evaluate(predictions, {mc_evaluator.metricName: "weightedRecall"}), 5)
-
-    # true, score, prediction
-    roc_dict = get_true_score_prediction(predictions, target)
-    true = roc_dict['true']
-    score = roc_dict['score']
-    pred = roc_dict['prediction']
-
-    metric_values.extend((area_roc, area_prc, acc, f1, prec, rec, true, score, pred))
-    metrics = dict(zip(metric_keys, metric_values))
-
-    return metrics
-
-
-def upload_file(bucket_name, source_file_name, destination_blob_name):
-    """
-    Upload file to bucket
-    Args:
-        bucket_name: bucket name
-        source_file_name: source file name
-        destination_blob_name: destination blob name
-    Returns:
-        None
-    """
-    storage_client = storage.Client()
-    bucket = storage_client.bucket(bucket_name)
-    blob = bucket.blob(destination_blob_name)
-    blob.upload_from_filename(source_file_name)
-
-
-def write_metrics(bucket_name, metrics, destination, dir='/tmp'):
-    """
-    Write metrics to file
-    Args:
-        bucket_name: bucket name
-        metrics: metrics
-        destination: destination
-        dir: directory to write file temporarily
-    Returns:
-        None
-    """
-    temp_dir = tempfile.TemporaryDirectory(dir=dir)
-    temp_metrics_file_path = str(path(temp_dir.name) / path(destination).name)
-    with open(temp_metrics_file_path, 'w') as temp_file:
-        json.dump(metrics, temp_file)
-    upload_file(bucket_name, temp_metrics_file_path, destination)
-    temp_dir.cleanup()
-
-
-# Main -------------------------------------------------------------------------------------
-
-def main(logger, args):
-    """
-    Main function
-    Args:
-        logger: logger
-        args: args
-    Returns:
-        None
-    """
-    train_path = args.train_path
-    model_path = args.model_path
-    metrics_path = args.metrics_path
-
-    try:
-        logger.info('initializing pipeline training.')
-        logger.info('start spark session.')
-        spark = (SparkSession.builder
-                 .master("local[*]")
-                 .appName("spark go live")
-                 .config('spark.ui.port', '4050')
-                 .getOrCreate())
-        logger.info(f'spark version: {spark.sparkContext.version}')
-        logger.info('start bulding pipeline.')
-        preprocessing_stages = build_preprocessing_components()
-        feature_engineering_stages = build_feature_engineering_components()
-        model_training_stage = build_training_model_component()
-        pipeline = build_pipeline(preprocessing_stages, feature_engineering_stages, model_training_stage)
-
-        logger.info(f'load train data from {train_path}.')
-        raw_data = (spark.read.format('csv')
-                    .option("header", "true")
-                    .schema(DATA_SCHEMA)
-                    .load(train_path))
-
-        logger.info(f'fit model pipeline.')
-        train, test = raw_data.randomSplit(RANDOM_QUOTAS, seed=RANDOM_SEED)
-        pipeline_model = pipeline.fit(train)
-        predictions = pipeline_model.transform(test)
-        metrics = get_metrics(predictions, TARGET, 'test')
-        for m, v in metrics.items():
-            print(f'{m}: {v}')
-
-        logger.info(f'load model pipeline in {model_path}.')
-        pipeline.write().overwrite().save(model_path)
-        if model_path.startswith('gs://'):
-            pipeline.write().overwrite().save(model_path)
-        else:
-            path(model_path).mkdir(parents=True, exist_ok=True)
-            pipeline.write().overwrite().save(model_path)
-
-        logger.info(f'Upload metrics under {metrics_path}.')
-        if metrics_path.startswith('gs://'):
-            bucket = urlparse(model_path).netloc
-            metrics_file_path = urlparse(metrics_path).path.strip('/')
-            write_metrics(bucket, metrics, metrics_file_path)
-        else:
-            metrics_version_path = path(metrics_path).parents[0]
-            metrics_version_path.mkdir(parents=True, exist_ok=True)
-            with open(metrics_path, 'w') as json_file:
-                json.dump(metrics, json_file)
-            json_file.close()
-    except RuntimeError as main_error:
-        logger.error(main_error)
-    else:
-        logger.info('model pipeline training successfully completed!')
-        return 0
-
-
-if __name__ == "__main__":
-    runtime_args = get_args()
-    runtime_logger = set_logger()
-    main(runtime_logger, runtime_args)

notebooks/official/sdk/SDK_AutoML_Video_Classification.ipynb

@@ -0,0 +1,998 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "modular-concentration"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2022 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "b88c5cede17b"
+      },
+      "source": [
+        "# Vertex AI SDK for Python: AutoML Video Classification Example\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "<td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/sdk/SDK_AutoML_Video_Classification.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/sdk/SDK_AutoML_Video_Classification.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/sdk/SDK_AutoML_Video_Classification.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "  </td>                                                                                               \n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "522e8eec0fcb"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "This notebook demonstrates how to create an AutoML Video Classification Model, with a Vertex AI video dataset, and how to serve the model for batch prediction. It requires you provide a bucket where the dataset will be stored.\n",
+        "\n",
+        "Note: you may incur charges for training, prediction, storage or usage of other GCP products in connection with testing this SDK."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "afc6017b7b45"
+      },
+      "source": [
+        "### Objective\n",
+        "\n",
+        "The objective of this notebook is to build a AutoML Video Classification Model. The following steps have been followed:  \n",
+        "This tutorial uses the following Google Cloud ML services :\n",
+        "- `Vertex AI Dataset` resource\n",
+        "- `AutoML Training`\n",
+        "- `Vertex AI Model` resource\n",
+        "- `Vertex AI Batch Prediction`\n",
+        "\n",
+        "\n",
+        "The steps performed include the following:\n",
+        "\n",
+        "- Set your task name, and GCS prefix\n",
+        "- Copy AutoML video demo train data for creating managed dataset\n",
+        "- Create a dataset on Vertex AI.\n",
+        "- Configure a training job\n",
+        "- Launch a training job and create a model on Vertex AI\n",
+        "- Copy AutoML Video Demo Prediction Data for creating batch prediction job\n",
+        "- Perform batch prediction job on the model\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5bba1b08cba7"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "##### HMDB: a large human motion database\n",
+        "Some training data and prediction data for the demo is prepared using the [HMDB Dataset](https://serre-lab.clps.brown.edu/resource/hmdb-a-large-human-motion-database).\n",
+        "\n",
+        "The HMDB Dataset is licensed under the Creative Commons Attribution 4.0 International License. To view a copy of this license, visit https://creativecommons.org/licenses/by/4.0/\n",
+        "\n",
+        "For more information about this dataset please visit: https://serre-lab.clps.brown.edu/resource/hmdb-a-large-human-motion-database/"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "248a51c68228"
+      },
+      "source": [
+        "## Costs\n",
+        "\n",
+        "This tutorial uses the following billable components of Google Cloud:\n",
+        "\n",
+        "- Vertex AI\n",
+        "- Cloud Storage\n",
+        "\n",
+        "\n",
+        "Learn about [Vertex AI\n",
+        "pricing](https://cloud.google.com/vertex-ai/pricing), [Cloud Storage\n",
+        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
+        "Calculator](https://cloud.google.com/products/calculator/)\n",
+        "to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5f9b7a53c13e"
+      },
+      "source": [
+        "### Set up your local development environment\n",
+        "\n",
+        "**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets\n",
+        "all the requirements to run this notebook. You can skip this step."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "544febf5376d"
+      },
+      "source": [
+        "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
+        "You need the following:\n",
+        "\n",
+        "* The Google Cloud SDK\n",
+        "* Git\n",
+        "* Python 3\n",
+        "* virtualenv\n",
+        "* Jupyter notebook running in a virtual environment with Python 3\n",
+        "\n",
+        "The Google Cloud guide to [Setting up a Python development\n",
+        "environment](https://cloud.google.com/python/setup) and the [Jupyter\n",
+        "installation guide](https://jupyter.org/install) provide detailed instructions\n",
+        "for meeting these requirements. The following steps provide a condensed set of\n",
+        "instructions:\n",
+        "\n",
+        "1. [Install and initialize the Cloud SDK.](https://cloud.google.com/sdk/docs/)\n",
+        "\n",
+        "1. [Install Python 3.](https://cloud.google.com/python/setup#installing_python)\n",
+        "\n",
+        "1. [Install\n",
+        "   virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv)\n",
+        "   and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
+        "\n",
+        "1. To install Jupyter, run `pip3 install jupyter` on the\n",
+        "command-line in a terminal shell.\n",
+        "\n",
+        "1. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
+        "\n",
+        "1. Open this notebook in the Jupyter Notebook Dashboard."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "a6c0a6e0a5d2"
+      },
+      "source": [
+        "## Install additional packages\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ef6579ee1c08"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
+        "\n",
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "USER_FLAG = \"\"\n",
+        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "coated-remark"
+      },
+      "outputs": [],
+      "source": [
+        "! pip3 install {USER_FLAG} --upgrade google-cloud-aiplatform"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d4f317591f55"
+      },
+      "source": [
+        "### Restart the kernel\n",
+        "\n",
+        "After you install the additional packages, you need to restart the notebook kernel so it can find the packages."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "f731803a16c0"
+      },
+      "outputs": [],
+      "source": [
+        "# Automatically restart kernel after installs\n",
+        "import os\n",
+        "\n",
+        "if not os.getenv(\"IS_TESTING\"):\n",
+        "    # Automatically restart kernel after installs\n",
+        "    import IPython\n",
+        "\n",
+        "    app = IPython.Application.instance()\n",
+        "    app.kernel.do_shutdown(True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3c2ae31f6491"
+      },
+      "source": [
+        "## Before you begin\n",
+        "\n",
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI, Cloud Storage, and Compute Engine APIs](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component,storage-component.googleapis.com). \n",
+        "\n",
+        "1. [Configure your Google Cloud project for Vertex Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/configure-project).\n",
+        "\n",
+        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands.\n",
+        "\n",
+        "### Set your project ID\n",
+        "\n",
+        "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "4e27466c7355"
+      },
+      "outputs": [],
+      "source": [
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "5a604eeffc32"
+      },
+      "outputs": [],
+      "source": [
+        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
+        "    # Get your GCP project id from gcloud\n",
+        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "    PROJECT_ID = shell_output[0]\n",
+        "    print(\"Project ID:\", PROJECT_ID)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "30e64c0eda41"
+      },
+      "outputs": [],
+      "source": [
+        "! gcloud config set project $PROJECT_ID"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "264543a144ad"
+      },
+      "source": [
+        "#### Region\n",
+        "\n",
+        "You can also change the `REGION` variable, which is used for operations\n",
+        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. It is recommended that you choose the region closest to you.\n",
+        "\n",
+        "- Americas: `us-central1`\n",
+        "- Europe: `europe-west4`\n",
+        "- Asia Pacific: `asia-east1`\n",
+        "\n",
+        "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
+        "\n",
+        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3281bedf6d3c"
+      },
+      "outputs": [],
+      "source": [
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "57dad372c81b"
+      },
+      "source": [
+        "#### UUID\n",
+        "\n",
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "4e166d927e36"
+      },
+      "outputs": [],
+      "source": [
+        "import random\n",
+        "import string\n",
+        "\n",
+        "\n",
+        "# Generate a uuid of a specifed length(default=8)\n",
+        "def generate_uuid(length: int = 8) -> str:\n",
+        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
+        "\n",
+        "\n",
+        "UUID = generate_uuid()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "1662d60ae8d2"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
+        "authenticated. Skip this step."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "a00567d0660a"
+      },
+      "source": [
+        "**If you are using Colab**, run the cell below and follow the instructions\n",
+        "when prompted to authenticate your account via oAuth.\n",
+        "\n",
+        "**Otherwise**, follow these steps:\n",
+        "\n",
+        "1. In the Cloud Console, go to the [**Create service account key**\n",
+        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
+        "\n",
+        "2. Click **Create service account**.\n",
+        "\n",
+        "3. In the **Service account name** field, enter a name, and\n",
+        "   click **Create**.\n",
+        "\n",
+        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
+        "into the filter box, and select\n",
+        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "\n",
+        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
+        "local environment.\n",
+        "\n",
+        "6. Enter the path to your service account key as the\n",
+        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "c9c2a9a5f992"
+      },
+      "outputs": [],
+      "source": [
+        "# If you are running this notebook in Colab, run this cell and follow the\n",
+        "# instructions to authenticate your GCP account. This provides access to your\n",
+        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
+        "# requests.\n",
+        "\n",
+        "import os\n",
+        "import sys\n",
+        "\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = \"google.colab\" in sys.modules\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
+        "    if \"google.colab\" in sys.modules:\n",
+        "        from google.colab import auth as google_auth\n",
+        "\n",
+        "        google_auth.authenticate_user()\n",
+        "\n",
+        "    # If you are running this notebook locally, replace the string below with the\n",
+        "    # path to your service account key and run this cell to authenticate your GCP\n",
+        "    # account.\n",
+        "    elif not os.getenv(\"IS_TESTING\"):\n",
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2577b7189f8b"
+      },
+      "source": [
+        "### Create a Cloud Storage bucket\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "Set the name of your Cloud Storage bucket below. It must be unique across all\n",
+        "Cloud Storage buckets.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2f6f0f6ec383"
+      },
+      "outputs": [],
+      "source": [
+        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "1f63fad70682"
+      },
+      "outputs": [],
+      "source": [
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
+        "    BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6ebc0bdb07af"
+      },
+      "source": [
+        "**Only if your bucket doesn't already exist**: Run the following cell to create your Cloud Storage bucket."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "f8a154fea495"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil mb -l $REGION $BUCKET_URI"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "c0fc6da3a36d"
+      },
+      "source": [
+        "**Finally**, validate access to your Cloud Storage bucket by examining its contents:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "a5dc9aab11fa"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil ls -al $BUCKET_URI"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "incorporated-edgar"
+      },
+      "source": [
+        "### Import libraries and define constants\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "hispanic-macedonia"
+      },
+      "outputs": [],
+      "source": [
+        "import json\n",
+        "\n",
+        "from google.cloud import aiplatform, storage\n",
+        "\n",
+        "MY_PROJECT = PROJECT_ID\n",
+        "MY_STAGING_BUCKET = BUCKET_URI"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "historical-consciousness"
+      },
+      "source": [
+        "### Set Your Task Name, and GCS Prefix\n",
+        "\n",
+        "If you want to centeralize all input and output files under the gcs location."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "organizational-salad"
+      },
+      "outputs": [],
+      "source": [
+        "TASK_TYPE = \"mbsdk_automl-video-training\"\n",
+        "PREDICTION_TYPE = \"classification\"\n",
+        "MODEL_TYPE = \"CLOUD\"\n",
+        "\n",
+        "TASK_NAME = f\"{TASK_TYPE}_{PREDICTION_TYPE}\"\n",
+        "BUCKET_NAME = MY_STAGING_BUCKET.split(\"gs://\")[1]\n",
+        "GCS_PREFIX = TASK_NAME\n",
+        "\n",
+        "print(f\"Bucket Name:    {BUCKET_NAME}\")\n",
+        "print(f\"Task Name:      {TASK_NAME}\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "professional-bulletin"
+      },
+      "source": [
+        "### Copy AutoML Video Demo Train Data for Creating Managed Dataset"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "accurate-producer"
+      },
+      "outputs": [],
+      "source": [
+        "automl_video_demo_train_data = (\n",
+        "    \"gs://automl-video-demo-data/hmdb_split1_5classes_all.csv\"\n",
+        ")\n",
+        "\n",
+        "\n",
+        "gcs_source_train = f\"gs://{BUCKET_NAME}/{TASK_NAME}/data/video_classification.csv\"\n",
+        "\n",
+        "!gsutil cp $automl_video_demo_train_data $gcs_source_train"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "rough-alert"
+      },
+      "source": [
+        "# Run AutoML Video Training with Vertex AI Video Dataset"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "adaptive-slovakia"
+      },
+      "source": [
+        "## Initialize Vertex AI SDK for Python\n",
+        "\n",
+        "Initialize the *client* for Vertex AI."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "figured-fellow"
+      },
+      "outputs": [],
+      "source": [
+        "aiplatform.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "pleasant-holmes"
+      },
+      "source": [
+        "## Create a Dataset on Vertex AI Dataset resource\n",
+        "We will now create a Vertex AI video dataset using the previously prepared csv files."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "uVBfL-0TTjNS"
+      },
+      "outputs": [],
+      "source": [
+        "dataset = aiplatform.VideoDataset.create(\n",
+        "    display_name=f\"temp-{TASK_NAME}\",\n",
+        "    gcs_source=gcs_source_train,\n",
+        "    import_schema_uri=aiplatform.schema.dataset.ioformat.video.classification,\n",
+        "    sync=False,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3x4xuyIbVR_N"
+      },
+      "outputs": [],
+      "source": [
+        "dataset.wait()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "mexican-spending"
+      },
+      "source": [
+        "## Launch a training job and create a model on Vertex AI"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "fdb1d50298ef"
+      },
+      "source": [
+        "To train an AutoML model, you perform two steps: 1) create a training pipeline, and 2) run the pipeline.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dynamic-piece"
+      },
+      "source": [
+        "### Configure a Training Job"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "a509028760a0"
+      },
+      "source": [
+        "An AutoML training pipeline is created with the `AutoMLVideoTrainingJob` class, with the following parameters:\n",
+        "\n",
+        "- `display_name`: The human readable name for the `TrainingJob` resource.\n",
+        "- `prediction_type`: The type task to train the model for.\n",
+        "  - `classification`: A video classification model.\n",
+        "  - `object_tracking`: A video object tracking model.\n",
+        "  - `action_recognition`: A video action recognition model."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "continuous-circular"
+      },
+      "outputs": [],
+      "source": [
+        "job = aiplatform.AutoMLVideoTrainingJob(\n",
+        "    display_name=f\"temp-{TASK_NAME}\",\n",
+        "    prediction_type=PREDICTION_TYPE,\n",
+        "    model_type=MODEL_TYPE,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "juvenile-parameter"
+      },
+      "source": [
+        "### Run the Training Job"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "a3b60132368e"
+      },
+      "source": [
+        "Next, you run the job to start the training job by invoking the method `run`, with the following parameters:\n",
+        "\n",
+        "- `dataset`: The `Dataset` resource to train the model.\n",
+        "- `model_display_name`: The human readable name for the trained model.\n",
+        "- `training_fraction_split`: The percentage of the dataset to use for training.\n",
+        "- `test_fraction_split`: The percentage of the dataset to use for test (holdout data).\n",
+        "- `sync`: If set to True, the call will block while waiting for the asynchronous batch job to complete.\n",
+        "\n",
+        "The `run` method when completed returns the `Model` resource.\n",
+        "\n",
+        "The execution of the training pipeline can take over 2 hours to complete."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "human-carrier"
+      },
+      "outputs": [],
+      "source": [
+        "model = job.run(\n",
+        "    dataset=dataset,\n",
+        "    training_fraction_split=0.8,\n",
+        "    test_fraction_split=0.2,\n",
+        "    model_display_name=f\"temp-{TASK_NAME}\",\n",
+        "    sync=False,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "abstract-textbook"
+      },
+      "outputs": [],
+      "source": [
+        "model.wait()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "noted-usage"
+      },
+      "source": [
+        "# Make a Batch Prediction request"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ruled-smith"
+      },
+      "source": [
+        "### Copy AutoML video demo prediction data for creating batch prediction job"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "polished-dispatch"
+      },
+      "outputs": [],
+      "source": [
+        "automl_video_demo_batch_prediction_data = (\n",
+        "    \"gs://automl-video-demo-data/hmdb_split1_predict.jsonl\"\n",
+        ")\n",
+        "\n",
+        "gcs_source_batch_prediction = (\n",
+        "    f\"gs://{BUCKET_NAME}/{TASK_NAME}/data/video_classification_batch_prediction.jsonl\"\n",
+        ")\n",
+        "gcs_destination_prefix_batch_prediction = (\n",
+        "    f\"gs://{BUCKET_NAME}/{TASK_NAME}/batch_prediction\"\n",
+        ")\n",
+        "\n",
+        "!gsutil cp $automl_video_demo_batch_prediction_data $gcs_source_batch_prediction"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2bd262d624e3"
+      },
+      "source": [
+        "### Perform batch prediction job on the model"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "b488cb43682a"
+      },
+      "source": [
+        "Now that your Model resource is trained, you can make a batch prediction by invoking the batch_predict() method, with the following parameters:\n",
+        "\n",
+        "- `job_display_name`: The human readable name for the batch prediction job.\n",
+        "- `gcs_source`: A list of one or more batch request input files.\n",
+        "- `gcs_destination_prefix`: The Cloud Storage location for storing the batch prediction resuls.\n",
+        "- `sync`: If set to True, the call will block while waiting for the asynchronous batch job to complete."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "piano-middle"
+      },
+      "outputs": [],
+      "source": [
+        "batch_predict_job = model.batch_predict(\n",
+        "    job_display_name=f\"temp-{TASK_NAME}\",\n",
+        "    gcs_source=gcs_source_batch_prediction,\n",
+        "    gcs_destination_prefix=gcs_destination_prefix_batch_prediction,\n",
+        "    sync=False,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "e06e39558a8e"
+      },
+      "outputs": [],
+      "source": [
+        "batch_predict_job.wait()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8359d586c1e8"
+      },
+      "source": [
+        "### Get the predictions\n",
+        "\n",
+        "Next, get the results from the completed batch prediction job.\n",
+        "\n",
+        "The results are written to the Cloud Storage output bucket you specified in the batch prediction request. You call the method iter_outputs() to get a list of each Cloud Storage file generated with the results. Each file contains one or more prediction requests in a JSON format:\n",
+        "\n",
+        "- `content`: The prediction request.\n",
+        "- `prediction`: The prediction response.\n",
+        "\n",
+        "Prediction response contains following fields\n",
+        "\n",
+        "- `ids`: The internal assigned unique identifiers for each prediction request.\n",
+        "- `displayNames`: The class names for each class label.\n",
+        "- `confidences`: The predicted confidence, between 0 and 1, per class label.\n",
+        "- `timeSegmentStart`: The time offset in the video to the start of the video sequence.\n",
+        "- `timeSegmentEnd`: The time offset in the video to the end of the video sequence."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "visible-scientist"
+      },
+      "outputs": [],
+      "source": [
+        "bp_iter_outputs = batch_predict_job.iter_outputs()\n",
+        "\n",
+        "prediction_results = list()\n",
+        "for blob in bp_iter_outputs:\n",
+        "    if blob.name.split(\"/\")[-1].startswith(\"prediction\"):\n",
+        "        prediction_results.append(blob.name)\n",
+        "client = storage.Client()\n",
+        "bucket = client.get_bucket(BUCKET_URI.replace(\"gs://\", \"\"))\n",
+        "for prediction_result in prediction_results:\n",
+        "    gfile_name = f\"{prediction_result}\"\n",
+        "    data = bucket.blob(gfile_name).download_as_string()\n",
+        "    data = json.loads(data)\n",
+        "    print(data)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0472cd54c140"
+      },
+      "source": [
+        "## Clean up\n",
+        "<a name=\"section-13\"></a>\n",
+        "\n",
+        "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
+        "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
+        "\n",
+        "Otherwise, you can delete the individual resources you created in this tutorial:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2f60a4fb2863"
+      },
+      "outputs": [],
+      "source": [
+        "# Delete the dataset using the Vertex dataset object\n",
+        "dataset.delete()\n",
+        "\n",
+        "# Delete the model using the Vertex model object\n",
+        "model.delete()\n",
+        "\n",
+        "# Delete the AutoML or Pipeline training job\n",
+        "job.delete()\n",
+        "\n",
+        "# Delete the batch prediction job using the Vertex batch prediction object\n",
+        "batch_predict_job.delete()\n",
+        "\n",
+        "# Delete the Cloud Storage bucket\n",
+        "\n",
+        "delete_bucket = False\n",
+        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "    ! gsutil -m rm -r $BUCKET_URI"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "SDK_AutoML_Video_Classification.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/official/workbench/ads_targetting/training-multi-class-classification-model-for-ads-targeting-usecase.ipynb

@@ -1,29 +1,55 @@
 {
   "cells": [
     {
-      "cell_type": "markdown",
+      "cell_type": "code",
+      "execution_count": null,
       "metadata": {
         "id": "503077811e70"
       },
+      "outputs": [],
+      "source": [
+        "# Copyright 2022 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e885ac09bc73"
+      },
       "source": [
         "# Train a multi-class classification model for ads-targeting\n",
+        "<table align=\"left\">\n",
         "\n",
-        "## Table of contents\n",
-        "\n",
-        "* [Overview](#section-1)\n",
-        "* [Dataset](#section-2)\n",
-        "* [Objective](#section-3)\n",
-        "* [Costs](#section-4)\n",
-        "* [Tutorial](#section-5)\n",
-        "\t- [Fetch the data from BigQuery](#section-5)\n",
-        "    - [Preprocess the data](#section-6)\n",
-        "    - [Train a TensorFlow model](#section-7)\n",
-        "    - [Run the model on test data](#section-8)\n",
-        "    - [Automating the execution of the notebook using executor](#section-9)\n",
-        "    - [Scheduled runs on executor](#section-10)\n",
-        "    - [Parameterizing the variables](#section-11)\n",
-        "* [Save the model to a Cloud Storage path](#section-12)\n",
-        "* [Clean up](#section-13)\n"
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/ads_targetting/training-multi-class-classification-model-for-ads-targeting-usecase.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/ads_targetting/training-multi-class-classification-model-for-ads-targeting-usecase.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/ads_targetting/training-multi-class-classification-model-for-ads-targeting-usecase.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "  </td>                                                                                               \n",
+        "</table>"
       ]
     },
     {
@@ -33,23 +59,19 @@
       },
       "source": [
         "## Overview\n",
-        "<a name=\"section-1\"></a>\n",
         "\n",
         "This tutorial demonstrates how to build a machine learning model for an ads-targeting use case. Ads-targeting is an advertisement technique where chosen or tailor-made ads are shown to the customers based on their past behavior and preferences. Targeted ads are meant to reach specific customers based on demographics, psychographics, behavior, and other second-order activities that are learned usually through data collected from the customers.\n",
         "\n",
-        "*Note: This notebook file was designed to run in a [Vertex AI Workbench managed notebooks](https://cloud.google.com/vertex-ai/docs/workbench/managed/create-instance) instance using the `TensorFlow 2 (Local)` kernel. Some components of this notebook may not work in other notebook environments.*\n",
-        "\n",
-        "## Dataset\n",
-        "<a name=\"section-2\"></a>\n",
-        "\n",
-        "This tutorial uses the `looker-private-demo.ecomm` dataset in BigQuery. The dataset consists of information about various advertisement campaigns including the demographics of users who have clicked and made some purchases after seeing the ads. For this tutorial, the top three campaigns from the USA are selected from this dataset and user information for those who have made purchases shall be used to train a model with the campaigns as the classes. The idea is to see if the advertisement and the user data can be used to identify which campaign is best-suited for the user.\n",
-        "\n",
-        "The dataset can be accessed by pinning the `looker-private-demo` project in BigQuery. Instead of going to the BigQuery user interface, this process can be performed from the JupyterLab user interface on a Vertex AI Workbench managed notebooks instance. Vertex AI Workbench managed notebooks instances support browsing through the datasets and tables from BigQuery through its BigQuery integration. \n",
-        "\n",
-        "<img src=\"images/Bigquery_UI_new.PNG\"></img>\n",
-        "\n",
+        "*Note: If you are using [Vertex AI Workbench managed notebooks](https://cloud.google.com/vertex-ai/docs/workbench/managed/create-instance) instance use the `TensorFlow 2 (Local)` kernel. Some components of this notebook may not work in other notebook environments.*\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "1bea2b6e9b25"
+      },
+      "source": [
         "## Objective\n",
-        "<a name=\"section-3\"></a>\n",
         "\n",
         "This tutorial demonstrates how to collect data from BigQuery, preprocess it, and train a multi-class classification model on an E-commerce dataset. The steps performed include the following:\n",
         "\n",
@@ -59,10 +81,31 @@
         "- Evaluate the loss for the trained model\n",
         "- Automate the notebook execution using the executor feature\n",
         "- Save the model to a Cloud Storage path\n",
-        "- Clean up the created resources\n",
+        "- Clean up the created resources"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "34d623e6dfa3"
+      },
+      "source": [
+        "## Dataset\n",
         "\n",
+        "This tutorial uses the `looker-private-demo.ecomm` dataset in BigQuery. The dataset consists of information about various advertisement campaigns including the demographics of users who have clicked and made some purchases after seeing the ads. For this tutorial, the top three campaigns from the USA are selected from this dataset and user information for those who have made purchases shall be used to train a model with the campaigns as the classes. The idea is to see if the advertisement and the user data can be used to identify which campaign is best-suited for the user.\n",
+        "\n",
+        "The dataset can be accessed by pinning the `looker-private-demo` project in BigQuery. If you are using Vertex AI Workbench managed notebooks instance, instead of going to the BigQuery user interface, this process can be performed from the JupyterLab user interface. Vertex AI Workbench managed notebooks instances support browsing through the datasets and tables from BigQuery through its BigQuery integration. \n",
+        "\n",
+        "<img src=\"images/Bigquery_UI_new.PNG\"></img>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ee02650bb7fd"
+      },
+      "source": [
         "### Costs \n",
-        "<a name=\"section-4\"></a>\n",
         "\n",
         "This tutorial uses billable components of Google Cloud:\n",
         "\n",
@@ -78,6 +121,121 @@
         "to generate a cost estimate based on your projected usage."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "y320EIk-kXT7"
+      },
+      "source": [
+        "### Set up your local development environment\n",
+        "\n",
+        "**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets\n",
+        "all the requirements to run this notebook. You can skip this step.\n",
+        "\n",
+        "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
+        "You need the following:\n",
+        "\n",
+        "* The Google Cloud SDK\n",
+        "* Git\n",
+        "* Python 3\n",
+        "* virtualenv\n",
+        "* Jupyter notebook running in a virtual environment with Python 3\n",
+        "\n",
+        "The Google Cloud guide to [Setting up a Python development\n",
+        "environment](https://cloud.google.com/python/setup) and the [Jupyter\n",
+        "installation guide](https://jupyter.org/install) provide detailed instructions\n",
+        "for meeting these requirements. The following steps provide a condensed set of\n",
+        "instructions:\n",
+        "\n",
+        "1. [Install and initialize the Cloud SDK.](https://cloud.google.com/sdk/docs/)\n",
+        "\n",
+        "1. [Install Python 3.](https://cloud.google.com/python/setup#installing_python)\n",
+        "\n",
+        "1. [Install\n",
+        "   virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv)\n",
+        "   and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
+        "\n",
+        "1. To install Jupyter, run `pip3 install jupyter` on the\n",
+        "command-line in a terminal shell.\n",
+        "\n",
+        "1. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
+        "\n",
+        "1. Open this notebook in the Jupyter Notebook Dashboard."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "1DouUvNOkXT8"
+      },
+      "source": [
+        "### Install additional packages\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Ayt1jhFXkXT9"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
+        "\n",
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "USER_FLAG = \"\"\n",
+        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "95826791kXT_"
+      },
+      "outputs": [],
+      "source": [
+        "! pip3 install {USER_FLAG} --upgrade pandas-gbq 'google-cloud-bigquery[bqstorage,pandas]' tensorflow sklearn protobuf==3.20.1 -q \\\n",
+        "                                     "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "aNeMRbpukXUA"
+      },
+      "source": [
+        "### Restart the kernel\n",
+        "\n",
+        "After you install the additional packages, you need to restart the notebook kernel so it can find the packages."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "dJ_yvi_9kXUB"
+      },
+      "outputs": [],
+      "source": [
+        "# Automatically restart kernel after installs\n",
+        "import os\n",
+        "\n",
+        "if not os.getenv(\"IS_TESTING\"):\n",
+        "    # Automatically restart kernel after installs\n",
+        "    import IPython\n",
+        "\n",
+        "    app = IPython.Application.instance()\n",
+        "    app.kernel.do_shutdown(True)"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -97,34 +255,67 @@
       },
       "outputs": [],
       "source": [
-        "PROJECT_ID = \"\"\n",
-        "\n",
-        "# Get your Google Cloud project ID from gcloud\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    shell_output=!gcloud config list --format 'value(core.project)' 2>/dev/null\n",
-        "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID: \", PROJECT_ID)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "d0058f55f8cf"
-      },
-      "source": [
-        "Otherwise, set your project ID here."
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "19579640c063"
+        "id": "5bf9979b96ff"
       },
       "outputs": [],
       "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
-        "    PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
+        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
+        "    # Get your GCP project id from gcloud\n",
+        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "    PROJECT_ID = shell_output[0]\n",
+        "    print(\"Project ID:\", PROJECT_ID)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "07-xo93jlC6l"
+      },
+      "outputs": [],
+      "source": [
+        "! gcloud config set project $PROJECT_ID"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "03d8d65b914d"
+      },
+      "source": [
+        "#### Region\n",
+        "\n",
+        "You can also change the `REGION` variable, which is used for operations\n",
+        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
+        "\n",
+        "- Americas: `us-central1`\n",
+        "- Europe: `europe-west4`\n",
+        "- Asia Pacific: `asia-east1`\n",
+        "\n",
+        "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
+        "\n",
+        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3281bedf6d3c"
+      },
+      "outputs": [],
+      "source": [
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -151,6 +342,74 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "OoPGk5KOkXUG"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
+        "authenticated. Skip this step.\n",
+        "\n",
+        "**If you are using Colab**, run the cell below and follow the instructions\n",
+        "when prompted to authenticate your account via oAuth.\n",
+        "\n",
+        "**Otherwise**, follow these steps:\n",
+        "\n",
+        "1. In the Cloud Console, go to the [**Create service account key**\n",
+        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
+        "\n",
+        "2. Click **Create service account**.\n",
+        "\n",
+        "3. In the **Service account name** field, enter a name, and\n",
+        "   click **Create**.\n",
+        "\n",
+        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
+        "into the filter box, and select\n",
+        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "\n",
+        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
+        "local environment.\n",
+        "\n",
+        "6. Enter the path to your service account key as the\n",
+        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Teyy6LGqkXUG"
+      },
+      "outputs": [],
+      "source": [
+        "# If you are running this notebook in Colab, run this cell and follow the\n",
+        "# instructions to authenticate your GCP account. This provides access to your\n",
+        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
+        "# requests.\n",
+        "\n",
+        "import os\n",
+        "import sys\n",
+        "\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = \"google.colab\" in sys.modules\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
+        "    if \"google.colab\" in sys.modules:\n",
+        "        from google.colab import auth as google_auth\n",
+        "\n",
+        "        google_auth.authenticate_user()\n",
+        "\n",
+        "    # If you are running this notebook locally, replace the string below with the\n",
+        "    # path to your service account key and run this cell to authenticate your GCP\n",
+        "    # account.\n",
+        "    elif not os.getenv(\"IS_TESTING\"):\n",
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -161,20 +420,8 @@
         "\n",
         "**The following steps are required, regardless of your notebook environment.**\n",
         "\n",
-        "When you submit a training job using the Cloud SDK, you upload a Python package\n",
-        "containing your training code to a Cloud Storage bucket. Vertex AI runs\n",
-        "the code from this package. In this tutorial, Vertex AI also saves the\n",
-        "trained model that results from your job in the same bucket. Using this model artifact, you can then\n",
-        "create Vertex AI model and endpoint resources in order to serve\n",
-        "online predictions.\n",
-        "\n",
         "Set the name of your Cloud Storage bucket below. It must be unique across all\n",
-        "Cloud Storage buckets.\n",
-        "\n",
-        "You may also change the `REGION` variable, which is used for operations\n",
-        "throughout the rest of this notebook. Make sure to [choose a region where Vertex AI services are\n",
-        "available](https://cloud.google.com/vertex-ai/docs/general/locations#available_regions). You may\n",
-        "not use a Multi-Regional Storage bucket for training with Vertex AI."
+        "Cloud Storage buckets.\n"
       ]
     },
     {
@@ -185,8 +432,8 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "REGION = \"[your-region]\"  # @param {type:\"string\"}"
+        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
     {
@@ -197,8 +444,9 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
+        "    BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
     {
@@ -218,7 +466,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_NAME"
+        "! gsutil mb -l $REGION $BUCKET_URI"
       ]
     },
     {
@@ -238,7 +486,36 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_NAME"
+        "! gsutil ls -al $BUCKET_URI"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "bmnMD2MjkXUJ"
+      },
+      "source": [
+        "### Import libraries and define constants"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "oqtZRqDEkXUJ"
+      },
+      "outputs": [],
+      "source": [
+        "import warnings\n",
+        "\n",
+        "import pandas as pd\n",
+        "from sklearn.model_selection import train_test_split\n",
+        "from sklearn.preprocessing import StandardScaler\n",
+        "from tensorflow.keras import Sequential\n",
+        "from tensorflow.keras.layers import Dense\n",
+        "from tensorflow.keras.utils import to_categorical\n",
+        "\n",
+        "warnings.filterwarnings(\"ignore\")"
       ]
     },
     {
@@ -249,8 +526,16 @@
       "source": [
         "## Tutorial\n",
         "\n",
-        "### Fetch the data from BigQuery \n",
-        "<a name=\"section-5\"></a>"
+        "### Fetch the data from BigQuery \n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5c07be8840ae"
+      },
+      "source": [
+        "If you are using ***Vertex AI Workbench managed notebooks instance***, below cell which starts with \"#@bigquery\" will be a SQL Query. If you are using Vertex AI Workbench user managed notebooks instance or Colab it will be a markdown cell."
       ]
     },
     {
@@ -331,7 +616,7 @@
         "id": "923fdd823683"
       },
       "source": [
-        "Once the results from BigQuery are displayed in the above cell, click the **Query and load as DataFrame** button and execute the generated code stub to fetch the data into the current notebook as a dataframe.\n",
+        "If you are using Vertex AI Workbench managed notebooks instance, once the results from BigQuery are displayed in the above cell, click the **Query and load as DataFrame** button and execute the generated code stub to fetch the data into the current notebook as a dataframe.\n",
         "\n",
         "*Note: By default the data is loaded into a `df` variable, though this can be changed before executing the cell if required.*"
       ]
@@ -348,7 +633,7 @@
         "# Comment out otherwise for speed-up.\n",
         "from google.cloud.bigquery import Client\n",
         "\n",
-        "client = Client()\n",
+        "client = Client(project=PROJECT_ID)\n",
         "\n",
         "query = \"\"\"WITH traindata AS (\n",
         "SELECT b.* except(ad_event_id, user_id), c.* except(id), d.* except(keyword_id, ad_id), a.amount, a.device_type, e.name\n",
@@ -379,44 +664,6 @@
       },
       "source": [
         "### Preprocess the data\n",
-        "<a name=\"section-6\"></a>"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "e8503e799eec"
-      },
-      "source": [
-        "Import the required libraries."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "5b11973ccf76"
-      },
-      "outputs": [],
-      "source": [
-        "import warnings\n",
-        "\n",
-        "import pandas as pd\n",
-        "from sklearn.model_selection import train_test_split\n",
-        "from sklearn.preprocessing import StandardScaler\n",
-        "from tensorflow.keras import Sequential\n",
-        "from tensorflow.keras.layers import Dense\n",
-        "from tensorflow.keras.utils import to_categorical\n",
-        "\n",
-        "warnings.filterwarnings(\"ignore\")"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "e48d156d8bb6"
-      },
-      "source": [
         "Select the necessary columns from the E-commerce data and divide them based on their type (numerical/categorical)."
       ]
     },
@@ -441,13 +688,22 @@
         "num_cols = [\"age\", \"cpc_bid_amount\", \"quality_score\", \"amount\"]"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9bd71de0d37e"
+      },
+      "source": [
+        "#### Select top three campaigns"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "ace612851261"
       },
       "source": [
-        "From the current dataset, only the top three camapigns will be chosen to target the users. All the relevant information about the advertisement and the user who purchased an item after seeing the advertisement is available in the dataframe already. "
+        "From the current dataset, only the top three campaigns will be chosen to target the users. All the relevant information about the advertisement and the user who purchased an item after seeing the advertisement is available in the dataframe already. "
       ]
     },
     {
@@ -481,13 +737,22 @@
         "df[\"name\"] = df[\"name\"].map({\"Tops & Tees\": 0, \"Active\": 1, \"Accessories\": 2})"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "c2d5338b1b95"
+      },
+      "source": [
+        "#### One-hot encode the categorical variables"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "8902f763d1ca"
       },
       "source": [
-        "One-hot encode the categorical variables. After one-hot encoding, the first level-column is dropped to avoid the [dummy-variable trap](https://en.wikipedia.org/wiki/Dummy_variable_(statistics)) scenario. This process is called *dummy-encoding*."
+        "After one-hot encoding, the first level-column is dropped to avoid the [dummy-variable trap](https://en.wikipedia.org/wiki/Dummy_variable_(statistics)) scenario. This process is called *dummy-encoding*."
       ]
     },
     {
@@ -521,7 +786,7 @@
         "id": "3abf027eda2d"
       },
       "source": [
-        "Split the data into train and test."
+        "#### Split the data into train and test."
       ]
     },
     {
@@ -546,7 +811,7 @@
         "id": "d1a32b9d9640"
       },
       "source": [
-        "Scale the data."
+        "#### Scale the data."
       ]
     },
     {
@@ -569,16 +834,7 @@
       },
       "source": [
         "### Train a TensorFlow model\n",
-        "<a name=\"section-7\"></a>"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "3e7656556a48"
-      },
-      "source": [
-        "Convert the target column to a categorical encoded colum (one-hot encoded)."
+        "#### Convert the target column to a categorical encoded colum (one-hot encoded)."
       ]
     },
     {
@@ -599,7 +855,7 @@
         "id": "3dd0014a7e1d"
       },
       "source": [
-        "Define hyperparameters for model training. \n",
+        "#### Define hyperparameters for model training. \n",
         "\n",
         "*Note: Comment or remove the parameters from the following cell if they are provided already as an input parameter through the executor feature.*"
       ]
@@ -624,7 +880,7 @@
         "id": "406b731f576b"
       },
       "source": [
-        "Define the architecture and compile the model."
+        "#### Define the architecture and compile the model."
       ]
     },
     {
@@ -664,7 +920,7 @@
         "id": "4ab12c34f258"
       },
       "source": [
-        "Fit the model."
+        "#### Fit the model."
       ]
     },
     {
@@ -684,8 +940,7 @@
         "id": "51a2d0b52df3"
       },
       "source": [
-        "### Run the model on test data\n",
-        "<a name=\"section-8\"></a>"
+        "### Run the model on test data\n"
       ]
     },
     {
@@ -694,7 +949,7 @@
         "id": "f08445f2cd02"
       },
       "source": [
-        "Evaluate the model on test data."
+        "#### Evaluate the model on test data."
       ]
     },
     {
@@ -709,16 +964,24 @@
         "print(f\"Test results - Loss: {test_results}\")"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "81ef0e081340"
+      },
+      "source": [
+        "**Please note that executor feature is available only in Vertex AI Workbench managed notebooks**"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "9769168778e8"
       },
       "source": [
-        "### Automating the execution of the notebook using executor\n",
-        "<a name=\"section-9\"></a>\n",
+        "### Automating the execution of the notebook using executor in Vertex AI Workbench managed notebooks instance\n",
         "\n",
-        "The executor can help you run a notebook file from start to end, with your choice of the environment, machine type, input parameters, and other characteristics. After setting up an execution, the notebook is executed as a job in Vertex AI custom training. Your jobs can be monitored from the <b>Notebook Executor</b> pane in the menu on the left.\n",
+        "If you are using Vertex AI Workbench managed notebooks instance, the executor can help you run a notebook file from start to end, with your choice of the environment, machine type, input parameters, and other characteristics. After setting up an execution, the notebook is executed as a job in Vertex AI custom training. Your jobs can be monitored from the <b>Notebook Executor</b> pane in the menu on the left.\n",
         "\n",
         "<img src=\"images/executor.png\"></img>\n",
         "\n",
@@ -731,10 +994,9 @@
         "id": "cf486c351581"
       },
       "source": [
-        "### Scheduled runs on executor\n",
-        "<a name=\"section-10\"></a>\n",
+        "### Scheduled runs on executor in Vertex AI Workbench managed notebooks instance\n",
         "\n",
-        "Notebook runs can also be scheduled recurringly with the executor. To do so, select <b>Schedule-based recurring executions</b> as the run type instead of <b>One-time execution</b>. The frequency of the job and the time when it executes is provided when you create the execution.\n",
+        "Vertex AI Workbench managed noteboook runs can also be scheduled recurringly with the executor. To do so, select <b>Schedule-based recurring executions</b> as the run type instead of <b>One-time execution</b>. The frequency of the job and the time when it executes is provided when you create the execution.\n",
         "\n",
         "<img src=\"images/executor_scheduled_runs2.png\"></img>"
       ]
@@ -746,9 +1008,8 @@
       },
       "source": [
         "### Parameterizing the variables\n",
-        "<a name=\"section-11\"></a>\n",
         "\n",
-        "Executor lets you run a notebook with different sets of input parameters. If required, constants in the notebook can be treated as arguments to a function, and when you submit the execution, you can provide those constants as input parameters.\n",
+        "If you are using Vertex AI Workbench managed notebooks instance, executor lets you run a notebook with different sets of input parameters. If required, constants in the notebook can be treated as arguments to a function, and when you submit the execution, you can provide those constants as input parameters.\n",
         "\n",
         "<img src=\"images/executor_input_parameters.png\"></img>\n",
         "\n",
@@ -762,7 +1023,6 @@
       },
       "source": [
         "### Save the model to a Cloud Storage path\n",
-        "<a name=\"section-12\"></a>\n",
         "\n",
         "TensorFlow's `model.save()` method supports Cloud Storage paths as well as the local file paths while writing the model object to a file. It needs to be ensured that the service account being used to run this notebook has `write` permissions to the specified Cloud Storage path."
       ]
@@ -775,7 +1035,7 @@
       },
       "outputs": [],
       "source": [
-        "GCS_PATH = \"gs://\" + BUCKET_NAME + \"/[path-to-save]/\"\n",
+        "GCS_PATH = BUCKET_URI + \"/path-to-save/\"\n",
         "model.save(GCS_PATH)"
       ]
     },
@@ -786,7 +1046,6 @@
       },
       "source": [
         "## Clean up\n",
-        "<a name=\"section-13\"></a>\n",
         "\n",
         "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
         "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
@@ -802,7 +1061,11 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil -m rm -r [cloud-storage-folder-path-to-delete]"
+        "# Delete the Cloud Storage bucket\n",
+        "\n",
+        "delete_bucket = False\n",
+        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "    ! gsutil -m rm -r $BUCKET_URI"
       ]
     }
   ],

notebooks/official/workbench/predictive_maintainance/predictive_maintenance_usecase.ipynb

@@ -1,17 +1,62 @@
 {
   "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "18ebbd838e32"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2022 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "aef73cfa8725"
       },
       "source": [
-        "# Predictive Maintenance \n",
+        "# Predictive Maintenance using Vertex AI\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/predictive_maintainance/predictive_maintenance_usecase.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "    <td>\n",
+        "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/predictive_maintainance/predictive_maintenance_usecase.ipynb\">\n",
+        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\\\" alt=\"Colab logo\"> Run in Colab\n",
+        "        </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/predictive_maintainance/predictive_maintenance_usecase.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "  </td>\n",
+        "</table>\n",
+        "<br/><br/><br/>\n",
+        "\n",
         "\n",
         "## Table of contents\n",
         "* [Overview](#section-1)\n",
-        "* [Dataset](#section-2)\n",
-        "* [Objective](#section-3)\n",
+        "* [Objective](#section-2)\n",
+        "* [Dataset](#section-3)\n",
         "* [Costs](#section-4)\n",
         "* [Data analysis](#section-5)\n",
         "* [Fit a regression model](#section-6)\n",
@@ -22,24 +67,32 @@
         "  * [Create an endpoint](#section-11)\n",
         "  * [Deploy the model to the created endpoint](#section-12)\n",
         "  * [Test calling the endpoint](#section-13)\n",
-        "* [Clean up](#section-14)\n",
-        "\n",
-        "\n",
+        "* [Clean up](#section-14)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e10c5167a061"
+      },
+      "source": [
         "## Overview\n",
         "<a name=\"section-1\"></a>\n",
         "\n",
-        "This notebook demonstrates how to perform predictive maintenance on industrial data using machine learning techniques, deploy the machine learning model on Vertex AI, and automate the workflow using the executor feature of Vertex AI Workbench.\n",
+        "In this notebook, you go through a predictive maintenance usecase on industrial data using machine learning techniques, deploy the machine learning model on Vertex AI, and automate the workflow using the executor feature of Vertex AI Workbench.\n",
         "\n",
-        "*Note: This notebook file was developed to run in a [Vertex AI Workbench managed notebooks](https://console.cloud.google.com/vertex-ai/workbench/list/managed) instance using the XGBoost (Local) kernel. Some components of this notebook may not work in other notebook environments.*\n",
-        "\n",
-        "## Dataset\n",
+        "*Note: This notebook file is developed to run in a [Vertex AI Workbench managed notebooks](https://console.cloud.google.com/vertex-ai/workbench/list/managed) instance using the XGBoost (Local) kernel. Some components of this notebook may not work in other notebook environments.*"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "fead9e83ebd7"
+      },
+      "source": [
+        "### Objective\n",
         "<a name=\"section-2\"></a>\n",
         "\n",
-        "The dataset used in this notebook is a part of the [NASA Turbofan Engine Degradation Simulation dataset](https://ti.arc.nasa.gov/tech/dash/groups/pcoe/prognostic-data-repository/), which consists of simulated time-series data for four sets of fleet engines under different combinations of operational conditions and fault modes. In this notebook, only one of the engine's simulated data (FD001) has been used to analyze and train a model that can predict the engine's remaining useful life.\n",
-        "\n",
-        "## Objectives\n",
-        "<a name=\"section-3\"></a>\n",
-        "\n",
         "The objectives of this notebook include:\n",
         "\n",
         "- Loading the required dataset from a Cloud Storage bucket.\n",
@@ -49,9 +102,28 @@
         "- Evaluating the model.\n",
         "- Running the notebook end-to-end as a training job using Executor.\n",
         "- Deploying the model on Vertex AI.\n",
-        "- Clean up.\n",
+        "- Clean up."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "a71f4d96bf80"
+      },
+      "source": [
+        "### Dataset\n",
+        "<a name=\"section-3\"></a>\n",
         "\n",
-        "## Costs\n",
+        "The dataset used in this notebook is a part of the [NASA Turbofan Engine Degradation Simulation dataset](https://ti.arc.nasa.gov/tech/dash/groups/pcoe/prognostic-data-repository/), which consists of simulated time-series data for four sets of fleet engines under different combinations of operational conditions and fault modes. A version of this dataset which is saved to a public Cloud Storage bucket is used in this notebook. In this notebook, one of the engine's simulated data (FD001) is used to analyze and train a model that can predict the engine's remaining useful life."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "36c53c95b4b9"
+      },
+      "source": [
+        "### Costs\n",
         "<a name=\"section-4\"></a>\n",
         "\n",
         "This tutorial uses the following billable components of Google Cloud:\n",
@@ -69,24 +141,126 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "5b15a97278df"
+        "id": "629f52f6efe1"
       },
       "source": [
-        "## Before you begin\n",
-        "\n",
         "### Kernel selection\n",
-        "Select <b>XGBoost</b> kernel while running this notebook on Vertex AI Workbench managed notebooks instances or ensure that the following libraries are installed in the environment where this notebook is being run.\n",
+        "Select <b>XGBoost</b> kernel while running this notebook on Vertex AI Workbench's managed instances. Otherwise, ensure that the following libraries are installed in the environment where this notebook is being run.\n",
         "- XGBoost\n",
         "- Pandas\n",
         "- Seaborn\n",
         "- Sklearn\n",
         "\n",
-        "Along with the above libraries, the following google-cloud libraries are also used in this notebook.\n",
+        "Along with the above libraries, th`e following google-cloud libraries are also used in this notebook.\n",
         "\n",
         "- google.cloud.aiplatform\n",
-        "- google.cloud.storage\n",
+        "- google.cloud.storage"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "16bee0754628"
+      },
+      "source": [
+        "## Installation\n",
         "\n",
-        "### Set your project ID\n",
+        "Install the following packages to run this notebook outside Vertex AI Workbench's managed instances."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "69520a67e54c"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
+        "\n",
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "USER_FLAG = \"\"\n",
+        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\"\n",
+        "    \n",
+        "! pip3 install {USER_FLAG} --upgrade google-cloud-aiplatform \\\n",
+        "                                    google-cloud-storage \\\n",
+        "                                    xgboost \\\n",
+        "                                    seaborn \\\n",
+        "                                    sklearn \\\n",
+        "                                    fsspec \\\n",
+        "                                    gcsfs \\\n",
+        "                                    pandas -q"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "eda79cca981d"
+      },
+      "source": [
+        "### Restart the kernel\n",
+        "\n",
+        "Once you've installed the additional packages, you need to restart the notebook kernel so it can find the packages."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "e200999cabe5"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "if not os.getenv(\"IS_TESTING\"):\n",
+        "    # Automatically restart kernel after installs\n",
+        "    import IPython\n",
+        "\n",
+        "    app = IPython.Application.instance()\n",
+        "    app.kernel.do_shutdown(True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5b15a97278df"
+      },
+      "source": [
+        "## Before you begin \n",
+        "\n",
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com).\n",
+        "\n",
+        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5aee4379e8e5"
+      },
+      "source": [
+        "#### Set your project ID\n",
         "\n",
         "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
       ]
@@ -99,36 +273,67 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
-        "PROJECT_ID = \"\"\n",
-        "\n",
-        "# Get your Google Cloud project ID from gcloud\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    shell_output = !gcloud config list --format 'value(core.project)' 2>/dev/null\n",
-        "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID: \", PROJECT_ID)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "750bf2883c2d"
-      },
-      "source": [
-        "Otherwise, set your project ID here."
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "3c6db1ca88b9"
+        "id": "5bf9979b96ff"
       },
       "outputs": [],
       "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
-        "    PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
+        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
+        "    # Get your GCP project id from gcloud\n",
+        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "    PROJECT_ID = shell_output[0]\n",
+        "    print(\"Project ID:\", PROJECT_ID)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "09021c90b34c"
+      },
+      "outputs": [],
+      "source": [
+        "! gcloud config set project $PROJECT_ID"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9658ecf524b1"
+      },
+      "source": [
+        "#### Region\n",
+        "\n",
+        "You can also change the `REGION` variable, which is used for operations\n",
+        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. It is recommended that you choose the region closest to you.\n",
+        "\n",
+        "- Americas: `us-central1`\n",
+        "- Europe: `europe-west4`\n",
+        "- Asia Pacific: `asia-east1`\n",
+        "\n",
+        "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
+        "\n",
+        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "5c615e53149f"
+      },
+      "outputs": [],
+      "source": [
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -137,9 +342,9 @@
         "id": "f66f96816fd0"
       },
       "source": [
-        "#### Timestamp\n",
+        "#### UUID\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append it onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
       ]
     },
     {
@@ -150,9 +355,84 @@
       },
       "outputs": [],
       "source": [
-        "from datetime import datetime\n",
+        "import random\n",
+        "import string\n",
         "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+        "\n",
+        "# Generate a uuid of a specifed length(default=8)\n",
+        "def generate_uuid(length: int = 8) -> str:\n",
+        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
+        "\n",
+        "\n",
+        "UUID = generate_uuid()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "df899ce9999c"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
+        "authenticated.\n",
+        "\n",
+        "**If you are using Colab**, run the cell below and follow the instructions\n",
+        "when prompted to authenticate your account via oAuth.\n",
+        "\n",
+        "**Otherwise**, follow these steps:\n",
+        "\n",
+        "1. In the Cloud Console, go to the [**Create service account key**\n",
+        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
+        "\n",
+        "2. Click **Create service account**.\n",
+        "\n",
+        "3. In the **Service account name** field, enter a name, and\n",
+        "   click **Create**.\n",
+        "\n",
+        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
+        "into the filter box, and select\n",
+        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "\n",
+        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
+        "local environment.\n",
+        "\n",
+        "6. Enter the path to your service account key as the\n",
+        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "201e8e760d22"
+      },
+      "outputs": [],
+      "source": [
+        "# If you are running this notebook in Colab, run this cell and follow the\n",
+        "# instructions to authenticate your GCP account. This provides access to your\n",
+        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
+        "# requests.\n",
+        "\n",
+        "import os\n",
+        "import sys\n",
+        "\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = \"google.colab\" in sys.modules\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
+        "    if \"google.colab\" in sys.modules:\n",
+        "        from google.colab import auth as google_auth\n",
+        "\n",
+        "        google_auth.authenticate_user()\n",
+        "\n",
+        "    # If you are running this notebook locally, replace the string below with the\n",
+        "    # path to your service account key and run this cell to authenticate your GCP\n",
+        "    # account.\n",
+        "    elif not os.getenv(\"IS_TESTING\"):\n",
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
       ]
     },
     {
@@ -161,11 +441,18 @@
         "id": "ea53caa30628"
       },
       "source": [
-        "## Select or Create a Cloud Storage Bucket for storing the model\n",
+        "### Create a Cloud Storage bucket\n",
         "\n",
-        "When you create a model resource on Vertex AI using the Cloud SDK, you need to give a Cloud Storage bucket URI of the model where the model is stored. Using the model saved, you can then create Vertex AI model and endpoint resources in order to serve online predictions.\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
         "\n",
-        "Set the name of your Cloud Storage bucket below. It must be unique across all Cloud Storage buckets. You may also change the `REGION` variable, which is used for operations throughout the rest of this notebook. Make sure to choose a region where Vertex AI services are available."
+        "\n",
+        "When you create a model in Vertex AI using the Cloud SDK, you give a Cloud Storage path where the trained model is saved. \n",
+        "In this tutorial, Vertex AI saves the trained model to a Cloud Storage bucket. Using this model artifact, you can then\n",
+        "create Vertex AI model and endpoint resources in order to serve\n",
+        "online predictions.\n",
+        "\n",
+        "Set the name of your Cloud Storage bucket below. It must be unique across all\n",
+        "Cloud Storage buckets."
       ]
     },
     {
@@ -176,9 +463,8 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"[your-bucket-name]\"\n",
-        "BUCKET_URI = f\"gs://{BUCKET_NAME}\"\n",
-        "REGION = \"us-central1\""
+        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
     {
@@ -189,13 +475,9 @@
       },
       "outputs": [],
       "source": [
-        "# Set a default bucketname in case bucket name is not given\n",
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None:\n",
-        "    from datetime import datetime\n",
-        "\n",
-        "    TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
-        "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
+        "    BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
     {
@@ -215,7 +497,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_NAME"
+        "! gsutil mb -l $REGION $BUCKET_URI"
       ]
     },
     {
@@ -235,7 +517,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_NAME"
+        "! gsutil ls -al $BUCKET_URI"
       ]
     },
     {
@@ -244,7 +526,7 @@
         "id": "4c0f6aac282a"
       },
       "source": [
-        "## Import the required libraries"
+        "### Import the required libraries"
       ]
     },
     {
@@ -287,7 +569,7 @@
       "outputs": [],
       "source": [
         "# load the data from the source\n",
-        "INPUT_PATH = \"gs://vertex_ai_managed_services_demo/mfg_predictive_maintenance/train_FD001.txt\"  # data source\n",
+        "INPUT_PATH = \"gs://cloud-samples-data/ai-platform-unified/datasets/tabular/predictive_maintenance.csv\"  # data source\n",
         "raw_data = pd.read_csv(INPUT_PATH, sep=\" \", header=None)\n",
         "# check the data\n",
         "print(raw_data.shape)\n",
@@ -492,7 +774,7 @@
         "id": "8197cdef2cff"
       },
       "source": [
-        "As the current objective is to predict the remaining useful life (RUL) of each unit (ID), the target variable needs to be identified. Since we're dealing with a timeseries data that represents the lifetime of a unit, remaining useful life of a unit can be calculated by subtracting the current cycle from the maximum cycle of that unit.\n",
+        "As the current objective is to predict the remaining useful life (RUL) of each unit (ID), the target variable needs to be identified. Since you're dealing with a timeseries data that represents the lifetime of a unit, remaining useful life of a unit can be calculated by subtracting the current cycle from the maximum cycle of that unit.\n",
         "\n",
         "\t\t\t\t\tRUL = Max. Cycle - Current Cycle    \n",
         "## RUL calculation and Feature selection"
@@ -810,6 +1092,7 @@
         "## Running a notebook end-to-end using executor\n",
         "<a name=\"section-9\"></a>\n",
         "\n",
+        "**Note:** This section can only be considered when running this notebook on Managed instances from Vertex AI Workbench.\n",
         "### Automating the notebook execution\n",
         "All the steps followed until now can be run as a training job without using any additional code using the Vertex AI Workbench executor. The executor can help you run a notebook file from start to end, with your choice of the environment, machine type, input parameters, and other characteristics. After setting up an execution, the notebook is executed as a job in Vertex AI custom training. Your jobs can be monitored from the Executor pane in the left sidebar.\n",
         "\n",
@@ -817,13 +1100,13 @@
         "\n",
         "The executor also lets you choose the environment and machine type while automating the runs similar to Vertex AI training jobs without switching to the training jobs UI. Apart from the custom container that replicates the existing kernel by default, pre-built environments like TensorFlow Enterprise, PyTorch, and others can also be selected to run the notebook. The required compute power can be specified by choosing from the list of machine types available, including GPUs.\n",
         "\n",
-        "## Scheduled runs on executor\n",
+        "### Scheduled runs on executor\n",
         "\n",
         "Notebook runs can also be scheduled recurringly with the executor. To do so, select Schedule-based recurring executions as the run type instead of One-time execution. The frequency of the job and the time when it executes is provided when you create the execution.\n",
         "\n",
         "<img src=\"https://storage.googleapis.com/gweb-cloudblog-publish/images/7_Vertex_AI_Workbench.max-1100x1100.jpg\">\n",
         "\n",
-        "## Parameterizing the variables\n",
+        "### Parameterizing the variables\n",
         "\n",
         "The executor lets you run a notebook with different sets of input parameters. If you’ve added parameter tags to any of your notebook cells, you can pass in your parameter values to the executor. More about how to use this feature can be found on this [blog](https://cloud.google.com/blog/products/ai-machine-learning/schedule-and-execute-notebooks-with-vertex-ai-workbench).\n",
         "\n",
@@ -855,6 +1138,37 @@
         "ARTIFACT_GCS_PATH = f\"gs://{BUCKET_NAME}/{BLOB_PATH}\""
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "1aa75b3d4616"
+      },
+      "source": [
+        "Give a display name to the Vertex AI model resource."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "02ca350dba6c"
+      },
+      "outputs": [],
+      "source": [
+        "# Set the model-dsiplay-name\n",
+        "MODEL_DISPLAY_NAME = \"[your-model-display-name]\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# Otherwise, use the default name\n",
+        "if (\n",
+        "    MODEL_DISPLAY_NAME == \"[your-model-display-name]\"\n",
+        "    or MODEL_DISPLAY_NAME is None\n",
+        "    or MODEL_DISPLAY_NAME == \"\"\n",
+        "):\n",
+        "    MODEL_DISPLAY_NAME = \"pred_maint_model_\" + UUID\n",
+        "\n",
+        "print(MODEL_DISPLAY_NAME)"
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -891,6 +1205,28 @@
         "Next, create an endpoint resource for deploying the model."
       ]
     },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "e1e0cd571992"
+      },
+      "outputs": [],
+      "source": [
+        "# Set the endpoint-dsiplay-name\n",
+        "ENDPOINT_DISPLAY_NAME = \"[your-endpoint-display-name]\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# Otherwise, use the default name\n",
+        "if (\n",
+        "    ENDPOINT_DISPLAY_NAME == \"[your-endpoint-display-name]\"\n",
+        "    or ENDPOINT_DISPLAY_NAME is None\n",
+        "    or ENDPOINT_DISPLAY_NAME == \"\"\n",
+        "):\n",
+        "    ENDPOINT_DISPLAY_NAME = \"pred_maint_endpoint_\" + UUID\n",
+        "\n",
+        "print(ENDPOINT_DISPLAY_NAME)"
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -899,6 +1235,7 @@
       },
       "outputs": [],
       "source": [
+        "# Create the Endpoint resource\n",
         "endpoint = aiplatform.Endpoint.create(display_name=ENDPOINT_DISPLAY_NAME)\n",
         "\n",
         "print(endpoint.display_name)\n",
@@ -915,18 +1252,11 @@
         "<a name=\"section-12\"></a>\n",
         "\n",
         "\n",
-        "Configure the deployment name, machine type, and other parameters for the deployment and deploy the model to the created endpoint."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "ca41cac871d6"
-      },
-      "outputs": [],
-      "source": [
-        "MACHINE_TYPE = \"n1-standard-2\""
+        "Configure the following parameters and deploy the model to the created endpoint.\n",
+        "\n",
+        "- `endpoint`: The `Endpoint` object created using Vertex AI SDK.\n",
+        "- `deployed_model_display_name`: A display-name for the deployment.\n",
+        "- `machine_type`: Type of the machine required for the deployment environment. See [here](https://cloud.google.com/vertex-ai/docs/predictions/configure-compute) for references."
       ]
     },
     {
@@ -940,8 +1270,8 @@
         "# deploy the model to the endpoint\n",
         "model.deploy(\n",
         "    endpoint=endpoint,\n",
-        "    deployed_model_display_name=DEPLOYED_MODEL_NAME,\n",
-        "    machine_type=MACHINE_TYPE,\n",
+        "    deployed_model_display_name=MODEL_DISPLAY_NAME + \"_deployment\",\n",
+        "    machine_type=\"n1-standard-2\",\n",
         ")\n",
         "\n",
         "model.wait()\n",
@@ -984,7 +1314,15 @@
         "## Clean up\n",
         "<a name=\"section-14\"></a>\n",
         "\n",
-        "Undeploy the model from the endpoint."
+        "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
+        "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
+        "\n",
+        "Otherwise, you can delete the individual resources you created in this tutorial:\n",
+        "* Vertex AI Model\n",
+        "* Vertex AI Endpoint\n",
+        "* Cloud Storage bucket\n",
+        "\n",
+        "Set `delete_bucket` to **True** to delete the Cloud Storage bucket."
       ]
     },
     {
@@ -995,68 +1333,19 @@
       },
       "outputs": [],
       "source": [
-        "DEPLOYED_MODEL_ID = \"\"\n",
-        "endpoint.undeploy(deployed_model_id=DEPLOYED_MODEL_ID)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "96e427b77791"
-      },
-      "source": [
-        "Delete the endpoint."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "ace028ac23ea"
-      },
-      "outputs": [],
-      "source": [
-        "endpoint.delete()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "4b77998d0512"
-      },
-      "source": [
-        "Delete the model."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "e034150a4c94"
-      },
-      "outputs": [],
-      "source": [
-        "model.delete()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "23cb2deb122d"
-      },
-      "source": [
-        "Remove the contents of the Cloud Storage bucket."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "98aaac27d85d"
-      },
-      "outputs": [],
-      "source": [
-        "! gsutil -m rm -r $BUCKET_URI"
+        "# Undeploy all the models from the endpoint\n",
+        "endpoint.undeploy_all()\n",
+        "\n",
+        "# Delete the endpoint resource\n",
+        "endpoint.delete()\n",
+        "\n",
+        "# Delete the model resource\n",
+        "model.delete()\n",
+        "\n",
+        "# Delete the Cloud Storage bucket\n",
+        "delete_bucket = False\n",
+        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "    ! gsutil -m rm -r $BUCKET_URI"
       ]
     }
   ],

notebooks/official/workbench/pricing_optimization/pricing-optimization.ipynb

@@ -1,16 +1,68 @@
 {
   "cells": [
     {
-      "cell_type": "markdown",
+      "cell_type": "code",
+      "execution_count": null,
       "metadata": {
         "id": "d1cc1c1fa076"
       },
+      "outputs": [],
+      "source": [
+        "# Copyright 2022 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9751bc48dbcb"
+      },
+      "source": [
+        "# Analysis of pricing optimization on CDM Pricing Data\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/pricing_optimization/pricing-optimization.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/pricing_optimization/pricing-optimization.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/workbench/pricing_optimization/pricing-optimization.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "  </td>                                                                                               \n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "cd1268747961"
+      },
       "source": [
-        "# Pricing Optimization \n",
         "## Table of contents\n",
         "* [Overview](#section-1)\n",
-        "* [Dataset](#section-2)\n",
-        "* [Objective](#section-3)\n",
+        "* [Objective](#section-2)\n",
+        "* [Dataset](#section-3)\n",
         "* [Costs](#section-4)\n",
         "* [Create a BigQuery dataset](#section-5)\n",
         "* [Load the dataset from Cloud Storage](#section-6)\n",
@@ -19,24 +71,41 @@
         "* [Train the model using BigQuery ML](#section-9)\n",
         "* [Generate forecasts from the model](#section-10)\n",
         "* [Interpret the results to choose the best price](#section-11)\n",
-        "* [Clean up](#section-12)\n",
-        "\n",
+        "* [Clean up](#section-12)\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8414ceb17c47"
+      },
+      "source": [
         "## Overview\n",
         "<a name=\"section-1\"></a>\n",
         "\n",
         "This notebook demonstrates analysis of pricing optimization on [CDM Pricing Data](https://github.com/trifacta/trifacta-google-cloud/tree/main/design-pattern-pricing-optimization) and automating the workflow using Vertex AI Workbench managed notebooks.\n",
         "\n",
-        "*Note: This notebook file was developed to run in a [Vertex AI Workbench managed notebooks](https://console.cloud.google.com/vertex-ai/workbench/list/managed) instance using the Python (Local) kernel. Some components of this notebook may not work in other notebook environments.*\n",
-        "\n",
-        "## Dataset\n",
+        "*Note: This notebook file was developed to run in a [Vertex AI Workbench managed notebooks](https://console.cloud.google.com/vertex-ai/workbench/list/managed) instance using the Python (Local) kernel. Some components of this notebook may not work in other notebook environments.*"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "71f69cfdff2b"
+      },
+      "source": [
+        "## Objective\n",
         "<a name=\"section-2\"></a>\n",
         "\n",
-        "The dataset used in this notebook is a part of the [CDM Pricing dataset](https://github.com/trifacta/trifacta-google-cloud/blob/main/design-pattern-pricing-optimization/CDM_Pricing_large_table.csv), which consists of product sales information on specified dates.\n",
+        "The objective of this notebook is to build a pricing optimization model using BigQuery ML. The following steps have been followed:  \n",
         "\n",
-        "## Objective\n",
-        "<a name=\"section-3\"></a>\n",
+        "This tutorial uses the following Google Cloud ML services and resources:\n",
         "\n",
-        "The objective of this notebook is to build a pricing optimization model using Vertex AI. The following steps have been followed:  \n",
+        "- Google Cloud Storage\n",
+        "- BigQuery\n",
+        "\n",
+        "\n",
+        "The steps performed include:\n",
         "\n",
         "- Load the required dataset from a Cloud Storage bucket.\n",
         "- Analyze the fields present in the dataset.\n",
@@ -44,8 +113,27 @@
         "- Build a BigQuery ML forecast model on the processed data.\n",
         "- Get forecasted values from the BigQuery ML model.\n",
         "- Interpret the forecasts to identify the best prices.\n",
-        "- Clean up.\n",
+        "- Clean up.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d20422a5c34d"
+      },
+      "source": [
+        "## Dataset\n",
+        "<a name=\"section-3\"></a>\n",
         "\n",
+        "The dataset used in this notebook is a part of the [CDM Pricing dataset](https://github.com/trifacta/trifacta-google-cloud/blob/main/design-pattern-pricing-optimization/CDM_Pricing_large_table.csv), which consists of product sales information on specified dates."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "c05bcd30859d"
+      },
+      "source": [
         "## Costs\n",
         "<a name=\"section-4\"></a>\n",
         "\n",
@@ -60,7 +148,121 @@
         "pricing](https://cloud.google.com/vertex-ai/pricing), [BigQuery pricing](https://cloud.google.com/bigquery/pricing) and [Cloud Storage\n",
         "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
         "Calculator](https://cloud.google.com/products/calculator/)\n",
-        "to generate a cost estimate based on your projected usage.\n"
+        "to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "f5494c42606e"
+      },
+      "source": [
+        "### Set up your local development environment\n",
+        "\n",
+        "**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets\n",
+        "all the requirements to run this notebook. You can skip this step.\n",
+        "\n",
+        "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
+        "You need the following:\n",
+        "\n",
+        "* The Google Cloud SDK\n",
+        "* Git\n",
+        "* Python 3\n",
+        "* virtualenv\n",
+        "* Jupyter notebook running in a virtual environment with Python 3\n",
+        "\n",
+        "The Google Cloud guide to [Setting up a Python development\n",
+        "environment](https://cloud.google.com/python/setup) and the [Jupyter\n",
+        "installation guide](https://jupyter.org/install) provide detailed instructions\n",
+        "for meeting these requirements. The following steps provide a condensed set of\n",
+        "instructions:\n",
+        "\n",
+        "1. [Install and initialize the Cloud SDK.](https://cloud.google.com/sdk/docs/)\n",
+        "\n",
+        "1. [Install Python 3.](https://cloud.google.com/python/setup#installing_python)\n",
+        "\n",
+        "1. [Install\n",
+        "   virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv)\n",
+        "   and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
+        "\n",
+        "1. To install Jupyter, run `pip3 install jupyter` on the\n",
+        "command-line in a terminal shell.\n",
+        "\n",
+        "1. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
+        "\n",
+        "1. Open this notebook in the Jupyter Notebook Dashboard."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2bed1491312f"
+      },
+      "source": [
+        "### Install additional packages\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "1fd00fa70a2a"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
+        "\n",
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "USER_FLAG = \"\"\n",
+        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "25fffcad67f0"
+      },
+      "outputs": [],
+      "source": [
+        "! pip3 install {USER_FLAG} --upgrade pandas-gbq 'google-cloud-bigquery[bqstorage,pandas]' seaborn fsspec gcsfs -q\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d3a26cb9b19d"
+      },
+      "source": [
+        "### Restart the kernel\n",
+        "\n",
+        "After you install the additional packages, you need to restart the notebook kernel so it can find the packages."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "c1464805870e"
+      },
+      "outputs": [],
+      "source": [
+        "# Automatically restart kernel after installs\n",
+        "import os\n",
+        "\n",
+        "if not os.getenv(\"IS_TESTING\"):\n",
+        "    # Automatically restart kernel after installs\n",
+        "    import IPython\n",
+        "\n",
+        "    app = IPython.Application.instance()\n",
+        "    app.kernel.do_shutdown(True)"
       ]
     },
     {
@@ -71,6 +273,25 @@
       "source": [
         "## Before you begin\n",
         "\n",
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI, Cloud Storage, and Compute Engine APIs](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component,storage-component.googleapis.com). \n",
+        "\n",
+        "1. [Configure your Google Cloud project for Vertex Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/configure-project).\n",
+        "\n",
+        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands.\n",
+        "\n",
         "### Set your project ID\n",
         "\n",
         "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
@@ -84,36 +305,139 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
-        "PROJECT_ID = \"\"\n",
-        "\n",
-        "# Get your Google Cloud project ID from gcloud\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    shell_output = !gcloud config list --format 'value(core.project)' 2>/dev/null\n",
-        "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID: \", PROJECT_ID)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "750bf2883c2d"
-      },
-      "source": [
-        "Otherwise, set your project ID here."
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "3c6db1ca88b9"
+        "id": "750bf2883c2d"
       },
       "outputs": [],
       "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
-        "    PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
+        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
+        "    # Get your GCP project id from gcloud\n",
+        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "    PROJECT_ID = shell_output[0]\n",
+        "    print(\"Project ID:\", PROJECT_ID)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "30e64c0eda41"
+      },
+      "outputs": [],
+      "source": [
+        "! gcloud config set project $PROJECT_ID"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0e5ca6c89ab7"
+      },
+      "source": [
+        "#### UUID\n",
+        "\n",
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "1105933b5528"
+      },
+      "outputs": [],
+      "source": [
+        "import random\n",
+        "import string\n",
+        "\n",
+        "\n",
+        "# Generate a uuid of a specifed length(default=8)\n",
+        "def generate_uuid(length: int = 8) -> str:\n",
+        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
+        "\n",
+        "\n",
+        "UUID = generate_uuid()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "67a2b5ee4efb"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
+        "authenticated. Skip this step."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e44201253746"
+      },
+      "source": [
+        "**If you are using Colab**, run the cell below and follow the instructions\n",
+        "when prompted to authenticate your account via oAuth.\n",
+        "\n",
+        "**Otherwise**, follow these steps:\n",
+        "\n",
+        "1. In the Cloud Console, go to the [**Create service account key**\n",
+        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
+        "\n",
+        "2. Click **Create service account**.\n",
+        "\n",
+        "3. In the **Service account name** field, enter a name, and\n",
+        "   click **Create**.\n",
+        "\n",
+        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
+        "into the filter box, and select\n",
+        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "\n",
+        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
+        "local environment.\n",
+        "\n",
+        "6. Enter the path to your service account key as the\n",
+        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "505a908f1d0e"
+      },
+      "outputs": [],
+      "source": [
+        "# If you are running this notebook in Colab, run this cell and follow the\n",
+        "# instructions to authenticate your GCP account. This provides access to your\n",
+        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
+        "# requests.\n",
+        "\n",
+        "import os\n",
+        "import sys\n",
+        "\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = \"google.colab\" in sys.modules\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
+        "    if \"google.colab\" in sys.modules:\n",
+        "        from google.colab import auth as google_auth\n",
+        "\n",
+        "        google_auth.authenticate_user()\n",
+        "\n",
+        "    # If you are running this notebook locally, replace the string below with the\n",
+        "    # path to your service account key and run this cell to authenticate your GCP\n",
+        "    # account.\n",
+        "    elif not os.getenv(\"IS_TESTING\"):\n",
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
       ]
     },
     {
@@ -140,6 +464,15 @@
         "from google.cloud.bigquery import Client"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3d5ff24d3194"
+      },
+      "source": [
+        "#### Set the BigQuery dataset ID and table ID"
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -148,8 +481,10 @@
       },
       "outputs": [],
       "source": [
-        "DATASET = \"[your-bigquery-dataset-id]\"  # set the BigQuery dataset-id\n",
-        "TRAINING_DATA_TABLE = \"[your-bigquery-table-id-to-store-the-training-data]\"  # set the BigQuery table-id to store the training data"
+        "DATASET = \"pricing_optimization\" + \"_\" + UUID  # set the BigQuery dataset-id\n",
+        "TRAINING_DATA_TABLE = (\n",
+        "    \"training_data_table\"  # set the BigQuery table-id to store the training data\n",
+        ")"
       ]
     },
     {
@@ -162,6 +497,15 @@
         "<a name=\"section-5\"></a>\n"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3a063f530682"
+      },
+      "source": [
+        "If you are using ***Vertex AI Workbench managed notebooks instance***, every cell which starts with \"#@bigquery\" will be a SQL Query. If you are using Vertex AI Workbench user managed notebooks instance or Colab it will be a markdown cell."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -171,12 +515,44 @@
         "#@bigquery\n",
         "-- create a dataset in BigQuery\n",
         "\n",
-        "CREATE SCHEMA pricing_optimization\n",
+        "CREATE SCHEMA [your-dataset-id]\n",
         "OPTIONS(\n",
         "  location=\"us\"\n",
         "  )"
       ]
     },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "00bd69008c92"
+      },
+      "outputs": [],
+      "source": [
+        "# Construct a BigQuery client object.\n",
+        "client = Client(project=PROJECT_ID)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "5f7acd204413"
+      },
+      "outputs": [],
+      "source": [
+        "query = \"\"\"\n",
+        "CREATE SCHEMA {DATASET}\n",
+        "OPTIONS(\n",
+        "  location=\"us\"\n",
+        "  )\n",
+        "\"\"\".format(\n",
+        "    DATASET=DATASET\n",
+        ")\n",
+        "query_job = client.query(query)\n",
+        "print(query_job.result())"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -207,7 +583,7 @@
         "id": "7b98d5f09842"
       },
       "source": [
-        "You will build a forecast model on this data and thus determine the best price for a product. For this type of model, you will not be using many fields: only the sales and price related ones. For the current execrcise, focus on the following fields:\n",
+        "You build a forecast model on this data and thus determine the best price for a product. For this type of model, you will not be using many fields: only the sales and price related ones. For the current execrcise, focus on the following fields:\n",
         "\n",
         "- `Product_ID`\n",
         "- `Customer_Hierarchy`\n",
@@ -221,7 +597,7 @@
         "\n",
         "First, explore the data and distributions.\n",
         "\n",
-        "Select the required columns from the dataframe."
+        "#### Select the required columns from the dataframe."
       ]
     },
     {
@@ -247,7 +623,7 @@
         "id": "3d780043ee5b"
       },
       "source": [
-        "Check the column types and null values in the dataframe."
+        "#### Check the column types and null values in the dataframe."
       ]
     },
     {
@@ -269,7 +645,7 @@
       "source": [
         "This data description reveals that there are no null values in the data. Also, the field `Fiscal_Date` which is a date field is loaded as an object type. \n",
         "\n",
-        "Change the type of the date field to datetime."
+        "#### Change the type of the date field to datetime."
       ]
     },
     {
@@ -289,7 +665,7 @@
         "id": "fb4778578064"
       },
       "source": [
-        "Plot the distributions for the categorical fields."
+        "#### Plot the distributions for the categorical fields."
       ]
     },
     {
@@ -312,7 +688,7 @@
         "id": "145deed255e0"
       },
       "source": [
-        "Plot the distributions for the numerical fields."
+        "#### Plot the distributions for the numerical fields."
       ]
     },
     {
@@ -338,7 +714,7 @@
         "id": "f9b9c2e58380"
       },
       "source": [
-        "Check the maximum date and minimum date in Fiscal_Date column."
+        "#### Check the maximum date and minimum date in Fiscal_Date column."
       ]
     },
     {
@@ -359,7 +735,7 @@
         "id": "4834f63e2e59"
       },
       "source": [
-        "Check the product distribution across each category."
+        "#### Check the product distribution across each category."
       ]
     },
     {
@@ -381,7 +757,7 @@
         "id": "01ed02b9c8fd"
       },
       "source": [
-        "Check the percentage changes in the orders based on the percentage changes in the price."
+        "#### Check the percentage changes in the orders based on the percentage changes in the price."
       ]
     },
     {
@@ -457,7 +833,7 @@
         "## Preprocess the data for training\n",
         "<a name=\"section-8\"></a>\n",
         "\n",
-        "Check which `Product_ID`'s  have the maximum orders."
+        "#### Check which `Product_ID`'s  have the maximum orders."
       ]
     },
     {
@@ -501,7 +877,7 @@
         "id": "2dbc0d64d157"
       },
       "source": [
-        "Check the various prices available for these `Product_ID`s."
+        "#### Check the various prices available for these `Product_ID`s."
       ]
     },
     {
@@ -543,9 +919,9 @@
         "id": "f023af578c0f"
       },
       "source": [
-        "In the publishing category, `Product_ID` `SKU 8` and `SKU 17` are less than or equal to two different prices in the entire data and so you will exclude them and consider the rest for building the forecast model. The idea here is to train a forecast model on the timeseries data for products with different prices.\n",
+        "In the publishing category, `Product_ID` `SKU 8` and `SKU 17` are less than or equal to two different prices in the entire data and so you exclude them and consider the rest for building the forecast model. The idea here is to train a forecast model on the timeseries data for products with different prices.\n",
         "\n",
-        "Join the data for all the `Product_ID`s into one dataframe and remove duplicate records."
+        "#### Join the data for all the `Product_ID`s into one dataframe and remove duplicate records."
       ]
     },
     {
@@ -579,7 +955,7 @@
         "id": "add5063df368"
       },
       "source": [
-        "Save the data to a BigQuery table."
+        "#### Save the data to a BigQuery table."
       ]
     },
     {
@@ -615,7 +991,7 @@
         "    \"{}.{}.{}\".format(PROJECT_ID, DATASET, TRAINING_DATA_TABLE),\n",
         "    job_config=job_config,\n",
         ")  # Make an API request.\n",
-        "job.result()  # Wait for the job to complete."
+        "print(job.result())  # Wait for the job to complete."
       ]
     },
     {
@@ -637,7 +1013,7 @@
       },
       "source": [
         "#@bigquery\n",
-        "create or replace model pricing_optimization.bqml_arima\n",
+        "create or replace model [your-dataset-id].bqml_arima\n",
         "options\n",
         " (model_type = 'ARIMA_PLUS',\n",
         "  time_series_timestamp_col = 'Fiscal_Date',\n",
@@ -649,7 +1025,35 @@
         " Concat(Product_ID,\"_\" ,Cast(List_Price_Converged as string)) as ID,\n",
         " Invoiced_quantity_in_Pieces\n",
         "from\n",
-        " pricing_optimization.TRAINING_DATA\n"
+        " [your-dataset-id].TRAINING_DATA\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "e25254d219b7"
+      },
+      "outputs": [],
+      "source": [
+        "query = \"\"\"\n",
+        "create or replace model `{PROJECT_ID}.{DATASET}.bqml_arima`\n",
+        "options\n",
+        " (model_type = 'ARIMA_PLUS',\n",
+        "  time_series_timestamp_col = 'Fiscal_Date',\n",
+        "  time_series_data_col = 'Invoiced_quantity_in_Pieces',\n",
+        "  time_series_id_col = 'ID'\n",
+        " ) as\n",
+        "select\n",
+        " Fiscal_Date,\n",
+        " Concat(Product_ID,\"_\" ,Cast(List_Price_Converged as string)) as ID,\n",
+        " Invoiced_quantity_in_Pieces\n",
+        "from\n",
+        " `{DATASET}.{TRAINING_DATA_TABLE}`\"\"\".format(\n",
+        "    PROJECT_ID=PROJECT_ID, DATASET=DATASET, TRAINING_DATA_TABLE=TRAINING_DATA_TABLE\n",
+        ")\n",
+        "query_job = client.query(query)\n",
+        "print(query_job.result())"
       ]
     },
     {
@@ -672,8 +1076,6 @@
       },
       "outputs": [],
       "source": [
-        "client = Client()\n",
-        "\n",
         "query = '''\n",
         "DECLARE HORIZON STRING DEFAULT \"30\"; #number of values to forecast\n",
         "DECLARE CONFIDENCE_LEVEL STRING DEFAULT \"0.90\"; ## required confidence level\n",
@@ -682,11 +1084,13 @@
         "    SELECT\n",
         "      *\n",
         "    FROM \n",
-        "      ML.FORECAST(MODEL pricing_optimization.bqml_arima, \n",
+        "      ML.FORECAST(MODEL {DATASET}.bqml_arima, \n",
         "                  STRUCT(%s AS horizon, \n",
         "                         %s AS confidence_level)\n",
         "                 )\n",
-        "    \"\"\",HORIZON,CONFIDENCE_LEVEL)'''\n",
+        "    \"\"\",HORIZON,CONFIDENCE_LEVEL)'''.format(\n",
+        "    DATASET=DATASET\n",
+        ")\n",
         "job = client.query(query)\n",
         "dfforecast = job.to_dataframe()\n",
         "dfforecast.head()"
@@ -701,7 +1105,7 @@
         "## Interpret the results to choose the best price\n",
         "<a name=\"section-11\"></a>\n",
         "\n",
-        "Calculate average forecast values for the forecast duration."
+        "#### Calculate average forecast values for the forecast duration."
       ]
     },
     {
@@ -723,7 +1127,7 @@
         "id": "5ce395d652a3"
       },
       "source": [
-        "Extract the ID and Price fields from the ID field."
+        "#### Extract the ID and Price fields from the ID field."
       ]
     },
     {
@@ -744,7 +1148,7 @@
         "id": "3cee67f4028f"
       },
       "source": [
-        "Plot the average forecasted sales vs. the price of the product."
+        "#### Plot the average forecasted sales vs. the price of the product."
       ]
     },
     {
@@ -773,9 +1177,15 @@
         "\n",
         "- SKU 107's price range can be from 4.44 - 4.73 units\n",
         "- SKU 140's price can be 1.95 units\n",
-        "- SKU 62's price can be 4.23 units\n",
-        "\n",
-        "\n",
+        "- SKU 62's price can be 4.23 units\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "01fdc73828af"
+      },
+      "source": [
         "## Clean Up\n",
         "<a name=\"section-12\"></a>\n",
         "\n",
@@ -792,11 +1202,8 @@
       },
       "outputs": [],
       "source": [
-        "# Construct a BigQuery client object.\n",
-        "client = bigquery.Client()\n",
-        "\n",
-        "# TODO(developer): Set model_id to the ID of the model to fetch.\n",
-        "dataset_id = \"{PROJECT}.{DATASET}\".format(PROJECT=PROJECT_ID, DATASET=DATASET)\n",
+        "# Set dataset_id to the ID of the dataset to fetch.\n",
+        "dataset_id = \"{PROJECT_ID}.{DATASET}\".format(PROJECT_ID=PROJECT_ID, DATASET=DATASET)\n",
         "\n",
         "# Use the delete_contents parameter to delete a dataset and its contents.\n",
         "# Use the not_found_ok parameter to not receive an error if the dataset has already been deleted.\n",