docs: fixing CODEOWNERS and instructions hyperlinks

When opening a PR, the CODEOWNERS and instructions hyperlinks throw a 404 error because they point to a URL that has been changed. Fixing these hyperlinks.

.cloud-build/cleanup/cleanup.py

@@ -1,13 +1,9 @@
 from typing import List
-from ratemate import RateLimit
-from resource_cleanup_manager import (
-    DatasetResourceCleanupManager,
-    ModelResourceCleanupManager,
-    EndpointResourceCleanupManager,
-    ResourceCleanupManager,
-)
 
-rate_limit = RateLimit(max_count=25, per=60, greedy=False)
+from resource_cleanup_manager import (DatasetResourceCleanupManager,
+                                      EndpointResourceCleanupManager,
+                                      ModelResourceCleanupManager,
+                                      ResourceCleanupManager)
 
 
 def run_cleanup_managers(managers: List[ResourceCleanupManager], is_dry_run: bool):
@@ -18,18 +14,17 @@ def run_cleanup_managers(managers: List[ResourceCleanupManager], is_dry_run: boo
         resources = manager.list()
         print(f"Found {len(resources)} {type_name}'s")
         for resource in resources:
-            try:
-                if not manager.is_deletable(resource):
-                    continue
+            if not manager.is_deletable(resource):
+                continue
 
-                if is_dry_run:
-                    resource_name = manager.resource_name(resource)
-                    print(f"Will delete '{type_name}': {resource_name}")
-                else:
-                    rate_limit.wait()  # wait before deleting
+            if is_dry_run:
+                resource_name = manager.resource_name(resource)
+                print(f"Will delete '{type_name}': {resource_name}")
+            else:
+                try:
                     manager.delete(resource)
-            except Exception as exception:
-                print(exception)
+                except Exception as exception:
+                    print(exception)
 
         print("")
 
@@ -43,7 +38,7 @@ if is_dry_run:
 managers = [
     DatasetResourceCleanupManager(),
     EndpointResourceCleanupManager(),
-    ModelResourceCleanupManager(),  # ModelResourceCleanupManager must follow EndpointResourceCleanupManager due to deployed models blocking model deletion.
+    ModelResourceCleanupManager(),
 ]
 
 run_cleanup_managers(managers=managers, is_dry_run=is_dry_run)

.cloud-build/cleanup/resource_cleanup_manager.py

@@ -1,5 +1,5 @@
 import abc
-from typing import Any, Type
+from typing import Any
 
 from google.cloud import aiplatform
 from google.cloud.aiplatform import base
@@ -41,7 +41,7 @@ class ResourceCleanupManager(abc.ABC):
         # Check that it wasn't created too recently, to prevent race conditions
         if time_difference <= RESOURCE_UPDATE_BUFFER_IN_SECONDS:
             print(
-                f"Skipping '{resource}' due to update_time being '{time_difference}', which is less than '{RESOURCE_UPDATE_BUFFER_IN_SECONDS}'."
+                f"Skipping '{resource}' due update_time being '{time_difference}', which is less than '{RESOURCE_UPDATE_BUFFER_IN_SECONDS}'."
             )
             return False
 
@@ -51,7 +51,7 @@ class ResourceCleanupManager(abc.ABC):
 class VertexAIResourceCleanupManager(ResourceCleanupManager):
     @property
     @abc.abstractmethod
-    def vertex_ai_resource(self) -> Type[base.VertexAiResourceNounWithFutureManager]:
+    def vertex_ai_resource(self) -> base.VertexAiResourceNounWithFutureManager:
         pass
 
     @property
@@ -61,9 +61,7 @@ class VertexAIResourceCleanupManager(ResourceCleanupManager):
     def list(self) -> Any:
         return self.vertex_ai_resource.list()
 
-    def resource_name(
-        self, resource: Type[base.VertexAiResourceNounWithFutureManager]
-    ) -> str:
+    def resource_name(self, resource: Any) -> str:
         return resource.display_name
 
     def delete(self, resource):
@@ -77,33 +75,12 @@ class VertexAIResourceCleanupManager(ResourceCleanupManager):
 
 class DatasetResourceCleanupManager(VertexAIResourceCleanupManager):
     vertex_ai_resource = aiplatform.datasets._Dataset
-    dataset_types = [
-        aiplatform.ImageDataset,
-        aiplatform.TabularDataset,
-        aiplatform.TextDataset,
-        aiplatform.TimeSeriesDataset,
-        aiplatform.VideoDataset,
-    ]
-
-    def list(self) -> Any:
-        return [
-            dataset
-            for dataset_type in self.dataset_types
-            for dataset in dataset_type.list()
-        ]
 
 
 class EndpointResourceCleanupManager(VertexAIResourceCleanupManager):
     vertex_ai_resource = aiplatform.Endpoint
 
     def delete(self, resource):
-        # TODO: Remove this once https://github.com/googleapis/python-aiplatform/issues/1441 is fixed
-        resource._sync_gca_resource()
-        for deployed_model_id in [
-            models.id for models in resource._gca_resource.deployed_models
-        ]:
-            resource._undeploy(deployed_model_id=deployed_model_id)
-
         resource.delete(force=True)
 
 

.cloud-build/execute_changed_notebooks_helper.py

@@ -17,7 +17,6 @@ import concurrent
 import dataclasses
 import datetime
 import functools
-import git
 import operator
 import os
 import pathlib
@@ -25,7 +24,6 @@ import re
 import subprocess
 from typing import List, Optional
 
-import execute_notebook_helper
 import execute_notebook_remote
 import nbformat
 from google.cloud.devtools.cloudbuild_v1.types import BuildOperationMetadata
@@ -233,40 +231,20 @@ def get_changed_notebooks(
 
     # Find notebooks
     notebooks = []
-
-    # Instantiate GitPython objects
-    repo = git.Repo(os.getcwd())
-    index = repo.index
-
     if base_branch:
-        # Get the point at which this branch branches off from main
-        branching_commits = repo.merge_base("HEAD", f"origin/{base_branch}")
-
-        if len(branching_commits) > 0:
-            branching_commit = branching_commits[0]
-            print(f"Looking for notebooks that changed from branch: {branching_commit}")
-
-            notebooks = [
-                diff.b_path
-                for diff in index.diff(branching_commit, paths=test_paths)
-                if diff.b_path is not None
-            ]
-        else:
-            notebooks = []
+        print(f"Looking for notebooks that changed from branch: {base_branch}")
+        notebooks = subprocess.check_output(
+            ["git", "diff", "--name-only", f"origin/{base_branch}..."] + test_paths
+        )
     else:
         print(f"Looking for all notebooks.")
         notebooks = subprocess.check_output(["git", "ls-files"] + test_paths)
-        notebooks = notebooks.decode("utf-8").split("\n")
 
+    notebooks = notebooks.decode("utf-8").split("\n")
     notebooks = [notebook for notebook in notebooks if notebook.endswith(".ipynb")]
     notebooks = [notebook for notebook in notebooks if len(notebook) > 0]
     notebooks = [notebook for notebook in notebooks if pathlib.Path(notebook).exists()]
 
-    if len(notebooks) > 0:
-        print(f"Found {len(notebooks)} notebooks:")
-        for notebook in notebooks:
-            print(f"\t{notebook}")
-
     return notebooks
 
 
@@ -309,15 +287,14 @@ def process_and_execute_notebooks(
         timeout (str):
             Required. Timeout string according to https://cloud.google.com/build/docs/build-config-file-schema#timeout.
     """
+    notebook_execution_results: List[NotebookExecutionResult] = []
 
     # Calculate deadline
     deadline = datetime.datetime.now() + datetime.timedelta(
         seconds=max(timeout - WORKER_TIMEOUT_BUFFER_IN_SECONDS, 0)
     )
 
-    if len(notebooks) > 1:
-        notebook_execution_results: List[NotebookExecutionResult] = []
-
+    if len(notebooks) > 0:
         print(f"Found {len(notebooks)} modified notebooks: {notebooks}")
 
         if should_parallelize and len(notebooks) > 1:
@@ -356,64 +333,43 @@ def process_and_execute_notebooks(
                 )
                 for notebook in notebooks
             ]
-
-        print("\n=== RESULTS ===\n")
-
-        results_sorted = sorted(
-            notebook_execution_results,
-            key=lambda result: result.is_pass,
-            reverse=True,
-        )
-
-        # Print results
-        print(
-            tabulate(
-                [
-                    [
-                        result.name,
-                        "PASSED" if result.is_pass else "FAILED",
-                        format_timedelta(result.duration),
-                        result.log_url,
-                        result.output_uri,
-                    ]
-                    for result in results_sorted
-                ],
-                headers=["build_tag", "status", "duration", "log_url", "output_url"],
-            )
-        )
-
-        print("\n=== END RESULTS===\n")
-
-        total_notebook_duration = functools.reduce(
-            operator.add,
-            [datetime.timedelta(seconds=0)]
-            + [result.duration for result in results_sorted],
-        )
-
-        print(
-            f"Cumulative notebook duration: {format_timedelta(total_notebook_duration)}"
-        )
-
-        # Raise error if any notebooks failed
-        if not all([result.is_pass for result in results_sorted]):
-            raise RuntimeError("Notebook failures detected. See logs for details")
-
-    elif len(notebooks) == 1:
-        notebook = notebooks[0]
-
-        # Pre-process notebook by substituting variable names
-        _process_notebook(
-            notebook_path=notebook,
-            variable_project_id=variable_project_id,
-            variable_region=variable_region,
-        )
-
-        execute_notebook_helper.execute_notebook(
-            notebook_source=notebook,
-            output_file_or_uri="/".join(
-                [artifacts_bucket, pathlib.Path(notebook).name]
-            ),
-            should_log_output=True,
-        )
     else:
         print("No notebooks modified in this pull request.")
+
+    print("\n=== RESULTS ===\n")
+
+    results_sorted = sorted(
+        notebook_execution_results,
+        key=lambda result: result.is_pass,
+        reverse=True,
+    )
+
+    # Print results
+    print(
+        tabulate(
+            [
+                [
+                    result.name,
+                    "PASSED" if result.is_pass else "FAILED",
+                    format_timedelta(result.duration),
+                    result.log_url,
+                ]
+                for result in results_sorted
+            ],
+            headers=["build_tag", "status", "duration", "log_url"],
+        )
+    )
+
+    print("\n=== END RESULTS===\n")
+
+    total_notebook_duration = functools.reduce(
+        operator.add,
+        [datetime.timedelta(seconds=0)]
+        + [result.duration for result in results_sorted],
+    )
+
+    print(f"Cumulative notebook duration: {format_timedelta(total_notebook_duration)}")
+
+    # Raise error if any notebooks failed
+    if not all([result.is_pass for result in results_sorted]):
+        raise RuntimeError("Notebook failures detected. See logs for details")

.cloud-build/notebook-execution-test-cloudbuild-single.yaml

@@ -10,37 +10,19 @@ steps:
     entrypoint: /bin/sh
     args:
     - -c
-    - python3 .cloud-build/CheckPythonVersion.py
-  # Create a virtual environment
-  - name: ${_PYTHON_IMAGE}
-    entrypoint: /bin/sh
-    args:
-    - -c
-    - python3 -m venv workspace/env
+    - 'python3 .cloud-build/CheckPythonVersion.py'
   # Install Python dependencies
-  - name: ${_PYTHON_IMAGE}
-    entrypoint: /bin/sh
-    args:
-    - -c 
-    - . workspace/env/bin/activate &&
-      python3 -m pip install -U pip &&
-      python3 -m pip install -U -r .cloud-build/requirements.txt
-  # pip freeze
   - name: ${_PYTHON_IMAGE}
     entrypoint: /bin/sh
     args:
     - -c
-    - |
-      . workspace/env/bin/activate &&
-      python3 -m pip freeze    
+    - 'python3 -m pip install -U pip && python3 -m pip install -U --user -r .cloud-build/requirements.txt'
   # Install Python dependencies and run testing script
   - name: ${_PYTHON_IMAGE}
     entrypoint: /bin/sh
     args:
     - -c
-    - | 
-      . workspace/env/bin/activate &&
-      python3 .cloud-build/execute_notebook_cli.py --notebook_source "${_NOTEBOOK_GCS_URI}" --output_file_or_uri "${_NOTEBOOK_OUTPUT_GCS_URI}"
+    - 'python3 -m pip install -U pip && python3 -m pip freeze && python3 .cloud-build/execute_notebook_cli.py --notebook_source "${_NOTEBOOK_GCS_URI}" --output_file_or_uri "${_NOTEBOOK_OUTPUT_GCS_URI}"'
     env:
     - 'IS_TESTING=1'
 timeout: 86400s

.cloud-build/notebook-execution-test-cloudbuild.yaml

@@ -4,47 +4,32 @@ steps:
     entrypoint: /bin/sh
     args:
     - -c
-    - gcloud config list
+    - 'gcloud config list'
   # Check the Python version
   - name: ${_PYTHON_IMAGE}
     entrypoint: /bin/sh
     args:
     - -c
-    - python3 .cloud-build/CheckPythonVersion.py
-  # Fetch full repo for diff purposes
-  - name: gcr.io/cloud-builders/git
-    args: [fetch, --unshallow]
-  # Create a virtual environment
+    - 'python3 .cloud-build/CheckPythonVersion.py'
+  # Fetch base branch if required
   - name: ${_PYTHON_IMAGE}
     entrypoint: /bin/sh
     args:
     - -c
-    - python3 -m venv workspace/env
+    - 'if [ -n "${_BASE_BRANCH}" ]; then git fetch origin "${_BASE_BRANCH}":refs/remotes/origin/"${_BASE_BRANCH}"; else echo "Skipping fetch."; fi'
   # Install Python dependencies
-  - name: ${_PYTHON_IMAGE}
-    entrypoint: /bin/sh
-    args:
-    - -c 
-    - . workspace/env/bin/activate &&
-      python3 -m pip install -U pip &&
-      python3 -m pip install -U -r .cloud-build/requirements.txt
-  # pip freeze
   - name: ${_PYTHON_IMAGE}
     entrypoint: /bin/sh
     args:
     - -c
-    - |
-      . workspace/env/bin/activate &&
-      python3 -m pip freeze
+    - 'python3 -m pip install -U pip && python3 -m pip install -U --user -r .cloud-build/requirements.txt'
   # Install Python dependencies and run testing script
   # TODO: Only pass in private_pool_id if it is set
   - name: ${_PYTHON_IMAGE}
     entrypoint: /bin/sh
     args:
     - -c
-    - | 
-      . workspace/env/bin/activate &&
-      python3 .cloud-build/execute_changed_notebooks_cli.py --test_paths_file "${_TEST_PATHS_FILE}" --base_branch "${_FORCED_BASE_BRANCH}" --container_uri ${_PYTHON_IMAGE} --staging_bucket ${_GCS_STAGING_BUCKET}  --artifacts_bucket ${_GCS_STAGING_BUCKET}/executed_notebooks/PR_${_PR_NUMBER}/BUILD_${BUILD_ID} --variable_project_id ${PROJECT_ID} --variable_region ${_GCP_REGION} `if [ ! -z "${_PRIVATE_POOL_NAME}" ]; then echo "--private_pool_id ${_PRIVATE_POOL_NAME}"; fi`
+    - 'python3 -m pip install -U pip && python3 -m pip freeze && python3 .cloud-build/execute_changed_notebooks_cli.py --test_paths_file "${_TEST_PATHS_FILE}" --base_branch "${_FORCED_BASE_BRANCH}" --container_uri ${_PYTHON_IMAGE} --staging_bucket ${_GCS_STAGING_BUCKET}  --artifacts_bucket ${_GCS_STAGING_BUCKET}/executed_notebooks/PR_${_PR_NUMBER}/BUILD_${BUILD_ID} --variable_project_id ${PROJECT_ID} --variable_region ${_GCP_REGION} `if [ ! -z "${_PRIVATE_POOL_NAME}" ]; then echo "--private_pool_id ${_PRIVATE_POOL_NAME}"; fi`'
     env:
     - 'IS_TESTING=1'
 timeout: 86400s

.cloud-build/requirements.txt

@@ -10,4 +10,3 @@ google-cloud-aiplatform
 google-cloud-storage
 google-cloud-build
 ratemate
-GitPython

.cloud-build/test_folders.txt

@@ -1,2 +1,3 @@
 notebooks/official
 notebooks/notebook_template.ipynb
+notebooks/community/ml_ops

.github/workflows/ci.yaml

@@ -7,9 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Set up Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: '3.x'      
+      uses: actions/setup-python@v3
     - name: Fetch pull request branch
       uses: actions/checkout@v3
       with:

.github/workflows/linter/requirements.txt

@@ -3,8 +3,7 @@ ipython
 jupyter
 nbconvert
 black==22.3.0
-pyupgrade==2.34.0
+pyupgrade==2.31.1
 isort==5.10.1
 flake8==4.0.1
 nbqa==1.3.1
-

CONTRIBUTING.md

@@ -48,8 +48,8 @@ then you will need to manually address them before submitting your PR.
 nbqa black "$notebook"
 nbqa pyupgrade "$notebook"
 nbqa isort "$notebook"
-nbqa flake8 "$notebook" --extend-ignore=W391,E501,F821,E402,F404,W503,E203,E722,W293,W291
 python3 -m tensorflow_docs.tools.nbfmt --remove_outputs "$notebook"
+nbqa flake8 "$notebook" --extend-ignore=W391,E501,F821,E402,F404,W503,E203,E722,W293,W291
 ```
 
 ## Code Reviews

community-content/tf_agents_bandits_movie_recommendation_with_kfp_and_vertex_sdk/mlops_pipeline_tf_agents_bandits_movie_recommendation/src/logger/requirements.txt

@@ -1,4 +1,4 @@
 google-cloud-bigquery==2.20.0
-tensorflow==2.7.2
+tensorflow==2.5.3
 pillow==9.0.1
 tf-agents==0.8.0

community-content/tf_agents_bandits_movie_recommendation_with_kfp_and_vertex_sdk/mlops_pipeline_tf_agents_bandits_movie_recommendation/src/prediction_container/requirements.txt

@@ -1,4 +1,4 @@
 google-cloud-pubsub==2.5.0
 pillow==9.0.1
 tf-agents==0.8.0
-tensorflow==2.7.2
+tensorflow==2.5.3

community-content/tf_agents_bandits_movie_recommendation_with_kfp_and_vertex_sdk/mlops_pipeline_tf_agents_bandits_movie_recommendation/src/simulator/requirements.txt

@@ -1,5 +1,5 @@
 dataclasses==0.6
 google-cloud-aiplatform==1.8.1
-tensorflow==2.7.2
+tensorflow==2.5.3
 pillow==9.0.1
 tf-agents==0.8.0

community-content/tf_keras_image_classification_distributed_multi_worker_with_vertex_sdk/trainer/requirements.txt

@@ -1 +1 @@
-tensorflow==2.7.2
+tensorflow==2.5.3

community-content/tf_keras_text_classification_distributed_single_worker_gpus_with_gcloud_local_run_and_vertex_sdk/trainer/requirements.txt

@@ -1 +1 @@
-tensorflow==2.7.2
+tensorflow==2.5.3

notebooks/README.md

@@ -1,5 +1,5 @@
-The [official](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/notebooks/official) folder contains notebooks organized by Google Cloud product. These are tested weekly and maintained by Google.
+The [official](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/notebooks/official) folder contains notebooks organized by Google Cloud product.
 
-The [community](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/notebooks/community) folder contains notebooks that may be created by Google or external contributors. They are not necessary maintained.
+The [community](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/notebooks/community) folder contains notebooks that aren't officially supported by Google.
 
 Contributions to the repo should use the [notebook template](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/notebook_template.ipynb) as a starting point.

notebooks/community/CODEOWNERS

@@ -12,17 +12,12 @@
 /managed_notebooks/
 /sdk/SDK_FBProphet_Forecasting_Online.ipynb @brianchunkang
 /pipelines/google_cloud_pipeline_components_TPU_model_train_upload_deploy.ipynb @brianchunkang
-/explainable_ai/SDK_Custom_Container_XAI.ipynb @brianchunkang
 /matching_engine/sdk_matching_engine_for_indexing.ipynb @ivanmkc
 /matching_engine/matching_engine_for_indexing.ipynb @yinghsienwu
 /sdk/pytorch_lightning_custom_container_training.ipynb @brianchunkang
 /tensorboard @yfang1
 /feature_store @nayaknishant @morgandu
-/prediction @googleapis/vertex-prediction-team
 /vertex_endpoints/tf_hub_obj_detection/deploy_tfhub_object_detection_on_vertex_endpoints.ipynb @entrpn
 /vertex_endpoints/nvidia-triton/nvidia-triton-custom-container-prediction.ipynb @RajeshThallam
 /vertex_endpoints/optimized_tensorflow_runtime @vlasenkoalexey
-/notebooks/community/ml_ops/stage2/get_started_with_visionapi_and_automl.ipynb @mansari
-/notebooks/community/neo4j/graph_paysim.ipynb @benofben @laeg
-/notebooks/community/ml_ops/stage1/get_started_with_visionapi_and_vertex_datasets.ipynb @mansari
-/notebooks/community/pipelines/google_cloud_pipeline_components_bqml_pipeline_demand_forecasting.ipynb @inardini
+/notebooks/community/ml_ops/stage2/get_started_with_visionapi_and_automl.ipynb @mansari

notebooks/community/feature_store/mobile_gaming/mobile_gaming_feature_store.ipynb

@@ -32,18 +32,18 @@
         "<table align=\"left\">\n",
         "\n",
         "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/vertex-ai-samples/blob/main/notebooks/community/feature_store/mobile_gaming/mobile_gaming_feature_store.ipynb\">\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/notebook_template.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/feature_store/mobile_gaming/mobile_gaming_feature_store.ipynb\">\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/notebook_template.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/feature_store/mobile_gaming/mobile_gaming_feature_store.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/notebook_template.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
@@ -54,52 +54,52 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "tvgnzT1CKxrO"
+        "id": "7FZeBEwdXS4d"
       },
       "source": [
         "## Overview\n",
-        " \n",
-        "Imagine you are a member of the Data Science team working on the same Mobile Gaming application reported in the [Churn prediction for game developers using Google Analytics 4 (GA4) and BigQuery ML](https://cloud.google.com/blog/topics/developers-practitioners/churn-prediction-game-developers-using-google-analytics-4-ga4-and-bigquery-ml) blog post.\n",
-        " \n",
-        "Business wants to use that information in real-time to take immediate intervention actions in-game to prevent churn. In particular, for each player, they want to provide gaming incentives like new items or bonus packs depending on the customer demographic,  behavioral information and the resulting propensity of return.\n",
-        " \n",
-        "Last year, Google Cloud announced Vertex AI, a managed machine learning (ML) platform that allows data science teams to accelerate the deployment and maintenance of ML models. One of the platform building blocks is Vertex AI Feature store which provides a managed service for low latency scalable feature serving. Also it is a centralized feature repository with easy APIs to search & discover features and feature monitoring capabilities to track drift and other quality issues.\n",
-        " \n",
-        "In this notebook, we will show how the role of Vertex AI Feature Store in a ready to production scenario when the user's activities within the first 24 hours of last engagement and the gaming platform would consume in order to improve UX. Below you can find the high level picture of the system\n",
-        " \n",
+        "\n",
+        "Imagine you are a member of the Data Science team working on the same Mobile Gaming application reported in the [Churn prediction for game developers using Google Analytics 4 (GA4) and BigQuery ML](https://cloud.google.com/blog/topics/developers-practitioners/churn-prediction-game-developers-using-google-analytics-4-ga4-and-bigquery-ml) blog post. \n",
+        "\n",
+        "Business wants to use that information in real-time to take immediate intervention actions in-game to prevent churn. In particular, for each player, they want to provide gaming incentives like new items or bonus packs depending on the customer demographic,  behavioral information and the resulting propensity of return. \n",
+        "\n",
+        "Last year, Google Cloud announced Vertex AI, a managed machine learning (ML) platform that allows data science teams to accelerate the deployment and maintenance of ML models. One of the platform building blocks is Vertex AI Feature store which provides a managed service for low latency scalable feature serving. Also it is a centralized feature repository with easy APIs to search & discover features and feature monitoring capabilities to track drift and other quality issues. \n",
+        "\n",
+        "In this notebook, we will show how the role of Vertex AI Feature Store in a ready to production scenario when the user's activities within the first 24 hours of last engagment and the gaming platform would consume in order to improver UX. Below you can find the high level picture of the system\n",
+        "\n",
         "<img src=\"./assets/mobile_gaming_architecture_1.png\">\n",
-        " \n",
-        " \n",
+        "\n",
+        "\n",
         "### Dataset\n",
-        " \n",
+        "\n",
         "The dataset is the public sample export data from an actual mobile game app called \"Flood It!\" (Android, iOS)\n",
-        " \n",
+        "\n",
         "### Objective\n",
-        " \n",
+        "\n",
         "In the following notebook, you will learn how Vertex AI Feature store\n",
-        " \n",
-        "1.  Provide a centralized feature repository with easy APIs to search & discover features and fetch them for training/serving.\n",
-        " \n",
-        "2.  Simplify deployments of models for Online Prediction, via low latency scalable feature serving.\n",
-        " \n",
-        "3.  Mitigate training serving skew and data leakage by performing point in time lookups to fetch historical data for training.\n",
-        " \n",
+        "\n",
+        "1.   Provide a centralized feature repository with easy APIs to search & discover features and fetch them for training/serving. \n",
+        "\n",
+        "2.   Simplify deployments of models for Online Prediction, via low latency scalable feature serving.\n",
+        "\n",
+        "3.   Mitigate training serving skew and data leakage by performing point in time lookups to fetch historical data for training.\n",
+        "\n",
         "**Notice that we assume that already know how to set up a Vertex AI Feature store. In case you are not, please check out [this detailed notebook](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/feature_store/gapic-feature-store.ipynb).**\n",
-        " \n",
-        " \n",
-        "### Costs\n",
-        " \n",
+        "\n",
+        "\n",
+        "### Costs \n",
+        "\n",
         "This tutorial uses billable components of Google Cloud:\n",
-        " \n",
+        "\n",
         "* Vertex AI\n",
         "* BigQuery\n",
         "* Cloud Storage\n",
-        " \n",
+        "\n",
         "Learn about [Vertex AI\n",
         "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
         "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
         "Calculator](https://cloud.google.com/products/calculator/)\n",
-        "to generate a cost estimate based on your projected usage.\n"
+        "to generate a cost estimate based on your projected usage."
       ]
     },
     {
@@ -110,7 +110,7 @@
       "source": [
         "### Set up your local development environment\n",
         "\n",
-        "**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets\n",
+        "**If you are using Colab or Google Cloud Notebooks**, your environment already meets\n",
         "all the requirements to run this notebook. You can skip this step."
       ]
     },
@@ -159,7 +159,7 @@
       "source": [
         "### Install additional packages\n",
         "\n",
-        "Install additional package dependencies not installed in your notebook environment, such as XGBoost. Use the latest major GA version of each package."
+        "Install additional package dependencies not installed in your notebook environment, such as {XGBoost, AdaNet, or TensorFlow Hub TODO: Replace with relevant packages for the tutorial}. Use the latest major GA version of each package."
       ]
     },
     {
@@ -172,15 +172,12 @@
       "source": [
         "import os\n",
         "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
+        "# The Google Cloud Notebook product has specific requirements\n",
+        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
         "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "# Google Cloud Notebook requires dependencies to be installed with '--user'\n",
         "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "if IS_GOOGLE_CLOUD_NOTEBOOK:\n",
         "    USER_FLAG = \"--user\""
       ]
     },
@@ -188,11 +185,11 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "_vr6BYED_5my"
+        "id": "SzEo6DeE2GOP"
       },
       "outputs": [],
       "source": [
-        "! pip3 install {USER_FLAG} --upgrade pip -q\n",
+        "! pip3 install {USER_FLAG} --upgrade pip\n",
         "! pip3 install {USER_FLAG} --upgrade google-cloud-aiplatform==1.11.0 -q --no-warn-conflicts\n",
         "! pip3 install {USER_FLAG} git+https://github.com/googleapis/python-aiplatform.git@main # For features monitoring\n",
         "! pip3 install {USER_FLAG} --upgrade google-cloud-bigquery==2.24.0 -q --no-warn-conflicts\n",
@@ -252,7 +249,7 @@
         "\n",
         "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
         "\n",
-        "1. [Enable the APIs](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,notebooks.googleapis.com, ). \n",
+        "1. [Enable the Vertex AI API and Compute Engine API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component). \n",
         "\n",
         "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
         "\n",
@@ -310,76 +307,18 @@
       "outputs": [],
       "source": [
         "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
-        "    PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
+        "    PROJECT_ID = \"\"  # @param {type:\"string\"}"
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "set_gcloud_project_id"
+        "id": "dEjRdjxBuDsi"
       },
       "outputs": [],
       "source": [
-        "! gcloud config set project $PROJECT_ID"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "23988890fef6"
-      },
-      "source": [
-        "#### Get your project number (Optional)\n",
-        "\n",
-        "Now that the project ID is set, you get your corresponding project number."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "2d6950574e1d"
-      },
-      "outputs": [],
-      "source": [
-        "shell_output = ! gcloud projects list --filter=\"PROJECT_ID:'{PROJECT_ID}'\" --format='value(PROJECT_NUMBER)'\n",
-        "PROJECT_NUMBER = shell_output[0]\n",
-        "print(\"Project Number:\", PROJECT_NUMBER)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "region"
-      },
-      "source": [
-        "#### Region\n",
-        "\n",
-        "You can also change the `REGION` variable, which is used for operations\n",
-        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
-        "\n",
-        "- Americas: `us-central1`\n",
-        "- Europe: `europe-west4`\n",
-        "- Asia Pacific: `asia-east1`\n",
-        "\n",
-        "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
-        "\n",
-        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "jIcZV7-C2RrX"
-      },
-      "outputs": [],
-      "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "!gcloud config set project $PROJECT_ID #change it"
       ]
     },
     {
@@ -414,7 +353,7 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
+        "**If you are using Google Cloud Notebooks**, your environment is already\n",
         "authenticated. Skip this step."
       ]
     },
@@ -437,13 +376,9 @@
         "3. In the **Service account name** field, enter a name, and\n",
         "   click **Create**.\n",
         "\n",
-        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list and add the following roles:\n",
-        "   - BigQuery Admin\n",
-        "   - Storage Admin\n",
-        "   - Storage Object Admin\n",
-        "   - Vertex AI Administrator\n",
-        "   - Vertex AI Feature Store Admin\n",
-        "\n",
+        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
+        "into the filter box, and select\n",
+        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
         "\n",
         "5. Click *Create*. A JSON file that contains your key downloads to your\n",
         "local environment.\n",
@@ -460,19 +395,19 @@
       },
       "outputs": [],
       "source": [
+        "import os\n",
+        "import sys\n",
+        "\n",
         "# If you are running this notebook in Colab, run this cell and follow the\n",
         "# instructions to authenticate your GCP account. This provides access to your\n",
         "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
         "# requests.\n",
         "\n",
-        "import os\n",
-        "import sys\n",
+        "# The Google Cloud Notebook product has specific requirements\n",
+        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
         "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = \"google.colab\" in sys.modules\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
+        "# If on Google Cloud Notebooks, then don't execute this code\n",
+        "if not IS_GOOGLE_CLOUD_NOTEBOOK:\n",
         "    if \"google.colab\" in sys.modules:\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
@@ -512,8 +447,8 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+        "BUCKET_URI = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "REGION = \"[your-region]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -524,9 +459,11 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"-aip-\" + TIMESTAMP\n",
-        "    BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"-aip-\" + TIMESTAMP\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -549,6 +486,26 @@
         "! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "994afa65eaa2"
+      },
+      "source": [
+        "Run the following cell to grant access to your Cloud Storage resources from Vertex AI Feature store"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "psP1rPU9TRnX"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil uniformbucketlevelaccess set on $BUCKET_URI"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -569,78 +526,6 @@
         "! gsutil ls -al $BUCKET_URI"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "set_service_account"
-      },
-      "source": [
-        "#### Service Account (Optional)\n",
-        "\n",
-        "If you do not want to use your project's Compute Engine service account, set `SERVICE_ACCOUNT` to another service account ID."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "MQVV9haf2Rra"
-      },
-      "outputs": [],
-      "source": [
-        "SERVICE_ACCOUNT = \"[your-service-account]\"  # @param {type:\"string\"}"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "autoset_service_account"
-      },
-      "outputs": [],
-      "source": [
-        "if (\n",
-        "    SERVICE_ACCOUNT == \"\"\n",
-        "    or SERVICE_ACCOUNT is None\n",
-        "    or SERVICE_ACCOUNT == \"[your-service-account]\"\n",
-        "):\n",
-        "    # Get your service account from gcloud\n",
-        "    if not IS_COLAB:\n",
-        "        shell_output = !gcloud auth list 2>/dev/null\n",
-        "        SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
-        "\n",
-        "    else:  # IS_COLAB:\n",
-        "        shell_output = ! gcloud projects describe  $PROJECT_ID\n",
-        "        project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n",
-        "        SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n",
-        "\n",
-        "    print(\"Service Account:\", SERVICE_ACCOUNT)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "set_service_account:pipelines"
-      },
-      "source": [
-        "#### Set service account access\n",
-        "\n",
-        "Run the following commands to grant your service account access. You only need to run this step once per service account."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "U4UpQThc2Rrb"
-      },
-      "outputs": [],
-      "source": [
-        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
-        "\n",
-        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -652,18 +537,6 @@
         "You create the BigQuery dataset to store the data along the demo."
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "8615339fa4ca"
-      },
-      "outputs": [],
-      "source": [
-        "BQ_DATASET = \"Mobile_Gaming\"  # @param {type:\"string\"}\n",
-        "LOCATION = \"US\""
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -672,6 +545,9 @@
       },
       "outputs": [],
       "source": [
+        "BQ_DATASET = \"Mobile_Gaming\"  # @param {type:\"string\"}\n",
+        "LOCATION = \"US\"\n",
+        "\n",
         "!bq mk --location=$LOCATION --dataset $PROJECT_ID:$BQ_DATASET"
       ]
     },
@@ -724,9 +600,12 @@
       "outputs": [],
       "source": [
         "# Data Engineering and Feature Engineering\n",
-        "TODAY = \"2022-06-16\"\n",
+        "TODAY = \"2018-10-03\"\n",
+        "TOMORROW = \"2018-10-04\"\n",
         "LABEL_TABLE = f\"label_table_{TODAY}\".replace(\"-\", \"\")\n",
-        "FEATURES_TABLE = f\"wide_features_table_{TODAY}\"  # @param {type:\"string\"}\n",
+        "FEATURES_TABLE = \"wide_features_table\"  # @param {type:\"string\"}\n",
+        "FEATURES_TABLE_TODAY = f\"wide_features_table_{TODAY}\".replace(\"-\", \"\")\n",
+        "FEATURES_TABLE_TOMORROW = f\"wide_features_table_{TOMORROW}\".replace(\"-\", \"\")\n",
         "FEATURESTORE_ID = \"mobile_gaming\"  # @param {type:\"string\"}\n",
         "ENTITY_TYPE_ID = \"user\"\n",
         "\n",
@@ -1068,37 +947,13 @@
         "You will cover those steps in details below."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "init_aip:mbsdk,all"
-      },
-      "source": [
-        "### Initialize Vertex AI SDK for Python\n",
-        "\n",
-        "Initialize the Vertex AI SDK for Python for your project and corresponding bucket."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "poLJ0fV52Rrc"
-      },
-      "outputs": [],
-      "source": [
-        "vertex_ai.init(project=PROJECT_ID, location=REGION, staging_bucket=BUCKET_URI)"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "4ffd54e97270"
       },
       "source": [
-        "### Initialize BigQuery SDK for Python\n",
-        "\n",
-        "Initialize the BigQuery AI SDK for Python for your project and corresponding bucket."
+        "## Initiate clients"
       ]
     },
     {
@@ -1109,53 +964,55 @@
       },
       "outputs": [],
       "source": [
-        "bq_client = bigquery.Client(project=PROJECT_ID, location=LOCATION)"
+        "bq_client = bigquery.Client(project=PROJECT_ID, location=LOCATION)\n",
+        "vertex_ai.init(project=PROJECT_ID, location=REGION, staging_bucket=BUCKET_URI)"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "WnUQO2IHC9pZ"
+        "id": "zmMWIpCwsET9"
       },
       "source": [
         "## Identify users and build your features\n",
-        " \n",
-        "This section we will have static features we want to fetch from Vertex AI Feature Store. In particular, we will cover the following steps:\n",
-        " \n",
+        "\n",
+        "This section we will static features we want to fetch from Vertex AI Feature Store. In particular, we will cover the following steps:\n",
+        "\n",
         "1. Identify users, process demographic features and process behavioral features within the last 24 hours using **BigQuery**\n",
-        " \n",
+        "\n",
         "2. Set up the feature store\n",
-        " \n",
+        "\n",
         "3. Register features using **Vertex AI Feature Store** and the SDK.\n",
-        " \n",
-        "Below you have a picture that shows the process.\n",
-        " \n",
+        "\n",
+        "Below you have a picture that shows the process. \n",
+        "\n",
         "<img src=\"./assets/feature_store_ingestion_2.png\">\n",
-        " \n",
-        " \n",
-        "The original dataset contains raw event data we cannot ingest in the feature store as they are. We need to pre-process the raw data in order to get user features.\n",
-        " \n",
-        "**Notice we simulate those transformations in different points of time (today and tomorrow).**\n"
+        "\n",
+        "\n",
+        "\n",
+        "The original dataset contains raw event data we cannot ingest in the feature store as they are. We need to pre-process the raw data in order to get user features. \n",
+        "\n",
+        "**Notice we simulate those transformations in different point of time (today and tomorrow).**\n"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "e9zIrwhpDF2q"
+        "id": "8avYy5QOv02s"
       },
       "source": [
         "### Label, Demographic and Behavioral Transformations\n",
-        " \n",
-        "This section is based on the [Churn prediction for game developers using Google Analytics 4 (GA4) and BigQuery ML](https://cloud.google.com/blog/topics/developers-practitioners/churn-prediction-game-developers-using-google-analytics-4-ga4-and-bigquery-ml?utm_source=linkedin&utm_medium=unpaidsoc&utm_campaign=FY21-Q2-Google-Cloud-Tech-Blog&utm_content=google-analytics-4&utm_term=-) blog article by Minhaz Kazi and Polong Lin.\n",
-        " \n",
-        "You will adapt it to turn a batch churn prediction (using features within the first 24h user of first engagement) into a real-time churn prediction (using features within the first 6h user of last engagement).\n"
+        "\n",
+        "This section is based on the [Churn prediction for game developers using Google Analytics 4 (GA4) and BigQuery ML](https://cloud.google.com/blog/topics/developers-practitioners/churn-prediction-game-developers-using-google-analytics-4-ga4-and-bigquery-ml?utm_source=linkedin&utm_medium=unpaidsoc&utm_campaign=FY21-Q2-Google-Cloud-Tech-Blog&utm_content=google-analytics-4&utm_term=-) blog article by Minhaz Kazi and Polong Lin. \n",
+        "\n",
+        "You will adapt it in order to turn a batch churn prediction (using features within the first 24h user of first engagment) in a real-time churn prediction (using features within the first 24h user of last engagment)."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "RQX5m8UiC_px"
+        "id": "YO28RAITh6L-"
       },
       "outputs": [],
       "source": [
@@ -1182,27 +1039,27 @@
         "  SELECT\n",
         "    event_timestamp,\n",
         "    user_pseudo_id,\n",
-        "    SUM(IF(event_name = 'user_engagement', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 21600000000 PRECEDING\n",
+        "    SUM(IF(event_name = 'user_engagement', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 86400000000 PRECEDING\n",
         "      AND CURRENT ROW ) AS cnt_user_engagement,\n",
-        "    SUM(IF(event_name = 'level_start_quickplay', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 21600000000 PRECEDING\n",
+        "    SUM(IF(event_name = 'level_start_quickplay', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 86400000000 PRECEDING\n",
         "      AND CURRENT ROW ) AS cnt_level_start_quickplay,\n",
-        "    SUM(IF(event_name = 'level_end_quickplay', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 21600000000 PRECEDING\n",
+        "    SUM(IF(event_name = 'level_end_quickplay', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 86400000000 PRECEDING\n",
         "      AND CURRENT ROW ) AS cnt_level_end_quickplay,\n",
-        "    SUM(IF(event_name = 'level_complete_quickplay', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 21600000000 PRECEDING\n",
+        "    SUM(IF(event_name = 'level_complete_quickplay', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 86400000000 PRECEDING\n",
         "      AND CURRENT ROW ) AS cnt_level_complete_quickplay,\n",
-        "    SUM(IF(event_name = 'level_reset_quickplay', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 21600000000 PRECEDING\n",
+        "    SUM(IF(event_name = 'level_reset_quickplay', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 86400000000 PRECEDING\n",
         "      AND CURRENT ROW ) AS cnt_level_reset_quickplay,\n",
-        "    SUM(IF(event_name = 'post_score', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 21600000000 PRECEDING\n",
+        "    SUM(IF(event_name = 'post_score', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 86400000000 PRECEDING\n",
         "      AND CURRENT ROW ) AS cnt_post_score,\n",
-        "    SUM(IF(event_name = 'spend_virtual_currency', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 21600000000 PRECEDING\n",
+        "    SUM(IF(event_name = 'spend_virtual_currency', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 86400000000 PRECEDING\n",
         "      AND CURRENT ROW ) AS cnt_spend_virtual_currency,\n",
-        "    SUM(IF(event_name = 'ad_reward', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 21600000000 PRECEDING\n",
+        "    SUM(IF(event_name = 'ad_reward', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 86400000000 PRECEDING\n",
         "      AND CURRENT ROW ) AS cnt_ad_reward,\n",
-        "    SUM(IF(event_name = 'challenge_a_friend', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 21600000000 PRECEDING\n",
+        "    SUM(IF(event_name = 'challenge_a_friend', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 86400000000 PRECEDING\n",
         "      AND CURRENT ROW ) AS cnt_challenge_a_friend,\n",
-        "    SUM(IF(event_name = 'completed_5_levels', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 21600000000 PRECEDING\n",
+        "    SUM(IF(event_name = 'completed_5_levels', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 86400000000 PRECEDING\n",
         "      AND CURRENT ROW ) AS cnt_completed_5_levels,\n",
-        "    SUM(IF(event_name = 'use_extra_steps', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 21600000000 PRECEDING\n",
+        "    SUM(IF(event_name = 'use_extra_steps', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 86400000000 PRECEDING\n",
         "      AND CURRENT ROW ) AS cnt_use_extra_steps,\n",
         "  FROM (\n",
         "    SELECT\n",
@@ -1214,7 +1071,7 @@
         "\n",
         "SELECT\n",
         "    -- PARSE_TIMESTAMP('%Y-%m-%d %H:%M:%S', CONCAT('{TODAY}', ' ', STRING(TIME_TRUNC(CURRENT_TIME(), SECOND))), 'UTC') as timestamp,\n",
-        "    TIMESTAMP_ADD(PARSE_TIMESTAMP('%Y-%m-%d %H:%M:%S', FORMAT_TIMESTAMP('%Y-%m-%d %H:%M:%S', TIMESTAMP_MICROS(beh.event_timestamp))), INTERVAL 1351 DAY) AS timestamp,\n",
+        "    PARSE_TIMESTAMP('%Y-%m-%d %H:%M:%S', FORMAT_TIMESTAMP('%Y-%m-%d %H:%M:%S', TIMESTAMP_MICROS(beh.event_timestamp))) AS timestamp,\n",
         "    dem.*,\n",
         "    CAST(IFNULL(beh.cnt_user_engagement, 0) AS FLOAT64)  AS cnt_user_engagement,\n",
         "    CAST(IFNULL(beh.cnt_level_start_quickplay, 0) AS FLOAT64) AS cnt_level_start_quickplay,\n",
@@ -1240,7 +1097,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "oGYxLCSnD068"
+        "id": "Z6CjIOmDsET-"
       },
       "outputs": [],
       "source": [
@@ -1250,31 +1107,31 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "xQLIlsTCD_nk"
+        "id": "Lx__2-assET-"
       },
       "source": [
         "## Create a Vertex AI Feature store and ingest your features\n",
-        " \n",
-        "Now you have a wide table of features. It is time to ingest them into the feature store.\n",
-        " \n",
+        "\n",
+        "Now you have the wide table of features. It is time to ingest them into the feature store. \n",
+        "\n",
         "Before to moving on, you may have a question: **Why do I need a feature store**\n",
         "in this scenario at that point?\n",
-        " \n",
-        "One of the reasons would be to make those features accessible across teams by calculating once and reuse them many times. And in order to make it possible you need also be able to monitor those features over time to guarantee freshness and in case have a new feature engineering run to refresh them.\n",
-        " \n",
-        "If it is not your case, I will give even more reasons about why you should consider a feature store in the following sections. Just keep following me for now.\n",
-        " \n",
-        "One of the most important things is related to its data model. As you can see in the picture below, Vertex AI Feature Store organizes resources hierarchically in the following order: `Featurestore -> EntityType -> Feature`. You must create these resources before you can ingest data into Vertex AI Feature Store.\n",
-        " \n",
+        "\n",
+        "One of the reason would be to make those features accessable across team by calculating once and reuse them many times. And in order to make it possible you need also be able to monitor those features over time to guarantee freshness and in case have a new feature engineerign run to refresh them. \n",
+        "\n",
+        "If it is not your case, I will give even more reasons about why you should consider feature store in the following sections. Just keep following me for now.\n",
+        "\n",
+        "One of the most important thing is related to its data model. As you can see in the picture below, Vertex AI Feature Store organizes resources hierarchically in the following order: `Featurestore -> EntityType -> Feature`. You must create these resources before you can ingest data into Vertex AI Feature Store.\n",
+        "\n",
         "<img src=\"./assets/feature_store_data_model_3.png\">\n",
-        " \n",
-        "In our case we are going to create **mobile_gaming** featurestore resource containing **user** entity type and all its associated **features** such as country or the number of times a user challenged a friend (cnt_challenge_a_friend).\n"
+        "\n",
+        "In our case we are going to create **mobile_gaming** featurestore resource containing **user** entity type and all its associated **features** such as country or the number of times a user challenged a friend (cnt_challenge_a_friend)."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "VR7BJEozED_Q"
+        "id": "8dNlxda2sET_"
       },
       "source": [
         "### Create featurestore, ```mobile_gaming```\n",
@@ -1286,7 +1143,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "vUFqtYU-EDTR"
+        "id": "t2en8I7TSe4b"
       },
       "outputs": [],
       "source": [
@@ -1307,7 +1164,7 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "mUlCwfdpEHJG"
+        "id": "rN-vlvPUsET_"
       },
       "source": [
         "### Create the ```User``` entity type and its features\n",
@@ -1319,7 +1176,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "PnCU1wBND3W7"
+        "id": "CbZ2RQ5XbuRq"
       },
       "outputs": [],
       "source": [
@@ -1337,7 +1194,7 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "bT9LXzu1EOvW"
+        "id": "B2PIAprPmnhB"
       },
       "source": [
         "### Set Feature Monitoring\n",
@@ -1351,7 +1208,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "8WBlYUkOERaI"
+        "id": "N6im2c3ymiwC"
       },
       "outputs": [],
       "source": [
@@ -1374,7 +1231,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "92X4-7PFETj5"
+        "id": "gp9xaLQXn0CS"
       },
       "outputs": [],
       "source": [
@@ -1397,18 +1254,18 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "hxAuZjt3EWFo"
+        "id": "ustwKOMle8Qp"
       },
       "source": [
         "### Create features\n",
         "\n",
-        "In order to ingest features, you need to provide feature configuration and create them as featurestore resources."
+        "In order to ingest features, you need to provide feature configuration and create them as featurestore resources.\n"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "hRXO2I5VEYwt"
+        "id": "ijeZCTKIfCRL"
       },
       "source": [
         "#### Create Feature configuration\n",
@@ -1421,7 +1278,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "K26NEYZIEbvE"
+        "id": "vX_uYmjUgd9x"
       },
       "outputs": [],
       "source": [
@@ -1502,7 +1359,7 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "FjzMd1XbEfdo"
+        "id": "ErkruXPJkPuy"
       },
       "source": [
         "#### Create features using `batch_create_features` method\n",
@@ -1514,7 +1371,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "nqlgCDI9pbCD"
+        "id": "ZsCAO_IfsEUC"
       },
       "outputs": [],
       "source": [
@@ -1531,19 +1388,19 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "7zpFV7wAppkC"
+        "id": "9WisJk18qqgs"
       },
       "source": [
         "### Search features\n",
         "\n",
-        "Vertex AI Feature store supports searching capabilities. Below you have a simple example that shows how to filter a feature based on its name. "
+        "Vertex AI Feature store supports serching capabilities. Below you have a simple example that show how to filter a feature based on its name. "
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "BJXYLLOfppCL"
+        "id": "JzqyarMZqvZS"
       },
       "outputs": [],
       "source": [
@@ -1555,7 +1412,7 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "is9C_6-QpxG3"
+        "id": "ugtBfW5gsEUD"
       },
       "source": [
         "## Ingest features \n",
@@ -1590,7 +1447,7 @@
         "        entity_id_field=ENTITY_ID_FIELD,\n",
         "        disable_online_serving=False,\n",
         "        worker_count=10,\n",
-        "        sync=False,\n",
+        "        sync=True,\n",
         "    )\n",
         "except RuntimeError as error:\n",
         "    print(error)"
@@ -1599,7 +1456,7 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "8lCMpDPGp-oQ"
+        "id": "3Yv8MenWXrRX"
       },
       "source": [
         "# Train and deploy a real-time churn ML model using Vertex AI Training and Endpoints\n",
@@ -1610,34 +1467,34 @@
         "\n",
         "<img src=\"./assets/train_model_4.png\">\n",
         "\n",
-        "Let's dive into each step of this process."
+        "Let's dive into each step of this process.\n"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "VMrvnuyjqGfY"
+        "id": "saZZ3zWKX1YK"
       },
       "source": [
-        "## Fetch training data with point-in-time query using BigQuery and Vertex AI Feature store \n",
-        " \n",
-        "As we mentioned above, in real time churn prediction, it is so important defining the label you want to predict with your model.\n",
-        " \n",
-        "Let's assume that you decide to predict the churn probability over the next hour. So now you have your label. Next step is to define your training sample. But let's think about that for a second.\n",
-        " \n",
-        "In that churn real time system, you have a high volume of transactions you could use to calculate those features which keep floating and are collected constantly over time. It implies that you always get fresh data to reconstruct features. And depending on when you decide to calculate one feature or another you can end up with a set of features that are not aligned in time.\n",
-        " \n",
+        "## Fetch training data with point-in-time query using BigQuery and Vertex AI Feature store  \n",
+        "\n",
+        "As we mentioned above, in real time churn prediction, it is so important defining the label you want to predict with your model. \n",
+        "\n",
+        "Let's assume that you decide to predict the churn probability over the last 24 hr. So now you have your label. Next step is to define your training sample. But let's think about that for a second. \n",
+        "\n",
+        "In that churn real time system, you have a high volume of transactions you could use to calculate those features which keep floating and are collected constantly over time. It implies that you always get fresh data to reconstruct features. And depending on when you decide to calculate one feature or another you can end up with a set of features that are not aligned in time. \n",
+        "\n",
         "When you have labels available, it would be incredibly difficult to say which set of features contains the most up to date historical information associated with the label you want to predict. And, when you are not able to guarantee that, the performance of your model would be badly affected because you serve no representative features of the data and the label from the field when it goes live. So you need a way to get the most updated features you calculated over time before the label becomes available in order to avoid this informational skew.\n",
-        " \n",
-        "**With the Vertex AI Feature store, you can fetch feature values corresponding to a particular timestamp thanks to point-in-time lookup capability.** In our case, it would be the timestamp associated with the label you want to predict with your model. In this way, you will avoid data leakage and you will get the most updated features to train your model.\n",
-        " \n",
-        "Let's see how to do that.\n"
+        "\n",
+        "**With the Vertex AI Feature store, you can fetch feature values corresponding to a particular timestamp thanks to point-in-time lookup capability.** In our case, it would be the timestamp associated to the label you want to predict with your model. In this way, you will avoid data leakage and you will get the most updated features to train your model. \n",
+        "\n",
+        "Let's see how to do that. \n"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "RE_Pvmu-qdDt"
+        "id": "YHNbIHqFcQiM"
       },
       "source": [
         "### Define query for reading instances at a specific point in time\n",
@@ -1649,7 +1506,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "bUDVw7l-qF2x"
+        "id": "DGUm0bYqhVV4"
       },
       "outputs": [],
       "source": [
@@ -1661,13 +1518,13 @@
         "  # get training threshold ----------------------------------------------------------------------------------\n",
         "  get_training_threshold AS (\n",
         "  SELECT\n",
-        "    (MAX(event_timestamp) - 10800000000) AS training_thrs\n",
+        "    (MAX(event_timestamp) - 86400000000) AS training_thrs\n",
         "  FROM\n",
         "    `firebase-public-project.analytics_153293282.events_*`\n",
         "  WHERE\n",
         "    event_name=\"user_engagement\"\n",
         "    AND\n",
-        "    TIMESTAMP_ADD(PARSE_TIMESTAMP('%Y-%m-%d %H:%M:%S', FORMAT_TIMESTAMP('%Y-%m-%d %H:%M:%S', TIMESTAMP_MICROS(event_timestamp))), INTERVAL 1351 DAY) < '{TODAY}'),\n",
+        "    PARSE_TIMESTAMP('%Y-%m-%d %H:%M:%S', FORMAT_TIMESTAMP('%Y-%m-%d %H:%M:%S', TIMESTAMP_MICROS(event_timestamp))) < '{TODAY}'),\n",
         "\n",
         "  # query to create label -----------------------------------------------------------------------------------\n",
         "  get_label AS (\n",
@@ -1692,7 +1549,7 @@
         "    WHERE\n",
         "      event_name=\"user_engagement\"\n",
         "    AND\n",
-        "    TIMESTAMP_ADD(PARSE_TIMESTAMP('%Y-%m-%d %H:%M:%S', FORMAT_TIMESTAMP('%Y-%m-%d %H:%M:%S', TIMESTAMP_MICROS(event_timestamp))), INTERVAL 1351 DAY) < '{TODAY}'\n",
+        "    PARSE_TIMESTAMP('%Y-%m-%d %H:%M:%S', FORMAT_TIMESTAMP('%Y-%m-%d %H:%M:%S', TIMESTAMP_MICROS(event_timestamp))) < '{TODAY}'\n",
         "    GROUP BY\n",
         "      user_pseudo_id )\n",
         "  GROUP BY\n",
@@ -1812,7 +1669,7 @@
       "source": [
         "!mkdir -m 777 -p trainer data/ingest data/raw model config\n",
         "!gsutil -m cp -r $GCS_DESTINATION_OUTPUT_URI/*.csv data/ingest\n",
-        "!head -n 2000 data/ingest/*.csv > data/raw/sample.csv"
+        "!head -n 1000 data/ingest/*.csv > data/raw/sample.csv"
       ]
     },
     {
@@ -2214,7 +2071,7 @@
       },
       "outputs": [],
       "source": [
-        "TRAIN_JOB_RESOURCE_NAME = \"[your-train-job-resource-name]\"  # @param {type:\"string\"}"
+        "TRAIN_JOB_RESOURCE_NAME = \"\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -2309,32 +2166,32 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "1TNzL_EGrVUm"
+        "id": "7c9330928aa1"
       },
       "source": [
         "# Serve ML features at scale with low latency\n",
-        " \n",
-        "At that time, you are ready **to deploy our simple model which would requires fetching preprocessed attributes as input features in real time**.\n",
-        " \n",
+        "\n",
+        "At that time, you are ready **to deploy our simple model which would requires fetching preprocessed attributes as input features in real time**. \n",
+        "\n",
         "Below you can see how it works\n",
-        " \n",
-        "<center><img src=\"./assets/online_serving_5.png\" width=\"800\"/></center>\n",
-        " \n",
-        "But think about those features for a second.\n",
-        " \n",
-        "Your behavioral features used to train your model, they cannot be computed when you are going to serve the model online.\n",
-        " \n",
-        "How could you compute the number of times a user challenged a friend within the last 24 hours on the fly?\n",
-        " \n",
-        "You need to be computed this feature on the server side and serve it with low latency. And because Bigquery is not optimized for those read operations, we need a different service that allows singleton lookup where the result is a single row with many columns.\n",
-        " \n",
-        "Also, even if it was not the case, when you deploy a model that requires preprocessing your data, you need to be sure to reproduce the same preprocessing steps you had when you trained it. If you are not able to do that a skew between training and serving data would happen and it will badly affect your model performance (and in the worst scenario break your serving system).\n",
-        " \n",
-        "You need a way to mitigate that in a way you don't need to implement those preprocessing steps online but just serve the same aggregated features you already have for training to generate online prediction.\n",
-        " \n",
-        "These are other valuable reasons to introduce Vertex AI Feature Store. With it, you have a service which helps you to serve features at scale with low latency as they were available at training time mitigating in that way possible training-serving skew.\n",
-        " \n",
-        "Now that you know **why you need a feature store**, let's conclude this journey by deploying your model using a feature store to retrieve features online, pass them to the endpoint and generate predictions.\n"
+        "\n",
+        "<img src=\"./assets/online_serving_5.png\" width=\"600\">\n",
+        "\n",
+        "But think about those features for a second. \n",
+        "\n",
+        "Your behavioral features used to trained your model, they cannot be computed when you are going to serve the model online. \n",
+        "\n",
+        "How could you compute the number of time a user challenged a friend withing the last 24 hours on the fly?\n",
+        "\n",
+        "You simply can't do that. You need to be computed this feature on the server side and serve it with low latency. And becuase Bigquery is not optimized for those read operations, we need a different service that allows singleton lookup where the result is a single row with many columns.\n",
+        "\n",
+        "Also, even if it was not the case, when you deploy a model that requires preprocessing your data, you need to be sure to reproduce the same preprocessing steps you had when you trained it. If you are not able to do that a skew between training and serving data would happen and it will affect badly your model performance (and in the worst scenario break your serving system). \n",
+        "\n",
+        "You need a way to mitigate that in a way you don't need to implement those preprocessing steps online but just serve the same aggregated features you already have for training to generate online prediction. \n",
+        "\n",
+        "These are other valuable reasons to introduce Vertex AI Feature Store. With it, you have a service which helps you to serve feature at scale with low latency as they were available at training time mitigating in that way possible training-serving skew.\n",
+        "\n",
+        "Now that you know **why you need a feature store**, let's closing this journey by deploying your model and use feature store to retrieve features online, pass them to endpoint and generate predictions.\n"
       ]
     },
     {
@@ -2364,13 +2221,13 @@
       },
       "outputs": [],
       "source": [
-        "simulate_prediction(endpoint=endpoint, n_requests=10, latency=1)"
+        "simulate_prediction(endpoint=endpoint, n_requests=1000, latency=1)"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "8d3S1d1urZOy"
+        "id": "TpV-iwP9qw9c"
       },
       "source": [
         "## Cleaning up\n",
@@ -2410,12 +2267,11 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "FMXT2akXrZOy"
+        "id": "sx_vKniMq9ZX"
       },
       "outputs": [],
       "source": [
         "# Delete bucket\n",
-        "delete_bucket = False\n",
         "if (delete_bucket or os.getenv(\"IS_TESTING\")) and \"BUCKET_URI\" in globals():\n",
         "    ! gsutil -m rm -r $BUCKET_URI"
       ]

notebooks/community/managed_notebooks/subscriber_churn_prediction/telecom-subscriber-churn-prediction.ipynb

@@ -1,56 +1,12 @@
 {
   "cells": [
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "18ebbd838e32"
-      },
-      "outputs": [],
-      "source": [
-        "# Copyright 2022 Google LLC\n",
-        "#\n",
-        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
-        "# you may not use this file except in compliance with the License.\n",
-        "# You may obtain a copy of the License at\n",
-        "#\n",
-        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
-        "#\n",
-        "# Unless required by applicable law or agreed to in writing, software\n",
-        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
-        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
-        "# See the License for the specific language governing permissions and\n",
-        "# limitations under the License."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "64f7165bd1ac"
       },
       "source": [
-        "# Telecom subscriber churn prediction on Vertex AI\n",
-        "\n",
-        "<table align=\"left\">\n",
-        "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/subscriber_churn_prediction/telecom-subscriber-churn-prediction.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
-        "      View on GitHub\n",
-        "    </a>\n",
-        "  </td>\n",
-        "    <td>\n",
-        "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/subscriber_churn_prediction/telecom-subscriber-churn-prediction.ipynb\">\n",
-        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\\\" alt=\"Colab logo\"> Run in Colab\n",
-        "        </a>\n",
-        "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/subscriber_churn_prediction/telecom-subscriber-churn-prediction.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
-        "    </a>\n",
-        "  </td>\n",
-        "</table>\n",
-        "<br/><br/><br/>"
+        "# Telecom subscriber churn prediction on Vertex AI"
       ]
     },
     {
@@ -83,7 +39,9 @@
         "## Overview\n",
         "<a name=\"section-1\"></a>\n",
         "\n",
-        "This example demonstrates building a subscriber churn prediction model on a [telecom customer churn dataset](https://www.kaggle.com/c/customer-churn-prediction-2020/overview). The generated churn model is further deployed to Vertex AI Endpoints and explanations are generated using the Explainable AI feature of Vertex AI. "
+        "This example demonstrates building a subscriber churn prediction model on a [telecom customer churn dataset](https://www.kaggle.com/c/customer-churn-prediction-2020/overview). The generated churn model is further deployed to Vertex AI Endpoints and explanations are generated using the Explainable AI feature of Vertex AI. \n",
+        "\n",
+        "*Note: This notebook file was designed to run in a [Vertex AI Workbench managed notebooks](https://cloud.google.com/vertex-ai/docs/workbench/managed/create-instance) instance using the `Python (Local)` kernel. Some components of this notebook may not work in other notebook environments.*"
       ]
     },
     {
@@ -95,7 +53,7 @@
         "## Dataset\n",
         "<a name=\"section-2\"></a>\n",
         "\n",
-        "The dataset used in this tutorial is Telecom-Customer Churn dataset publicly available on Kaggle. See [Customer Churn Prediction 2020](https://www.kaggle.com/c/customer-churn-prediction-2020/data). This dataset is used to build and deploy a churn prediction model using Vertex AI in this notebook."
+        "The dataset used in this tutorial is publicly available at Kaggle. See [Customer Churn Prediction 2020](https://www.kaggle.com/c/customer-churn-prediction-2020/data). "
       ]
     },
     {
@@ -107,7 +65,7 @@
         "## Objective\n",
         "<a name=\"section-3\"></a>\n",
         "\n",
-        "This tutorial shows you how to do exploratory data analysis, preprocess data, train, deploy and get predictions from a churn prediction model on a tabular churn dataset. The objectives of this tutorial are as follows:\n",
+        "This tutorial shows you how to do exploratory data analysis, preprocess data, and train a churn prediction model on a tabular churn dataset. The steps include the following:\n",
         "\n",
         "- Load data from a Cloud Storage path\n",
         "- Perform exploratory data analysis (EDA)\n",
@@ -149,9 +107,7 @@
         "id": "44b8ae8e2d19"
       },
       "source": [
-        "## Installation\n",
-        "\n",
-        "Install the following packages to run this notebook."
+        "## Installation"
       ]
     },
     {
@@ -173,6 +129,17 @@
         "    USER_FLAG = \"--user\""
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "606337930991"
+      },
+      "source": [
+        "Install the latest version of the Vertex AI client library.\n",
+        "\n",
+        "Run the following command in your virtual environment to install the Vertex SDK for Python:"
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -181,43 +148,67 @@
       },
       "outputs": [],
       "source": [
-        "! pip install {USER_FLAG} --upgrade google-cloud-aiplatform \\\n",
-        "                                    google-cloud-storage \\\n",
-        "                                    category_encoders \\\n",
-        "                                    seaborn \\\n",
-        "                                    sklearn \\\n",
-        "                                    pandas \\\n",
-        "                                    fsspec \\\n",
-        "                                    gcsfs -q"
+        "! pip install {USER_FLAG} --upgrade google-cloud-aiplatform"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "b24902cde81b"
+        "id": "e67139e68463"
       },
       "source": [
-        "### Restart the kernel\n",
-        "\n",
-        "Once you've installed the additional packages, you need to restart the notebook kernel so it can find the packages."
+        "Install the Cloud Storage library:"
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "c61d171395d7"
+        "id": "2ad918f94f5d"
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    # Automatically restart kernel after installs\n",
-        "    import IPython\n",
-        "\n",
-        "    app = IPython.Application.instance()\n",
-        "    app.kernel.do_shutdown(True)"
+        "! pip install {USER_FLAG} --upgrade google-cloud-storage"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "eb0c1e24a8f0"
+      },
+      "source": [
+        "Install the `category_encoders` library:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "deb95a7f2104"
+      },
+      "outputs": [],
+      "source": [
+        "! pip install --upgrade category_encoders"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "184560c1b742"
+      },
+      "source": [
+        "Install the `seaborn` library for the EDA step. If a Vertex AI Workbench managed notebooks instance is being used, this step is optional as the library is already available in the `Python (Local)` kernel."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "0d99cdcdc470"
+      },
+      "outputs": [],
+      "source": [
+        "! pip install --upgrade seaborn"
       ]
     },
     {
@@ -252,7 +243,7 @@
         "id": "96ff17f75e21"
       },
       "source": [
-        "### Set your project ID\n",
+        "#### Set your project ID\n",
         "\n",
         "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
       ]
@@ -265,13 +256,11 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
         "PROJECT_ID = \"\"\n",
         "\n",
         "# Get your Google Cloud project ID from gcloud\n",
         "if not os.getenv(\"IS_TESTING\"):\n",
-        "    shell_output = !gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "    shell_output=!gcloud config list --format 'value(core.project)' 2>/dev/null\n",
         "    PROJECT_ID = shell_output[0]\n",
         "    print(\"Project ID: \", PROJECT_ID)"
       ]
@@ -297,58 +286,13 @@
         "    PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "f2e3c0f2cbfb"
-      },
-      "outputs": [],
-      "source": [
-        "! gcloud config set project $PROJECT_ID"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "60d535f443ac"
-      },
-      "source": [
-        "### Region\n",
-        "\n",
-        "You can also change the `REGION` variable, which is used for operations\n",
-        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
-        "\n",
-        "- Americas: `us-central1`\n",
-        "- Europe: `europe-west4`\n",
-        "- Asia Pacific: `asia-east1`\n",
-        "\n",
-        "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
-        "\n",
-        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "3aaadaaf9b30"
-      },
-      "outputs": [],
-      "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "e663bd062c6f"
       },
       "source": [
-        "### Timestamp\n",
+        "#### Timestamp\n",
         "\n",
         "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append it onto the name of resources you create in this tutorial."
       ]
@@ -366,63 +310,6 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "3ffa6b6c7cdb"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Google Cloud Notebooks**, your environment is already authenticated. Skip this step.\n",
-        "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
-        "\n",
-        "1. **Click Create service account**.\n",
-        "\n",
-        "2. In the **Service account name** field, enter a name, and click **Create**.\n",
-        "\n",
-        "3. In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex AI\" into the filter box, and select **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "4. Click Create. A JSON file that contains your key downloads to your local environment.\n",
-        "\n",
-        "5. Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "2b72272258fc"
-      },
-      "outputs": [],
-      "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Google Cloud Notebook, then don't execute this code\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -440,7 +327,12 @@
         "online predictions.\n",
         "\n",
         "Set the name of your Cloud Storage bucket below. It must be unique across all\n",
-        "Cloud Storage buckets."
+        "Cloud Storage buckets.\n",
+        "\n",
+        "You may also change the `REGION` variable, which is used for operations\n",
+        "throughout the rest of this notebook. Make sure to [choose a region where Vertex AI services are\n",
+        "available](https://cloud.google.com/vertex-ai/docs/general/locations#available_regions). You may\n",
+        "not use a Multi-Regional Storage bucket for training with Vertex AI."
       ]
     },
     {
@@ -451,8 +343,8 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "REGION = \"[your-region]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -463,9 +355,8 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
-        "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
       ]
     },
     {
@@ -485,7 +376,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_URI"
+        "! gsutil mb -l $REGION $BUCKET_NAME"
       ]
     },
     {
@@ -505,7 +396,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_URI"
+        "! gsutil ls -al $BUCKET_NAME"
       ]
     },
     {
@@ -559,13 +450,7 @@
         "id": "e37354341588"
       },
       "source": [
-        "### Load data from Cloud Storage using Pandas\n",
-        "\n",
-        "The Telecom-Customer Churn dataset from [Kaggle](https://www.kaggle.com/c/customer-churn-prediction-2020/overview) is made available on a public Cloud Storage bucket at: \n",
-        "\n",
-        "```gs://cloud-samples-data/vertex-ai/managed_notebooks/telecom_churn_prediction/train.csv```\n",
-        "\n",
-        "Use Pandas to read data directly from the URI."
+        "### Load data from Cloud Storage path using Pandas"
       ]
     },
     {
@@ -1231,8 +1116,6 @@
         "    \"[your-blob-path]\"  # leave blank if no folders inside the bucket are needed.\n",
         ")\n",
         "\n",
-        "if BLOB_PATH == (\"[your-blob-path]\"):\n",
-        "    BLOB_PATH = \"\"\n",
         "\n",
         "BLOB_NAME = BLOB_PATH + FILE_NAME\n",
         "\n",
@@ -1250,9 +1133,7 @@
         "## Create a model with Explainable AI support in Vertex AI\n",
         "<a name=\"section-9\"></a>\n",
         "\n",
-        "Before creating a model, configure the explanations for the model. For further details, see [Configuring explanations in Vertex AI](https://cloud.google.com/vertex-ai/docs/explainable-ai/configuring-explanations#scikit-learn-and-xgboost-pre-built-containers).\n",
-        "\n",
-        "Set a display name for the model resource."
+        "Before creating a model, configure the explanations for the model. For further details, see [Configuring explanations in Vertex AI](https://cloud.google.com/vertex-ai/docs/explainable-ai/configuring-explanations#scikit-learn-and-xgboost-pre-built-containers)."
       ]
     },
     {
@@ -1263,13 +1144,10 @@
       },
       "outputs": [],
       "source": [
-        "# Set the model display name\n",
-        "MODEL_DISPLAY_NAME = \"[your-model-display-name]\"  # @param {type:\"string\"}\n",
-        "\n",
-        "if MODEL_DISPLAY_NAME == \"[your-model-display-name]\":\n",
-        "    MODEL_DISPLAY_NAME = \"subscriber_churn_model\"\n",
-        "\n",
+        "MODEL_DISPLAY_NAME = \"[your-model-display-name]\"\n",
         "ARTIFACT_GCS_PATH = f\"gs://{BUCKET_NAME}/{BLOB_PATH}\"\n",
+        "PROJECT = \"[your-project-id]\"\n",
+        "LOCATION = REGION\n",
         "\n",
         "# Feature-name(Inp_feature) and Output-name(Model_output) can be arbitrary\n",
         "exp_metadata = {\"inputs\": {\"Inp_feature\": {}}, \"outputs\": {\"Model_output\": {}}}"
@@ -1283,20 +1161,17 @@
       },
       "outputs": [],
       "source": [
-        "from google.cloud.aiplatform_v1.types import SampledShapleyAttribution\n",
         "# Create a Vertex AI model resource with support for explanations\n",
-        "from google.cloud.aiplatform_v1.types.explanation import ExplanationParameters\n",
         "\n",
-        "aiplatform.init(project=PROJECT_ID, location=REGION)\n",
+        "aiplatform.init(project=PROJECT, location=LOCATION)\n",
+        "explanation_parameters = {\"sampledShapleyAttribution\": {\"pathCount\": 25}}\n",
         "\n",
         "model = aiplatform.Model.upload(\n",
         "    display_name=MODEL_DISPLAY_NAME,\n",
         "    artifact_uri=ARTIFACT_GCS_PATH,\n",
-        "    serving_container_image_uri=\"us-docker.pkg.dev/vertex-ai/prediction/sklearn-cpu.1-0:latest\",\n",
+        "    serving_container_image_uri=\"us-docker.pkg.dev/vertex-ai/prediction/sklearn-cpu.0-24:latest\",\n",
         "    explanation_metadata=exp_metadata,\n",
-        "    explanation_parameters=ExplanationParameters(\n",
-        "        sampled_shapley_attribution=SampledShapleyAttribution(path_count=25)\n",
-        "    ),\n",
+        "    explanation_parameters=explanation_parameters,\n",
         ")\n",
         "\n",
         "model.wait()\n",
@@ -1317,7 +1192,7 @@
         "gcloud beta ai models upload \\\n",
         "  --region=$REGION \\\n",
         "  --display-name=$MODEL_DISPLAY_NAME \\\n",
-        "  --container-image-uri=\"us-docker.pkg.dev/vertex-ai/prediction/sklearn-cpu.1-0:latest\" \\\n",
+        "  --container-image-uri=\"us-docker.pkg.dev/vertex-ai/prediction/sklearn-cpu.0-24:latest\" \\\n",
         "  --artifact-uri=$ARTIFACT_GCS_PATH \\\n",
         "  --explanation-method=sampled-shapley \\\n",
         "  --explanation-path-count=25 \\\n",
@@ -1342,9 +1217,7 @@
       },
       "outputs": [],
       "source": [
-        "ENDPOINT_DISPLAY_NAME = \"[your-endpoint-display-name]\"  # @param {type:\"string\"}\n",
-        "if ENDPOINT_DISPLAY_NAME == \"[your-endpoint-display-name]\":\n",
-        "    ENDPOINT_DISPLAY_NAME = \"subsc_churn_endpoint\""
+        "ENDPOINT_DISPLAY_NAME = \"[your-endpoint-display-name]\""
       ]
     },
     {
@@ -1356,13 +1229,33 @@
       "outputs": [],
       "source": [
         "endpoint = aiplatform.Endpoint.create(\n",
-        "    display_name=ENDPOINT_DISPLAY_NAME, project=PROJECT_ID, location=REGION\n",
+        "    display_name=ENDPOINT_DISPLAY_NAME, project=PROJECT, location=LOCATION\n",
         ")\n",
         "\n",
         "print(endpoint.display_name)\n",
         "print(endpoint.resource_name)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ae4c69ef8a8c"
+      },
+      "source": [
+        "Save the endpoint ID after the endpoint is created."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6aa73d9a88d3"
+      },
+      "outputs": [],
+      "source": [
+        "ENDPOINT_ID = \"[your-endpoint-id]\""
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -1382,11 +1275,8 @@
       },
       "outputs": [],
       "source": [
-        "DEPLOYED_MODEL_NAME = \"[deployment-model-name]\"  # @param {type:\"string\"}\n",
-        "MACHINE_TYPE = \"n1-standard-4\"\n",
-        "\n",
-        "if DEPLOYED_MODEL_NAME == \"[deployment-model-name]\":\n",
-        "    DEPLOYED_MODEL_NAME = \"subsc_churn_deployment\""
+        "DEPLOYED_MODEL_NAME = \"[deployment-model-name]\"\n",
+        "MACHINE_TYPE = \"n1-standard-4\""
       ]
     },
     {
@@ -1416,7 +1306,7 @@
         "id": "359c43e630cb"
       },
       "source": [
-        "To ensure the model is deployed, the ID of the deployed model can be checked using the `endpoint.list_models()` method."
+        "Save the ID of the deployed model. The ID of the deployed model can also checked using the `endpoint.list_models()` method."
       ]
     },
     {
@@ -1427,7 +1317,7 @@
       },
       "outputs": [],
       "source": [
-        "endpoint.list_models()"
+        "DEPLOYED_MODEL_ID = \"[your-deployed-model-id]\""
       ]
     },
     {
@@ -1446,7 +1336,7 @@
         "id": "7b50c31e0552"
       },
       "source": [
-        "Get explanations for a test instance from the hosted model."
+        "Get explanations for some test instances from the hosted model."
       ]
     },
     {
@@ -1457,8 +1347,8 @@
       },
       "outputs": [],
       "source": [
-        "# format a test instance as the request's payload\n",
-        "test_json = [X_test.iloc[0].tolist()]"
+        "# format the top 2 test instances as the request's payload\n",
+        "test_json = {\"instances\": [X_test.iloc[0].tolist(), X_test.iloc[1].tolist()]}"
       ]
     },
     {
@@ -1495,13 +1385,15 @@
         "    return\n",
         "\n",
         "\n",
-        "def explain_tabular_sample(project: str, location: str, endpoint, instances: list):\n",
+        "def explain_tabular_sample(\n",
+        "    project: str, location: str, endpoint_id: str, instances: list\n",
+        "):\n",
         "    \"\"\"\n",
         "    Function to make an explanation request for the specified payload and generate feature attribution plots\n",
         "    \"\"\"\n",
         "    aiplatform.init(project=project, location=location)\n",
         "\n",
-        "    # endpoint = aiplatform.Endpoint(endpoint_id)\n",
+        "    endpoint = aiplatform.Endpoint(endpoint_id)\n",
         "\n",
         "    response = endpoint.explain(instances=instances)\n",
         "    print(\"#\" * 10 + \"Explanations\" + \"#\" * 10)\n",
@@ -1530,8 +1422,8 @@
         "    return response\n",
         "\n",
         "\n",
-        "# Get explanations for the test instance\n",
-        "prediction = explain_tabular_sample(PROJECT_ID, REGION, endpoint, test_json)"
+        "test_json = [X_test.iloc[0].tolist(), X_test.iloc[1].tolist()]\n",
+        "prediction = explain_tabular_sample(PROJECT, LOCATION, ENDPOINT_ID, test_json)"
       ]
     },
     {
@@ -1546,12 +1438,7 @@
         "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
         "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
         "\n",
-        "Otherwise, you can delete the individual resources you created in this tutorial:\n",
-        "* Vertex AI Model\n",
-        "* Vertex AI Endpoint\n",
-        "* Cloud Storage bucket\n",
-        "\n",
-        "Set `delete_bucket` to *True* to delete the Cloud Storage bucket."
+        "Otherwise, you can delete the individual resources you created in this tutorial:"
       ]
     },
     {
@@ -1562,8 +1449,8 @@
       },
       "outputs": [],
       "source": [
-        "# Undeploy model\n",
-        "endpoint.undeploy_all()"
+        "# undeploy the model\n",
+        "endpoint.undeploy(deployed_model_id=DEPLOYED_MODEL_ID)"
       ]
     },
     {
@@ -1574,7 +1461,7 @@
       },
       "outputs": [],
       "source": [
-        "# Delete the endpoint\n",
+        "# delete the endpoint\n",
         "endpoint.delete()"
       ]
     },
@@ -1586,7 +1473,7 @@
       },
       "outputs": [],
       "source": [
-        "# Delete the model\n",
+        "# delete the model\n",
         "model.delete()"
       ]
     },
@@ -1598,10 +1485,8 @@
       },
       "outputs": [],
       "source": [
-        "# Delete the Cloud Storage bucket\n",
-        "delete_bucket = True\n",
-        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
-        "    ! gsutil -m rm -r $BUCKET_URI"
+        "# remove the contents of the Cloud Storage bucket\n",
+        "! gsutil -m rm -r $BUCKET_NAME"
       ]
     }
   ],

notebooks/community/matching_engine/sdk_matching_engine_for_indexing.ipynb

@@ -97,9 +97,11 @@
       "source": [
         "## Before you begin\n",
         "\n",
-        "* **Prepare a VPC network**.  To reduce any network overhead that might lead to unnecessary increase in overhead latency, it is best to call the ANN endpoints from your VPC via a direct [VPC Peering](https://cloud.google.com/vertex-ai/docs/general/vpc-peering) connection. \n",
-        "  * The following section describes how to setup a VPC Peering connection if you don't have one. \n",
-        "  * This is a one-time initial setup task. You can also reuse existing VPC network and skip this section."
+        "* **Prepare a VPC network**.  To reduce any network overhead that might lead to unnecessary increase in overhead latency, it is best to call the ANN endpoints from your VPC via a direct [VPC Peering](https://cloud.google.com/vertex-ai/docs/general/vpc-peering) connection. The following section describes how to setup a VPC Peering connection if you don't have one. This is a one-time initial setup task. You can also reuse existing VPC network and skip this section.\n",
+        "* **WARNING:** The MatchingIndexEndpoint.match method (to create online queries against your deployed index) has to be executed in a Vertex AI Workbench notebook instance that is created with the following requirements:\n",
+        "  * **In the same region as where your ANN service is deployed** (for example, if you set `REGION = \"us-central1\"` as same as the tutorial, the notebook instance has to be in `us-central1`).\n",
+        "  * **Make sure you select the VPC network you created for ANN service** (instead of using the \"default\" one). That is, you will have to create the VPC network below and then create a new notebook instance that uses that VPC.  \n",
+        "  * If you run it in the colab or a Vertex AI Workbench notebook instance in a different VPC network or region, the gRPC API will fail to peer the network (InactiveRPCError)."
       ]
     },
     {
@@ -110,11 +112,11 @@
       },
       "outputs": [],
       "source": [
-        "PROJECT_ID = \"python-docs-samples-tests\"  # @param {type:\"string\"}\n",
+        "PROJECT_ID = \"<your_project_id>\"  # @param {type:\"string\"}\n",
         "\n",
-        "NETWORK_NAME = \"ann-vpc-network\"  # @param {type:\"string\"}\n",
+        "NETWORK_NAME = \"my-vpc-network\"  # @param {type:\"string\"}\n",
         "\n",
-        "PEERING_RANGE_NAME = \"ann-haystack-range\""
+        "PEERING_RANGE_NAME = \"my-haystack-range\""
       ]
     },
     {
@@ -141,7 +143,6 @@
         "! gcloud compute addresses create {PEERING_RANGE_NAME} --global --prefix-length=16 --network={NETWORK_NAME} --purpose=VPC_PEERING --project={PROJECT_ID} --description=\"peering range\"\n",
         "\n",
         "# Set up peering with service networking\n",
-        "# Your account must have the \"Compute Network Admin\" role to run the following.\n",
         "! gcloud services vpc-peerings connect --service=servicenetworking.googleapis.com --network={NETWORK_NAME} --ranges={PEERING_RANGE_NAME} --project={PROJECT_ID}"
       ]
     },
@@ -154,20 +155,6 @@
         "* Authentication: Rerun the `gcloud auth login` command in the Vertex AI Workbench notebook terminal when you are logged out and need the credential again."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "d5de53b31bf1"
-      },
-      "source": [
-        "## Make sure the following cells are run from inside the VPC network that you created in the previous step.\n",
-        "\n",
-        "* **WARNING:** The MatchingIndexEndpoint.match method (to create online queries against your deployed index) has to be executed in a Vertex AI Workbench notebook instance that is created with the following requirements:\n",
-        "  * **In the same region as where your ANN service is deployed** (for example, if you set `REGION = \"us-central1\"` as same as the tutorial, the notebook instance has to be in `us-central1`).\n",
-        "  * **Make sure you select the VPC network you created for ANN service** (instead of using the \"default\" one). That is, you will have to create the VPC network below and then create a new notebook instance that uses that VPC.  \n",
-        "  * If you run it in the colab or a Vertex AI Workbench notebook instance in a different VPC network or region, \"Create Online Queries\" section will fail."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -284,7 +271,7 @@
       "source": [
         "import os\n",
         "\n",
-        "PROJECT_ID = \"python-docs-samples-tests\"\n",
+        "PROJECT_ID = \"\"\n",
         "\n",
         "# Get your Google Cloud project ID from gcloud\n",
         "if not os.getenv(\"IS_TESTING\"):\n",
@@ -747,28 +734,6 @@
         "INDEX_RESOURCE_NAME"
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "0f1a9fbecabb"
-      },
-      "outputs": [],
-      "source": [
-        "Using the resource name, you can retrieve an existing MatchingEngineIndex."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "1ddb70647d98"
-      },
-      "outputs": [],
-      "source": [
-        "tree_ah_index = aiplatform.MatchingEngineIndex(INDEX_RESOURCE_NAME)"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -777,7 +742,7 @@
       "source": [
         "### Create Brute Force Index (for Ground Truth)\n",
         "\n",
-        "The brute force index uses a naive brute force method to find the nearest neighbors. This method is not fast or efficient. Hence brute force indices are not recommended for production usage. They are to be used to find the \"ground truth\" set of neighbors, so that the \"ground truth\" set can be used to measure recall of the indices being tuned for production usage. To ensure an apples to apples comparison, the `distanceMeasureType` and `dimensions` of the brute force index should match those of the production indices being tuned.\n",
+        "The brute force index uses a naive brute force method to find the nearest neighbors. This method is not fast or efficient. Hence brute force indices are not recommended for production usage. They are to be used to find the \"ground truth\" set of neighbors, so that the \"ground truth\" set can be used to measure recall of the indices being tuned for production usage. To ensure an apples to apples comparison, the `distanceMeasureType` and `featureNormType`, `dimensions` of the brute force index should match those of the production indices being tuned.\n",
         "\n",
         "Create the brute force index configuration:"
       ]
@@ -812,19 +777,6 @@
         "INDEX_BRUTE_FORCE_RESOURCE_NAME"
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "865fcad494d7"
-      },
-      "outputs": [],
-      "source": [
-        "brute_force_index = aiplatform.MatchingEngineIndex(\n",
-        "    \"projects/1012616486416/locations/us-central1/indexes/6738176690918260736\"\n",
-        ")"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -989,7 +941,7 @@
       },
       "outputs": [],
       "source": [
-        "DEPLOYED_INDEX_ID = f\"tree_ah_glove_deployed_{TIMESTAMP}\""
+        "DEPLOYED_INDEX_ID = \"tree_ah_glove_deployed\""
       ]
     },
     {
@@ -1024,7 +976,7 @@
       },
       "outputs": [],
       "source": [
-        "DEPLOYED_BRUTE_FORCE_INDEX_ID = f\"glove_brute_force_deployed_{TIMESTAMP}\""
+        "DEPLOYED_BRUTE_FORCE_INDEX_ID = \"glove_brute_force_deployed\""
       ]
     },
     {
@@ -1071,13 +1023,344 @@
       "outputs": [],
       "source": [
         "# Test query\n",
+        "query = [\n",
+        "    -0.11333,\n",
+        "    0.48402,\n",
+        "    0.090771,\n",
+        "    -0.22439,\n",
+        "    0.034206,\n",
+        "    -0.55831,\n",
+        "    0.041849,\n",
+        "    -0.53573,\n",
+        "    0.18809,\n",
+        "    -0.58722,\n",
+        "    0.015313,\n",
+        "    -0.014555,\n",
+        "    0.80842,\n",
+        "    -0.038519,\n",
+        "    0.75348,\n",
+        "    0.70502,\n",
+        "    -0.17863,\n",
+        "    0.3222,\n",
+        "    0.67575,\n",
+        "    0.67198,\n",
+        "    0.26044,\n",
+        "    0.4187,\n",
+        "    -0.34122,\n",
+        "    0.2286,\n",
+        "    -0.53529,\n",
+        "    1.2582,\n",
+        "    -0.091543,\n",
+        "    0.19716,\n",
+        "    -0.037454,\n",
+        "    -0.3336,\n",
+        "    0.31399,\n",
+        "    0.36488,\n",
+        "    0.71263,\n",
+        "    0.1307,\n",
+        "    -0.24654,\n",
+        "    -0.52445,\n",
+        "    -0.036091,\n",
+        "    0.55068,\n",
+        "    0.10017,\n",
+        "    0.48095,\n",
+        "    0.71104,\n",
+        "    -0.053462,\n",
+        "    0.22325,\n",
+        "    0.30917,\n",
+        "    -0.39926,\n",
+        "    0.036634,\n",
+        "    -0.35431,\n",
+        "    -0.42795,\n",
+        "    0.46444,\n",
+        "    0.25586,\n",
+        "    0.68257,\n",
+        "    -0.20821,\n",
+        "    0.38433,\n",
+        "    0.055773,\n",
+        "    -0.2539,\n",
+        "    -0.20804,\n",
+        "    0.52522,\n",
+        "    -0.11399,\n",
+        "    -0.3253,\n",
+        "    -0.44104,\n",
+        "    0.17528,\n",
+        "    0.62255,\n",
+        "    0.50237,\n",
+        "    -0.7607,\n",
+        "    -0.071786,\n",
+        "    0.0080131,\n",
+        "    -0.13286,\n",
+        "    0.50097,\n",
+        "    0.18824,\n",
+        "    -0.54722,\n",
+        "    -0.42664,\n",
+        "    0.4292,\n",
+        "    0.14877,\n",
+        "    -0.0072514,\n",
+        "    -0.16484,\n",
+        "    -0.059798,\n",
+        "    0.9895,\n",
+        "    -0.61738,\n",
+        "    0.054169,\n",
+        "    0.48424,\n",
+        "    -0.35084,\n",
+        "    -0.27053,\n",
+        "    0.37829,\n",
+        "    0.11503,\n",
+        "    -0.39613,\n",
+        "    0.24266,\n",
+        "    0.39147,\n",
+        "    -0.075256,\n",
+        "    0.65093,\n",
+        "    -0.20822,\n",
+        "    -0.17456,\n",
+        "    0.53571,\n",
+        "    -0.16537,\n",
+        "    0.13582,\n",
+        "    -0.56016,\n",
+        "    0.016964,\n",
+        "    0.1277,\n",
+        "    0.94071,\n",
+        "    -0.22608,\n",
+        "    -0.021106,\n",
+        "]\n",
+        "\n",
         "response = my_index_endpoint.match(\n",
-        "    deployed_index_id=DEPLOYED_INDEX_ID, queries=test[:1], num_neighbors=NUM_NEIGHBOURS\n",
+        "    deployed_index_id=DEPLOYED_INDEX_ID, queries=[query], num_neighbors=NUM_NEIGHBOURS\n",
         ")\n",
         "\n",
         "response"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "_mNwdU9_B_Ez"
+      },
+      "source": [
+        "### Batch Query\n",
+        "\n",
+        "You can run multiple queries in a single match call:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "A0XL0PJ1GoM9"
+      },
+      "outputs": [],
+      "source": [
+        "# Test query\n",
+        "queries = [\n",
+        "    [\n",
+        "        -0.11333,\n",
+        "        0.48402,\n",
+        "        0.090771,\n",
+        "        -0.22439,\n",
+        "        0.034206,\n",
+        "        -0.55831,\n",
+        "        0.041849,\n",
+        "        -0.53573,\n",
+        "        0.18809,\n",
+        "        -0.58722,\n",
+        "        0.015313,\n",
+        "        -0.014555,\n",
+        "        0.80842,\n",
+        "        -0.038519,\n",
+        "        0.75348,\n",
+        "        0.70502,\n",
+        "        -0.17863,\n",
+        "        0.3222,\n",
+        "        0.67575,\n",
+        "        0.67198,\n",
+        "        0.26044,\n",
+        "        0.4187,\n",
+        "        -0.34122,\n",
+        "        0.2286,\n",
+        "        -0.53529,\n",
+        "        1.2582,\n",
+        "        -0.091543,\n",
+        "        0.19716,\n",
+        "        -0.037454,\n",
+        "        -0.3336,\n",
+        "        0.31399,\n",
+        "        0.36488,\n",
+        "        0.71263,\n",
+        "        0.1307,\n",
+        "        -0.24654,\n",
+        "        -0.52445,\n",
+        "        -0.036091,\n",
+        "        0.55068,\n",
+        "        0.10017,\n",
+        "        0.48095,\n",
+        "        0.71104,\n",
+        "        -0.053462,\n",
+        "        0.22325,\n",
+        "        0.30917,\n",
+        "        -0.39926,\n",
+        "        0.036634,\n",
+        "        -0.35431,\n",
+        "        -0.42795,\n",
+        "        0.46444,\n",
+        "        0.25586,\n",
+        "        0.68257,\n",
+        "        -0.20821,\n",
+        "        0.38433,\n",
+        "        0.055773,\n",
+        "        -0.2539,\n",
+        "        -0.20804,\n",
+        "        0.52522,\n",
+        "        -0.11399,\n",
+        "        -0.3253,\n",
+        "        -0.44104,\n",
+        "        0.17528,\n",
+        "        0.62255,\n",
+        "        0.50237,\n",
+        "        -0.7607,\n",
+        "        -0.071786,\n",
+        "        0.0080131,\n",
+        "        -0.13286,\n",
+        "        0.50097,\n",
+        "        0.18824,\n",
+        "        -0.54722,\n",
+        "        -0.42664,\n",
+        "        0.4292,\n",
+        "        0.14877,\n",
+        "        -0.0072514,\n",
+        "        -0.16484,\n",
+        "        -0.059798,\n",
+        "        0.9895,\n",
+        "        -0.61738,\n",
+        "        0.054169,\n",
+        "        0.48424,\n",
+        "        -0.35084,\n",
+        "        -0.27053,\n",
+        "        0.37829,\n",
+        "        0.11503,\n",
+        "        -0.39613,\n",
+        "        0.24266,\n",
+        "        0.39147,\n",
+        "        -0.075256,\n",
+        "        0.65093,\n",
+        "        -0.20822,\n",
+        "        -0.17456,\n",
+        "        0.53571,\n",
+        "        -0.16537,\n",
+        "        0.13582,\n",
+        "        -0.56016,\n",
+        "        0.016964,\n",
+        "        0.1277,\n",
+        "        0.94071,\n",
+        "        -0.22608,\n",
+        "        -0.021106,\n",
+        "    ],\n",
+        "    [\n",
+        "        -0.99544,\n",
+        "        -2.3651,\n",
+        "        -0.24332,\n",
+        "        -1.0321,\n",
+        "        0.42052,\n",
+        "        -1.1817,\n",
+        "        -0.16451,\n",
+        "        -1.683,\n",
+        "        0.49673,\n",
+        "        -0.27258,\n",
+        "        -0.025397,\n",
+        "        0.34188,\n",
+        "        1.5523,\n",
+        "        1.3532,\n",
+        "        0.33297,\n",
+        "        -0.0056677,\n",
+        "        -0.76525,\n",
+        "        0.49587,\n",
+        "        1.2211,\n",
+        "        0.83394,\n",
+        "        -0.20031,\n",
+        "        -0.59657,\n",
+        "        0.38485,\n",
+        "        -0.23487,\n",
+        "        -1.0725,\n",
+        "        0.95856,\n",
+        "        0.16161,\n",
+        "        -1.2496,\n",
+        "        1.6751,\n",
+        "        0.73899,\n",
+        "        0.051347,\n",
+        "        -0.42702,\n",
+        "        0.16257,\n",
+        "        -0.16772,\n",
+        "        0.40146,\n",
+        "        0.29837,\n",
+        "        0.96204,\n",
+        "        -0.36232,\n",
+        "        -0.47848,\n",
+        "        0.78278,\n",
+        "        0.14834,\n",
+        "        1.3407,\n",
+        "        0.47834,\n",
+        "        -0.39083,\n",
+        "        -1.037,\n",
+        "        -0.24643,\n",
+        "        -0.75841,\n",
+        "        0.7669,\n",
+        "        -0.37363,\n",
+        "        0.52741,\n",
+        "        0.018563,\n",
+        "        -0.51301,\n",
+        "        0.97674,\n",
+        "        0.55232,\n",
+        "        1.1584,\n",
+        "        0.73715,\n",
+        "        1.3055,\n",
+        "        -0.44743,\n",
+        "        -0.15961,\n",
+        "        0.85006,\n",
+        "        -0.34092,\n",
+        "        -0.67667,\n",
+        "        0.2317,\n",
+        "        1.5582,\n",
+        "        1.2308,\n",
+        "        -0.62213,\n",
+        "        -0.032801,\n",
+        "        0.1206,\n",
+        "        -0.25899,\n",
+        "        -0.02756,\n",
+        "        -0.52814,\n",
+        "        -0.93523,\n",
+        "        0.58434,\n",
+        "        -0.24799,\n",
+        "        0.37692,\n",
+        "        0.86527,\n",
+        "        0.069626,\n",
+        "        1.3096,\n",
+        "        0.29975,\n",
+        "        -1.3651,\n",
+        "        -0.32048,\n",
+        "        -0.13741,\n",
+        "        0.33329,\n",
+        "        -1.9113,\n",
+        "        -0.60222,\n",
+        "        -0.23921,\n",
+        "        0.12664,\n",
+        "        -0.47961,\n",
+        "        -0.89531,\n",
+        "        0.62054,\n",
+        "        0.40869,\n",
+        "        -0.08503,\n",
+        "        0.6413,\n",
+        "        -0.84044,\n",
+        "        -0.74325,\n",
+        "        -0.19426,\n",
+        "        0.098722,\n",
+        "        0.32648,\n",
+        "        -0.67621,\n",
+        "        -0.62692,\n",
+        "    ],\n",
+        "]"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -1086,7 +1369,7 @@
       "source": [
         "### Compute Recall\n",
         "\n",
-        "Use the deployed brute force Index as the ground truth to calculate the recall of ANN Index. Note that you can run multiple queries in a single match call."
+        "Use deployed brute force Index as the ground truth to calculate the recall of ANN Index:"
       ]
     },
     {
@@ -1119,20 +1402,18 @@
       "outputs": [],
       "source": [
         "# Calculate recall by determining how many neighbors were correctly retrieved as compared to the brute-force option.\n",
-        "recalled_neighbors = 0\n",
+        "correct_neighbors = 0\n",
         "for tree_ah_neighbors, brute_force_neighbors in zip(\n",
         "    tree_ah_response_test, brute_force_response_test\n",
         "):\n",
         "    tree_ah_neighbor_ids = [neighbor.id for neighbor in tree_ah_neighbors]\n",
         "    brute_force_neighbor_ids = [neighbor.id for neighbor in brute_force_neighbors]\n",
         "\n",
-        "    recalled_neighbors += len(\n",
+        "    correct_neighbors += len(\n",
         "        set(tree_ah_neighbor_ids).intersection(brute_force_neighbor_ids)\n",
         "    )\n",
         "\n",
-        "recall = recalled_neighbors / len(\n",
-        "    [neighbor for neighbors in brute_force_response_test for neighbor in neighbors]\n",
-        ")\n",
+        "recall = correct_neighbors / (len(test) * NUM_NEIGHBOURS)\n",
         "\n",
         "print(\"Recall: {}\".format(recall))"
       ]

notebooks/community/ml_ops/stage1/README.md

@@ -22,7 +22,7 @@ The first stage in MLOps is the collection and preparation for the purpose of de
 - Data is preprocessed for training and evaluation using Dataflow.
 - Data augmentation is performed on-the-fly and is coupled with model feeding.
 
-<img src='stage1v2.png'>
+<img src='stage1.jpg'>
 
 ## Notebooks
 
@@ -76,7 +76,7 @@ The steps performed include:
     - image data
 ```
 
-[Get Started with Data Labeling](get_started_with_data_labeling.ipynb)
+[Get Started with Data Labeling](get_started_data_labeling.ipynb)
 
 ```
 The steps performed include:
@@ -88,17 +88,6 @@ The steps performed include:
 - Cancel a data labeling job.
 ```
 
-[Get Started with Vision API and Vertex AI Datasets](get_started_with_visionapi_and_vertex_datasets.ipynb)
-
-```
-The steps performed include:
-
-- Using Vision API to perform Optical Character Recognition (OCR) to extract text from PDF files.
-- Processing the results and saving them to text files.
-- Generating a Vertex AI Dataset import file.
-- Creating a new unlabelled text entity extraction Vertex AI Dataset resource in Vertex AI.
-```
-
 ### E2E Stage Example
 
 [Stage 1: Data Management](mlops_data_management.ipynb)

notebooks/community/ml_ops/stage1/get_started_bq_datasets.ipynb

@@ -174,7 +174,7 @@
         "import os\n",
         "\n",
         "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
         "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
         "    \"/opt/deeplearning/metadata/env_version\"\n",
         ")\n",
@@ -389,7 +389,7 @@
       },
       "outputs": [],
       "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
+        "i  # If you are running this notebook in Colab, run this cell and follow the\n",
         "# instructions to authenticate your GCP account. This provides access to your\n",
         "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
         "# requests.\n",
@@ -1132,7 +1132,7 @@
       "source": [
         "dataframe[\"station_number\"] = pd.to_numeric(dataframe[\"station_number\"])\n",
         "labels = dataframe[\"mean_temp\"]\n",
-        "data = dataframe.drop([\"mean_temp\"], axis=1)\n",
+        "data = dataframe.drop(4)\n",
         "\n",
         "dtrain = xgb.DMatrix(data, label=labels)"
       ]

notebooks/community/ml_ops/stage1/get_started_dataflow.ipynb

@@ -159,7 +159,7 @@
         "import os\n",
         "\n",
         "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
         "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
         "    \"/opt/deeplearning/metadata/env_version\"\n",
         ")\n",

notebooks/community/ml_ops/stage1/get_started_vertex_datasets.ipynb

@@ -171,7 +171,7 @@
         "import os\n",
         "\n",
         "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
         "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
         "    \"/opt/deeplearning/metadata/env_version\"\n",
         ")\n",
@@ -188,7 +188,7 @@
         "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG -q\n",
         "! pip3 install --upgrade google-cloud-bigquery $USER_FLAG -q\n",
         "! pip3 install -U tensorflow-io==0.18 $USER_FLAG -q\n",
-        "! pip3 install --upgrade db-dtypes $USER_FLAG -q! pip3 install --upgrade future $USER_FLAG -q"
+        "! pip3 install --upgrade future $USER_FLAG -q"
       ]
     },
     {
@@ -408,11 +408,12 @@
         "import sys\n",
         "\n",
         "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = \"google.colab\" in sys.modules\n",
+        "IS_COLAB = False\n",
         "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
         "    \"DL_ANACONDA_HOME\"\n",
         "):\n",
         "    if \"google.colab\" in sys.modules:\n",
+        "        IS_COLAB = True\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
         "        google_auth.authenticate_user()\n",
@@ -557,7 +558,7 @@
       },
       "outputs": [],
       "source": [
-        "aip.init(project=PROJECT_ID, location=REGION, staging_bucket=BUCKET_URI)"
+        "aip.init(project=PROJECT_ID, location=REGION)"
       ]
     },
     {
@@ -611,13 +612,26 @@
         "Learn more about [All dataset documentation](https://cloud.google.com/vertex-ai/docs/datasets/datasets)"
       ]
     },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "import_file:flowers,csv,icn"
+      },
+      "outputs": [],
+      "source": [
+        "IMPORT_FILE = (\n",
+        "    \"gs://cloud-samples-data/vision/automl_classification/flowers/all_data_v2.csv\"\n",
+        ")"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "create_dataset:image,icn"
       },
       "source": [
-        "### Create an Image Dataset\n",
+        "### Create the Dataset\n",
         "\n",
         "Next, create the `Dataset` resource using the `create` method for the `ImageDataset` class, which takes the following parameters:\n",
         "\n",
@@ -632,19 +646,6 @@
         "Learn more about [ImageDataset](https://cloud.google.com/vertex-ai/docs/datasets/prepare-image)."
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "import_file:flowers,csv,icn"
-      },
-      "outputs": [],
-      "source": [
-        "IMPORT_FILE = (\n",
-        "    \"gs://cloud-samples-data/vision/automl_classification/flowers/all_data_v2.csv\"\n",
-        ")"
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -662,13 +663,24 @@
         "print(dataset.resource_name)"
       ]
     },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "import_file:hmdb,csv,vcn"
+      },
+      "outputs": [],
+      "source": [
+        "IMPORT_FILE = \"gs://automl-video-demo-data/hmdb_split1_5classes_train_inf.csv\""
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "create_dataset:video,vcn"
       },
       "source": [
-        "### Create a Video Dataset\n",
+        "### Create the Dataset\n",
         "\n",
         "Next, create the `Dataset` resource using the `create` method for the `VideoDataset` class, which takes the following parameters:\n",
         "\n",
@@ -682,17 +694,6 @@
         "Learn more about [VideoDataset](https://cloud.google.com/vertex-ai/docs/datasets/prepare-video)."
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "import_file:hmdb,csv,vcn"
-      },
-      "outputs": [],
-      "source": [
-        "IMPORT_FILE = \"gs://automl-video-demo-data/hmdb_split1_5classes_train_inf.csv\""
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -710,13 +711,24 @@
         "print(dataset.resource_name)"
       ]
     },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "import_file:happydb,csv,tcn"
+      },
+      "outputs": [],
+      "source": [
+        "IMPORT_FILE = \"gs://cloud-ml-data/NL-classification/happiness.csv\""
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "create_dataset:text,tcn"
       },
       "source": [
-        "### Create a Text Dataset\n",
+        "### Create the Dataset\n",
         "\n",
         "Next, create the `Dataset` resource using the `create` method for the `TextDataset` class, which takes the following parameters:\n",
         "\n",
@@ -731,17 +743,6 @@
         "Learn more about [TextDataset](https://cloud.google.com/vertex-ai/docs/datasets/prepare-text)."
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "import_file:happydb,csv,tcn"
-      },
-      "outputs": [],
-      "source": [
-        "IMPORT_FILE = \"gs://cloud-ml-data/NL-classification/happiness.csv\""
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -759,24 +760,6 @@
         "print(dataset.resource_name)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "create_dataset:tabular,bq,lrg,v2"
-      },
-      "source": [
-        "### Create a Tabular Dataset\n",
-        "\n",
-        "#### CSV input data\n",
-        "\n",
-        "Next, create the `Dataset` resource using the `create` method for the `TabularDataset` class for CSV input data, which takes the following parameters:\n",
-        "\n",
-        "- `display_name`: The human readable name for the `Dataset` resource.\n",
-        "- `gcs_source`: A list of one or more dataset index files to import the data items into the `Dataset` resource.\n",
-        "\n",
-        "Learn more about [TabularDataset from CSV files](https://cloud.google.com/vertex-ai/docs/datasets/create-dataset-api#aiplatform_create_dataset_tabular_gcs_sample-python)"
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -785,50 +768,27 @@
       },
       "outputs": [],
       "source": [
-        "IMPORT_FILE = \"gs://cloud-samples-data/tables/iris_1000.csv\""
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "create_dataset:tabular,bq,lrg,v2"
-      },
-      "outputs": [],
-      "source": [
-        "dataset = aip.TabularDataset.create(\n",
-        "    display_name=\"example\" + \"_\" + TIMESTAMP, gcs_source=[IMPORT_FILE]\n",
-        ")\n",
-        "\n",
-        "print(dataset.resource_name)"
+        "IMPORT_FILE = \"bq://bigquery-public-data.samples.gsod\"\n",
+        "BQ_TABLE = \"bigquery-public-data.samples.gsod\""
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "854dd1e0195c"
+        "id": "create_dataset:tabular,bq,lrg,v2"
       },
       "source": [
-        "#### BigQuery input data\n",
+        "### Create the Dataset\n",
         "\n",
-        "Next, create the `Dataset` resource using the `create` method for the `TabularDataset` class for BigQuery table input, which takes the following parameters:\n",
+        "#### CSV input data\n",
+        "\n",
+        "Next, create the `Dataset` resource using the `create` method for the `TabularDataset` class, which takes the following parameters:\n",
         "\n",
         "- `display_name`: The human readable name for the `Dataset` resource.\n",
-        "- `bq_source`: A list of one or more BigQuery tables to import the data items into the `Dataset` resource.\n",
+        "- `gcs_source`: A list of one or more dataset index files to import the data items into the `Dataset` resource.\n",
+        "- `labels`: User defined metadata. In this example, you store the location of the Cloud Storage bucket containing the user defined data.\n",
         "\n",
-        "Learn more about [TabularDataset from BigQuery table](https://cloud.google.com/vertex-ai/docs/datasets/create-dataset-api#aiplatform_create_dataset_tabular_bigquery_sample-pythonn)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "86343c146300"
-      },
-      "outputs": [],
-      "source": [
-        "IMPORT_FILE = \"bq://bigquery-public-data.samples.gsod\"\n",
-        "BQ_TABLE = \"bigquery-public-data.samples.gsod\""
+        "Learn more about [TabularDataset from CSV files](https://cloud.google.com/vertex-ai/docs/datasets/create-dataset-api#aiplatform_create_dataset_tabular_gcs_sample-python)"
       ]
     },
     {
@@ -846,82 +806,6 @@
         "print(dataset.resource_name)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "82e9fe20ce71"
-      },
-      "source": [
-        "#### Dataframe input data\n",
-        "\n",
-        "Next, create the `Dataset` resource using the `create_from_dataframe` method for the `TabularDataset` class for pandas dataframe input, which takes the following parameters:\n",
-        "\n",
-        "- `display_name`: The human readable name for the `Dataset` resource.\n",
-        "- `df_source`: The pandas dataframe to import the data items into the `Dataset` resource.\n",
-        "- `staging_path`: The BigQuery table to store the imported data."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "3805f945ffdd"
-      },
-      "outputs": [],
-      "source": [
-        "# Download the table.\n",
-        "table = bigquery.TableReference.from_string(BQ_TABLE)\n",
-        "\n",
-        "rows = bqclient.list_rows(\n",
-        "    table,\n",
-        "    max_results=10000,\n",
-        "    selected_fields=[\n",
-        "        bigquery.SchemaField(\"station_number\", \"STRING\"),\n",
-        "        bigquery.SchemaField(\"year\", \"INTEGER\"),\n",
-        "        bigquery.SchemaField(\"month\", \"INTEGER\"),\n",
-        "        bigquery.SchemaField(\"day\", \"INTEGER\"),\n",
-        "        bigquery.SchemaField(\"mean_temp\", \"FLOAT\"),\n",
-        "    ],\n",
-        ")\n",
-        "\n",
-        "dataframe = rows.to_dataframe()\n",
-        "print(dataframe.head())"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "create_dataset:tabular,bq,lrg,v2"
-      },
-      "outputs": [],
-      "source": [
-        "dataset = aip.TabularDataset.create_from_dataframe(\n",
-        "    display_name=\"example\" + \"_\" + TIMESTAMP,\n",
-        "    df_source=dataframe,\n",
-        "    staging_path=f\"bq://{PROJECT_ID}.samples.gsod\",\n",
-        ")\n",
-        "\n",
-        "print(dataset.resource_name)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "create_dataset:tabular,forecast,v2"
-      },
-      "source": [
-        "### Create a Time Series Dataset\n",
-        "\n",
-        "Next, create the `Dataset` resource using the `create` method for the `TimeSeriesDataset` class, which takes the following parameters:\n",
-        "\n",
-        "- `display_name`: The human readable name for the `Dataset` resource.\n",
-        "- `gcs_source`: A list of one or more dataset index files to import the data items into the `Dataset` resource.\n",
-        "- `bq_source`: Alternatively, import data items from a BigQuery table into the `Dataset` resource.\n",
-        "\n",
-        "Learn more about [TimeSeriesDataset](https://cloud.google.com/vertex-ai/docs/datasets/prepare-tabular)."
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -933,6 +817,23 @@
         "IMPORT_FILE = \"gs://cloud-samples-data/ai-platform/covid/bigquery-public-covid-nyt-us-counties-train.csv\""
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "create_dataset:tabular,forecast,v2"
+      },
+      "source": [
+        "### Create the Dataset\n",
+        "\n",
+        "Next, create the `Dataset` resource using the `create` method for the `TimeSeriesDataset` class, which takes the following parameters:\n",
+        "\n",
+        "- `display_name`: The human readable name for the `Dataset` resource.\n",
+        "- `gcs_source`: A list of one or more dataset index files to import the data items into the `Dataset` resource.\n",
+        "- `bq_source`: Alternatively, import data items from a BigQuery table into the `Dataset` resource.\n",
+        "\n",
+        "Learn more about [TimeSeriesDataset](https://cloud.google.com/vertex-ai/docs/datasets/prepare-tabular)."
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -1378,7 +1279,7 @@
         "import os\n",
         "import sys\n",
         "\n",
-        "# If on Workbench AI Notebook, then don't execute this code\n",
+        "# If on Google Cloud Notebook, then don't execute this code\n",
         "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
         "    if \"google.colab\" in sys.modules:\n",
         "        ! pip3 install fsspec\n",
@@ -1719,15 +1620,11 @@
       },
       "outputs": [],
       "source": [
-        "delete_bucket = False\n",
-        "\n",
         "# Delete the dataset using the Vertex dataset object\n",
-        "datasets = aip.TabularDataset.list(filter=f'display_name=\"example_{TIMESTAMP}\"')\n",
-        "for dataset in datasets:\n",
-        "    dataset.delete()\n",
+        "dataset.delete()\n",
         "\n",
         "# Delete the bucket\n",
-        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "if os.getenv(\"IS_TESTING\"):\n",
         "    ! gsutil rm -r $BUCKET_URI"
       ]
     }

notebooks/community/ml_ops/stage1/get_started_with_data_labeling.ipynb

@@ -146,7 +146,7 @@
         "import os\n",
         "\n",
         "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
         "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
         "    \"/opt/deeplearning/metadata/env_version\"\n",
         ")\n",

notebooks/community/ml_ops/stage1/get_started_with_visionapi_and_vertex_datasets.ipynb

@@ -1,994 +0,0 @@
-{
-  "cells": [
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "copyright"
-      },
-      "outputs": [],
-      "source": [
-        "# Copyright 2022 Google LLC\n",
-        "#\n",
-        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
-        "# you may not use this file except in compliance with the License.\n",
-        "# You may obtain a copy of the License at\n",
-        "#\n",
-        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
-        "#\n",
-        "# Unless required by applicable law or agreed to in writing, software\n",
-        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
-        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
-        "# See the License for the specific language governing permissions and\n",
-        "# limitations under the License."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "4JIDiHvGasba"
-      },
-      "source": [
-        "This notebook was contributed by [Mohammad Al-Ansari](https://github.com/Mansari)."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "2xDiUNIZINWp"
-      },
-      "source": [
-        "# E2E ML on GCP: MLOps stage 1 : data management: create an unlabelled Vertex AI AutoML text entity extraction dataset from PDFs using Vision API\n",
-        "\n",
-        "<table align=\"left\">\n",
-        "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_with_visionapi_and_vertex_datasets.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
-        "    </a>\n",
-        "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_with_visionapi_and_vertex_datasets.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
-        "      View on GitHub\n",
-        "    </a>\n",
-        "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_with_visionapi_and_vertex_datasets.ipynb\">\n",
-        "     <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
-        "    </a>\n",
-        "  </td>\n",
-        "</table>\n",
-        "<br/><br/><br/>"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "H0alLPo_A-LK"
-      },
-      "source": [
-        "## Overview\n",
-        "\n",
-        "This notebook will create an unlabelled `Vertex AI AutoML` text entity extraction dataset based on a collection of PDF files stored in a Cloud Storage bucket. \n",
-        "\n",
-        "The notebook can be modified to create different types of text datasets including sentiment analysis and classification."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "W4IBLTKOA5nl"
-      },
-      "source": [
-        "### Dataset\n",
-        "\n",
-        "The dataset used for this tutorial is the [Patent PDF Samples with Extracted Structured Data](https://console.cloud.google.com/marketplace/product/global-patents/labeled-patents) from Google Public Data Sets. \n",
-        "\n",
-        "This dataset includes data extracted from over 300 patent documents issued in the US and EU. The dataset includes links to Cloud Storage blobs for the first page of each patent, in addition to a number of extracted entities. \n",
-        "\n",
-        "The data is published as a [public dataset](https://cloud.google.com/bigquery/public-data) on `BigQuery`."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "3f8c2f702ccd"
-      },
-      "source": [
-        "### Objective\n",
-        "\n",
-        "In this tutorial, you learn to use `Vision API` to extract text from PDF files stored on a Cloud Storage bucket. You will then process the results and create an unlabelled `Vertex AI Dataset`, compatible with `AutoML`, for text entity extraction.\n",
-        "\n",
-        "You can then either use Google Cloud console to annotate / label the dataset, or create a labelling job as demonstrated in [this notebook](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_with_data_labeling.ipynb).\n",
-        "\n",
-        "This tutorial uses the following Google Cloud services:\n",
-        "\n",
-        "- `Vision AI`\n",
-        "- `Vertex AI AutoML`\n",
-        "\n",
-        "The steps performed include:\n",
-        "\n",
-        "1. Using `Vision API` to perform Optical Character Recognition (OCR) to extract text from PDF files.\n",
-        "2. Processing the results and saving them to text files.\n",
-        "3. Generating a `Vertex AI Dataset` import file.\n",
-        "4. Creating a new unlabelled text entity extraction `Vertex AI Dataset` resource in `Vertex AI`."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "CgLDJ419LPJs"
-      },
-      "source": [
-        "### Costs\n",
-        "\n",
-        "This tutorial uses billable components of Google Cloud:\n",
-        "\n",
-        "* Vision API\n",
-        "* Vertex AI\n",
-        "* Cloud Storage\n",
-        "\n",
-        "Learn about [Vertex AI\n",
-        "pricing](https://cloud.google.com/vertex-ai/pricing), [Vision API pricing](https://cloud.google.com/vision/pricing), [Cloud Storage\n",
-        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
-        "Calculator](https://cloud.google.com/products/calculator/)\n",
-        "to generate a cost estimate based on your projected usage."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "va2g7m9wLTjA"
-      },
-      "source": [
-        "### Set up your local development environment\n",
-        "\n",
-        "If you are using Colab or Vertex AI Workbench Notebooks, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
-        "\n",
-        "Otherwise, make sure your environment meets this notebook's requirements. You need the following:\n",
-        "\n",
-        "- The Vision API SDK\n",
-        "- The Vertex AI SDK\n",
-        "- The Cloud Storage SDK\n",
-        "- Git\n",
-        "- Python 3\n",
-        "- virtualenv\n",
-        "- Jupyter notebook running in a virtual environment with Python 3\n",
-        "\n",
-        "The Cloud Storage guide to [Setting up a Python development environment](https://cloud.google.com/python/setup) and the [Jupyter installation guide](https://jupyter.org/install) provide detailed instructions for meeting these requirements. The following steps provide a condensed set of instructions:\n",
-        "\n",
-        "1. [Install and initialize the SDKs](https://cloud.google.com/sdk/docs/).\n",
-        "\n",
-        "2. [Install Python 3](https://cloud.google.com/python/setup#installing_python).\n",
-        "\n",
-        "3. [Install virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv) and create a virtual environment that uses Python 3.  Activate the virtual environment.\n",
-        "\n",
-        "4. To install Jupyter, run `pip3 install jupyter` on the command-line in a terminal shell.\n",
-        "\n",
-        "5. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
-        "\n",
-        "6. Open this notebook in the Jupyter Notebook Dashboard.\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "X2tZAmugAe6h"
-      },
-      "source": [
-        "## Installation\n",
-        "\n",
-        "Install the packages required for executing this notebook. You can ignore errors for the `pip` dependecy resolver as they do not impact this notebook."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "BQOsJ1hZAZu0"
-      },
-      "outputs": [],
-      "source": [
-        "import os\n",
-        "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
-        "! pip3 install --upgrade google-cloud-storage google-cloud-vision google-cloud-aiplatform $USER_FLAG -q"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "yzvvcmCuAon3"
-      },
-      "source": [
-        "### Restart the kernel\n",
-        "\n",
-        "Once you've installed the additional packages, you need to restart the notebook kernel so it can find the packages."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "6qEonzbuAoI_"
-      },
-      "outputs": [],
-      "source": [
-        "import os\n",
-        "\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    # Automatically restart kernel after installs\n",
-        "    import IPython\n",
-        "\n",
-        "    app = IPython.Application.instance()\n",
-        "    app.kernel.do_shutdown(True)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "pGbbyN7rAuRM"
-      },
-      "source": [
-        "## Before you begin\n",
-        "\n",
-        "### GPU runtime\n",
-        "\n",
-        "*Make sure you're running this notebook in a GPU runtime if you have that option. In Colab, select* **Runtime > Change Runtime Type > GPU**\n",
-        "\n",
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "2. [Make sure that billing is enabled for your project.](https://cloud.google.com/billing/docs/how-to/modify-project)\n",
-        "\n",
-        "3. [Enable the following APIs: Vision API, Vertex AI APIs, Compute Engine APIs, and Cloud Storage.](https://console.cloud.google.com/flows/enableapi?apiid=vision.googleapis.com,aiplatform.googleapis.com,compute_component,storage-component.googleapis.com)\n",
-        "\n",
-        "4. If you are running this notebook locally, you will need to install the [Cloud SDK]((https://cloud.google.com/sdk)).\n",
-        "\n",
-        "5. Enter your project ID in the cell below. Then run the  cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$`."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "AE97adtnAzrr"
-      },
-      "outputs": [],
-      "source": [
-        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "nWlzLu5ELxWd"
-      },
-      "outputs": [],
-      "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
-        "    # Get your GCP project id from gcloud\n",
-        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
-        "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID:\", PROJECT_ID)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "GB5b27r0LxqE"
-      },
-      "outputs": [],
-      "source": [
-        "! gcloud config set project $PROJECT_ID"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "pMJdU1K5xG7D"
-      },
-      "source": [
-        "### Regions\n",
-        "\n",
-        "#### Vision AI\n",
-        "\n",
-        "You can now specify continent-level data storage and Optical Character Regonition (OCR) processing by setting the `VISION_AI_REGION` variable. You can select one of the following options:\n",
-        "\n",
-        "* USA country only: `us`\n",
-        "* The European Union: `eu`\n",
-        "\n",
-        "Learn more about [Vision AI regions for OCR](https://cloud.google.com/vision/docs/pdf#regionalization)\n",
-        "\n",
-        "#### Vertex AI\n",
-        "\n",
-        "You can also change the `VERTEX_AI_REGION` variable, which is used for operations throughout the rest of this notebook.  Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
-        "\n",
-        "- Americas: `us-central1`\n",
-        "- Europe: `europe-west4`\n",
-        "- Asia Pacific: `asia-east1`\n",
-        "\n",
-        "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
-        "\n",
-        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "5EhEAOK5xIKc"
-      },
-      "outputs": [],
-      "source": [
-        "VISION_AI_REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if VISION_AI_REGION == \"[your-region]\":\n",
-        "    VISION_AI_REGION = \"us\"\n",
-        "\n",
-        "VERTEX_AI_REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if VERTEX_AI_REGION == \"[your-region]\":\n",
-        "    VERTEX_AI_REGION = \"us-central1\""
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "xkgvWoXkxM1r"
-      },
-      "source": [
-        "### Timestamp\n",
-        "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append onto the name of resources which will be created in this tutorial."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "gr0HTpQZxNy4"
-      },
-      "outputs": [],
-      "source": [
-        "from datetime import datetime\n",
-        "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "AA-ns5CcBA9U"
-      },
-      "source": [
-        "### Vertex AI dataset import schema\n",
-        "\n",
-        "This constant tells Vertex AI the schema for importing the dataset. In this tutorial you are going to use the value for text extraction, but you can also change it to any of the values below for other use cases:\n",
-        "\n",
-        "- \n",
-        "`aiplatform.schema.dataset.ioformat.text.single_label_classification`\n",
-        "\n",
-        "- \n",
-        "`aiplatform.schema.dataset.ioformat.text.multi_label_classification`\n",
-        "\n",
-        "- \n",
-        "`aiplatform.schema.dataset.ioformat.text.extraction`\n",
-        "\n",
-        "- \n",
-        "`aiplatform.schema.dataset.ioformat.text.sentiment`\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "jnOb6Pp-4w5P"
-      },
-      "outputs": [],
-      "source": [
-        "from google.cloud import aiplatform\n",
-        "\n",
-        "DATASET_IMPORT_SCHEMA = aiplatform.schema.dataset.ioformat.text.extraction"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "ekbg-G7UA-bK"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Vertex AI Workbench**, your environment is already authenticated. Skip this step. If you receive errors still, you may have to grant the service account that is your Workbench notebook is running under access to the services listed below.\n",
-        "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
-        "\n",
-        "**Click Create service account**.\n",
-        "\n",
-        "In the **Service account name** field, enter a name, and click **Create**.\n",
-        "\n",
-        "In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex\" into the filter box, and select **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "Click Create. A JSON file that contains your key downloads to your local environment.\n",
-        "\n",
-        "Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "lCRrULxKBAfa"
-      },
-      "outputs": [],
-      "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = \"google.colab\" in sys.modules\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "rHB6fbonMMbI"
-      },
-      "source": [
-        "### Create a Cloud Storage bucket\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "When you initialize the Vertex AI SDK for Python, you specify a Cloud Storage staging bucket. The staging bucket is where all the data associated with your dataset and model resources are retained across sessions. This bucket will be also used to store the output of the Vision API SDK PDF-to-text conversion process.\n",
-        "\n",
-        "Set the name of your Cloud Storage bucket below. Bucket names must be globally unique across all Google Cloud projects, including those outside of your organization."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "ZSM5j0nfMOVK"
-      },
-      "outputs": [],
-      "source": [
-        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "i6H2iQX2MP-s"
-      },
-      "outputs": [],
-      "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
-        "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "AOsnYE5cMQX4"
-      },
-      "source": [
-        "**Only if your bucket doesn't already exist**: Run the following cell to create your Cloud Storage bucket."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "33RgSjhyMR6C"
-      },
-      "outputs": [],
-      "source": [
-        "! gsutil mb -l $VERTEX_AI_REGION -p $PROJECT_ID $BUCKET_URI"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "UpKfi0VfMTwe"
-      },
-      "source": [
-        "Finally, validate access to your Cloud Storage bucket by examining its contents:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "G9dMjMnkMVNt"
-      },
-      "outputs": [],
-      "source": [
-        "! gsutil ls -al $BUCKET_URI"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "k2qH7YCI0vnG"
-      },
-      "source": [
-        "### Set up variables\n",
-        "\n",
-        "Next, set up some variables used throughout the tutorial.\n",
-        "\n",
-        "### Import libraries and define constants"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "TB5-_2Xh01NH"
-      },
-      "outputs": [],
-      "source": [
-        "import os\n",
-        "\n",
-        "from google.cloud import aiplatform, storage, vision"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "-v7gY_KABIn8"
-      },
-      "source": [
-        "### Initialize Vision API SDK for Python\n",
-        "\n",
-        "Initialize the `Vision AI` SDK for Python for your project and region."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "DRbf--kWBLpx"
-      },
-      "outputs": [],
-      "source": [
-        "vision_client_options = {\n",
-        "    \"quota_project_id\": PROJECT_ID,\n",
-        "    \"api_endpoint\": f\"{VISION_AI_REGION}-vision.googleapis.com\",\n",
-        "}\n",
-        "vision_client = vision.ImageAnnotatorClient(client_options=vision_client_options)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "CA4nNVbBZ25d"
-      },
-      "source": [
-        "### Initialize Vertex AI SDK for Python\n",
-        "\n",
-        "Initialize the `Vertex AI` SDK for Python for your project, region and corresponding bucket."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "awWpNW1vZ6uV"
-      },
-      "outputs": [],
-      "source": [
-        "aiplatform.init(\n",
-        "    project=PROJECT_ID, location=VERTEX_AI_REGION, staging_bucket=BUCKET_URI\n",
-        ")"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "debBBljMDqkM"
-      },
-      "source": [
-        "### Initialize Cloud Storage SDK for Python\n",
-        "\n",
-        "Initialize the `Cloud Storage` SDK for Python for your project."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "ZtzmI9tpDr4e"
-      },
-      "outputs": [],
-      "source": [
-        "storage_client = storage.Client(project=PROJECT_ID)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "mvD0BxVXMtJe"
-      },
-      "source": [
-        "## Tutorial\n",
-        "\n",
-        "Now you are ready to start creating an unlabelled `Vertex AI Dataset` text entity extraction dataset from PDF files."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "EurEFM3GBap9"
-      },
-      "source": [
-        "### Convert PDF files to text using Vision API\n",
-        "\n",
-        "First, you make a `Vision API` request to OCR to text the PDFs from the Patent samples stored in the Cloud Storage bucket.\n",
-        "\n",
-        "*Note:* `Visions API` only allows batches of 100 document submissions at a time."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "uXVPOvjTBeK3"
-      },
-      "outputs": [],
-      "source": [
-        "ORIGIN_BUCKET_NAME = \"gcs-public-data--labeled-patents\"\n",
-        "# You can add a path if needed\n",
-        "ORIGIN_BUCKET_PATH = \"\"\n",
-        "\n",
-        "DESTINATION_BUCKET_NAME = BUCKET_NAME\n",
-        "DESTINATION_BUCKET_PATH = \"ocr-output\"\n",
-        "\n",
-        "gcs_destination_uri = f\"gs://{DESTINATION_BUCKET_NAME}/{DESTINATION_BUCKET_PATH}\"\n",
-        "\n",
-        "# Specify the feature for the Vision API processor\n",
-        "feature = vision.Feature(type_=vision.Feature.Type.DOCUMENT_TEXT_DETECTION)\n",
-        "\n",
-        "# Retrieve a list of all files in the bucket and path\n",
-        "blobs = storage_client.list_blobs(\n",
-        "    ORIGIN_BUCKET_NAME, prefix=ORIGIN_BUCKET_PATH, delimiter=\"/\"\n",
-        ")\n",
-        "\n",
-        "# Create a collection of requests. The SDK requires a separate request per each\n",
-        "# file that we want to extract text from\n",
-        "async_requests = []\n",
-        "\n",
-        "# Visions API only supports processing up to 100 documents at a time\n",
-        "# so we will process the first 100 elements only\n",
-        "sliced_blob_list = list(blobs)[:100]\n",
-        "\n",
-        "# Loop through the source bucket and create a request for each file there\n",
-        "for blob in sliced_blob_list:\n",
-        "    # Build input_config\n",
-        "    # Ensure we are only processing PDF files\n",
-        "    if blob.name.endswith(\".pdf\"):\n",
-        "        gcs_source = vision.GcsSource(uri=f\"gs://{ORIGIN_BUCKET_NAME}/{blob.name}\")\n",
-        "        input_config = vision.InputConfig(\n",
-        "            gcs_source=gcs_source, mime_type=\"application/pdf\"\n",
-        "        )\n",
-        "\n",
-        "        # Build output config\n",
-        "        # Get file name\n",
-        "        file_name = os.path.splitext(os.path.basename(blob.name))[0]\n",
-        "        gcs_destination = vision.GcsDestination(\n",
-        "            uri=f\"{gcs_destination_uri}/{file_name}-\"\n",
-        "        )\n",
-        "        output_config = vision.OutputConfig(gcs_destination=gcs_destination)\n",
-        "\n",
-        "        # Build request object and add to the collection\n",
-        "        async_request = vision.AsyncAnnotateFileRequest(\n",
-        "            features=[feature], input_config=input_config, output_config=output_config\n",
-        "        )\n",
-        "\n",
-        "        async_requests.append(async_request)\n",
-        "\n",
-        "print(f\"Created {len(async_requests)} requests\")\n",
-        "\n",
-        "# Submit the batch OCR job\n",
-        "\n",
-        "operation = vision_client.async_batch_annotate_files(requests=async_requests)\n",
-        "print(\"Submitting the batch OCR job\")\n",
-        "\n",
-        "print(\"Waiting for the operation to finish... this will take a short while\")\n",
-        "\n",
-        "response = operation.result(timeout=420)\n",
-        "\n",
-        "print(\"Completed!\")"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "7b15473e1937"
-      },
-      "source": [
-        "#### Quick peek at extracted annotated JSON files\n",
-        "\n",
-        "Next, you take a peek at the contents of one of the extracted JSON annotated files."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "4366442c1373"
-      },
-      "outputs": [],
-      "source": [
-        "json_files = ! gsutil ls {gcs_destination_uri}\n",
-        "\n",
-        "example = json_files[0]\n",
-        "! gsutil cat {example} | head -n 1"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "QWmeHWPIHako"
-      },
-      "source": [
-        "### Process results and build the import file\n",
-        "\n",
-        "The `Vision API` output is in JSON format, and contains detailed text extraction data. You only need the full text output, so you will processs the JSON results, extract the text output, and save it in new text files to be used later in the tutorial."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "WDLtiejKHug6"
-      },
-      "outputs": [],
-      "source": [
-        "import json\n",
-        "\n",
-        "print(\"Extracting text from Vision API output and saving it to text files\")\n",
-        "\n",
-        "ocr_blobs = storage_client.list_blobs(\n",
-        "    DESTINATION_BUCKET_NAME, prefix=DESTINATION_BUCKET_PATH\n",
-        ")\n",
-        "\n",
-        "output_bucket = storage_client.bucket(DESTINATION_BUCKET_NAME)\n",
-        "\n",
-        "# begin building the import file content\n",
-        "import_file_entries = []\n",
-        "\n",
-        "for ocr_blob in ocr_blobs:\n",
-        "    # Only process .json files, in case we previously processed files and had .txt files\n",
-        "    if ocr_blob.name.endswith(\".json\"):\n",
-        "        print(f\"Extracting text from {ocr_blob.name}\")\n",
-        "        # read each blob into a stream\n",
-        "        contents = ocr_blob.download_as_string()\n",
-        "        # load as JSON\n",
-        "        json_object = json.loads(contents)\n",
-        "        # extract text\n",
-        "        full_text = \"\"\n",
-        "        for response in json_object[\"responses\"]:\n",
-        "            if response[\"fullTextAnnotation\"]:\n",
-        "                full_text += response[\"fullTextAnnotation\"][\"text\"] + \"\\r\\n\"\n",
-        "\n",
-        "        # save as a blob\n",
-        "        output_blob_name = f\"{ocr_blob.name}.txt\"\n",
-        "        import_file_blob = output_bucket.blob(output_blob_name)\n",
-        "        import_file_blob.upload_from_string(full_text)\n",
-        "\n",
-        "        # create import file listing\n",
-        "        import_file_entry = {\n",
-        "            \"textGcsUri\": f\"gs://{DESTINATION_BUCKET_NAME}/{output_blob_name}\"\n",
-        "        }\n",
-        "\n",
-        "        import_file_entries.append(import_file_entry)\n",
-        "\n",
-        "print(\"Extraction completed!\")"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "0a5aae0eab44"
-      },
-      "source": [
-        "#### Quick peek at extracted text files\n",
-        "\n",
-        "Next, you take a peek at the contents of one of the extracted text files."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "76ce5f57b1ae"
-      },
-      "outputs": [],
-      "source": [
-        "example = import_file_entries[0][\"textGcsUri\"]\n",
-        "\n",
-        "! gsutil cat {example}"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "hqTLS_AmLWQP"
-      },
-      "source": [
-        "### Generate and save import file to be used in `Vertex AI Dataset` resource\n",
-        "\n",
-        "You will now build the import file that will be used to create the `Vertex AI Dataset` resource."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "_xFvOdQ_LWne"
-      },
-      "outputs": [],
-      "source": [
-        "IMPORT_FILE_PATH = \"import_file\"\n",
-        "\n",
-        "# Convert import file entries to JSON Lines format\n",
-        "import_file_content = \"\"\n",
-        "for entry in import_file_entries:\n",
-        "    import_file_content += json.dumps(entry) + \"\\n\"\n",
-        "\n",
-        "print(f\"Created import file based on {len(import_file_entries)} annotations\")\n",
-        "\n",
-        "# Upload content to GCS to be used in our next step\n",
-        "gcs_annotation_file_name = f\"{IMPORT_FILE_PATH}/import_file_{TIMESTAMP}.jsonl\"\n",
-        "import_file_blob = output_bucket.blob(gcs_annotation_file_name)\n",
-        "import_file_blob.upload_from_string(import_file_content)\n",
-        "\n",
-        "print(f\"Uploaded import file to {output_bucket.name}/{gcs_annotation_file_name}\")"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "6dVjFftOaKdw"
-      },
-      "source": [
-        "### Create an unlabelled `Vertex AI Dataset` resource\n",
-        "\n",
-        "Next, you create the `Dataset` resource using the `create` method for the `TextDataset` class, which takes the following parameters:\n",
-        "\n",
-        "- `display_name`: The human readable name for the `Dataset` resource.\n",
-        "- `gcs_source`: A list of one or more dataset index files to import the data items into the `Dataset` resource.\n",
-        "- `import_schema_uri`: The data labeling schema for the data items.\n",
-        "\n",
-        "This operation may take ten to twenty minutes."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "ciM9HLGCaOTJ"
-      },
-      "outputs": [],
-      "source": [
-        "print(\"Creating dataset ...\")\n",
-        "\n",
-        "dataset = aiplatform.TextDataset.create(\n",
-        "    display_name=\"Text Dataset \" + TIMESTAMP,\n",
-        "    gcs_source=[f\"gs://{output_bucket.name}/{gcs_annotation_file_name}\"],\n",
-        "    import_schema_uri=DATASET_IMPORT_SCHEMA,\n",
-        ")\n",
-        "\n",
-        "print(\"Completed!\")\n",
-        "\n",
-        "print(dataset.resource_name)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "2vagHf5T6Jd4"
-      },
-      "source": [
-        "**Congratulations, your dataset is now ready for annotations!**\n",
-        "\n",
-        "You have two options:\n",
-        "\n",
-        "*   Use Google Cloud Console to manually annotate the dataset in `Vertex AI`. Checkout [this link](https://cloud.google.com/vertex-ai/docs/datasets/label-using-console#entity-extraction) for more details on how to do so.\n",
-        "*   Create a labelling job to request data labelling. Check out [this link](https://cloud.google.com/vertex-ai/docs/datasets/data-labeling-job) and [this notebook](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_with_data_labeling.ipynb) for more details and examples.\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "cleanup:migration,new"
-      },
-      "source": [
-        "# Cleaning up\n",
-        "\n",
-        "To clean up all GCP resources used in this project, you can [delete the GCP\n",
-        "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
-        "\n",
-        "Otherwise, you can delete the individual resources you created in this tutorial.\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "aoJ18d8Y_jAy"
-      },
-      "outputs": [],
-      "source": [
-        "# Set this to true only if you'd like to delete your bucket\n",
-        "delete_bucket = False\n",
-        "\n",
-        "# Delete the dataset using the Vertex AI fully qualified identifier for the dataset\n",
-        "dataset.delete()\n",
-        "\n",
-        "# Delete the bucket created\n",
-        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
-        "    ! gsutil rm -r $BUCKET_URI"
-      ]
-    }
-  ],
-  "metadata": {
-    "accelerator": "GPU",
-    "colab": {
-      "collapsed_sections": [],
-      "name": "get_started_with_visionapi_and_vertex_datasets.ipynb",
-      "toc_visible": true
-    },
-    "kernelspec": {
-      "display_name": "Python 3",
-      "name": "python3"
-    }
-  },
-  "nbformat": 4,
-  "nbformat_minor": 0
-}

notebooks/community/ml_ops/stage1/mlops_data_management.ipynb

@@ -142,7 +142,7 @@
         "import os\n",
         "\n",
         "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
         "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
         "    \"/opt/deeplearning/metadata/env_version\"\n",
         ")\n",

notebooks/community/ml_ops/stage2/README.md

@@ -25,46 +25,24 @@ The second stage in MLOps is experimenting in developing one or more baseline mo
 - Use the What-if-Tool (WIT) to explore how the trained model would make predictions in different scenarios.
 
 
-<img src='stage2v3.png'>
-<br/>
-<br/>
-<br/>
-<img src='stage2.2v1.png'>
+<img src='stage2.png'>
 
 ## Notebooks
 
 ### Get Started
 
-[Get Started with Logging](get_started_with_logging.ipynb)
-
-```
-The steps performed include:
-
-- Use Python logging to log training configuration/results locally.
-- Use Google Cloud Logging to log training configuration/results in cloud storage.
-```
-
 [Get Started with Vertex Experiments and Vertex ML Metadata](get_started_vertex_experiments.ipynb)
 
 ```
 The steps performed include:
 
-- Local (notebook) Training
-    - Create an experiment
-    - Create a first run in the experiment
-    - Log parameters and metrics
-    - Create artifact lineage
-    - Visualize the experiment results
-    - Execute a second run
-    - Compare the two runs in the experiment
-- Cloud (`Vertex AI`) Training
-    - Within the training script:
-        - Create an experiment
-        - Log parameters and metrics
-        - Create artifact lineage
-    - Create a `Vertex AI Training` custom job
-    - Execute the custom job
-    - Visualize the experiment results
+- Use Python logging to log training configuration/results locally.
+- Use Google Cloud Logging to log training configuration/results in cloud storage.
+- Create a Vertex AI `Experiment` resource.
+- Instantiate an experiment run.
+- Log parameters for the run.
+- Log metrics for the run.
+- Display the logged experiment run.
 ```
 
 [Get Started with Vertex TensorBoard](get_started_vertex_tensorboard.ipynb)
@@ -137,32 +115,6 @@ The steps performed include:
 - Train a R model using `Vertex AI Trainingh` service with the R-to-Python training package.
 ```
 
-[Get Started with Custom Training Packages (R) and Deployment in R environment](get_started_vertex_training_r_using_r_kernel.ipynb)
-```
-The steps performed include:
-
-- Create a custom R training script
-- Create a custom R serving script
-- Create a custom R deployment (serving) container.
-- Train the model using `Vertex AI` custom training.
-- Create an `Endpoint` resource.
-- Deploy the `Model` resource (trained R model) to the `Endpoint` resource.
-- Make an online prediction.
-```
-
-[Get Started with Custom Training Packages (LightGBM)](get_started_vertex_training_lightgbm.ipynb)
-
-```
-The steps performed include:
-
-- Training using a Python package.
-- Save the model artifacts to Cloud Storage using GCSFuse.
-- Construct a FastAPI prediction server.
-- Construct a Dockerfile deployment image.
-- Test the deployment image locally.
-- Create a `Vertex AI Model` resource.
-```
-
 [Get Started with Distributed Training](get_started_vertex_distributed_training.ipynb)
 
 ```
@@ -253,6 +205,19 @@ The steps performed include:
 ```
 The steps performed include:
 
+- Get the training data.
+- Configure training parameters for the Vertex AI TabNet container.
+- Train the model using Vertex AI Training using CSV data.
+- Upload the model as a Vertex AI Model resource.
+- Deploy the Vertex AI Model resource to a Vertex AI Endpoint resource.
+- Make a prediction with the deployed model.
+- Hyperparameter tuning the Vertex AI TabNet model.
+- Train the model using Vertex AI Training using BigQuery table.
+```
+[Get Started with Vertex AI TabNet builtin algorithm](get_started_with_tabnet.ipynb)
+```
+The steps performed include:
+
 - Get the training data.
 - Configure training parameters for the Vertex AI TabNet container.
 - Train the model using Vertex AI Training using CSV data.

notebooks/community/ml_ops/stage2/get_started_automl_training.ipynb

@@ -40,7 +40,7 @@
         "  </td>\n",
         "  <td>\n",
         "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_automl_training.ipynb\">\n",
-        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\\\" alt=\"Colab logo\"> Run in Colab\n",
         "        </a>\n",
         "  </td>\n",
         "  <td>\n",
@@ -187,7 +187,7 @@
         "import os\n",
         "\n",
         "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
         "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
         "    \"/opt/deeplearning/metadata/env_version\"\n",
         ")\n",

notebooks/community/ml_ops/stage2/get_started_bqml_training.ipynb

@@ -40,7 +40,7 @@
         "  </td>\n",
         "    <td>\n",
         "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_bqml_training.ipynb\">\n",
-        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\\\" alt=\"Colab logo\"> Run in Colab\n",
         "        </a>\n",
         "  </td>\n",
         "  <td>\n",
@@ -134,7 +134,7 @@
         "import os\n",
         "\n",
         "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
         "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
         "\n",
         "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
@@ -442,54 +442,6 @@
         "! gsutil ls -al $BUCKET_URI"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "set_service_account"
-      },
-      "source": [
-        "#### Service Account\n",
-        "\n",
-        "You use a service account to create Vertex AI Pipeline jobs. If you do not want to use your project's Compute Engine service account, set `SERVICE_ACCOUNT` to another service account ID."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "set_service_account"
-      },
-      "outputs": [],
-      "source": [
-        "SERVICE_ACCOUNT = \"[your-service-account]\"  # @param {type:\"string\"}"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "autoset_service_account"
-      },
-      "outputs": [],
-      "source": [
-        "if (\n",
-        "    SERVICE_ACCOUNT == \"\"\n",
-        "    or SERVICE_ACCOUNT is None\n",
-        "    or SERVICE_ACCOUNT == \"[your-service-account]\"\n",
-        "):\n",
-        "    # Get your service account from gcloud\n",
-        "    if not IS_COLAB:\n",
-        "        shell_output = !gcloud auth list 2>/dev/null\n",
-        "        SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
-        "\n",
-        "    if IS_COLAB:\n",
-        "        shell_output = ! gcloud projects describe  $PROJECT_ID\n",
-        "        project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n",
-        "        SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n",
-        "\n",
-        "    print(\"Service Account:\", SERVICE_ACCOUNT)"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -778,6 +730,32 @@
         "print(\"{} created in {}\".format(tblname, job.ended - job.started))"
       ]
     },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3b3aa4481cd7"
+      },
+      "outputs": [],
+      "source": [
+        "MODEL_QUERY = f\"\"\"\n",
+        "DROP MODEL `{BQ_DATASET_NAME}.{MODEL_NAME}`\n",
+        "\"\"\"\n",
+        "\n",
+        "job = bqclient.query(MODEL_QUERY)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ef9e14b91475"
+      },
+      "outputs": [],
+      "source": [
+        "job"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -1207,21 +1185,24 @@
         "\n",
         "### Setting permissions to automatically register the model\n",
         "\n",
-        "You need to set some additional IAM permissions for BigQuery ML to automatically upload and register the model after training. Depending on your service account, the setting of the permissions below may fail. In this case, we recommend executing the permissions in a Cloud Shell.\n",
-        "\n",
-        "Learn more about [Setting permissions for Model Registry](https://cloud.google.com/bigquery-ml/docs/managing-models-vertex)\n"
+        "You need to set some additional IAM permissions for BigQuery ML to automatically upload and register the model after training. Depending on your service account, the setting of the permissions below may fail. In this case, we recommend executing the permissions in a Cloud Shell."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "29229f72d13d"
+        "id": "0472e888105e"
       },
       "outputs": [],
       "source": [
         "! gcloud projects add-iam-policy-binding $PROJECT_ID \\\n",
-        "    --member=serviceAccount:$SERVICE_ACCOUNT --role=roles/aiplatform.admin --condition=None"
+        "      --member='serviceAccount:cloud-dataengine@system.gserviceaccount.com' \\\n",
+        "      --role='roles/aiplatform.admin'\n",
+        "\n",
+        "! gcloud projects add-iam-policy-binding $PROJECT_ID \\\n",
+        "      --member='user:cloud-dataengine@prod.google.com' \\\n",
+        "      --role='roles/aiplatform.admin'"
       ]
     },
     {
@@ -1302,20 +1283,6 @@
         "print(model.gca_resource)"
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "48e6ef5d5ffa"
-      },
-      "outputs": [],
-      "source": [
-        "models = aiplatform.Model.list()\n",
-        "for model in models:\n",
-        "    if model.gca_resource.display_name.startswith(\"bqml\"):\n",
-        "        print(model.gca_resource.display_name)"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {

notebooks/community/ml_ops/stage2/get_started_vertex_distributed_training.ipynb

@@ -160,7 +160,7 @@
         "import os\n",
         "\n",
         "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
         "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
         "    \"/opt/deeplearning/metadata/env_version\"\n",
         ")\n",

notebooks/community/ml_ops/stage2/get_started_vertex_feature_store.ipynb

@@ -41,7 +41,7 @@
         "    \n",
         "  <td>\n",
         "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_feature_store.ipynb\">\n",
-        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\\\" alt=\"Colab logo\"> Run in Colab\n",
         "        </a>\n",
         "  </td>\n",
         "    \n",
@@ -145,7 +145,7 @@
         "import os\n",
         "\n",
         "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
         "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
         "    \"/opt/deeplearning/metadata/env_version\"\n",
         ")\n",
@@ -362,11 +362,12 @@
         "import sys\n",
         "\n",
         "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = \"google.colab\" in sys.modules\n",
+        "IS_COLAB = False\n",
         "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
         "    \"DL_ANACONDA_HOME\"\n",
         "):\n",
         "    if \"google.colab\" in sys.modules:\n",
+        "        IS_COLAB = True\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
         "        google_auth.authenticate_user()\n",
@@ -489,7 +490,7 @@
       "outputs": [],
       "source": [
         "# Represents featurestore resource path.\n",
-        "FEATURESTORE_NAME = \"movies_\" + TIMESTAMP\n",
+        "FEATURESTORE_NAME = \"movies\"\n",
         "\n",
         "featurestore = aiplatform.Featurestore.create(\n",
         "    featurestore_id=FEATURESTORE_NAME,\n",

notebooks/community/ml_ops/stage2/get_started_vertex_tensorboard.ipynb

@@ -194,7 +194,7 @@
         "import os\n",
         "\n",
         "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
         "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
         "    \"/opt/deeplearning/metadata/env_version\"\n",
         ")\n",

notebooks/community/ml_ops/stage2/get_started_vertex_training.ipynb

@@ -40,7 +40,7 @@
         "  </td>\n",
         "  <td>\n",
         "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training.ipynb\">\n",
-        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\\\" alt=\"Colab logo\"> Run in Colab\n",
         "        </a>\n",
         "  </td>\n",
         "  <td>\n",
@@ -147,7 +147,7 @@
         "import os\n",
         "\n",
         "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
         "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
         "    \"/opt/deeplearning/metadata/env_version\"\n",
         ")\n",

notebooks/community/ml_ops/stage2/get_started_vertex_training_pytorch.ipynb

@@ -191,7 +191,7 @@
         "import os\n",
         "\n",
         "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
         "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
         "    \"/opt/deeplearning/metadata/env_version\"\n",
         ")\n",

notebooks/community/ml_ops/stage2/get_started_vertex_training_r.ipynb

@@ -40,7 +40,7 @@
         "  </td>\n",
         "  <td>\n",
         "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_r.ipynb\">\n",
-        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\\\" alt=\"Colab logo\"> Run in Colab\n",
         "        </a>\n",
         "  </td>\n",
         "  <td>\n",
@@ -148,7 +148,7 @@
         "import os\n",
         "\n",
         "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
         "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
         "    \"/opt/deeplearning/metadata/env_version\"\n",
         ")\n",

notebooks/community/ml_ops/stage2/get_started_vertex_training_sklearn.ipynb

@@ -50,6 +50,9 @@
         "    </a>\n",
         "  </td>\n",
         "</table>\n",
+        "    </a>\n",
+        "  </td>\n",
+        "</table>\n",
         "<br/><br/><br/>"
       ]
     },
@@ -183,7 +186,7 @@
         "import os\n",
         "\n",
         "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
         "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
         "    \"/opt/deeplearning/metadata/env_version\"\n",
         ")\n",

notebooks/community/ml_ops/stage2/get_started_vertex_training_xgboost.ipynb

@@ -136,7 +136,7 @@
         "import os\n",
         "\n",
         "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
         "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
         "    \"/opt/deeplearning/metadata/env_version\"\n",
         ")\n",

notebooks/community/ml_ops/stage2/get_started_vertex_vizier.ipynb

@@ -40,7 +40,7 @@
         "  </td>\n",
         "  <td>\n",
         "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_vizier.ipynb\">\n",
-        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\\\" alt=\"Colab logo\"> Run in Colab\n",
         "        </a>\n",
         "  </td>\n",
         "  <td>\n",
@@ -156,7 +156,7 @@
         "import os\n",
         "\n",
         "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
         "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
         "    \"/opt/deeplearning/metadata/env_version\"\n",
         ")\n",

notebooks/community/ml_ops/stage2/get_started_with_cmek_training.ipynb

@@ -39,7 +39,7 @@
         "  </td>\n",
         "  <td>\n",
         "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_with_cmek_training.ipynb\">\n",
-        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\\\" alt=\"Colab logo\"> Run in Colab\n",
         "        </a>\n",
         "  </td>\n",
         "  <td>\n",
@@ -139,7 +139,7 @@
         "import os\n",
         "\n",
         "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
         "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
         "    \"/opt/deeplearning/metadata/env_version\"\n",
         ")\n",

notebooks/community/ml_ops/stage2/get_started_with_logging.ipynb

@@ -1,684 +0,0 @@
-{
-  "cells": [
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "copyright"
-      },
-      "outputs": [],
-      "source": [
-        "# Copyright 2022 Google LLC\n",
-        "#\n",
-        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
-        "# you may not use this file except in compliance with the License.\n",
-        "# You may obtain a copy of the License at\n",
-        "#\n",
-        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
-        "#\n",
-        "# Unless required by applicable law or agreed to in writing, software\n",
-        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
-        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
-        "# See the License for the specific language governing permissions and\n",
-        "# limitations under the License."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "title:generic,gcp"
-      },
-      "source": [
-        "# E2E ML on GCP: MLOps stage 2 : experimentation: get started with Logging\n",
-        "\n",
-        "<table align=\"left\">\n",
-        "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_with_logging.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
-        "      View on GitHub\n",
-        "    </a>\n",
-        "  </td>\n",
-        "    <td>\n",
-        "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_with_logging.ipynb\">\n",
-        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
-        "        </a>\n",
-        "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage2/get_started_with_logging.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
-        "    </a>\n",
-        "  </td>\n",
-        "</table>\n",
-        "<br/><br/><br/>"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "overview:mlops"
-      },
-      "source": [
-        "## Overview\n",
-        "\n",
-        "\n",
-        "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 2 : experimentation: get started with Logging."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "objective:mlops,stage2,get_started_vertex_experiments"
-      },
-      "source": [
-        "### Objective\n",
-        "\n",
-        "In this tutorial, you learn how to use Python and Cloud logging awhen training with `Vertex AI`.\n",
-        "\n",
-        "This tutorial uses the following Google Cloud ML services:\n",
-        "\n",
-        "- `Cloud Logging`\n",
-        "\n",
-        "The steps performed include:\n",
-        "\n",
-        "- Use Python logging to log training configuration/results locally.\n",
-        "- Use Google Cloud Logging to log training configuration/results in cloud storage."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "recommendation:mlops,stage2,logging"
-      },
-      "source": [
-        "### Recommendations\n",
-        "\n",
-        "When doing E2E MLOps on Google Cloud, the following are some of the best practices for logging data when experimenting or formally training a model.\n",
-        "\n",
-        "#### Python Logging\n",
-        "\n",
-        "Use Python's logging package when doing ad-hoc training locally.\n",
-        "\n",
-        "#### Cloud Logging\n",
-        "\n",
-        "Use `Google Cloud Logging` when doing training on the cloud.\n",
-        "\n",
-        "#### Experiments\n",
-        "\n",
-        "Use Vertex AI Experiments in conjunction with logging when performing experiments to compare results for different experiment configurations.\n",
-        "\n",
-        "### Costs\n",
-        "This tutorial uses billable components of Google Cloud:\n",
-        "\n",
-        "- Vertex AI\n",
-        "\n",
-        "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing) and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "install_mlops"
-      },
-      "source": [
-        "## Installations\n",
-        "\n",
-        "Install the following packages for executing this notebook."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "install_mlops"
-      },
-      "outputs": [],
-      "source": [
-        "import os\n",
-        "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
-        "! pip3 install --upgrade google-cloud-logging $USER_FLAG -q"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "restart"
-      },
-      "source": [
-        "### Restart the kernel\n",
-        "\n",
-        "Once you've installed the additional packages, you need to restart the notebook kernel so it can find the packages."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "restart"
-      },
-      "outputs": [],
-      "source": [
-        "import os\n",
-        "\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    # Automatically restart kernel after installs\n",
-        "    import IPython\n",
-        "\n",
-        "    app = IPython.Application.instance()\n",
-        "    app.kernel.do_shutdown(True)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "project_id"
-      },
-      "source": [
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI, Compute Engine, Cloud Storage and Cloud Logging APIs](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component,storage_component,logging).\n",
-        "\n",
-        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands.\n",
-        "\n",
-        "\n",
-        "#### Set your project ID\n",
-        "\n",
-        "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "set_project_id"
-      },
-      "outputs": [],
-      "source": [
-        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "autoset_project_id"
-      },
-      "outputs": [],
-      "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
-        "    # Get your GCP project id from gcloud\n",
-        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
-        "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID:\", PROJECT_ID)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "set_gcloud_project_id"
-      },
-      "outputs": [],
-      "source": [
-        "! gcloud config set project $PROJECT_ID"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "region"
-      },
-      "source": [
-        "#### Region\n",
-        "\n",
-        "You can also change the `REGION` variable, which is used for operations\n",
-        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
-        "\n",
-        "- Americas: `us-central1`\n",
-        "- Europe: `europe-west4`\n",
-        "- Asia Pacific: `asia-east1`\n",
-        "\n",
-        "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
-        "\n",
-        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "region"
-      },
-      "outputs": [],
-      "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "timestamp"
-      },
-      "source": [
-        "#### Timestamp\n",
-        "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "timestamp"
-      },
-      "outputs": [],
-      "source": [
-        "from datetime import datetime\n",
-        "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "f3bd8c0d0469"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
-        "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
-        "\n",
-        "1. **Click Create service account**.\n",
-        "\n",
-        "2. In the **Service account name** field, enter a name, and click **Create**.\n",
-        "\n",
-        "3. In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex AI\" into the filter box, and select **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "4. Click Create. A JSON file that contains your key downloads to your local environment.\n",
-        "\n",
-        "5. Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "e0953a00668e"
-      },
-      "outputs": [],
-      "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = \"google.colab\" in sys.modules\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "setup_vars"
-      },
-      "source": [
-        "### Set up variables\n",
-        "\n",
-        "Next, set up some variables used throughout the tutorial.\n",
-        "### Import libraries"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "import_aip:mbsdk"
-      },
-      "outputs": [],
-      "source": [
-        "import logging\n",
-        "\n",
-        "import google.cloud.aiplatform as aiplatform"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "init_aip:mbsdk,region"
-      },
-      "source": [
-        "### Initialize Vertex AI SDK for Python\n",
-        "\n",
-        "Initialize the Vertex AI SDK for Python for your project and corresponding bucket."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "init_aip:mbsdk,region"
-      },
-      "outputs": [],
-      "source": [
-        "aiplatform.init(project=PROJECT_ID, location=REGION)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "python_logging"
-      },
-      "source": [
-        "## Python Logging\n",
-        "\n",
-        "The Python logging package is widely used for logging within Python scripts. Commonly used features:\n",
-        "\n",
-        "- Set logging levels.\n",
-        "- Send log output to console.\n",
-        "- Send log output to a file.\n",
-        "\n",
-        "### Logging Levels in Python Logging\n",
-        "\n",
-        "The logging levels in order (from least to highest) and each level inclusive of the previous level are :\n",
-        "\n",
-        "1. Informational\n",
-        "2. Warnings\n",
-        "3. Errors\n",
-        "4. Debugging\n",
-        "\n",
-        "By default, the logging level is set to error level.\n",
-        "\n",
-        "### Logging output to console\n",
-        "\n",
-        "By default, the Python logging package outputs to the console. Note, in the example the debug log message is not outputted since the default logging level is set to error."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "python_logging"
-      },
-      "outputs": [],
-      "source": [
-        "def logging_examples():\n",
-        "    logging.info(\"Model training started...\")\n",
-        "    logging.warning(\"Using older version of package ...\")\n",
-        "    logging.error(\"Training was terminated ...\")\n",
-        "    logging.debug(\"Hyperparameters were ...\")\n",
-        "\n",
-        "\n",
-        "logging_examples()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "python_logging_level"
-      },
-      "source": [
-        "### Setting logging level\n",
-        "\n",
-        "To set the logging level, you get the logging handler using `getLogger()`. You can have multiple logging handles. When `getLogger()` is called without any arguments, it gets the default handler named ROOT. With the handler, you set the logging level with the method `setLevel()`."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "python_logging_level"
-      },
-      "outputs": [],
-      "source": [
-        "logging.getLogger().setLevel(logging.DEBUG)\n",
-        "\n",
-        "logging_examples()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "python_logging_remove"
-      },
-      "source": [
-        "### Clearing handlers\n",
-        "\n",
-        "At times, you may desire to reconfigure your logging. A common practice in this case is to first remove all existing logging handles for a fresh start."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "python_logging_remove"
-      },
-      "outputs": [],
-      "source": [
-        "for handler in logging.root.handlers[:]:\n",
-        "    logging.root.removeHandler(handler)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "python_logging_file"
-      },
-      "source": [
-        "### Output to a local file\n",
-        "\n",
-        "You can preserve your logging output to a file that is local to where the Python script is running with the method `BasicConfig()`, that takes the following parameters:\n",
-        "\n",
-        "- `filename`: The file path to the local file to write the log output to.\n",
-        "- `level`: Sets the level of logging that is written to the logging file.\n",
-        "\n",
-        "*Note:* You cannot use a Cloud Storage bucket as the output file."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "python_logging_file"
-      },
-      "outputs": [],
-      "source": [
-        "logging.basicConfig(filename=\"mylog.log\", level=logging.DEBUG)\n",
-        "\n",
-        "logging_examples()\n",
-        "\n",
-        "! cat mylog.log"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "cloud_logging"
-      },
-      "source": [
-        "## Logging with Google Cloud Logging\n",
-        "\n",
-        "You can preserve and retrieve your logging output to `Google Cloud Logging` service. Commonly used features:\n",
-        "\n",
-        "- Set logging levels.\n",
-        "- Send log output to storage.\n",
-        "- Retrieve log output from storage.\n",
-        "\n",
-        "### Logging Levels in Cloud Logging\n",
-        "\n",
-        "The logging levels in order (from least to highest) are, with each level inclusive of the previous level:\n",
-        "\n",
-        "1. Informational\n",
-        "2. Warnings\n",
-        "3. Errors\n",
-        "4. Debugging\n",
-        "\n",
-        "By default, the logging level is set to warning level.\n",
-        "\n",
-        "### Configurable and storing log data.\n",
-        "\n",
-        "To use the `Google Cloud Logging` service, you do the following steps:\n",
-        "\n",
-        "1. Create a client to the service.\n",
-        "2. Obtain a handler for the service.\n",
-        "3. Create a logger instance and set logging level.\n",
-        "4. Attach logger instance to the service.\n",
-        "\n",
-        "Learn more about [Logging client libraries](https://cloud.google.com/logging/docs/reference/libraries)."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "cloud_logging"
-      },
-      "outputs": [],
-      "source": [
-        "import google.cloud.logging\n",
-        "from google.cloud.logging.handlers import CloudLoggingHandler\n",
-        "\n",
-        "# Connect to the Cloud Logging service\n",
-        "cl_client = google.cloud.logging.Client(project=PROJECT_ID)\n",
-        "handler = CloudLoggingHandler(cl_client, name=\"mylog\")\n",
-        "\n",
-        "# Create a logger instance and logging level\n",
-        "cloud_logger = logging.getLogger(\"cloudLogger\")\n",
-        "cloud_logger.setLevel(logging.INFO)\n",
-        "\n",
-        "# Attach the logger instance to the service.\n",
-        "cloud_logger.addHandler(handler)\n",
-        "\n",
-        "# Log something\n",
-        "cloud_logger.error(\"bad news\")"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "cloud_logging_write"
-      },
-      "source": [
-        "### Logging output\n",
-        "\n",
-        "Logging output at specific levels is identical to Python logging with respect to method and method names. The only difference is that you use your instance of the cloud logger in place of logging."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "cloud_logging_write"
-      },
-      "outputs": [],
-      "source": [
-        "cloud_logger.info(\"Model training started...\")\n",
-        "cloud_logger.warning(\"Using older version of package ...\")\n",
-        "cloud_logger.error(\"Training was terminated ...\")\n",
-        "cloud_logger.debug(\"Hyperparameters were ...\")"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "cloud_logging_list"
-      },
-      "source": [
-        "### Get logging entries\n",
-        "\n",
-        "To get the logged output, you:\n",
-        "\n",
-        "1. Retrieve the log handle to the service.\n",
-        "2. Using the handle, call the method `list_entries()`.\n",
-        "3. Iterate through the entries."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "cloud_logging_list"
-      },
-      "outputs": [],
-      "source": [
-        "logger = cl_client.logger(\"mylog\")\n",
-        "\n",
-        "for entry in logger.list_entries():\n",
-        "    timestamp = entry.timestamp.isoformat()\n",
-        "    print(\"* {}: {}: {}\".format(timestamp, entry.severity, entry.payload))"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "cleanup:mbsdk"
-      },
-      "source": [
-        "# Cleaning up\n",
-        "\n",
-        "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
-        "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
-        "\n",
-        "Otherwise, you can delete the individual resources you created in this tutorial."
-      ]
-    }
-  ],
-  "metadata": {
-    "colab": {
-      "name": "get_started_with_logging.ipynb",
-      "toc_visible": true
-    },
-    "kernelspec": {
-      "display_name": "Python 3",
-      "name": "python3"
-    }
-  },
-  "nbformat": 4,
-  "nbformat_minor": 0
-}

notebooks/community/ml_ops/stage2/get_started_with_tabnet.ipynb

@@ -133,7 +133,7 @@
         "import os\n",
         "\n",
         "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
         "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
         "    \"/opt/deeplearning/metadata/env_version\"\n",
         ")\n",

notebooks/community/ml_ops/stage2/get_started_with_tfhub_models.ipynb

@@ -40,7 +40,7 @@
         "  </td>\n",
         "  <td>\n",
         "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_with_tfhub_models.ipynb\">\n",
-        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\\\" alt=\"Colab logo\"> Run in Colab\n",
         "        </a>\n",
         "  </td>\n",
         "  <td>\n",
@@ -145,7 +145,7 @@
         "import os\n",
         "\n",
         "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
         "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
         "    \"/opt/deeplearning/metadata/env_version\"\n",
         ")\n",

notebooks/community/ml_ops/stage2/get_started_with_visionapi_and_automl.ipynb

@@ -84,11 +84,11 @@
       "source": [
         "### Dataset\n",
         "\n",
-        "The dataset used for this tutorial is the [Patent PDF Samples with Extracted Structured Data](https://console.cloud.google.com/marketplace/product/global-patents/labeled-patents) from Google Public Data Sets. \n",
+        "The dataset used for this tutorial is the [Patent PDF Samples with Extracted Structured Data](https://pantheon.corp.google.com/marketplace/details/global-patents/labeled-patents?project=kudos-333820) from Google Public Data Sets. \n",
         "\n",
         "This dataset includes data extracted from over 300 patent documents issued in the US and EU. The dataset includes links to Google Cloud Storage blobs for the first page of each patent, in addition to a number of extracted entities. \n",
         "\n",
-        "The data is published as a [public dataset](https://cloud.google.com/bigquery/public-data) on `BigQuery`."
+        "The data is published as a public dataset on `BigQuery`."
       ]
     },
     {
@@ -201,7 +201,7 @@
         "import os\n",
         "\n",
         "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
         "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
         "    \"/opt/deeplearning/metadata/env_version\"\n",
         ")\n",

notebooks/community/ml_ops/stage2/mlops_experimentation.ipynb

@@ -40,7 +40,7 @@
         "  </td>\n",
         "  <td>\n",
         "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/mlops_experimentation.ipynb\">\n",
-        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\\\" alt=\"Colab logo\"> Run in Colab\n",
         "        </a>\n",
         "  </td>\n",
         "  <td>\n",
@@ -182,7 +182,7 @@
         "import os\n",
         "\n",
         "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
         "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
         "    \"/opt/deeplearning/metadata/env_version\"\n",
         ")\n",

notebooks/community/ml_ops/stage3/README.md

@@ -27,7 +27,7 @@ The third stage in MLOps is formalization to develop an automated pipeline proce
 - Use early stop procedure in training script to detect failure to achieve training objective.
 - Store the results of the trained model evaluation in Vertex AI ML Metadata.
 
-<img src='stage3v3.png'>
+<img src='stage3.png'>
 
 ## Notebooks
 
@@ -166,50 +166,6 @@ The steps performed in this tutorial include:
 - Execute pipeline using customjob-level settings for machine resources
 ```
 
-[Get Started with Apache Airflow and Vertex AI Pipelines](get_started_with_airflow_and_vertex_pipelines.ipynb)
-
-```
-The steps performed in this tutorial include:
-
-- Create Cloud Composer environment.
-- Upload Airflow DAG to Composer environment that performs data processing -- i.e., creates a BigQuery table from a CSV file.
-- Create a Vertex Pipeline that triggers the Airflow DAG.
-- Execute the `Vertex AI Pipeline`.
-```
-
-
-[Get Started with Vertex AI Model Registry](get_started_with_model_registry.ipynb)
-
-```
-The steps performed in this tutorial include:
-
-- Create and register a first version of a model to `Vertex AI Model Registry`
-- Create and register a second version of a model to `Vertex AI Model Registry`
-- List all versions of a `Model` resource.
-- Change the default version of a `Model` resource`
-- Deploy the default version of a `Model` resource.
-- Delete a model version from a `Model` resource.
-- Delete a `Model` resource along with all model versions.
-```
-
-[Get Started with AutoML Tabular Pipeline Workflow](get_started_with_automl_tabular_pipeline_workflow.ipynb)
-
-```
-The steps performed in this tutorial include:
-
-- Define training specification.
-    - Dataset specification
-    - Hyperparameter overide specification
-    - machine specifications
-- Construct tabular workflow pipeline.
-- Compile and execute pipeline.
-- View evaluation metrics artifact.
-- Export AutoML model as an OSS TF model.
-- Create `Endpoint` resource.
-- Deploy exported OSS TF model.
-- Make a prediction.
-```
-
 ### E2E Stage Example
 
 [Stage 3: Formalization](mlops_formalization.ipynb)

notebooks/community/ml_ops/stage3/get_started_with_automl_pipeline_components.ipynb

@@ -40,7 +40,7 @@
         "  </td>\n",
         "    <td>\n",
         "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_automl_pipeline_components.ipynb\">\n",
-        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\\\" alt=\"Colab logo\"> Run in Colab\n",
         "        </a>\n",
         "  </td>\n",
         "  <td>\n",
@@ -133,7 +133,7 @@
         "import os\n",
         "\n",
         "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
         "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
         "    \"/opt/deeplearning/metadata/env_version\"\n",
         ")\n",

notebooks/community/ml_ops/stage3/get_started_with_bq_tfdv_pipeline_components.ipynb

@@ -143,7 +143,7 @@
         "import os\n",
         "\n",
         "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
         "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
         "    \"/opt/deeplearning/metadata/env_version\"\n",
         ")\n",

notebooks/community/ml_ops/stage3/get_started_with_bqml_pipeline_components.ipynb

@@ -149,7 +149,7 @@
         "import os\n",
         "\n",
         "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
         "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
         "    \"/opt/deeplearning/metadata/env_version\"\n",
         ")\n",

notebooks/community/ml_ops/stage3/get_started_with_custom_training_pipeline_components.ipynb

@@ -153,7 +153,7 @@
         "import os\n",
         "\n",
         "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
         "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
         "    \"/opt/deeplearning/metadata/env_version\"\n",
         ")\n",

notebooks/community/ml_ops/stage4/README.md

@@ -36,7 +36,7 @@ This stage may be done entirely by MLOps. We recommend:
 
 
 
-<img src='stage4v3.png'>
+<img src='stage4.png'>
 
 ## Notebooks
 
@@ -75,25 +75,6 @@ The steps performed include:
     - Query your pipeline run metadata.
 ```
 
-[Get started with Vertex ML Metadata and AutoML](get_started_with_vertex_ml_metadata_and_automl.ipynb)
-
-```
-The steps performed include:
-
-- Create a `Dataset` resource.
-- Create a corresponding `google.VertexDataset` artifact.
-- Train a model using `AutoML`.
-- Create a corresponding `google.VertexModel` artifact.
-- Create an `Endpoint` resource.
-- Create a corresponding `google.Endpoint` artifact.
-- Deploy the train model to the `Endpoint`.
-- Create an execution and context for the `AutoML` training job and deployment.
-- Add the corresponding artifacts and context to the execution.
-- Add artifact links (event) to the execution.
-- Display the execution graph.
-```
-
-
 Get started with custom model evaluation
 
 Get started with A/B Testing

notebooks/community/ml_ops/stage4/get_started_with_vertex_ml_metadata.ipynb

@@ -576,7 +576,7 @@
     "\n",
     "Setup up the following constants for Vertex AI:\n",
     "\n",
-    "- `API_ENDPOINT`: The Vertex AI API service endpoint for `ML Metadata` services."
+    "- `API_ENDPOINT`: The Vertex AI API service endpoint for `FeatureStore` services."
    ]
   },
   {

notebooks/community/ml_ops/stage4/get_started_with_vertex_xai.ipynb

@@ -29,7 +29,7 @@
         "id": "title"
       },
       "source": [
-        "# Vertex SDK: E2E ML on GCP: MLOps stage 4 : evaluation: get started with Vertex AI Explanations\n",
+        "# Vertex SDK: E2E ML on GCP: MLOps stage 4 : formalization: get started with Vertex AI Explanations\n",
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",

notebooks/community/ml_ops/stage5/README.md

@@ -2,23 +2,11 @@
 
 ## Purpose
 
-Configure compute and networking requirements for containerized serving binaries for a production load.
-
 
 ## Recommendations  
 
-The fifth stage in MLOps is deployment to production of the blessed model, which will replace the previous blessed model in production. This stage may be done entirely by MLOps. We recommend:
 
-- Deploy the blessed model from the Vertex Model Registry.
-- Use the Google Container Registry for the deployment container.
-- Attach, if any, serving function from the Vertex Model Registry to the deployed model.
-- Use Vertex Pipelines for the deployment.
-- For cloud models, deploy within the Google Cloud infrastructure.
-- Use Vertex Prediction traffic split for production rollout.
-- Use Vertex Prediction to set your criteria for scaling and load balancing.
-
-
-<img src='stage5v3.png'>
+<img src='stage5.png'>
 
 ## Notebooks
 
@@ -57,35 +45,3 @@ The steps performed include:
 - Deploying a `Model` resource to a `Private Endpoint` resource.
 - Send a prediction request to a `Private Endpoint`
 ```
-
-[Get started with Vertex AI Endpoints and co-hosting models on shared VM](get_started_with_vertex_endpoint_and_shared_vm.ipynb)
-
-```
-The steps performed include:
-
-- Upload a pre-trained image classification model as a `Model` resource (model A).
-- Upload a pre-trained text sentence encoder model as a `Model` resource (model B).
-- Create a shared VM deployment resource pool.
-- List shared VM deployment resource pools.
-- Create two `Endpoint` resources.
-- Deploy first model (model A) to first `Endpoint` resource using shared VM deployment resource pool.
-- Deploy second model (model B) to second `Endpoint` resource using shared VM deployment resource pool.
-- Make a prediction request with first deployed model (model A).
-- Make a prediction request with second deployed model (model B).
-```
-
-[Get started with Auto-Scaling for Vertex AI Endpoints](get_started_with_autoscaling.ipynb)
-
-```
-The steps performed include:
-
-- Download a pretrained image classification model from TensorFlow Hub.
-- Upload the pretrained model as a `Model` resource.
-- Create an `Endpoint` resource.
-- Deploy `Model` resource for no-scaling (single node).
-- Deploy `Model` resource for manual scaling.
-- Deploy `Model` resource for auto-scaling.
-- Fine-tune scaling thresholds for CPU utilization.
-- Fine-tune scaling thresholds for GPU utilization.
-- Deploy mix of CPU and GPU model instances with auto-scaling to an `Endpoint` resource.
-```

notebooks/community/ml_ops/stage5/get_started_with_vertex_endpoints.ipynb

@@ -29,7 +29,7 @@
         "id": "title:generic,gcp"
       },
       "source": [
-        "# E2E ML on GCP: MLOps stage 5 : deployment: Get started with Vertex AI Endpoints\n",
+        "# E2E ML on GCP: MLOps stage 5 : Get started with Vertex AI Endpoints\n",
         "<table align=\"left\">\n",
         "  <td>\n",
         "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage5/get_started_with_vertex_endpoints.ipynb\">\n",

notebooks/community/ml_ops/stage5/get_started_with_vertex_private_endpoints.ipynb

@@ -171,6 +171,20 @@
         "! pip3 install --upgrade tensorflow $USER_FLAG -q"
       ]
     },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "38379eb00a31"
+      },
+      "outputs": [],
+      "source": [
+        "# Temporary, until feature pushed to Pypi\n",
+        "! pip3 uninstall google-cloud-aiplatform -y\n",
+        "\n",
+        "! pip install --user git+https://github.com/googleapis/python-aiplatform.git@private-ep"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {

notebooks/community/ml_ops/stage6/README.md

@@ -23,7 +23,9 @@ This stage may be done entirely by MLOps. We recommend:
 - Features that dynamically change per example (e.g., bank balance) are stored in Vertex Feature Store.
 
 
-<img src='stage6v2.png'>
+<img src='stage6a.png'>
+<img src='stage6b.png'>
+<img src='stage6c.png'>
 
 ## Notebooks
 
@@ -175,30 +177,4 @@ The steps performed include:
 
 ```
 The steps performed include:
-
-1. Train the `Swivel` algorithm to generate embeddings (encoder) for the dataset.
-2. Hyperparameter tune the trained `Swivel` encoder.
-3. Make example predictions (embeddings) from then trained encoder.
-4. Generate embeddings using the trained `Swivel` builtin algorithm.
-5. Store embeddings to format supported by `Matching Engine`.
-6. Create a `Matching Engine Index` for the embeddings.
-7. Deploy the `Matching Engine Index` to a `Index Endpoint`.
-8. Make a matching engine prediction request.
-```
-
-[Get started with Explainable AI and custom model server](get_started_with_xai_and_custom_server.ipynb)
-
-```
-The steps performed include:
-
-- Locally train a Pytorch tabular classifier.
-- Locally test the trained model.
-- Build a HTTP server using FastAPI.
-- Create a custom serving container with the trained model and FastAPI server.
-- Locally test the custom serving container.
-- Push the custom serving container to the Artifact Registry.
-- Upload the custom serving container as a `Model` resource.
-- Deploy the `Model` resource to an `Endpoint` resource.
-- Make a prediction request to the deployed custom serving container.
-- Make an explanation request to the deployed custom serving container.
 ```

notebooks/community/ml_ops/stage6/get_started_with_matching_engine_twotowers.ipynb

@@ -1951,6 +1951,15 @@
         "full_network_name = f\"projects/{PROJECT_NUMBER}/global/networks/{NETWORK}\""
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Z4f5yEmatu8P"
+      },
+      "source": [
+        "The following function calls the deployed Vertex Prediction model using the sample query object input file. Note that it uses the model resource directly and doesn't require a deployed endpoint. Once you start the job, you can track its status on the [Cloud Console](https://console.cloud.google.com/vertex-ai/batch-predictions)."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {

notebooks/community/ml_ops/stage7/README.md

@@ -1,36 +0,0 @@
-# Stage 7: Monitoring
-
-## Purpose
-
-Monitor predict requests to detect model degradation and alert or trigger degradation response procedures. 
-
-## Recommendations  
-
-Degradation includes, but not limited to:
-
-1. Health - deterioration in the operational performance of the serving binary.
-2. Latency - deterioration in the elapsed time to transmit a prediction response from the serving binary.
-3. Serving skew - detection of a distribution difference between the training data and the data seen at serving. This may be either or both the features of the input or the prediction of the output.
-4. Data drift - detection of a change of distribution in the input features of the serving data over time.
-5. Concept drift - degradation of the business objective.
-
-For cases of skew and drift, random samples of the serving requests/responses are collected in the serving binary. Another process continuously inspects the distribution of the random collected samples. This process may either alter or initiate retraining of the model once skew or drift exceeds pre-specified thresholds.
-
-In the case of concept drift, one may initiate a rollback of the blessed model and/or renewed A/B testing of the blessed model and previous blessed models.
-
-This stage may be done entirely by MLOps. We recommend:
-
-- When manually inspecting the operation of the serving binary, attach to the serving binary using the Vertex Serving Binary Debugger.
-- Use Google Cloud network monitoring to monitor the operational health of the serving binary.
-- Store network monitoring logs in Cloud Storage and view logs using StackDriver.
-- Use Vertex AI Model Monitoring to random sample prediction requests/responses and to measure distributions for skew and drift.
-- Store sampled prediction requests/responses in Big Query.
-- Use Vertex ML Metadata to periodically record serving distribution statistics.
-- Use Vertex Explainable AI to manually inspect for concept drift in business objectives.
-- Use Cloud Pub/Sub to automatically trigger re-training pipeline.
-
-<img src='stage7v2.png'>
-
-## Notebooks
-
-### Get Started

notebooks/community/neo4j/graph_paysim.ipynb

@@ -8,7 +8,7 @@
       },
       "outputs": [],
       "source": [
-        "# Copyright 2022 Google LLC\n",
+        "# Copyright 2021 Google LLC\n",
         "#\n",
         "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
         "# you may not use this file except in compliance with the License.\n",
@@ -42,11 +42,6 @@
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/laeg/vertex-ai-samples/main/notebooks/community/neo4j/graph_paysim.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">Open in Vertex AI Workbench\n",
-        "    </a>\n",
-        "</td>\n",
         "</table>"
       ]
     },
@@ -148,7 +143,7 @@
       },
       "outputs": [],
       "source": [
-        "!pip install --quiet --upgrade graphdatascience==1.0.0"
+        "!pip install --quiet --upgrade neo4j"
       ]
     },
     {
@@ -260,7 +255,7 @@
       "outputs": [],
       "source": [
         "import pandas as pd\n",
-        "from graphdatascience import GraphDataScience"
+        "from neo4j import GraphDatabase"
       ]
     },
     {
@@ -271,19 +266,7 @@
       },
       "outputs": [],
       "source": [
-        "# If you are connecting the client to an AuraDS instance, you can get the recommended non-default configuration settings of the Python Driver applied automatically. To achieve this, set the constructor argument aura_ds=True\n",
-        "gds = GraphDataScience(DB_URL, auth=(DB_USER, DB_PASS), aura_ds=True)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "f14915ddd1fb"
-      },
-      "outputs": [],
-      "source": [
-        "gds.set_database(DB_NAME)"
+        "driver = GraphDatabase.driver(DB_URL, auth=(DB_USER, DB_PASS))"
       ]
     },
     {
@@ -304,16 +287,19 @@
       "outputs": [],
       "source": [
         "# node labels\n",
-        "result = gds.run_cypher(\n",
+        "with driver.session(database=DB_NAME) as session:\n",
+        "    result = session.read_transaction(\n",
+        "        lambda tx: tx.run(\n",
+        "            \"\"\"\n",
+        "    CALL db.labels() YIELD label\n",
+        "    CALL apoc.cypher.run('MATCH (:`'+label+'`) RETURN count(*) as freq', {})\n",
+        "    YIELD value\n",
+        "    RETURN label, value.freq AS freq\n",
         "    \"\"\"\n",
-        "CALL db.labels() YIELD label\n",
-        "CALL apoc.cypher.run('MATCH (:`'+label+'`) RETURN count(*) as freq', {})\n",
-        "YIELD value\n",
-        "RETURN label, value.freq AS freq\n",
-        "\"\"\"\n",
-        ")\n",
-        "\n",
-        "display(result)"
+        "        ).data()\n",
+        "    )\n",
+        "df = pd.DataFrame(result)\n",
+        "display(df)"
       ]
     },
     {
@@ -325,17 +311,20 @@
       "outputs": [],
       "source": [
         "# relationship types\n",
-        "result = gds.run_cypher(\n",
-        "    \"\"\"\n",
-        "CALL db.relationshipTypes() YIELD relationshipType as type\n",
-        "CALL apoc.cypher.run('MATCH ()-[:`'+type+'`]->() RETURN count(*) as freq', {})\n",
-        "YIELD value\n",
-        "RETURN type AS relationshipType, value.freq AS freq\n",
-        "ORDER by freq DESC\n",
-        "\"\"\"\n",
-        ")\n",
-        "\n",
-        "display(result)"
+        "with driver.session(database=DB_NAME) as session:\n",
+        "    result = session.read_transaction(\n",
+        "        lambda tx: tx.run(\n",
+        "            \"\"\"\n",
+        "      CALL db.relationshipTypes() YIELD relationshipType as type\n",
+        "      CALL apoc.cypher.run('MATCH ()-[:`'+type+'`]->() RETURN count(*) as freq', {})\n",
+        "      YIELD value\n",
+        "      RETURN type AS relationshipType, value.freq AS freq\n",
+        "      ORDER by freq DESC\n",
+        "      \"\"\"\n",
+        "        ).data()\n",
+        "    )\n",
+        "df = pd.DataFrame(result)\n",
+        "display(df)"
       ]
     },
     {
@@ -347,20 +336,23 @@
       "outputs": [],
       "source": [
         "# transaction types\n",
-        "result = gds.run_cypher(\n",
+        "with driver.session(database=DB_NAME) as session:\n",
+        "    result = session.read_transaction(\n",
+        "        lambda tx: tx.run(\n",
+        "            \"\"\"\n",
+        "    MATCH (t:Transaction)\n",
+        "    WITH sum(t.amount) AS globalSum, count(t) AS globalCnt\n",
+        "    WITH *, 10^3 AS scaleFactor\n",
+        "    UNWIND ['CashIn', 'CashOut', 'Payment', 'Debit', 'Transfer'] AS txType\n",
+        "      CALL apoc.cypher.run('MATCH (t:' + txType + ')\n",
+        "        RETURN sum(t.amount) as txAmount, count(t) AS txCnt', {})\n",
+        "      YIELD value\n",
+        "    RETURN txType,value.txAmount AS TotalMarketValue\n",
         "    \"\"\"\n",
-        "      MATCH (t:Transaction)\n",
-        "      WITH sum(t.amount) AS globalSum, count(t) AS globalCnt\n",
-        "      WITH *, 10^3 AS scaleFactor\n",
-        "      UNWIND ['CashIn', 'CashOut', 'Payment', 'Debit', 'Transfer'] AS txType\n",
-        "        CALL apoc.cypher.run('MATCH (t:' + txType + ')\n",
-        "          RETURN sum(t.amount) as txAmount, count(t) AS txCnt', {})\n",
-        "        YIELD value\n",
-        "      RETURN txType,value.txAmount AS TotalMarketValue\n",
-        "    \"\"\"\n",
-        ")\n",
-        "\n",
-        "display(result)"
+        "        ).data()\n",
+        "    )\n",
+        "df = pd.DataFrame(result)\n",
+        "display(df)"
       ]
     },
     {
@@ -383,14 +375,18 @@
       },
       "outputs": [],
       "source": [
-        "# We get a tuple back with an object that represents the graph projection and the results of the GDS call\n",
-        "G, results = gds.graph.project.cypher(\n",
-        "    \"client_graph\",\n",
-        "    \"MATCH (c:Client) RETURN id(c) as id, c.num_transactions as num_transactions, c.total_transaction_amnt as total_transaction_amnt, c.is_fraudster as is_fraudster\",\n",
-        "    'MATCH (c:Client)-[:PERFORMED]->(t:Transaction)-[:TO]->(c2:Client) return id(c) as source, id(c2) as target, sum(t.amount) as amount, \"TRANSACTED_WITH\" as type ',\n",
-        ")\n",
-        "\n",
-        "display(results)"
+        "with driver.session(database=DB_NAME) as session:\n",
+        "    result = session.read_transaction(\n",
+        "        lambda tx: tx.run(\n",
+        "            \"\"\"\n",
+        "    CALL gds.graph.create.cypher('client_graph', \n",
+        "      'MATCH (c:Client) RETURN id(c) as id, c.num_transactions as num_transactions, c.total_transaction_amnt as total_transaction_amnt, c.is_fraudster as is_fraudster',\n",
+        "      'MATCH (c:Client)-[:PERFORMED]->(t:Transaction)-[:TO]->(c2:Client) return id(c) as source, id(c2) as target, sum(t.amount) as amount, \"TRANSACTED_WITH\" as type ')\n",
+        "    \"\"\"\n",
+        "        ).data()\n",
+        "    )\n",
+        "df = pd.DataFrame(result)\n",
+        "display(df)"
       ]
     },
     {
@@ -410,19 +406,25 @@
       },
       "outputs": [],
       "source": [
-        "results = gds.fastRP.mutate(\n",
-        "    G,\n",
-        "    relationshipWeightProperty=\"amount\",\n",
-        "    iterationWeights=[0.0, 1.00, 1.00, 0.80, 0.60],\n",
-        "    featureProperties=[\"num_transactions\", \"total_transaction_amnt\"],\n",
-        "    propertyRatio=0.25,\n",
-        "    nodeSelfInfluence=0.15,\n",
-        "    embeddingDimension=16,\n",
-        "    randomSeed=1,\n",
-        "    mutateProperty=\"embedding\",\n",
-        ")\n",
-        "\n",
-        "display(result)"
+        "with driver.session(database=DB_NAME) as session:\n",
+        "    result = session.read_transaction(\n",
+        "        lambda tx: tx.run(\n",
+        "            \"\"\"\n",
+        "    CALL gds.fastRP.mutate('client_graph',{\n",
+        "      relationshipWeightProperty:'amount',\n",
+        "      iterationWeights: [0.0, 1.00, 1.00, 0.80, 0.60],\n",
+        "      featureProperties: ['num_transactions', 'total_transaction_amnt'],\n",
+        "      propertyRatio: 0.25, \n",
+        "      nodeSelfInfluence: 0.15,\n",
+        "      embeddingDimension: 16,\n",
+        "      randomSeed: 1, \n",
+        "      mutateProperty:'embedding'\n",
+        "    })\n",
+        "    \"\"\"\n",
+        "        ).data()\n",
+        "    )\n",
+        "df = pd.DataFrame(result)\n",
+        "display(df)"
       ]
     },
     {
@@ -442,11 +444,19 @@
       },
       "outputs": [],
       "source": [
-        "node_properties = gds.graph.streamNodeProperties(\n",
-        "    G, [\"embedding\", \"num_transactions\", \"total_transaction_amnt\", \"is_fraudster\"]\n",
-        ")\n",
-        "\n",
-        "node_properties.head()"
+        "with driver.session(database=DB_NAME) as session:\n",
+        "    result = session.read_transaction(\n",
+        "        lambda tx: tx.run(\n",
+        "            \"\"\"\n",
+        "    CALL gds.graph.streamNodeProperties\n",
+        "    ('client_graph', ['embedding', 'num_transactions', 'total_transaction_amnt', 'is_fraudster'])\n",
+        "    YIELD nodeId, nodeProperty, propertyValue\n",
+        "    RETURN nodeId, nodeProperty, propertyValue\n",
+        "    \"\"\"\n",
+        "        ).data()\n",
+        "    )\n",
+        "df = pd.DataFrame(result)\n",
+        "df.head()"
       ]
     },
     {
@@ -466,9 +476,7 @@
       },
       "outputs": [],
       "source": [
-        "x = node_properties.pivot(\n",
-        "    index=\"nodeId\", columns=\"nodeProperty\", values=\"propertyValue\"\n",
-        ")\n",
+        "x = df.pivot(index=\"nodeId\", columns=\"nodeProperty\", values=\"propertyValue\")\n",
         "x = x.reset_index()\n",
         "x.columns.name = None\n",
         "x.head()"
@@ -691,8 +699,8 @@
         "id": "ArK3cfKsdT1x"
       },
       "source": [
-        "## Train and deploy a model with Vertex AI\n",
-        "We'll use the engineered features to train an AutoML Tabular Data, then deploy it to an endpoint"
+        "## Train and deploy a model on GCP\n",
+        "We'll use the engineered features to train an AutoML Tables model, then deploy it to an endpoint"
       ]
     },
     {
@@ -774,8 +782,8 @@
         "id": "-NnDaATyWY7z"
       },
       "source": [
-        "## Loading Data into Vertex AI Feature Store\n",
-        "In this section, we'll take our dataframe with newly engineered features and load that into Vertex AI Feature Store."
+        "## Loading Data into GCP Feature Store\n",
+        "In this section, we'll take our dataframe with newly engineered features and load that into GCP feature store."
       ]
     },
     {
@@ -1075,7 +1083,14 @@
       },
       "outputs": [],
       "source": [
-        "gds.graph.drop(G)"
+        "with driver.session(database=DB_NAME) as session:\n",
+        "    result = session.read_transaction(\n",
+        "        lambda tx: tx.run(\n",
+        "            \"\"\"\n",
+        "        CALL gds.graph.drop('client_graph')\n",
+        "        \"\"\"\n",
+        "        ).data()\n",
+        "    )"
       ]
     },
     {

notebooks/community/sdk/SDK_BigQuery_Custom_Container_Training.ipynb

@@ -8,7 +8,7 @@
       },
       "outputs": [],
       "source": [
-        "# Copyright 2022 Google LLC\n",
+        "# Copyright 2021 Google LLC\n",
         "#\n",
         "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
         "# you may not use this file except in compliance with the License.\n",
@@ -26,99 +26,25 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "et4hRnB9mrau"
+        "id": "976753012196"
       },
       "source": [
-        "<table align=\"left\">\n",
-        "\n",
-        "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/sdk/SDK_BigQuery_Custom_Container_Training.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
-        "    </a>\n",
-        "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/sdk/SDK_BigQuery_Custom_Container_Training.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
-        "      View on GitHub\n",
-        "    </a>\n",
-        "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/sdk/SDK_BigQuery_Custom_Container_Training.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
-        "    </a>\n",
-        "  </td>                                                                                               \n",
-        "</table>"
+        "# Feedback or issues?\n",
+        "For any feedback or questions, please open an [issue](https://github.com/googleapis/python-aiplatform/issues)."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "wLMxmUTwn1td"
+        "id": "8c3048cd1427"
       },
       "source": [
-        "###  Overview \n",
-        "To use this Jupyter notebook, copy the notebook to a Google Cloud Notebooks instance and open it. You can run each step, or cell, and see its results. To run a cell, use Shift+Enter. Jupyter automatically displays the return value of the last line in each cell. For more information about running notebooks in Google Cloud Notebook, see the Google Cloud Notebook guide.. \n",
-        "\n",
-        "### Objective \n",
+        "# Vertex SDK for Python: BigQuery Custom Container Training Example\n",
+        "To use this Jupyter notebook, copy the notebook to a Google Cloud Notebooks instance and open it. You can run each step, or cell, and see its results. To run a cell, use Shift+Enter. Jupyter automatically displays the return value of the last line in each cell. For more information about running notebooks in Google Cloud Notebook, see the [Google Cloud Notebook guide](https://cloud.google.com/vertex-ai/docs/general/notebooks).\n",
         "\n",
         "This notebook demonstrate how to create a Custom Model using Custom Container Training and a Big Query Dataset. It will require you provide a bucket where the dataset will be stored.\n",
         "\n",
-        "Costs  \n",
-        "This tutorial uses billable components of Google Cloud: \n",
-        "\n",
-        "Vertex AI\n",
-        "Cloud Storage\n",
-        "Learn about Vertex AI  pricing and Cloud Storage pricing, and use the Pricing Calculator to generate a cost estimate based on your projected usage."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "g5dkyDy1obku"
-      },
-      "source": [
-        "### Set up your local development environment\n",
-        "\n",
-        "**If you are using Colab or Google Cloud Notebooks**, your environment already meets\n",
-        "all the requirements to run this notebook. You can skip this step."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "gyja44LCozU_"
-      },
-      "source": [
-        "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
-        "You need the following:\n",
-        "\n",
-        "* The Google Cloud SDK\n",
-        "* Git\n",
-        "* Python 3\n",
-        "* virtualenv\n",
-        "* Jupyter notebook running in a virtual environment with Python 3\n",
-        "\n",
-        "The Google Cloud guide to [Setting up a Python development\n",
-        "environment](https://cloud.google.com/python/setup) and the [Jupyter\n",
-        "installation guide](https://jupyter.org/install) provide detailed instructions\n",
-        "for meeting these requirements. The following steps provide a condensed set of\n",
-        "instructions:\n",
-        "\n",
-        "1. [Install and initialize the Cloud SDK.](https://cloud.google.com/sdk/docs/)\n",
-        "\n",
-        "1. [Install Python 3.](https://cloud.google.com/python/setup#installing_python)\n",
-        "\n",
-        "1. [Install\n",
-        "   virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv)\n",
-        "   and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
-        "\n",
-        "1. To install Jupyter, run `pip3 install jupyter` on the\n",
-        "command-line in a terminal shell.\n",
-        "\n",
-        "1. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
-        "\n",
-        "1. Open this notebook in the Jupyter Notebook Dashboard."
+        "Note: you may incur charges for training, prediction, storage or usage of other GCP products in connection with testing this SDK."
       ]
     },
     {
@@ -139,7 +65,7 @@
         "id": "xOMNWzTbftDr"
       },
       "source": [
-        "# Install Vertex AI SDK for Python\n",
+        "# Install Vertex SDK for Python\n",
         "\n",
         "\n",
         "After the SDK installation the kernel will be automatically restarted."
@@ -180,223 +106,10 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
-        "PROJECT_ID = \"\"\n",
-        "\n",
-        "# Get your Google Cloud project ID from gcloud\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    shell_output = !gcloud config list --format 'value(core.project)' 2>/dev/null\n",
-        "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID: \", PROJECT_ID)\n",
-        "\n",
+        "MY_PROJECT = \"YOUR PROJECT ID\"\n",
         "MY_STAGING_BUCKET = \"gs://YOUR BUCKET\"  # bucket should be in same region as ucaip"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "_HRQg6eXiolk"
-      },
-      "source": [
-        "Otherwise, set your project ID here."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "Sg6AbQRviolk"
-      },
-      "outputs": [],
-      "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
-        "    PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}\n",
-        "    print(\"Project ID: \", PROJECT_ID)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "6x6CSodKjMmg"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Google Cloud Notebooks**, your environment is already\n",
-        "authenticated. Skip this step"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "ZaQd5jNwjP_0"
-      },
-      "source": [
-        "**If you are using Colab**, run the cell below and follow the instructions\n",
-        "when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "1. In the Cloud Console, go to the [**Create service account key**\n",
-        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
-        "\n",
-        "2. Click **Create service account**.\n",
-        "\n",
-        "3. In the **Service account name** field, enter a name, and\n",
-        "   click **Create**.\n",
-        "\n",
-        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
-        "into the filter box, and select\n",
-        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
-        "local environment.\n",
-        "\n",
-        "6. Enter the path to your service account key as the\n",
-        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "idpLDmlyjWyU"
-      },
-      "outputs": [],
-      "source": [
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "# The Google Cloud Notebook product has specific requirements\n",
-        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
-        "\n",
-        "# If on Google Cloud Notebooks, then don't execute this code\n",
-        "if not IS_GOOGLE_CLOUD_NOTEBOOK:\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "r2lr6-MVpXLP"
-      },
-      "source": [
-        "#### Timestamp\n",
-        "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append it onto the name of resources you create in this tutorial."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "he2lcG3Jpdxu"
-      },
-      "outputs": [],
-      "source": [
-        "from datetime import datetime\n",
-        "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "VxkQeJLyjgyr"
-      },
-      "source": [
-        "### Create a Cloud Storage bucket\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "When you submit a training job using the Cloud SDK, you upload a Python package\n",
-        "containing your training code to a Cloud Storage bucket. Vertex AI runs\n",
-        "the code from this package. In this tutorial, Vertex AI also saves the\n",
-        "trained model that results from your job in the same bucket. Using this model artifact, you can then\n",
-        "create Vertex AI model and endpoint resources in order to serve\n",
-        "online predictions.\n",
-        "\n",
-        "Set the name of your Cloud Storage bucket below. It must be unique across all\n",
-        "Cloud Storage buckets.\n",
-        "\n",
-        "You may also change the `REGION` variable, which is used for operations\n",
-        "throughout the rest of this notebook. We suggest that you [choose a region where Vertex AI services are\n",
-        "available](https://cloud.google.com/vertex-ai/docs/general/locations#available_regions)."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "GF076Vmoioll"
-      },
-      "outputs": [],
-      "source": [
-        "BUCKET_URI = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "REGION = \"[your-region]\"  # @param {type:\"string\"}\n",
-        "\n",
-        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "ElTizrkXiolm"
-      },
-      "source": [
-        "**Only if your bucket doesn't already exist**: Run the following cell to create your Cloud Storage bucket."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "3ar8qPT6iolm"
-      },
-      "outputs": [],
-      "source": [
-        "! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "T_Hq3rtPiolm"
-      },
-      "source": [
-        "Finally, validate access to your Cloud Storage bucket by examining its contents:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "_KfwVmnIiolm"
-      },
-      "outputs": [],
-      "source": [
-        "! gsutil ls -al $BUCKET_URI"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -426,8 +139,8 @@
       "source": [
         "import os\n",
         "\n",
-        "os.environ[\"GOOGLE_CLOUD_PROJECT\"] = PROJECT_ID\n",
-        "!bq mk {PROJECT_ID}:ml_datasets"
+        "os.environ[\"GOOGLE_CLOUD_PROJECT\"] = MY_PROJECT\n",
+        "!bq mk {MY_PROJECT}:ml_datasets"
       ]
     },
     {
@@ -447,7 +160,7 @@
       },
       "outputs": [],
       "source": [
-        "!bq cp -n --project_id={PROJECT_ID} bigquery-public-data:ml_datasets.iris {PROJECT_ID}:ml_datasets.iris "
+        "!bq cp -n bigquery-public-data:ml_datasets.iris {MY_PROJECT}:ml_datasets.iris"
       ]
     },
     {
@@ -492,9 +205,9 @@
       "source": [
         "cloudbuild_yaml = \"\"\"steps:\n",
         "- name: 'gcr.io/cloud-builders/docker'\n",
-        "  args: [ 'build', '-t', 'gcr.io/{PROJECT_ID}/test-custom-container', '.' ]\n",
-        "images: ['gcr.io/{PROJECT_ID}/test-custom-container']\"\"\".format(\n",
-        "    PROJECT_ID=PROJECT_ID\n",
+        "  args: [ 'build', '-t', 'gcr.io/{MY_PROJECT}/test-custom-container', '.' ]\n",
+        "images: ['gcr.io/{MY_PROJECT}/test-custom-container']\"\"\".format(\n",
+        "    MY_PROJECT=MY_PROJECT\n",
         ")\n",
         "\n",
         "with open(f\"{CONTAINER_ARTIFACTS_DIR}/cloudbuild.yaml\", \"w\") as fp:\n",
@@ -507,7 +220,7 @@
         "id": "gQ_GUCtZftDz"
       },
       "source": [
-        "### Write The Dockerfile"
+        "### Write Dockerfile"
       ]
     },
     {
@@ -540,7 +253,7 @@
         "id": "9dfrLShaftDz"
       },
       "source": [
-        "### Write the entrypoint script to invoke trainer"
+        "### Write entrypoint script to invoke trainer"
       ]
     },
     {
@@ -644,7 +357,7 @@
         "id": "6LYlV4D2ftD0"
       },
       "source": [
-        "### Build The Container"
+        "### Build Container"
       ]
     },
     {
@@ -655,7 +368,7 @@
       },
       "outputs": [],
       "source": [
-        "!gcloud builds submit --project={PROJECT_ID} --config {CONTAINER_ARTIFACTS_DIR}/cloudbuild.yaml {CONTAINER_ARTIFACTS_DIR}"
+        "!gcloud builds submit --config {CONTAINER_ARTIFACTS_DIR}/cloudbuild.yaml {CONTAINER_ARTIFACTS_DIR}"
       ]
     },
     {
@@ -664,7 +377,7 @@
         "id": "Pf0pugbvftD1"
       },
       "source": [
-        "# Run The Custom Container Training"
+        "# Run Custom Container Training"
       ]
     },
     {
@@ -673,7 +386,7 @@
         "id": "7ee691569d8d"
       },
       "source": [
-        "## Initialize The Vertex AI SDK for Python\n",
+        "## Initialize Vertex SDK for Python\n",
         "\n",
         "Initialize the *client* for Vertex AI"
       ]
@@ -688,7 +401,7 @@
       "source": [
         "from google.cloud import aiplatform\n",
         "\n",
-        "aiplatform.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
+        "aiplatform.init(project=MY_PROJECT, staging_bucket=MY_STAGING_BUCKET)"
       ]
     },
     {
@@ -711,7 +424,7 @@
       "outputs": [],
       "source": [
         "ds = aiplatform.TabularDataset.create(\n",
-        "    display_name=\"bq_iris_dataset\", bq_source=f\"bq://{PROJECT_ID}.ml_datasets.iris\"\n",
+        "    display_name=\"bq_iris_dataset\", bq_source=f\"bq://{MY_PROJECT}.ml_datasets.iris\"\n",
         ")"
       ]
     },
@@ -721,7 +434,7 @@
         "id": "ee242cc1f74c"
       },
       "source": [
-        "# Launch The Training Job to Create a Model\n",
+        "# Launch a Training Job to Create a Model\n",
         "\n",
         "We will train a model with the container we built above."
       ]
@@ -736,14 +449,14 @@
       "source": [
         "job = aiplatform.CustomContainerTrainingJob(\n",
         "    display_name=\"train-bq-iris\",\n",
-        "    container_uri=f\"gcr.io/{PROJECT_ID}/test-custom-container:latest\",\n",
+        "    container_uri=f\"gcr.io/{MY_PROJECT}/test-custom-container:latest\",\n",
         "    model_serving_container_image_uri=\"gcr.io/cloud-aiplatform/prediction/tf2-cpu.2-2:latest\",\n",
         ")\n",
         "model = job.run(\n",
         "    ds,\n",
         "    replica_count=1,\n",
         "    model_display_name=\"bq-iris-model\",\n",
-        "    bigquery_destination=f\"bq://{PROJECT_ID}\",\n",
+        "    bigquery_destination=f\"bq://{MY_PROJECT}\",\n",
         ")"
       ]
     },
@@ -753,7 +466,7 @@
         "id": "a7fa9b59f919"
       },
       "source": [
-        "# Deploy The Model\n",
+        "# Deploy Your Model\n",
         "\n",
         "Deploy your model, then wait until the model FINISHES deployment before proceeding to prediction."
       ]
@@ -775,7 +488,7 @@
         "id": "4dbd6c650a03"
       },
       "source": [
-        "# Make a prediction\n"
+        "# Predict on the Endpoint"
       ]
     },
     {
@@ -790,46 +503,12 @@
         "    [{\"sepal_length\": 5.1, \"sepal_width\": 2.5, \"petal_length\": 3.0, \"petal_width\": 1.1}]\n",
         ")"
       ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "MaoIczP8qu--"
-      },
-      "source": [
-        "## Cleaning up\n",
-        "\n",
-        "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
-        "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
-        "\n",
-        "Otherwise, you can delete the individual resources you created in this tutorial:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "-UaP-qoKqzc1"
-      },
-      "outputs": [],
-      "source": [
-        "import os\n",
-        "\n",
-        "# Delete endpoint resource\n",
-        "! gcloud ai endpoints delete $ENDPOINT_NAME --quiet --region $REGION_NAME\n",
-        "\n",
-        "# Delete Cloud Storage objects that were created\n",
-        "! gsutil -m rm -r $JOB_DIR\n",
-        "\n",
-        "if os.getenv(\"IS_TESTING\"):\n",
-        "! gsutil -m rm -r $BUCKET_URI  "
-      ]
     }
   ],
   "metadata": {
     "colab": {
       "collapsed_sections": [],
-      "name": "SDK_BigQuery_Custom_Container_Training.ipynb",
+      "name": "AI_Platform_(Unified)_SDK_BigQuery_Custom_Container_Training.ipynb",
       "toc_visible": true
     },
     "kernelspec": {