Merge branch 'main' into fix_is_testing

.cloud-build/cleanup/cleanup-requirements.txt

@@ -1,2 +1 @@
-ratemate
 google-cloud-aiplatform

.cloud-build/cleanup/cleanup.py

@@ -1,49 +1,45 @@
 from typing import List
-from ratemate import RateLimit
 from resource_cleanup_manager import (
-    DatasetResourceCleanupManager,
-    ModelResourceCleanupManager,
-    EndpointResourceCleanupManager,
-    ResourceCleanupManager,
+	ResourceCleanupManager,
+	DatasetResourceCleanupManager,
+	EndpointResourceCleanupManager,
+	ModelResourceCleanupManager,
 )
 
-rate_limit = RateLimit(max_count=25, per=60, greedy=False)
-
 
 def run_cleanup_managers(managers: List[ResourceCleanupManager], is_dry_run: bool):
-    for manager in managers:
-        type_name = manager.type_name
+	for manager in managers:
+		type_name = manager.type_name
 
-        print(f"Fetching {type_name}'s...")
-        resources = manager.list()
-        print(f"Found {len(resources)} {type_name}'s")
-        for resource in resources:
-            try:
-                if not manager.is_deletable(resource):
-                    continue
+		print(f"Fetching {type_name}'s...")
+		resources = manager.list()
+		print(f"Found {len(resources)} {type_name}'s")
+		for resource in resources:
+			if not manager.is_deletable(resource):
+				continue
 
-                if is_dry_run:
-                    resource_name = manager.resource_name(resource)
-                    print(f"Will delete '{type_name}': {resource_name}")
-                else:
-                    rate_limit.wait()  # wait before deleting
-                    manager.delete(resource)
-            except Exception as exception:
-                print(exception)
+			if is_dry_run:
+				resource_name = manager.resource_name(resource)
+				print(f"Will delete '{type_name}': {resource_name}")
+			else:
+				try:
+					manager.delete(resource)
+				except Exception as exception:
+					print(exception)
 
-        print("")
+		print("")
 
 
 is_dry_run = False
 
 if is_dry_run:
-    print("Starting cleanup in dry run mode...")
+	print("Starting cleanup in dry run mode...")
 
 # List of all cleanup managers
 managers = [
-    DatasetResourceCleanupManager(),
-    EndpointResourceCleanupManager(),
-    ModelResourceCleanupManager(),  # ModelResourceCleanupManager must follow EndpointResourceCleanupManager due to deployed models blocking model deletion.
+	DatasetResourceCleanupManager(),
+	EndpointResourceCleanupManager(),
+	ModelResourceCleanupManager(),
 ]
 
 run_cleanup_managers(managers=managers, is_dry_run=is_dry_run)

.cloud-build/cleanup/resource_cleanup_manager.py

@@ -1,9 +1,8 @@
 import abc
-from typing import Any, Type
-
 from google.cloud import aiplatform
-from google.cloud.aiplatform import base
+from typing import Any
 from proto.datetime_helpers import DatetimeWithNanoseconds
+from google.cloud.aiplatform import base
 
 # If a resource was updated within this number of seconds, do not delete.
 RESOURCE_UPDATE_BUFFER_IN_SECONDS = 60 * 60 * 8
@@ -41,7 +40,7 @@ class ResourceCleanupManager(abc.ABC):
         # Check that it wasn't created too recently, to prevent race conditions
         if time_difference <= RESOURCE_UPDATE_BUFFER_IN_SECONDS:
             print(
-                f"Skipping '{resource}' due to update_time being '{time_difference}', which is less than '{RESOURCE_UPDATE_BUFFER_IN_SECONDS}'."
+                f"Skipping '{resource}' due update_time being '{time_difference}', which is less than '{RESOURCE_UPDATE_BUFFER_IN_SECONDS}'."
             )
             return False
 
@@ -51,7 +50,7 @@ class ResourceCleanupManager(abc.ABC):
 class VertexAIResourceCleanupManager(ResourceCleanupManager):
     @property
     @abc.abstractmethod
-    def vertex_ai_resource(self) -> Type[base.VertexAiResourceNounWithFutureManager]:
+    def vertex_ai_resource(self) -> base.VertexAiResourceNounWithFutureManager:
         pass
 
     @property
@@ -61,9 +60,7 @@ class VertexAIResourceCleanupManager(ResourceCleanupManager):
     def list(self) -> Any:
         return self.vertex_ai_resource.list()
 
-    def resource_name(
-        self, resource: Type[base.VertexAiResourceNounWithFutureManager]
-    ) -> str:
+    def resource_name(self, resource: Any) -> str:
         return resource.display_name
 
     def delete(self, resource):
@@ -77,33 +74,12 @@ class VertexAIResourceCleanupManager(ResourceCleanupManager):
 
 class DatasetResourceCleanupManager(VertexAIResourceCleanupManager):
     vertex_ai_resource = aiplatform.datasets._Dataset
-    dataset_types = [
-        aiplatform.ImageDataset,
-        aiplatform.TabularDataset,
-        aiplatform.TextDataset,
-        aiplatform.TimeSeriesDataset,
-        aiplatform.VideoDataset,
-    ]
-
-    def list(self) -> Any:
-        return [
-            dataset
-            for dataset_type in self.dataset_types
-            for dataset in dataset_type.list()
-        ]
 
 
 class EndpointResourceCleanupManager(VertexAIResourceCleanupManager):
     vertex_ai_resource = aiplatform.Endpoint
 
     def delete(self, resource):
-        # TODO: Remove this once https://github.com/googleapis/python-aiplatform/issues/1441 is fixed
-        resource._sync_gca_resource()
-        for deployed_model_id in [
-            models.id for models in resource._gca_resource.deployed_models
-        ]:
-            resource._undeploy(deployed_model_id=deployed_model_id)
-
         resource.delete(force=True)
 
 

.cloud-build/execute_changed_notebooks_cli.py

@@ -17,7 +17,6 @@
 
 import argparse
 import pathlib
-
 import execute_changed_notebooks_helper
 
 
@@ -62,12 +61,6 @@ parser.add_argument(
     help="The GCP region. This is used to inject a variable value into the notebook before running.",
     required=True,
 )
-parser.add_argument(
-    "--variable_service_account",
-    type=str,
-    help="A service account. This is used to inject a variable value into the notebook before running. This is not the account that will run the notebook.",
-    required=True,
-)
 parser.add_argument(
     "--staging_bucket",
     type=str,
@@ -80,13 +73,6 @@ parser.add_argument(
     help="The GCP directory for storing executed notebooks.",
     required=True,
 )
-parser.add_argument(
-    "--timeout",
-    type=int,
-    help="Timeout in seconds",
-    default=86400,
-    required=False,
-)
 parser.add_argument(
     "--private_pool_id",
     type=str,
@@ -116,8 +102,6 @@ execute_changed_notebooks_helper.process_and_execute_notebooks(
     artifacts_bucket=args.artifacts_bucket,
     variable_project_id=args.variable_project_id,
     variable_region=args.variable_region,
-    variable_service_account=args.variable_service_account,
-    private_pool_id=args.private_pool_id,
+    private_pool_id=args.private_pool_id if not "default" else None,
     should_parallelize=args.should_parallelize,
-    timeout=args.timeout,
 )

.cloud-build/execute_changed_notebooks_helper.py

@@ -17,24 +17,18 @@ import concurrent
 import dataclasses
 import datetime
 import functools
-import git
-import operator
 import os
 import pathlib
+import nbformat
 import re
 import subprocess
 from typing import List, Optional
-
-import execute_notebook_helper
-import execute_notebook_remote
-import nbformat
-from google.cloud.devtools.cloudbuild_v1.types import BuildOperationMetadata
-from ratemate import RateLimit
 from tabulate import tabulate
-from utils import NotebookProcessors, util
+import operator
 
-# A buffer so that workers finish before the orchestrating job
-WORKER_TIMEOUT_BUFFER_IN_SECONDS: int = 60 * 60
+import execute_notebook_remote
+from utils import util, NotebookProcessors
+from google.cloud.devtools.cloudbuild_v1.types import BuildOperationMetadata
 
 
 def format_timedelta(delta: datetime.timedelta) -> str:
@@ -67,20 +61,12 @@ class NotebookExecutionResult:
     output_uri: str
     build_id: str
     error_message: Optional[str]
-    
-    @property
-    def output_uri_web(self) -> Optional[str]:
-        if self.output_uri.startswith("gs://"):
-            return f"https://storage.googleapis.com/{self.output_uri[5:]}"
-        else:
-            return None
 
 
 def _process_notebook(
     notebook_path: str,
     variable_project_id: str,
     variable_region: str,
-    variable_service_account: str,
 ):
     # Read notebook
     with open(notebook_path) as f:
@@ -92,7 +78,6 @@ def _process_notebook(
         replacement_map={
             "PROJECT_ID": variable_project_id,
             "REGION": variable_region,
-            "SERVICE_ACCOUNT": variable_service_account,
         },
     )
 
@@ -118,23 +103,16 @@ def _create_tag(filepath: str) -> str:
     return tag
 
 
-rate_limit = RateLimit(max_count=50, per=60, greedy=True)
-
-
 def process_and_execute_notebook(
     container_uri: str,
     staging_bucket: str,
     artifacts_bucket: str,
     variable_project_id: str,
     variable_region: str,
-    variable_service_account: str,
     private_pool_id: Optional[str],
-    deadline: datetime,
     notebook: str,
     should_get_tail_logs: bool = False,
 ) -> NotebookExecutionResult:
-    rate_limit.wait()  # wait before creating the task
-
     print(f"Running notebook: {notebook}")
 
     # Create paths
@@ -162,26 +140,19 @@ def process_and_execute_notebook(
             notebook_path=notebook,
             variable_project_id=variable_project_id,
             variable_region=variable_region,
-            variable_service_account=variable_service_account,
         )
 
         # Upload the pre-processed code to a GCS bucket
         code_archive_uri = util.archive_code_and_upload(staging_bucket=staging_bucket)
 
-        # Calculate timeout in seconds
-        timeout_in_seconds = max(
-            int((deadline - datetime.datetime.now()).total_seconds()), 1
-        )
-
         operation = execute_notebook_remote.execute_notebook_remote(
             code_archive_uri=code_archive_uri,
             notebook_uri=notebook,
             notebook_output_uri=notebook_output_uri,
             container_uri=container_uri,
             tag=tag,
+            region=variable_region,
             private_pool_id=private_pool_id,
-            private_pool_region=variable_region,
-            timeout_in_seconds=timeout_in_seconds,
         )
 
         operation_metadata = BuildOperationMetadata(mapping=operation.metadata)
@@ -244,40 +215,20 @@ def get_changed_notebooks(
 
     # Find notebooks
     notebooks = []
-
-    # Instantiate GitPython objects
-    repo = git.Repo(os.getcwd())
-    index = repo.index
-
     if base_branch:
-        # Get the point at which this branch branches off from main
-        branching_commits = repo.merge_base("HEAD", f"origin/{base_branch}")
-
-        if len(branching_commits) > 0:
-            branching_commit = branching_commits[0]
-            print(f"Looking for notebooks that changed from branch: {branching_commit}")
-
-            notebooks = [
-                diff.b_path
-                for diff in index.diff(branching_commit, paths=test_paths)
-                if diff.b_path is not None
-            ]
-        else:
-            notebooks = []
+        print(f"Looking for notebooks that changed from branch: {base_branch}")
+        notebooks = subprocess.check_output(
+            ["git", "diff", "--name-only", f"origin/{base_branch}..."] + test_paths
+        )
     else:
         print(f"Looking for all notebooks.")
         notebooks = subprocess.check_output(["git", "ls-files"] + test_paths)
-        notebooks = notebooks.decode("utf-8").split("\n")
 
+    notebooks = notebooks.decode("utf-8").split("\n")
     notebooks = [notebook for notebook in notebooks if notebook.endswith(".ipynb")]
     notebooks = [notebook for notebook in notebooks if len(notebook) > 0]
     notebooks = [notebook for notebook in notebooks if pathlib.Path(notebook).exists()]
 
-    if len(notebooks) > 0:
-        print(f"Found {len(notebooks)} notebooks:")
-        for notebook in notebooks:
-            print(f"\t{notebook}")
-
     return notebooks
 
 
@@ -288,10 +239,8 @@ def process_and_execute_notebooks(
     artifacts_bucket: str,
     variable_project_id: str,
     variable_region: str,
-    variable_service_account: str,
     private_pool_id: Optional[str],
     should_parallelize: bool,
-    timeout: int,
 ):
     """
     Run the notebooks that exist under the folders defined in the test_paths_file.
@@ -318,27 +267,17 @@ def process_and_execute_notebooks(
             Required. The value for REGION to inject into notebooks.
         should_parallelize (bool):
             Required. Should run notebooks in parallel using a thread pool as opposed to in sequence.
-        timeout (str):
-            Required. Timeout string according to https://cloud.google.com/build/docs/build-config-file-schema#timeout.
     """
+    notebook_execution_results: List[NotebookExecutionResult] = []
 
-    # Calculate deadline
-    deadline = datetime.datetime.now() + datetime.timedelta(
-        seconds=max(timeout - WORKER_TIMEOUT_BUFFER_IN_SECONDS, 0)
-    )
-
-    if len(notebooks) > 1:
-        notebook_execution_results: List[NotebookExecutionResult] = []
-
+    if len(notebooks) > 0:
         print(f"Found {len(notebooks)} modified notebooks: {notebooks}")
 
         if should_parallelize and len(notebooks) > 1:
             print(
                 "Running notebooks in parallel, so no logs will be displayed. Please wait..."
             )
-            with concurrent.futures.ThreadPoolExecutor(max_workers=100) as executor:
-                print(f"Max workers: {executor._max_workers}")
-
+            with concurrent.futures.ThreadPoolExecutor(max_workers=None) as executor:
                 notebook_execution_results = list(
                     executor.map(
                         functools.partial(
@@ -348,9 +287,7 @@ def process_and_execute_notebooks(
                             artifacts_bucket,
                             variable_project_id,
                             variable_region,
-                            variable_service_account,
                             private_pool_id,
-                            deadline,
                         ),
                         notebooks,
                     )
@@ -363,73 +300,48 @@ def process_and_execute_notebooks(
                     artifacts_bucket=artifacts_bucket,
                     variable_project_id=variable_project_id,
                     variable_region=variable_region,
-                    variable_service_account=variable_service_account,
                     private_pool_id=private_pool_id,
-                    deadline=deadline,
                     notebook=notebook,
                 )
                 for notebook in notebooks
             ]
-
-        print("\n=== RESULTS ===\n")
-
-        results_sorted = sorted(
-            notebook_execution_results,
-            key=lambda result: result.is_pass,
-            reverse=True,
-        )
-
-        # Print results
-        print(
-            tabulate(
-                [
-                    [
-                        result.name,
-                        "PASSED" if result.is_pass else "FAILED",
-                        format_timedelta(result.duration),
-                        result.log_url,
-                        result.output_uri,
-                        result.output_uri_web
-                    ]
-                    for result in results_sorted
-                ],
-                headers=["build_tag", "status", "duration", "log_url", "output_uri", "output_uri_web"],
-            )
-        )
-
-        print("\n=== END RESULTS===\n")
-
-        total_notebook_duration = functools.reduce(
-            operator.add,
-            [datetime.timedelta(seconds=0)]
-            + [result.duration for result in results_sorted],
-        )
-
-        print(
-            f"Cumulative notebook duration: {format_timedelta(total_notebook_duration)}"
-        )
-
-        # Raise error if any notebooks failed
-        if not all([result.is_pass for result in results_sorted]):
-            raise RuntimeError("Notebook failures detected. See logs for details")
-
-    elif len(notebooks) == 1:
-        notebook = notebooks[0]
-
-        # Pre-process notebook by substituting variable names
-        _process_notebook(
-            notebook_path=notebook,
-            variable_project_id=variable_project_id,
-            variable_region=variable_region,
-            variable_service_account=variable_service_account,
-        )
-
-        execute_notebook_helper.execute_notebook(
-            notebook_source=notebook,
-            output_file_or_uri="/".join(
-                [artifacts_bucket, pathlib.Path(notebook).name]
-            ),
-            should_log_output=True,
-        )
     else:
         print("No notebooks modified in this pull request.")
+
+    print("\n=== RESULTS ===\n")
+
+    results_sorted = sorted(
+        notebook_execution_results,
+        key=lambda result: result.is_pass,
+        reverse=True,
+    )
+
+    # Print results
+    print(
+        tabulate(
+            [
+                [
+                    result.name,
+                    "PASSED" if result.is_pass else "FAILED",
+                    format_timedelta(result.duration),
+                    result.log_url,
+                ]
+                for result in results_sorted
+            ],
+            headers=["build_tag", "status", "duration", "log_url"],
+        )
+    )
+
+    print("\n=== END RESULTS===\n")
+
+    total_notebook_duration = functools.reduce(
+        operator.add,
+        [datetime.timedelta(seconds=0)]
+        + [result.duration for result in results_sorted],
+    )
+
+    print(f"Cumulative notebook duration: {format_timedelta(total_notebook_duration)}")
+
+    # Raise error if any notebooks failed
+    if not all([result.is_pass for result in results_sorted]):
+        raise RuntimeError("Notebook failures detected. See logs for details")

.cloud-build/execute_notebook_cli.py

@@ -16,7 +16,6 @@
 """A CLI to download (optional) and run a single notebook locally"""
 
 import argparse
-
 import execute_notebook_helper
 
 parser = argparse.ArgumentParser(description="Run a single notebook locally.")

.cloud-build/execute_notebook_helper.py

@@ -15,14 +15,14 @@
 
 """Methods to run a notebook locally"""
 
-import errno
-import os
-import shutil
 import sys
-
+import os
+import errno
 import papermill as pm
-from google.cloud.aiplatform import utils
+import shutil
+
 from utils import util
+from google.cloud.aiplatform import utils
 
 # This script is used to execute a notebook and write out the output notebook.
 
@@ -50,20 +50,6 @@ def execute_notebook(
 
     execution_exception = None
 
-
-    print("\n=== DOWNLOAD EXECUTED NOTEBOOK ===\n")
-    print(
-                f"Please debug the executed notebook by downloading the executed notebook:"
-            )
-
-    print("Option 1. Using gsutil. Run the following command in your terminal.")
-    print(f"\tgsutil cp \"{output_file_or_uri}\" .")
-
-    print("Option 2. Using this link.")
-    print(f"\thttps://storage.googleapis.com/{output_file_or_uri[5:]}")
-
-    print("\n======\n")
-
     # Execute notebook
     try:
         # Execute notebook
@@ -85,6 +71,10 @@ def execute_notebook(
             util.upload_file(notebook_source, remote_file_path=output_file_or_uri)
 
             print("\n=== EXECUTION FINISHED ===\n")
+            print(
+                f"Please debug the executed notebook by downloading: {output_file_or_uri}"
+            )
+            print("\n======\n")
         else:
             # Create directories if they don't exist
             if not os.path.exists(os.path.dirname(output_file_or_uri)):

.cloud-build/execute_notebook_remote.py

@@ -16,18 +16,22 @@
 """Methods to run a notebook on Google Cloud Build"""
 
 from re import sub
-from typing import Optional
-
-import google.auth
-import yaml
-from google.api_core import client_options, operation
-from google.cloud.aiplatform import utils
-from google.cloud.devtools import cloudbuild_v1
-from google.cloud.devtools.cloudbuild_v1.types import Source, StorageSource
 from google.protobuf import duration_pb2
 from yaml.loader import FullLoader
 
+import google.auth
+from google.cloud.devtools import cloudbuild_v1
+from google.cloud.devtools.cloudbuild_v1.types import Source, StorageSource
+
+from typing import Optional
+import yaml
+
+from google.cloud.aiplatform import utils
+from google.api_core import operation, client_options
+
+
 CLOUD_BUILD_FILEPATH = ".cloud-build/notebook-execution-test-cloudbuild-single.yaml"
+TIMEOUT_IN_SECONDS = 86400
 SERVICE_BASE_PATH = "cloudbuild.googleapis.com"
 
 
@@ -36,14 +40,12 @@ def execute_notebook_remote(
     notebook_uri: str,
     notebook_output_uri: str,
     container_uri: str,
+    region: str,
     private_pool_id: Optional[str],
-    private_pool_region: Optional[str],
     tag: Optional[str],
-    timeout_in_seconds: Optional[int] = None,
 ) -> operation.Operation:
     """Create and execute a single notebook on Google Cloud Build"""
     # Load build steps from YAML
-
     cloudbuild_config = yaml.load(open(CLOUD_BUILD_FILEPATH), Loader=FullLoader)
 
     substitutions = {
@@ -55,14 +57,13 @@ def execute_notebook_remote(
     build = cloudbuild_v1.Build()
 
     options: Optional[client_options.ClientOptions] = None
-    if private_pool_id and private_pool_region:
-        # substitutions["_PRIVATE_POOL_NAME"] = private_pool_id
-        build.options = cloudbuild_config.get("options")
-        build.options.pool = {"name": private_pool_id}
+    if private_pool_id:
+        substitutions["_PRIVATE_POOL_NAME"] = private_pool_id
+        build.options = cloudbuild_config["options"]
 
         # Switch to the regional endpoint of the pool
         options = client_options.ClientOptions(
-            api_endpoint=f"{private_pool_region}-{SERVICE_BASE_PATH}"
+            api_endpoint=f"{region}-{SERVICE_BASE_PATH}"
         )
 
     # Authorize the client with Google defaults
@@ -84,8 +85,8 @@ def execute_notebook_remote(
 
     build.steps = cloudbuild_config["steps"]
     build.substitutions = substitutions
-    build.timeout = duration_pb2.Duration(seconds=timeout_in_seconds)
-    build.queue_ttl = duration_pb2.Duration(seconds=timeout_in_seconds)
+    build.timeout = duration_pb2.Duration(seconds=TIMEOUT_IN_SECONDS)
+    build.queue_ttl = duration_pb2.Duration(seconds=TIMEOUT_IN_SECONDS)
 
     if tag:
         build.tags = [tag]

.cloud-build/notebook-execution-test-cloudbuild-single.yaml

@@ -4,35 +4,28 @@ steps:
     entrypoint: /bin/sh
     args:
     - -c
-    - 'gcloud config list --quiet'
+    - 'gcloud config list'
   # Check the Python version
   - name: ${_PYTHON_IMAGE}
     entrypoint: /bin/sh
     args:
     - -c
-    - python3 .cloud-build/CheckPythonVersion.py -q
-  # Create a virtual environment
-  - name: ${_PYTHON_IMAGE}
-    entrypoint: /bin/sh
-    args:
-    - -c
-    - python3 -m venv workspace/env
+    - 'python3 .cloud-build/CheckPythonVersion.py'
   # Install Python dependencies
   - name: ${_PYTHON_IMAGE}
     entrypoint: /bin/sh
     args:
     - -c
-    - . workspace/env/bin/activate &&
-      python3 -m pip -q install -U pip &&
-      python3 -m pip -q install -U -r .cloud-build/requirements.txt
+    - 'python3 -m pip install -U pip && python3 -m pip install -U --user -r .cloud-build/requirements.txt'
   # Install Python dependencies and run testing script
   - name: ${_PYTHON_IMAGE}
     entrypoint: /bin/sh
     args:
     - -c
-    - |
-      . workspace/env/bin/activate &&
-      python3 .cloud-build/execute_notebook_cli.py --notebook_source "${_NOTEBOOK_GCS_URI}" --output_file_or_uri "${_NOTEBOOK_OUTPUT_GCS_URI}"
+    - 'python3 -m pip install -U pip && python3 -m pip freeze && python3 .cloud-build/execute_notebook_cli.py --notebook_source "${_NOTEBOOK_GCS_URI}" --output_file_or_uri "${_NOTEBOOK_OUTPUT_GCS_URI}"'
     env:
     - 'IS_TESTING=1'
 timeout: 86400s
+options:
+  pool:
+    name: ${_PRIVATE_POOL_NAME}

.cloud-build/notebook-execution-test-cloudbuild.yaml

@@ -4,42 +4,35 @@ steps:
     entrypoint: /bin/sh
     args:
     - -c
-    - gcloud config list --quiet
+    - 'gcloud config list'
   # Check the Python version
   - name: ${_PYTHON_IMAGE}
     entrypoint: /bin/sh
     args:
     - -c
-    - python3 .cloud-build/CheckPythonVersion.py -q
-  # Fetch full repo for diff purposes
-  - name: gcr.io/cloud-builders/git
-    args: [fetch, --unshallow, --quiet]
-  # Create a virtual environment
+    - 'python3 .cloud-build/CheckPythonVersion.py'
+  # Fetch base branch if required
   - name: ${_PYTHON_IMAGE}
     entrypoint: /bin/sh
     args:
     - -c
-    - python3 -m venv workspace/env
+    - 'if [ -n "${_BASE_BRANCH}" ]; then git fetch origin "${_BASE_BRANCH}":refs/remotes/origin/"${_BASE_BRANCH}"; else echo "Skipping fetch."; fi'
   # Install Python dependencies
   - name: ${_PYTHON_IMAGE}
     entrypoint: /bin/sh
     args:
     - -c
-    - . workspace/env/bin/activate &&
-      python3 -m pip -q install -U pip &&
-      python3 -m pip -q install -U -r .cloud-build/requirements.txt
+    - 'python3 -m pip install -U pip && python3 -m pip install -U --user -r .cloud-build/requirements.txt'
   # Install Python dependencies and run testing script
   # TODO: Only pass in private_pool_id if it is set
   - name: ${_PYTHON_IMAGE}
     entrypoint: /bin/sh
     args:
     - -c
-    - |
-      . workspace/env/bin/activate &&
-      python3 .cloud-build/execute_changed_notebooks_cli.py --test_paths_file "${_TEST_PATHS_FILE}" --base_branch "${_FORCED_BASE_BRANCH}" --container_uri ${_PYTHON_IMAGE} --staging_bucket ${_GCS_STAGING_BUCKET}  --artifacts_bucket ${_GCS_STAGING_BUCKET}/executed_notebooks/PR_${_PR_NUMBER}/BUILD_${BUILD_ID} --variable_project_id ${PROJECT_ID} --variable_region ${_GCP_REGION} --variable_service_account ${_GCP_SERVICE_ACCOUNT} `if [ ! -z "${_PRIVATE_POOL_NAME}" ]; then echo "--private_pool_id ${_PRIVATE_POOL_NAME}"; fi`
+    - 'python3 -m pip install -U pip && python3 -m pip freeze && python3 .cloud-build/execute_changed_notebooks_cli.py --test_paths_file "${_TEST_PATHS_FILE}" --base_branch "${_FORCED_BASE_BRANCH}" --container_uri ${_PYTHON_IMAGE} --staging_bucket ${_GCS_STAGING_BUCKET}  --artifacts_bucket ${_GCS_STAGING_BUCKET}/executed_notebooks/PR_${_PR_NUMBER}/BUILD_${BUILD_ID} --variable_project_id ${PROJECT_ID} --variable_region ${_GCP_REGION} `if [ ! -z "${_PRIVATE_POOL_NAME}" ]; then echo "--private_pool_id ${_PRIVATE_POOL_NAME}"; fi`'
     env:
     - 'IS_TESTING=1'
 timeout: 86400s
 options:
   pool:
-    name: ${_PRIVATE_POOL_NAME}
+    name: ${_PRIVATE_POOL_NAME}

.cloud-build/requirements.txt

@@ -9,5 +9,4 @@ tabulate
 google-cloud-aiplatform
 google-cloud-storage
 google-cloud-build
-ratemate
-GitPython
+gcloud

.cloud-build/test_notebook_vm.txt

@@ -1,4 +1,5 @@
 notebooks/official/vizier/gapic-vizier-multi-objective-optimization.ipynb
 notebooks/official/pipelines/lightweight_functions_component_io_kfp.ipynb
+notebooks/official/matching_engine/intro-swivel.ipynb
 notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-locally-trained-models.ipynb
 notebooks/official/pipelines/metrics_viz_run_compare_kfp.ipynb

.cloud-build/test_single_notebook_vm.txt

@@ -1 +0,0 @@
-notebooks/official/pipelines/metrics_viz_run_compare_kfp.ipynb

.cloud-build/utils/NotebookProcessors.py

@@ -13,10 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Dict
-
 from nbconvert.preprocessors import Preprocessor
-
+from typing import Dict
 from . import UpdateNotebookVariables as update_notebook_variables
 
 
@@ -62,4 +60,4 @@ class UpdateVariablesPreprocessor(Preprocessor):
 
             executable_cells.append(cell)
         notebook.cells = executable_cells
-        return notebook, resources
+        return notebook, resources

.cloud-build/utils/UpdateNotebookVariables.py

@@ -35,8 +35,8 @@ Variables in conditionals can also be replaced:
 
 def get_updated_value(content: str, variable_name: str, variable_value: str) -> str:
     return re.sub(
-        rf"({variable_name}.*? = .*?[\",\'])\[.+?\]([\",\'].*?)",
-        rf"\g<1>{variable_value}\g<2>",
+        rf"({variable_name}.*?=.*?[\",\'])\[.+?\]([\",\'].*?)",
+        rf"\1{variable_value}\2",
         content,
         flags=re.M,
     )
@@ -78,27 +78,4 @@ def test_region():
         variable_name="REGION",
         variable_value="us-central1",
     )
-    assert new_content == 'REGION = "us-central1"  # @param {type:"string"}'
-
-
-def test_region_equal_equals_ignore():
-    # Tests that == is ignored
-    new_content = get_updated_value(
-        content='REGION == "[your-region]"  # @param {type:"string"}',
-        variable_name="REGION",
-        variable_value="us-central1",
-    )
-    assert new_content == 'REGION == "[your-region]"  # @param {type:"string"}'
-
-
-def test_service_account():
-    # Tests that == is ignored
-    new_content = get_updated_value(
-        content='SERVICE_ACCOUNT = "[your-service-account]"  # @param {type:"string"}',
-        variable_name="SERVICE_ACCOUNT",
-        variable_value="12345-compute@developer.gserviceaccount.com",
-    )
-    assert (
-        new_content
-        == 'SERVICE_ACCOUNT = "12345-compute@developer.gserviceaccount.com"  # @param {type:"string"}'
-    )
+    assert new_content == 'REGION = "us-central1"  # @param {type:"string"}'

.cloud-build/utils/util.py

@@ -1,13 +1,13 @@
+from datetime import datetime
+from typing import Optional
+from google.cloud import storage
+from google.cloud.aiplatform import utils
+from google.auth import credentials as auth_credentials
 import os
+
 import subprocess
 import tarfile
 import uuid
-from datetime import datetime
-from typing import Optional
-
-from google.auth import credentials as auth_credentials
-from google.cloud import storage
-from google.cloud.aiplatform import utils
 
 
 def download_file(bucket_name: str, blob_name: str, destination_file: str) -> str:
@@ -57,4 +57,4 @@ def archive_code_and_upload(staging_bucket: str):
 
     print(f"Uploaded source code archive to {source_archived_file_gcs}")
 
-    return source_archived_file_gcs
+    return source_archived_file_gcs

.github/PULL_REQUEST_TEMPLATE.md

@@ -2,17 +2,17 @@ If you are opening a PR for `Official Notebooks` under the [notebooks/official](
 - [ ] Use the [notebook template](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/notebook_template.ipynb) as a starting point.
 - [ ] Follow the style and grammar rules outlined in the above notebook template.
 - [ ] Verify the notebook runs successfully in Colab since the automated tests cannot guarantee this even when it passes.
-- [ ] Passes all the required automated checks. You can locally test for formatting and linting with these [instructions](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/CONTRIBUTING.md#code-quality-checks).
+- [ ] Passes all the required automated checks. You can locally test for formatting and linting with these [instructions](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/docs/contributing.md#code-quality-checks).
 - [ ] You have consulted with a tech writer to see if tech writer review is necessary. If so, the notebook has been reviewed by a tech writer, and they have approved it.
-- [ ] This notebook has been added to the [CODEOWNERS](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/CODEOWNERS) file under the `Official Notebooks` section, pointing to the author or the author's team.
+- [ ] This notebook has been added to the [CODEOWNERS](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/docs/CODEOWNERS) file under `# Official Notebooks` section, pointing to the author or the author's team.
 - [ ] The Jupyter notebook cleans up any artifacts it has created (datasets, ML models, endpoints, etc) so as not to eat up unnecessary resources.
 
 
 If you are opening a PR for `Community Notebooks` under the [notebooks/community](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/notebooks/community) folder:
-- [ ] This notebook has been added to the [CODEOWNERS](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/CODEOWNERS) file under the `Community Notebooks` section, pointing to the author or the author's team.
-- [ ] Passes all the required formatting and linting checks. You can locally test with these [instructions](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/CONTRIBUTING.md#code-quality-checks).
+- [ ] This notebook has been added to the [CODEOWNERS](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/docs/CODEOWNERS) file under the `# Community Notebooks` section, pointing to the author or the author's team.
+- [ ] Passes all the required formatting and linting checks. You can locally test with these [instructions](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/docs/contributing.md#code-quality-checks).
 
 If you are opening a PR for `Community Content` under the [community-content](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/community-content) folder:
 - [ ] Make sure your main `Content Directory Name` is descriptive, informative, and includes some of the key products and attributes of your content, so that it is differentiable from other content
-- [ ] The main content directory has been added to the [CODEOWNERS](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/community-content/CODEOWNERS) file under the `Community Content` section, pointing to the author or the author's team.
-- [ ] Passes all the required formatting and linting checks. You can locally test with these [instructions](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/CONTRIBUTING.md#code-quality-checks).
+- [ ] The main content directory has been added to the [CODEOWNERS](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/docs/CODEOWNERS) file under the `# Community Content` section, pointing to the author or the author's team.
+- [ ] Passes all the required formatting and linting checks. You can locally test with these [instructions](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/docs/contributing.md#code-quality-checks).

.github/workflows/ci.yaml

@@ -7,9 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Set up Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: '3.x'      
+      uses: actions/setup-python@v3
     - name: Fetch pull request branch
       uses: actions/checkout@v3
       with:

.github/workflows/linter/requirements.txt

@@ -2,9 +2,8 @@ git+https://github.com/tensorflow/docs
 ipython
 jupyter
 nbconvert
-black==22.6.0
-pyupgrade==2.34.0
+black==22.1.0
+pyupgrade==2.31.1
 isort==5.10.1
 flake8==4.0.1
-nbqa==1.4.0
-
+nbqa==1.3.1

.github/workflows/linter/run_linter.sh

@@ -68,7 +68,7 @@ if [ ${#notebooks[@]} -gt 0 ]; then
 
             if [ "$is_test" = true ]; then
                 echo "Running nbfmt..."
-                python3 -m tensorflow_docs.tools.nbfmt --test "$notebook"
+                python3 -m tensorflow_docs.tools.nbfmt --remove_outputs --test "$notebook"
                 NBFMT_RTN=$?
                 # echo "Running black..."
                 # python3 -m nbqa black "$notebook" --check
@@ -93,7 +93,7 @@ if [ ${#notebooks[@]} -gt 0 ]; then
                 python3 -m nbqa isort "$notebook"
                 ISORT_RTN=$?
                 echo "Running nbfmt..."
-                python3 -m tensorflow_docs.tools.nbfmt "$notebook"
+                python3 -m tensorflow_docs.tools.nbfmt --remove_outputs "$notebook"
                 NBFMT_RTN=$?
                 echo "Running flake8..."
                 python3 -m nbqa flake8 "$notebook" --show-source --extend-ignore=W391,E501,F821,E402,F404,W503,E203,E722,W293,W291

CONTRIBUTING.md

@@ -48,8 +48,8 @@ then you will need to manually address them before submitting your PR.
 nbqa black "$notebook"
 nbqa pyupgrade "$notebook"
 nbqa isort "$notebook"
-nbqa flake8 "$notebook" --extend-ignore=W391,E501,F821,E402,F404,W503,E203,E722,W293,W291
 python3 -m tensorflow_docs.tools.nbfmt --remove_outputs "$notebook"
+nbqa flake8 "$notebook" --extend-ignore=W391,E501,F821,E402,F404,W503,E203,E722,W293,W291
 ```
 
 ## Code Reviews

community-content/CODEOWNERS

@@ -3,5 +3,3 @@
 /pytorch_text_classification_using_vertex_sdk_and_gcloud  @RajeshThallam
 /pytorch_text_classification_using_vertex_sdk_and_gcloud		@RajeshThallam @ultrons
 /sklearn_text_classification_from_script_using_vertex_sdk       @maxhardt
-/pluto_on_workbench @wkharold
-/cpr-examples @samthrasher

community-content/cpr-examples/timm_serving/.dockerignore

@@ -1,5 +0,0 @@
-testdata/*
-build.py
-test.py
-state_dict.pth
-config.json

community-content/cpr-examples/timm_serving/.gitignore

@@ -1,5 +0,0 @@
-cpr_model_server.py
-entrypoint.py
-state_dict.pth
-config.json
-**/__pycache__

community-content/cpr-examples/timm_serving/README.md

@@ -1,93 +0,0 @@
-# CPR Example: PyTorch Image Models (timm)
-
-## About CPR
-
-CPR ([custom prediction routines](https://github.com/googleapis/python-aiplatform/blob/custom-prediction-routine/google/cloud/aiplatform/prediction/README.md)) is a framework designed by Google Cloud developers to make it easier to combine machine learning models with custom preprocessing and postprocessing logic in a real-time serving application. 
-
-## Using this example
-
-This code is a self-contained example of a custom model server project built using CPR.
-
-As is, you can use it to serve the ViT-Small image classification model from Ross Wightman's [`timm`](https://github.com/rwightman/pytorch-image-models) library of image model implementations in PyTorch. Both CPU and GPU are supported.
-
-You can also consider using the code here as a template for your own CPR project if you want to use a different model from `timm`, a different PyTorch model, or an entirely different framework.
-
-### Requirements
-
-In order to use this example, you'll need Docker and Python 3 installed on your system.
-
-To get started, first create a virtual environment in an empty directory:
-```sh
-mkdir cpr-example
-python3 -m venv cpr-example
-cd cpr-example && source bin/activate
-``` 
-
-Then, clone the [vertex-ai-samples repo](https://github.com/GoogleCloudPlatform/vertex-ai-samples) in that directory:
-```sh
-git clone https://github.com/GoogleCloudPlatform/vertex-ai-samples.git
-cd vertex-ai-samples/community-content/cpr-examples/timm_serving
-```
-
-Finally, install the Python modules required to build and run the model server:
-```sh
-pip install -r requirements.txt
-```
-
-### Predictor
-
-The `TimmPredictor` class in `timm_serving/predictor.py` implements most of the important logic for the server.
-
-- `load(artifacts_dir)`: The predictor's `load` method is called when the server starts up in order to set up the predictor, usually by loading model weights and any artifacts needed for preprocessing and postprocessing. In this example, we initialize the saved model from the `state_dict.pth` file located inside the `artifacts_dir` folder and create the preprocessing transform from the model config.
-
-- `preprocess`, `predict`, `postprocess`: These methods are applied in sequence to the deserialized JSON data from each request. 
-    - `preprocess` decodes images from base64 and apply cropping, scaling and normalizing transforms. 
-    - `predict` runs the ViT-Small model on the preprocessed images and returns class scores.
-    - `postprocess` finds the top five classes and packs the class names, probabilities, and indices in a serializable result.
-
-### Building the container
-
-To build the model server locally, run the build command:
-```sh
-python build.py build
-```
-
-You can edit configuration values such as the model server's base image, the name and tag assigned to the image, and the path where model weights are stored locally.
-
-When you run the build command, model weights are downloaded and the model server container is built.
-
-### Running local tests
-
-`test.py` contains a suite of unit tests for the predictor as well as end-to-end tests for the model server. 
-
-To run the tests:
-```sh
-python test.py
-```
-
-All of the test images are public domain.
-- [Cat](https://commons.wikimedia.org/wiki/File:Stray_cat_on_wall.jpg)
-- [Airplane](https://commons.wikimedia.org/wiki/File:Airplanes_jets.jpg)
-- The infamous [mandrill](https://commons.wikimedia.org/wiki/File:Wikipedia-sipi-image-db-mandrill-4.2.03.png)
-
-### Deploying to Vertex AI
-
-Before uploading or deploying the container, you'll need to modify `config.py` to set appropriate values for: 
-- `project_id`: Your GCP project id.
-- `region`: Region where the model will be uploaded and deployed.
-- `repository`: [Artifact Registry repository](https://cloud.google.com/artifact-registry/docs/repositories/create-repos) in your project where the container image will be uploaded. 
-- `artifacts_gcs_dir`: Folder in a [Google Cloud Storage bucket](https://cloud.google.com/storage/docs/creating-buckets) where the model weights will be uploaded.
-
-Once this is done, first upload the model:
-```sh
-python build.py upload
-```
-
-Then deploy it:
-```sh
-python build.py deploy
-```
-
-If you run the deploy command again, it will create a new endpoint. If you want to undeploy the model, you can do so using the Vertex AI dashboard on the Google Cloud console, or use `gcloud ai endpoints undeploy` from the command line.
-
-After deploying successfully, you can run `python build.py probe` to send a sample request to the deployed model. 

community-content/cpr-examples/timm_serving/build.py

@@ -1,117 +0,0 @@
-# Copyright 2022 Google LLC
-
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-
-#     https://www.apache.org/licenses/LICENSE-2.0
-
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Build the model server container."""
-import json
-import logging
-import os
-import pathlib
-from typing import Sequence
-
-from absl import app
-from absl import logging
-from config import CPRConfig
-from google.cloud import aiplatform
-from google.cloud.aiplatform import prediction as cpr
-import smart_open
-import timm
-from timm_serving import predictor
-import torch
-
-
-def build_container(config: CPRConfig, tag: str) -> cpr.LocalModel:
-    """Build the model server container.
-
-    Args:
-      tag: Output image tag.
-
-    Returns:
-      LocalModel exposing the built model server.
-    """
-    return cpr.LocalModel.build_cpr_model(
-        src_dir=os.path.join(os.getcwd()),
-        output_image_uri=tag,
-        base_image=config.base_image,
-        predictor=predictor.TimmPredictor,
-        requirements_path=os.path.join(os.getcwd(), "requirements.txt"),
-    )
-
-
-def save_model_artifact(destination: str) -> None:
-    """Save a copy of the model state dict."""
-    model = timm.create_model(predictor.TimmPredictor.TIMM_MODEL_NAME, pretrained=True)
-    dest_file = os.path.join(destination, predictor.TimmPredictor.WEIGHTS_FILE)
-    with smart_open.open(dest_file, "wb") as f:
-        torch.save(model, f)
-    logging.info("Saved model to %s", dest_file)
-    logging.info("%s parameters", sum(p.numel() for p in model.parameters()))
-
-
-def upload_model(config: CPRConfig) -> aiplatform.Model:
-    """Tag and upload the model server."""
-    ar_tag = (
-        f"{config.region}-docker.pkg.dev/{config.project_id}"
-        f"/{config.repository}/{config.image}"
-    )
-    local_model = build_container(config, tag=ar_tag)
-    aiplatform.init(project=config.project_id, location=config.region)
-    local_model.push_image()
-    aip_model = aiplatform.Model.upload(
-        local_model=local_model,
-        display_name=predictor.TimmPredictor.TIMM_MODEL_NAME,
-        artifact_uri=config.artifact_gcs_dir,
-    )
-    config.model_name = aip_model.resource_name
-    config.save()
-    return aip_model
-
-
-def deploy_model(config: CPRConfig) -> aiplatform.Endpoint:
-    """Deploy the model server to a Vertex Prediction endpoint."""
-    aiplatform.init(project=config.project_id, location=config.region)
-    aip_model = aiplatform.Model(model_name=config.model_name)
-    endpoint = aip_model.deploy(machine_type=config.machine_type)
-    config.endpoint_name = endpoint.resource_name
-    config.save()
-    return endpoint
-
-
-def probe_prediction(config: CPRConfig, request_path: str) -> None:
-    """Send a sample prediction request to the Vertex Prediction endpoint."""
-    aiplatform.init(project=config.project_id, location=config.region)
-    aip_endpoint = aiplatform.Endpoint(endpoint_name=config.endpoint_name)
-    with open(request_path) as f:
-        logging.info(aip_endpoint.predict(**json.load(f)))
-
-
-def main(argv: Sequence[str]):
-    config = CPRConfig()
-    if pathlib.Path(config.config_file).exists():
-        config.load()
-
-    actions = set(argv[1:])
-    if "build" in actions:
-        build_container(config, config.image)
-        save_model_artifact(config.artifact_local_dir)
-    if "upload" in actions:
-        save_model_artifact(config.artifact_gcs_dir)
-        upload_model(config)
-    if "deploy" in actions:
-        deploy_model(config)
-    if "probe" in actions:
-        probe_prediction(config, request_path="sample_request.json")
-
-
-if __name__ == "__main__":
-    app.run(main)

community-content/cpr-examples/timm_serving/config.py

@@ -1,76 +0,0 @@
-# Copyright 2022 Google LLC
-
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-
-#     https://www.apache.org/licenses/LICENSE-2.0
-
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import dataclasses
-import json
-
-
-@dataclasses.dataclass
-class CPRConfig(object):
-    """Configure the build process by editing the default values here.
-
-    config_file: File path used to save values in this config. (Some
-      values, such as the model name, are generated at build time and
-      depended on by future steps, so saving it allows this script to
-      deploy the model without re-uploading it, for example.)
-
-    base_image: Base Docker image on top of which the model server will
-      be built. By default, a Debian-based Python 3 image without GPU
-      support will be used.
-
-    image: Name and tag assigned to the built model server image.
-
-    artifact_local_dir: Local directory where a copy of the pretrained model weights
-      will be saved.
-
-    region: Google Cloud Region where the model will be uploaded during the
-      build process.
-
-    project_id: Google Cloud project ID.
-
-    repository: Name of the Artifact Registry repository where the container
-      will be uploaded.
-
-    artifact_gcs_dir: Location on GCS where a copy of the pretrained model
-      weights will be uploaded.
-
-    model_name: Full resource path of the uploaded model. This is a write-only
-      field, the value is generated by Vertex AI when the model is uploaded.
-
-    endpoint_name: Full resource path of the created endpoint. This is a
-      write-only field, the value is generated by Vertex AI when the model is
-      deployed to an endpoint.
-
-    machine_type: Machine type to use when deploying the model.
-    """
-
-    config_file: str = "config.json"
-    base_image: str = "python:3.10-bullseye"
-    image: str = "timm_predictor:latest"
-    artifact_local_dir: str = ""
-    region: str = "us-central1"
-    project_id: str = "samthrasher-experimental"
-    repository: str = "cpr-images"
-    artifact_gcs_dir: str = "gs://samthrasher-cpr-example/timm-vit224/"
-    model_name: str = ""
-    endpoint_name: str = ""
-    machine_type: str = "n1-standard-2"
-
-    def save(self):
-        with open(self.config_file, "w") as f:
-            json.dump(dataclasses.asdict(self), f, indent=2)
-
-    def load(self):
-        with open(self.config_file) as f:
-            self.__init__(**json.load(f))

community-content/cpr-examples/timm_serving/requirements.txt

@@ -1,8 +0,0 @@
-absl-py==1.1.0
-fastapi==0.75.2
-uvicorn==0.18.2
-timm==0.5.4
-smart_open==6.0.0
-
-google-cloud-storage>=1.26.0,<2.0.0dev
-google-cloud-aiplatform[prediction] @ git+https://github.com/googleapis/python-aiplatform.git@custom-prediction-routine

community-content/cpr-examples/timm_serving/test.py

@@ -1,249 +0,0 @@
-"""Test the timm_serving predictor."""
-import base64
-import json
-import logging
-import os
-import pickle
-from typing import List, Dict
-
-from absl import flags
-from absl import logging
-from absl.testing import absltest
-from config import CPRConfig
-import fastapi
-from google.cloud import aiplatform
-from google.cloud.aiplatform import prediction as cpr
-import PIL
-from timm_serving import predictor
-import torch
-
-VIT_SMALL_PARAMS = 22878952
-
-
-def b64_encode_file(path: str) -> str:
-    """Encode a file's contents as base64.
-
-    Args:
-      path: Path to the file.
-
-    Returns:
-      Base64-encoded contents of the file.
-    """
-    with open(path, "rb") as f:
-        return str(base64.b64encode(f.read()), encoding="utf-8")
-
-
-def make_instance_dict(
-    image_paths: List[str], base64_encodings: List[str]
-) -> Dict[str, List[str]]:
-    """Generate a dictionary similar to a parsed prediction server request.
-
-    Args:
-      image_paths: Paths to image files to include.
-      base64_encodings: Pre-encoded base64 strings.
-
-    Returns:
-      Dictionary of instances in the format accepted by the preprocessor.
-    """
-    instances = [s for s in base64_encodings]
-    for path in image_paths:
-        instances.append(b64_encode_file(path))
-    return {"instances": instances}
-
-
-def count_parameters(model: torch.nn.Module):
-    """Count the parameters in a Pytorch model.
-
-    Args:
-      model: Pytorch model (nn.Module).
-
-    Returns:
-      Number of parameters in the model.
-
-    """
-    return sum(p.numel() for p in model.parameters())
-
-
-class PredictorUnitTests(absltest.TestCase):
-    """Unit tests for timm_serving.predictor."""
-
-    def setUp(self):
-        super().setUp()
-        self.config = CPRConfig()
-        self.config.load()
-        self.predictor = predictor.TimmPredictor()
-
-    def test_load_from_saved_state_dict_ok(self):
-        self.predictor.load(self.config.artifact_local_dir)
-        self.assertEqual(count_parameters(self.predictor._model), VIT_SMALL_PARAMS)
-
-    def test_load_bad_path(self):
-        with self.assertRaises(FileNotFoundError):
-            self.predictor.load("testdata/")
-        with self.assertRaisesRegex(ValueError, "not a directory"):
-            self.predictor.load("blah")
-
-    def test_load_bad_data(self):
-        with self.assertRaises(pickle.UnpicklingError):
-            self.predictor.load("testdata/bad_model_1")
-        with self.assertRaisesRegex(RuntimeError, "Invalid magic number"):
-            self.predictor.load("testdata/bad_model_2")
-
-    def test_preprocess_ok(self):
-        self.predictor.load(self.config.artifact_local_dir)
-        instance_dict = make_instance_dict(
-            base64_encodings=[],
-            image_paths=[
-                "testdata/airplane.jpg",
-                "testdata/mandrill.tiff",
-                "testdata/mandrill.tiff",
-                "testdata/cat_alpha.png",
-            ],
-        )
-        result = self.predictor.preprocess(instance_dict)
-        self.assertEqual(result.size(), torch.Size([4, 3, 224, 224]))
-        self.assertEqual(result.dtype, torch.float32)
-
-    def test_preprocess_no_instances(self):
-        self.predictor.load(self.config.artifact_local_dir)
-        with self.assertRaises(fastapi.HTTPException) as ctx:
-            self.predictor.preprocess({})
-        self.assertEqual(ctx.exception.status_code, 400)
-        self.assertRegex(ctx.exception.detail, 'must contain "instances"')
-
-    def test_preprocess_wrong_shape_instances(self):
-        self.predictor.load(self.config.artifact_local_dir)
-        instance_dict = {"instances": [[b64_encode_file("testdata/mandrill.tiff")]]}
-        with self.assertRaises(fastapi.HTTPException) as ctx:
-            self.predictor.preprocess(instance_dict)
-        self.assertEqual(ctx.exception.status_code, 400)
-        self.assertRegex(ctx.exception.detail, "not 'list'")
-
-    def test_preprocess_bad_base64(self):
-        self.predictor.load(self.config.artifact_local_dir)
-        instance_dict = make_instance_dict(base64_encodings=["!@#$"], image_paths=[])
-        with self.assertRaises(fastapi.HTTPException) as ctx:
-            self.predictor.preprocess(instance_dict)
-        self.assertEqual(ctx.exception.status_code, 400)
-        self.assertRegex(ctx.exception.detail, "[Bb]ase64")
-
-    def test_preprocess_not_image_data(self):
-        self.predictor.load(self.config.artifact_local_dir)
-        instance_dict = make_instance_dict(
-            base64_encodings=[], image_paths=["testdata/bad.jpg"]
-        )
-        with self.assertRaises(fastapi.HTTPException) as ctx:
-            self.predictor.preprocess(instance_dict)
-        self.assertEqual(ctx.exception.status_code, 400)
-        self.assertRegex(ctx.exception.detail, "image file")
-
-    def test_predict_ok(self):
-        self.predictor.load(self.config.artifact_local_dir)
-        inputs = torch.zeros(size=[2, 3, 224, 224], dtype=torch.float32)
-        if torch.cuda.device_count() > 0:
-            inputs = inputs.cuda()
-        result = self.predictor.predict(inputs)
-        self.assertEqual(result.size(), torch.Size([2, 1000]))
-        self.assertEqual(result.dtype, torch.float32)
-
-    def test_postprocess_ok(self):
-        class_probs = torch.zeros(size=[2, 1000])
-        class_probs[0, 0] = 1
-        class_probs[1, 123] = 1
-        result = self.predictor.postprocess(class_probs)
-        predictions = result["predictions"]
-        self.assertLen(predictions[0]["class_names"], 5)
-        self.assertLen(predictions[0]["indices"], 5)
-        self.assertLen(predictions[0]["probabilities"], 5)
-        self.assertLen(predictions[1]["class_names"], 5)
-        self.assertLen(predictions[1]["indices"], 5)
-        self.assertLen(predictions[1]["probabilities"], 5)
-        self.assertContainsSubsequence(predictions[0]["class_names"][0], "tench")
-        self.assertContainsSubsequence(
-            predictions[1]["class_names"][0], "spiny lobster"
-        )
-
-
-class ServerEndToEndTests(absltest.TestCase):
-    """End-to-end tests for the model server, using LocalEndpoint."""
-
-    def setUp(self):
-        super().setUp()
-        self.config = CPRConfig()
-        self.config.load()
-        self.local_model = cpr.LocalModel(
-            serving_container_spec=aiplatform.gapic.ModelContainerSpec(
-                image_uri=self.config.image
-            )
-        )
-
-        self.local_endpoint = self.local_model.deploy_to_local_endpoint(
-            artifact_uri=self.config.artifact_local_dir or os.getcwd()
-        )
-        self.local_endpoint.serve()
-
-    def tearDown(self):
-        self.local_endpoint.stop()
-        super().tearDown()
-
-    def test_e2e_healthcheck_ok(self):
-        health_check_response = self.local_endpoint.run_health_check()
-        self.assertEqual(health_check_response.status_code, 200)
-        self.assertEqual(health_check_response.content, b"{}")
-
-    def test_e2e_predict_ok(self):
-        predict_request = json.dumps(
-            make_instance_dict(
-                base64_encodings=[],
-                image_paths=[
-                    "testdata/mandrill.tiff",
-                ],
-            )
-        )
-        response = self.local_endpoint.predict(
-            request=predict_request, headers={"Content-Type": "application/json"}
-        )
-        logging.info(response.content)
-        self.assertEqual(response.status_code, 200)
-        predictions = response.json()["predictions"]
-        self.assertContainsSubsequence(predictions[0]["class_names"][0], "baboon")
-
-    def test_e2e_predict_bad_json_returns_400(self):
-        predict_request = "blah"
-        response = self.local_endpoint.predict(
-            request=predict_request, headers={"Content-Type": "application/json"}
-        )
-        logging.info(response.content)
-        self.assertEqual(response.status_code, 400)
-
-    def test_e2e_predict_no_instances_returns_400(self):
-        predict_request = json.dumps({})
-        response = self.local_endpoint.predict(
-            request=predict_request, headers={"Content-Type": "application/json"}
-        )
-        logging.info(response.content)
-        self.assertEqual(response.status_code, 400)
-
-    def test_e2e_predict_bad_base64_returns_400(self):
-        predict_request = json.dumps(
-            make_instance_dict(base64_encodings=["blah"], image_paths=[])
-        )
-        response = self.local_endpoint.predict(
-            request=predict_request, headers={"Content-Type": "application/json"}
-        )
-        logging.info(response.content)
-        self.assertEqual(response.status_code, 400)
-
-    def test_e2e_predict_bad_image_returns_400(self):
-        predict_request = json.dumps(
-            make_instance_dict(base64_encodings=[], image_paths=["testdata/bad.jpg"])
-        )
-        response = self.local_endpoint.predict(
-            request=predict_request, headers={"Content-Type": "application/json"}
-        )
-        logging.info(response.content)
-        self.assertEqual(response.status_code, 400)
-
-
-if __name__ == "__main__":
-    absltest.main()

community-content/cpr-examples/timm_serving/testdata/bad.jpg

@@ -1 +0,0 @@
-some non-image data

community-content/cpr-examples/timm_serving/testdata/bad.txt

@@ -1 +0,0 @@
-some non-image data

community-content/cpr-examples/timm_serving/timm_serving/predictor.py

@@ -1,178 +0,0 @@
-# Copyright 2022 Google LLC
-
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-
-#     https://www.apache.org/licenses/LICENSE-2.0
-
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Adapts a pretrained TIMM image classification model to the CPR framework.
-
-Documentation for the TIMM (Torch IMage Models) library is here:
-  https://rwightman.github.io/pytorch-image-models/
-
-Its source can also be found here:
-  https://github.com/rwightman/pytorch-image-models
-"""
-
-import base64
-import binascii
-import io
-import os
-from typing import Dict, List, Union
-
-from fastapi import HTTPException
-from google.cloud.aiplatform import prediction as cpr
-from pathlib import Path
-import PIL
-import smart_open
-import timm
-import torch
-import torch.nn.functional as F
-
-with open(Path(__file__).parent.absolute().joinpath("imagenet.txt")) as f:
-    IMAGENET_CLASSES = f.read().splitlines()
-
-
-class TimmPredictor(cpr.predictor.Predictor):
-    """Predictor class for image models based on TIMM."""
-
-    TIMM_MODEL_NAME = os.getenv("TIMM_MODEL_NAME", default="vit_small_patch32_224")
-    WEIGHTS_FILE = "state_dict.pth"
-    NUM_TOP_CLASSES_TO_RETURN = 5
-
-    def __init__(self):
-        self._cuda = torch.cuda.device_count() > 0
-
-    def load(self, artifacts_uri: str = ""):
-        """Initializes the model and preprocessing transforms.
-
-        Args:
-          artifacts_uri: Directory where state dict is stored. Can be a
-          GCS URI or local path.
-        """
-        if artifacts_uri:
-            artifact_path = os.path.join(artifacts_uri)
-            if not (os.path.isdir(artifact_path) or artifact_path.startswith("gs://")):
-                raise ValueError("Provided artifact_uri is not a directory.")
-        else:
-            artifact_path = os.getcwd()
-
-        artifact_path = os.path.join(artifact_path, self.WEIGHTS_FILE)
-        with smart_open.open(artifact_path, "rb") as f:
-            self._model = torch.load(f)
-
-        if self._cuda:
-            self._model.cuda()
-
-        config = timm.data.resolve_data_config(model=self.TIMM_MODEL_NAME, args=[])
-        self._transform = timm.data.create_transform(
-            is_training=False, use_prefetcher=False, **config
-        )
-
-    def preprocess(self, request_dict: Dict[str, List[str]]) -> torch.Tensor:
-        """Performs preprocessing.
-
-        By default, the server expects a request body consisting of a valid JSON
-        object. This will be parsed by the handler before it's evaluated by the
-        preprocess method.
-
-        Args:
-          request_dict: Parsed request body. We expect that the input consists of
-            a list of base64-encoded image files under the "instances" key. (Any
-            image format that PIL.image.open can handle is okay.)
-
-        Returns:
-          torch.Tensor containing the preprocessed images as a batch. If GPU is
-          available, the result tensor will be stored on GPU.
-        """
-
-        if "instances" not in request_dict:
-            raise HTTPException(
-                status_code=400,
-                detail='Request must contain "instances" as a top-level key.',
-            )
-
-        tensors = []
-
-        for (i, image) in enumerate(request_dict["instances"]):
-            # We use Base64 encoding to handle image data.
-            # This is probably the best we can do while still using JSON input.
-            # Overriding the input format requires building a custom Handler.
-            try:
-                image_bytes = base64.b64decode(image, validate=True)
-            except (binascii.Error, TypeError) as e:
-                raise HTTPException(
-                    status_code=400,
-                    detail=f"Base64 decoding of the input image at index {i} failed:"
-                    f" {str(e)}",
-                )
-
-            try:
-                pil_image = PIL.Image.open(io.BytesIO(image_bytes)).convert("RGB")
-            except PIL.UnidentifiedImageError:
-                raise HTTPException(
-                    status_code=400,
-                    detail=f"The input image at index {i} could not be identified as an"
-                    " image file.",
-                )
-
-            tensors.append(self._transform(pil_image))
-
-        with torch.inference_mode():
-            result = torch.stack(tensors)
-            if self._cuda:
-                result = result.cuda()
-        return result
-
-    def predict(self, instances: torch.Tensor) -> torch.Tensor:
-        """Performs prediction.
-
-        Args:
-          instances: torch.Tensor with type torch.float32 and shape
-            [?, 3, 224, 224], containing the pre-processed input images.
-
-        Returns:
-          Vector of scores with type torch.float32 and shape [?, 1000],
-            representing the model's estimate of the likelihood that the
-            input belongs to the Imagenet class with that index.
-        """
-        with torch.inference_mode():
-            class_scores = self._model(instances)
-        return class_scores
-
-    def postprocess(
-        self, class_scores: torch.Tensor
-    ) -> Dict[str, List[Dict[str, Union[str, int, float]]]]:
-        """Translate the model output into a classification result.
-
-        Args:
-          class_scores: torch.Tensor with type torch.float32 and shape
-            [?, 1000], containing the scores assigned to each class by
-            the model.
-
-        Returns:
-          Dictionary containing the list of classification results. Each
-            classification result contains the probabilities, class names, and
-            class indices of the classes with the top class scores as reported by
-            the model.
-        """
-        class_probs = F.softmax(class_scores, dim=1)
-        top_k = class_probs.topk(self.NUM_TOP_CLASSES_TO_RETURN)
-        top_k_values = top_k.values.numpy().tolist()
-        top_k_indices = top_k.indices.numpy().tolist()
-        predictions = [
-            dict(
-                probabilities=values,
-                indices=indices,
-                class_names=[IMAGENET_CLASSES[int(class_num)] for class_num in indices],
-            )
-            for (values, indices) in zip(top_k_values, top_k_indices)
-        ]
-        return {"predictions": predictions}

community-content/pluto_on_workbench/README.md

@@ -1,52 +0,0 @@
-# Overview
-*Pluto* is a programming environment for Julia, designed to be interactive and helpful. It provides a familiar notebook interface but it is not a Jupyter notebook. The biggest difference is that Pluto notebooks are reactive, changing a variable or function in one cell causes the cells that depend on that variable or function to be reevaluated. Pluto also provides useful interaction mechanisms that allow users to dynamically interact with the notebooks computation state.
-
-The JuliaCon 2020 presentation: [Interactive notebooks ~ Pluto.jl]() provides a good introduction to Pluto. The source is at [fonsp/Pluto.jl]()
-
-# Install Pluto
-
-## Create a Vertex AI JupyterLab Instance
-
-1. From the [GCP console](https://console.cloud.google.com) "hamburger menu"
-
-   select Vertex AI > Workbench
-2. Click NEW NOTEBOOK
-   
-   * Choose Python 3 if you won't be using a GPU
-   * Choose Python 3 (CUDA Toolkit xx.y) if you do want use a GPU
-3. Give the notebook an appropriate name
-4. Edit Notebook properties if you have special requirements otherwise accept the defaults and click CREATE
-5. When the notebook instance is ready click OPEN JUPYTERLAB
-
-## Configure JupyterLab
-
-1. Open a terminal by clicking the Terminal icon.
-1. Install the plutoserver
-pip3 install git+https://github.com/fonsp/pluto-on-jupyterlab.git
-1. In a browser go to [julialang.org/downloads](https://julialang.org/downloads/)
-1. In the Current stable release right click on the `Generic Linux on x86 / 64-bit (glibc)` link
-Select copy link address
-1. Back in the terminal switch to root via
-sudo -i
-1. Download the release to /opt and install julia in /usr/local/bin
-    ```bash
-    cd /opt
-    wget <paste the release link address>
-    tar xf <name of the downloaded tar file>
-    ln -s /opt/<julia-x.y.z>/bin/julia /usr/local/bin
-    ^d
-    ```
-1. Add the Pluto package to Julia
-   ```bash
-    julia
-    julia> ]add Pluto
-    julia> bksp
-    julia> using Pluto
-    julia> ^d
-    ```
-1. From the JupyterLab menu bar select File > Shut Down
-
-# Start Pluto
-1. Click OPEN JUPYTERLAB in the Workbench
-1. In the Notebook section of the Launcher click Pluto.jl
-1. The welcome to Pluto.jl screen should appear

community-content/tf_agents_bandits_movie_recommendation_with_kfp_and_vertex_sdk/mlops_pipeline_tf_agents_bandits_movie_recommendation/src/logger/requirements.txt

@@ -1,4 +1,4 @@
 google-cloud-bigquery==2.20.0
-tensorflow==2.7.2
+tensorflow==2.5.3
 pillow==9.0.1
 tf-agents==0.8.0

community-content/tf_agents_bandits_movie_recommendation_with_kfp_and_vertex_sdk/mlops_pipeline_tf_agents_bandits_movie_recommendation/src/prediction_container/requirements.txt

@@ -1,4 +1,4 @@
 google-cloud-pubsub==2.5.0
 pillow==9.0.1
 tf-agents==0.8.0
-tensorflow==2.7.2
+tensorflow==2.5.3

community-content/tf_agents_bandits_movie_recommendation_with_kfp_and_vertex_sdk/mlops_pipeline_tf_agents_bandits_movie_recommendation/src/simulator/requirements.txt

@@ -1,5 +1,5 @@
 dataclasses==0.6
 google-cloud-aiplatform==1.8.1
-tensorflow==2.7.2
+tensorflow==2.5.3
 pillow==9.0.1
 tf-agents==0.8.0

community-content/tf_agents_bandits_movie_recommendation_with_kfp_and_vertex_sdk/step_by_step_sdk_tf_agents_bandits_movie_recommendation/step_by_step_sdk_tf_agents_bandits_movie_recommendation.ipynb

@@ -706,8 +706,8 @@
         "    else:\n",
         "        data_spec = training_data_spec_transformation_fn(\n",
         "            agent.policy.trajectory_spec)\n",
-        "    replay_buffer = trainer.get_replay_buffer(data_spec, environment.batch_size,\n",
-        "                                              steps_per_loop)\n",
+        "        replay_buffer = trainer.get_replay_buffer(data_spec, environment.batch_size,\n",
+        "                                                  steps_per_loop)\n",
         "\n",
         "    # `step_metric` records the number of individual rounds of bandit interaction;\n",
         "    # that is, (number of trajectories) * batch_size.\n",

community-content/tf_keras_image_classification_distributed_multi_worker_with_vertex_sdk/trainer/requirements.txt

@@ -1 +1 @@
-tensorflow==2.7.2
+tensorflow==2.5.3

community-content/tf_keras_text_classification_distributed_single_worker_gpus_with_gcloud_local_run_and_vertex_sdk/trainer/requirements.txt

@@ -1 +1 @@
-tensorflow==2.7.2
+tensorflow==2.5.3

notebooks/README.md

@@ -1,5 +1,5 @@
-The [official](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/notebooks/official) folder contains notebooks organized by Google Cloud product. These are tested weekly and maintained by Google.
+The [official](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/notebooks/official) folder contains notebooks organized by Google Cloud product.
 
-The [community](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/notebooks/community) folder contains notebooks that may be created by Google or external contributors. They are not necessary maintained.
+The [community](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/notebooks/community) folder contains notebooks that aren't officially supported by Google.
 
 Contributions to the repo should use the [notebook template](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/notebook_template.ipynb) as a starting point.

notebooks/community/CODEOWNERS

@@ -5,26 +5,17 @@
 
 /sdk/sdk_* @andrewferlitsch
 /gapic @andrewferlitsch
-/gapic/custom/showcase_custom_image_classification_online_explain_example_based_api.ipynb @inardini
 /ml_ops @andrewferlitsch
 /model_monitoring/* @mco-gh
 /structured_data/rapid_prototyping_* @rafael-carvalho
 
 /managed_notebooks/
-/bigquery_ml/ @polong
 /sdk/SDK_FBProphet_Forecasting_Online.ipynb @brianchunkang
 /pipelines/google_cloud_pipeline_components_TPU_model_train_upload_deploy.ipynb @brianchunkang
-/explainable_ai/SDK_Custom_Container_XAI.ipynb @brianchunkang
 /matching_engine/sdk_matching_engine_for_indexing.ipynb @ivanmkc
 /matching_engine/matching_engine_for_indexing.ipynb @yinghsienwu
 /sdk/pytorch_lightning_custom_container_training.ipynb @brianchunkang
 /tensorboard @yfang1
 /feature_store @nayaknishant @morgandu
-/prediction @googleapis/vertex-prediction-team
 /vertex_endpoints/tf_hub_obj_detection/deploy_tfhub_object_detection_on_vertex_endpoints.ipynb @entrpn
 /vertex_endpoints/nvidia-triton/nvidia-triton-custom-container-prediction.ipynb @RajeshThallam
-/vertex_endpoints/optimized_tensorflow_runtime @vlasenkoalexey
-/notebooks/community/ml_ops/stage2/get_started_with_visionapi_and_automl.ipynb @mansari
-/notebooks/community/neo4j/graph_paysim.ipynb @benofben @laeg
-/notebooks/community/ml_ops/stage1/get_started_with_visionapi_and_vertex_datasets.ipynb @mansari
-/notebooks/community/pipelines/google_cloud_pipeline_components_bqml_pipeline_demand_forecasting.ipynb @inardini

notebooks/community/feature_store/sdk-feature-store-pandas.ipynb

@@ -30,25 +30,17 @@
       },
       "source": [
         "<table align=\"left\">\n",
-        "    <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/feature_store/sdk-feature-store-pandas.ipynb\">\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/feature_store/sdk-feature-store-pandas.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/feature_store/sdk-feature-store-pandas.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
-        "    \n",
-        "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/feature_store/sdk-feature-store-pandas.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> \n",
-        "        Run in Colab\n",
-        "    </a>\n",
-        "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/feature_store/sdk-feature-store-pandas.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
-        "    </a>\n",
-        "  </td>\n",
         "</table>"
       ]
     },
@@ -60,7 +52,7 @@
       "source": [
         "## Overview\n",
         "\n",
-        "This notebook introduces Pandas support for Feature Store using Vertex AI SDK. For pre-requisites and introduction on Vertex AI SDK and Feature Store native support, please go through this [Colab notebook](https://colab.sandbox.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/feature_store/sdk-feature-store.ipynb). "
+        "This Colab introduces Pandas support of Vertex AI SDK Feature Store. For pre-requisite and introduction for Vertex AI SDK Feature Store native support, please see this [Colab](https://colab.sandbox.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/feature_store/sdk-feature-store.ipynb). "
       ]
     },
     {
@@ -71,7 +63,7 @@
       "source": [
         "### Dataset\n",
         "\n",
-        "This tutorial uses a movie recommendation dataset as an example throughout all the notebooks including this one. The original task is to train a model to predict if a user is going to watch a movie and serve the model online."
+        "This Colab uses a movie recommendation dataset as an example throughout all the sessions. The task is to train a model to predict if a user is going to watch a movie and serve this model online."
       ]
     },
     {
@@ -82,16 +74,16 @@
       "source": [
         "### Objective\n",
         "\n",
-        "In this notebook, you learn how to:\n",
+        "In this notebook, you will learn how to:\n",
         "\n",
-        "- Ingest Feature values from Pandas DataFrame into Feature Store's Entity types.\n",
-        "- Read Entity Feature values from Online Feature Store into Pandas DataFrame.\n",
-        "- Batch serve Feature values from your Feature Store into Pandas DataFrame.\n",
+        "    * Ingest Feature Values from Pandas DataFrame into featurestore's entity types.\n",
+        "    * Read Entity Feature Values from Online Feature Store into Pandas DataFrame.\n",
+        "    * Batch Serve Feature Values from your featurestore to Pandas DataFrame.\n",
         "\n",
-        "You also learn how Vertex AI Feature Store can be useful in the below scenarios:\n",
+        "We will also discuss how Vertex AI Feature Store can be useful in the below scenarios:\n",
         "\n",
-        "- Online serving with updated feature values.\n",
-        "- Point-in-time correctness to fetch feature values for training."
+        "    * online serving with updated feature values\n",
+        "    * point-in-time correctness to fetch feature values for training"
       ]
     },
     {
@@ -105,9 +97,11 @@
         "This tutorial uses billable components of Google Cloud:\n",
         "\n",
         "* Vertex AI\n",
+        "* Cloud BigQuery\n",
         "\n",
         "Learn about [Vertex AI\n",
-        "pricing](https://cloud.google.com/vertex-ai/pricing) and use the [Pricing\n",
+        "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
+        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
         "Calculator](https://cloud.google.com/products/calculator/)\n",
         "to generate a cost estimate based on your projected usage."
       ]
@@ -129,7 +123,7 @@
       "source": [
         "### Install additional packages\n",
         "\n",
-        "To run this notebook, you need to install the following packages for Python."
+        "For this Colab, you need the Vertex SDK for Python."
       ]
     },
     {
@@ -148,14 +142,35 @@
         "# Google Cloud Notebook requires dependencies to be installed with '--user'\n",
         "USER_FLAG = \"\"\n",
         "if IS_GOOGLE_CLOUD_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "    \n",
-        "! pip install -U {USER_FLAG} --upgrade google-cloud-aiplatform \\\n",
-        "                                        google-cloud-bigquery \\\n",
-        "                                        google-cloud-bigquery-storage \\\n",
-        "                                        avro \\\n",
-        "                                        pyarrow \\\n",
-        "                                        pandas -q"
+        "    USER_FLAG = \"--user\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Kd0kgDqVZyRe"
+      },
+      "outputs": [],
+      "source": [
+        "! pip uninstall {USER_FLAG} -y google-cloud-aiplatform\n",
+        "! pip uninstall {USER_FLAG} -y google-cloud-bigquery\n",
+        "! pip uninstall {USER_FLAG} -y google-cloud-bigquery-storage\n",
+        "! pip uninstall {USER_FLAG} -y google-cloud-aiplatform"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "wUswAmpiN2l-"
+      },
+      "outputs": [],
+      "source": [
+        "! pip install {USER_FLAG} --upgrade google-cloud-aiplatform\n",
+        "! pip install {USER_FLAG} --upgrade google-cloud-bigquery\n",
+        "! pip install {USER_FLAG} --upgrade google-cloud-bigquery-storage\n",
+        "! pip install {USER_FLAG} avro"
       ]
     },
     {
@@ -166,7 +181,7 @@
       "source": [
         "### Restart the kernel\n",
         "\n",
-        "After you install the packages, you need to restart the notebook kernel so that it can find the packages."
+        "After you install the SDK, you need to restart the notebook kernel so it can find the packages. You can restart kernel from *Kernel -> Restart Kernel*, or running the following:"
       ]
     },
     {
@@ -223,17 +238,6 @@
         "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "dcdfccf50581"
-      },
-      "outputs": [],
-      "source": [
-        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -242,56 +246,37 @@
       },
       "outputs": [],
       "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
-        "    # Get your GCP project id from gcloud\n",
-        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "import os\n",
+        "\n",
+        "PROJECT_ID = \"\"\n",
+        "\n",
+        "# Get your Google Cloud project ID from gcloud\n",
+        "if not os.getenv(\"IS_TESTING\"):\n",
+        "    shell_output = !gcloud config list --format 'value(core.project)' 2>/dev/null\n",
         "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID:\", PROJECT_ID)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "09021c90b34c"
-      },
-      "outputs": [],
-      "source": [
-        "! gcloud config set project $PROJECT_ID"
+        "    print(\"Project ID: \", PROJECT_ID)"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "f41eda68c379"
+        "id": "qJYoRfYng0XZ"
       },
       "source": [
-        "#### Region\n",
-        "\n",
-        "You can also change the `REGION` variable, which is used for operations\n",
-        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
-        "\n",
-        "- Americas: `us-central1`\n",
-        "- Europe: `europe-west4`\n",
-        "- Asia Pacific: `asia-east1`\n",
-        "\n",
-        "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
-        "\n",
-        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)"
+        "Otherwise, set your project ID here."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "5c615e53149f"
+        "id": "riG_qUokg0XZ"
       },
       "outputs": [],
       "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
+        "    PROJECT_ID = \"\"  # @param {type:\"string\"}\n",
+        "print(\"Project ID: \", PROJECT_ID)"
       ]
     },
     {
@@ -391,6 +376,8 @@
         "import pandas as pd\n",
         "from google.cloud import aiplatform\n",
         "\n",
+        "REGION = \"\"  # @param {type:\"string\"}\n",
+        "\n",
         "aiplatform.init(project=PROJECT_ID, location=REGION)"
       ]
     },
@@ -409,11 +396,11 @@
         "id": "buQBIv3ZL3A0"
       },
       "source": [
-        "### Create Feature Store\n",
+        "### Create Featurestore\n",
         "\n",
-        "The method to create a Feature Store returns a\n",
+        "The method to create a Featurestore returns a\n",
         "[long-running operation](https://google.aip.dev/151) (LRO). An LRO starts an asynchronous job. LROs are returned for other API\n",
-        "methods too, such as updating or deleting a featurestore. Running the code cell creates a featurestore and prints the process logs."
+        "methods too, such as updating or deleting a featurestore. Running the code cell will create a featurestore and print the process log."
       ]
     },
     {
@@ -438,7 +425,7 @@
       "source": [
         "### Create Entity Types\n",
         "\n",
-        "Entity types can be created within the Featurestore class. Below, you create the `Users` entity type and `Movies` entity type. Process logs are printed in the output for each cell."
+        "Entity types can be created within the Featurestore class. Below, create the Users entity type and Movies entity type. A process log will be printed out."
       ]
     },
     {
@@ -476,7 +463,7 @@
       },
       "source": [
         "### Create Features\n",
-        "Features can be created within each entity type. Add defining features to the `Users` entity type and `Movies` entity type by using the following methods."
+        "Features can be created within each entity type. Add defining features to the Users entity type and Movies entity type by using the following methods."
       ]
     },
     {
@@ -560,7 +547,7 @@
         "id": "BlqJ-QdTcs6W"
       },
       "source": [
-        "#### Get data from source files"
+        "#### Entity Type Source Files"
       ]
     },
     {
@@ -656,7 +643,7 @@
         "id": "bgb0WGwX5OW6"
       },
       "source": [
-        "#### Ingest Feature Values into _Users_ Entity Type"
+        "#### Ingest Feature Values into Users Entity Type"
       ]
     },
     {
@@ -681,7 +668,7 @@
         "id": "PCAdQ3cF5OW6"
       },
       "source": [
-        "#### Ingest Feature Values into _Movies_ Entity Type"
+        "#### Ingest Feature Values into Movies Entity Type"
       ]
     },
     {
@@ -747,9 +734,9 @@
         "id": "AK2Glzkq5OW7"
       },
       "source": [
-        "## Batch Serve Feature Values from Vertex AI Feature Store\n",
+        "## Batch Serve Featurestore's Feature Values from Vertex AI Feature Store\n",
         "\n",
-        "Batch Serving is used to fetch a large batch of feature values for high-throughput, and is typically used for training a model or batch prediction. In this section, you learn how to prepare training examples by using the Feature Store's batch serve function."
+        "Batch Serving is used to fetch a large batch of feature values for high-throughput, and is typically used for training a model or batch prediction. In this section, you will learn how to prepare for training examples by using the Featurestore's batch serve function."
       ]
     },
     {
@@ -758,7 +745,7 @@
         "id": "hxsotHUe5OW7"
       },
       "source": [
-        "#### Read instances from source file"
+        "#### Read Instances Source File"
       ]
     },
     {
@@ -769,8 +756,7 @@
       },
       "outputs": [],
       "source": [
-        "GCS_READ_INSTANCES_CSV_URI = \"gs://cloud-samples-data-us-central1/vertex-ai/feature-store/datasets/movie_prediction.csv\"\n",
-        "READ_INSTANCES_CSV_FN = \"data.csv\""
+        "GCS_READ_INSTANCES_CSV_URI = \"gs://cloud-samples-data-us-central1/vertex-ai/feature-store/datasets/movie_prediction.csv\""
       ]
     },
     {
@@ -790,7 +776,7 @@
         "id": "T5DW1MFt5OW7"
       },
       "source": [
-        "#### Load CSV file into a Pandas DataFrame"
+        "#### Load Csv File into a Pandas DataFrame"
       ]
     },
     {
@@ -801,7 +787,7 @@
       },
       "outputs": [],
       "source": [
-        "read_instances_df = pd.read_csv(READ_INSTANCES_CSV_FN)\n",
+        "read_instances_df = pd.read_csv(read_instances_csv_fn)\n",
         "print(read_instances_df)"
       ]
     },
@@ -833,7 +819,7 @@
         "id": "ao1dC5Pc5OW8"
       },
       "source": [
-        "#### Batch Serve Feature Values from Movie Predictions Feature Store"
+        "#### Batch Serve Feature Values from Movie Predictions Featurestore"
       ]
     },
     {
@@ -869,8 +855,7 @@
         "id": "XN84znoI5OW8"
       },
       "source": [
-        "#### Feature Values from last ingestion\n",
-        "Recall read from the Entity Type shows Feature Values from the last ingestion."
+        "#### Recall Read from the Entity Type Shows Feature Values from the Last Ingestion"
       ]
     },
     {
@@ -890,7 +875,7 @@
         "id": "feTUJjqG5OW9"
       },
       "source": [
-        "#### Ingest updated Feature Values"
+        "#### Ingest Updated Feature Values"
       ]
     },
     {
@@ -930,8 +915,7 @@
         "id": "s47WCIvL5OW9"
       },
       "source": [
-        "#### Latest Feature Values\n",
-        "Read from the Entity Type shows updated Feature values from the latest ingestion."
+        "#### Read from the Entity Type Shows Updated Feature Values from the Latest Ingestion"
       ]
     },
     {
@@ -964,8 +948,7 @@
         "id": "R1YGRNsW5OW9"
       },
       "source": [
-        "#### Missing data\n",
-        "Recall Batch Serve from the last ingestion has some missing data in it."
+        "#### Recall Batch Serve From the Last Ingestion Has Missing Data"
       ]
     },
     {
@@ -985,7 +968,7 @@
         "id": "abQRF6mx5OW-"
       },
       "source": [
-        "#### Backfill/Correct point-in-time data"
+        "#### Backfill/Correct Point-in-Time Data"
       ]
     },
     {
@@ -1026,7 +1009,7 @@
         "id": "WXb4JUhu5OW-"
       },
       "source": [
-        "#### Ingest backfilled/corrected point-in-time data from dataframe"
+        "#### Ingest Backfill/Correct Point-in-Time Data"
       ]
     },
     {
@@ -1067,8 +1050,7 @@
         "id": "1e62Ku6W5OW_"
       },
       "source": [
-        "#### Latest ingestion with imputed missing data\n",
-        "Batch Serve from the latest ingestion with backfill/correction has reduced missing data."
+        "#### Batch Serve From the Latest Ingestion with Backfill/Correction Has Reduced Missing Data"
       ]
     },
     {

notebooks/community/managed_notebooks/chicago_taxi_fare_prediction/chicago_taxi_fare_prediction.ipynb

@@ -1,71 +1,13 @@
 {
   "cells": [
     {
-      "cell_type": "code",
-      "execution_count": null,
+      "cell_type": "markdown",
       "metadata": {
         "id": "c8c4e360024a"
       },
-      "outputs": [],
       "source": [
-        "# Copyright 2022 Google LLC\n",
-        "#\n",
-        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
-        "# you may not use this file except in compliance with the License.\n",
-        "# You may obtain a copy of the License at\n",
-        "#\n",
-        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
-        "#\n",
-        "# Unless required by applicable law or agreed to in writing, software\n",
-        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
-        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
-        "# See the License for the specific language governing permissions and\n",
-        "# limitations under the License."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "d6728c7e34d2"
-      },
-      "source": [
-        "<table align=\"left\">\n",
+        "# Taxi fare prediction using the Chicago Taxi Trips dataset\n",
         "\n",
-        "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/chicago_taxi_fare_prediction/chicago_taxi_fare_prediction.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
-        "    </a>\n",
-        "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/chicago_taxi_fare_prediction/chicago_taxi_fare_prediction.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
-        "      View on GitHub\n",
-        "    </a>\n",
-        "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/chicago_taxi_fare_prediction/chicago_taxi_fare_prediction.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
-        "    </a>\n",
-        "  </td>                                                                                               \n",
-        "</table>"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "28451b7c3d4b"
-      },
-      "source": [
-        "# Taxi fare prediction using the Chicago Taxi Trips dataset"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "eaf8ba4eeea8"
-      },
-      "source": [
         "## Table of contents\n",
         "\n",
         "* [Overview](#section-1)\n",
@@ -82,9 +24,9 @@
         "## Overview\n",
         "<a name=\"section-1\"></a>\n",
         "\n",
-        "This notebook demonstrates analysis, feature selection, model building, and deployment with Explainable AI configured on Vertex AI, using a subset of the Chicago Taxi Trips dataset for taxi-fare prediction.\n",
+        "This notebook demonstrates analysis, feature selection, model building, and deployment with Vertex Explainable AI configured on Vertex AI, using a subset of the Chicago Taxi Trips dataset for taxi-fare prediction.\n",
         "\n",
-        "*Note: This notebook is developed to run in a [Vertex AI Workbench managed notebooks](https://console.cloud.google.com/vertex-ai/workbench/list/managed) instance using the Python (Local) kernel. Some components of this notebook may not work in other notebook environments.*\n",
+        "*Note: This notebook file was developed to run in a [Vertex AI Workbench managed notebooks](https://console.cloud.google.com/vertex-ai/workbench/list/managed) instance using the Python (Local) kernel. Some components of this notebook may not work in other notebook environments.*\n",
         "\n",
         "## Dataset\n",
         "<a name=\"section-2\"></a>\n",
@@ -96,7 +38,7 @@
         "## Objective\n",
         "<a name=\"section-3\"></a>\n",
         "\n",
-        "The goal of this notebook is to provide an overview on the latest Vertex AI features like **Explainable AI** and **BigQuery in Notebooks** by trying to solve a taxi fare prediction problem. The steps followed in this notebook include: \n",
+        "The goal of this notebook is to provide an overview on the latest Vertex AI features like Explainable AI and \"BigQuery in Notebooks\" by trying to solve a taxi fare prediction problem. The steps followed in this notebook include: \n",
         "\n",
         "- Loading the dataset using \"BigQuery in Notebooks\".\n",
         "- Performing exploratory data analysis on the dataset.\n",
@@ -124,131 +66,6 @@
         "to generate a cost estimate based on your projected usage."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "5178273783dd"
-      },
-      "source": [
-        "### Set up your local development environment\n",
-        "\n",
-        "**If you are using Colab or Google Cloud Notebooks**, your environment already meets\n",
-        "all the requirements to run this notebook. You can skip this step."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "f5494c42606e"
-      },
-      "source": [
-        "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
-        "You need the following:\n",
-        "\n",
-        "* The Google Cloud SDK\n",
-        "* Git\n",
-        "* Python 3\n",
-        "* virtualenv\n",
-        "* Jupyter notebook running in a virtual environment with Python 3\n",
-        "\n",
-        "The Google Cloud guide to [Setting up a Python development\n",
-        "environment](https://cloud.google.com/python/setup) and the [Jupyter\n",
-        "installation guide](https://jupyter.org/install) provide detailed instructions\n",
-        "for meeting these requirements. The following steps provide a condensed set of\n",
-        "instructions:\n",
-        "\n",
-        "1. [Install and initialize the Cloud SDK.](https://cloud.google.com/sdk/docs/)\n",
-        "\n",
-        "1. [Install Python 3.](https://cloud.google.com/python/setup#installing_python)\n",
-        "\n",
-        "1. [Install\n",
-        "   virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv)\n",
-        "   and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
-        "\n",
-        "1. To install Jupyter, run `pip3 install jupyter` on the\n",
-        "command-line in a terminal shell.\n",
-        "\n",
-        "1. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
-        "\n",
-        "1. Open this notebook in the Jupyter Notebook Dashboard."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "23976b1be293"
-      },
-      "source": [
-        "### Install additional packages"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "1fd00fa70a2a"
-      },
-      "outputs": [],
-      "source": [
-        "import os\n",
-        "\n",
-        "# The Google Cloud Notebook product has specific requirements\n",
-        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
-        "\n",
-        "# Google Cloud Notebook requires dependencies to be installed with '--user'\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_GOOGLE_CLOUD_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\""
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "a50fd443a6ce"
-      },
-      "outputs": [],
-      "source": [
-        "! pip install {USER_FLAG} --upgrade google-cloud-bigquery \\\n",
-        "                                    google-cloud-aiplatform \\\n",
-        "                                    google-cloud-storage \\\n",
-        "                                    seaborn \\\n",
-        "                                    sklearn \\\n",
-        "                                    pandas \\\n",
-        "                                    fsspec \\\n",
-        "                                    pyarrow"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "d3a26cb9b19d"
-      },
-      "source": [
-        "### Restart the kernel\n",
-        "\n",
-        "After you install the additional packages, you need to restart the notebook kernel so it can find the packages."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "c1464805870e"
-      },
-      "outputs": [],
-      "source": [
-        "# Automatically restart kernel after installs\n",
-        "import os\n",
-        "\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    # Automatically restart kernel after installs\n",
-        "    import IPython\n",
-        "\n",
-        "    app = IPython.Application.instance()\n",
-        "    app.kernel.do_shutdown(True)"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -257,31 +74,7 @@
       "source": [
         "## Before you begin\n",
         "\n",
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com).\n",
-        "\n",
-        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "5aee4379e8e5"
-      },
-      "source": [
-        "#### Set your project ID\n",
+        "### Set your project ID\n",
         "\n",
         "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
       ]
@@ -326,175 +119,17 @@
         "    PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "30e64c0eda41"
-      },
-      "outputs": [],
-      "source": [
-        "! gcloud config set project $PROJECT_ID"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "7d43ac19ea91"
+        "id": "fed4b24ea061"
       },
       "source": [
-        "### Region\n",
+        "## Select or create a Cloud Storage bucket for storing the model\n",
         "\n",
-        "You can also change the `REGION` variable, which is used for operations\n",
-        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
+        "When you create a model resource on Vertex AI using the Cloud SDK, you need to give a Cloud Storage bucket uri of the model where the model is stored. Using the model saved, you can then create a Vertex AI model and endpoint resources in order to serve online predictions.\n",
         "\n",
-        "- Americas: `us-central1`\n",
-        "- Europe: `europe-west4`\n",
-        "- Asia Pacific: `asia-east1`\n",
-        "\n",
-        "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
-        "\n",
-        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "3281bedf6d3c"
-      },
-      "outputs": [],
-      "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "0c9906f72b18"
-      },
-      "source": [
-        "### UUID\n",
-        "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "8940c46120e6"
-      },
-      "outputs": [],
-      "source": [
-        "import random\n",
-        "import string\n",
-        "\n",
-        "\n",
-        "# Generate a uuid of length 8\n",
-        "def generate_uuid():\n",
-        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=8))\n",
-        "\n",
-        "\n",
-        "UUID = generate_uuid()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "648aa9824ac6"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Google Cloud Notebooks**, your environment is already\n",
-        "authenticated. Skip this step."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "fc52bba17ee3"
-      },
-      "source": [
-        "**If you are using Colab**, run the cell below and follow the instructions\n",
-        "when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "1. In the Cloud Console, go to the [**Create service account key**\n",
-        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
-        "\n",
-        "2. Click **Create service account**.\n",
-        "\n",
-        "3. In the **Service account name** field, enter a name, and\n",
-        "   click **Create**.\n",
-        "\n",
-        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
-        "into the filter box, and select\n",
-        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
-        "local environment.\n",
-        "\n",
-        "6. Enter the path to your service account key as the\n",
-        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "535223fa4b84"
-      },
-      "outputs": [],
-      "source": [
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "# The Google Cloud Notebook product has specific requirements\n",
-        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
-        "\n",
-        "# If on Google Cloud Notebooks, then don't execute this code\n",
-        "if not IS_GOOGLE_CLOUD_NOTEBOOK:\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "0474cb91d91f"
-      },
-      "source": [
-        "### Create a Cloud Storage bucket\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "\n",
-        "When you create a model in Vertex AI using the Cloud SDK, you give a Cloud Storage path where the trained model is saved. \n",
-        "In this tutorial, Vertex AI saves the trained model to a Cloud Storage bucket. Using this model artifact, you can then\n",
-        "create Vertex AI model and endpoint resources in order to serve\n",
-        "online predictions.\n",
-        "\n",
-        "Set the name of your Cloud Storage bucket below. It must be unique across all\n",
-        "Cloud Storage buckets."
+        "Set the name of your Cloud Storage bucket below. It must be unique across all Cloud Storage buckets. You may also change the `LOCATION` variable, which is used for operations throughout the rest of this notebook. Make sure to choose a region where Vertex AI services are available."
       ]
     },
     {
@@ -505,8 +140,9 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+        "BUCKET_NAME = \"[your-bucket-name]\"\n",
+        "BUCKET_URI = f\"gs://{BUCKET_NAME}\"\n",
+        "LOCATION = \"us-central1\""
       ]
     },
     {
@@ -517,8 +153,13 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
+        "from datetime import datetime\n",
+        "\n",
+        "# Set a default bucket name in case bucket name is not given\n",
+        "if BUCKET_NAME == \"\" or BUCKET_NAME == \"[your-bucket-name]\" or BUCKET_NAME is None:\n",
+        "\n",
+        "    TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
         "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
       ]
     },
@@ -539,7 +180,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_URI"
+        "! gsutil mb -l $REGION $BUCKET_NAME"
       ]
     },
     {
@@ -559,7 +200,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_URI"
+        "! gsutil ls -al $BUCKET_NAME"
       ]
     },
     {
@@ -568,7 +209,7 @@
         "id": "2e52fd6d4854"
       },
       "source": [
-        "### Import libraries"
+        "## Import the required libraries and define constants"
       ]
     },
     {
@@ -597,11 +238,11 @@
         "id": "5166f42557ad"
       },
       "source": [
-        "The original dataset considered for this tutorial is a large and noisy one and so data from a specific date range will be used. Based on various online resources, the data from around May 2018 gave some really good results compared to the other date ranges. While there are also some complicated models proposed for the same problem, like considering the weather data, holidays and seasons, the current notebook only explores a simple linear regression model. Our main objective is to demonstrate the model deployment with Vertex Explainable AI configured on Vertex AI.\n",
+        "The dataset is quite a large and noisy one, so data from a specific date range will be used. Based on various blogs and resources that are available online, many of them seem to have used the data from around May 2018 which gave some really good results compared to the other date ranges. While there are also some complicated research models proposed for the same problem, like considering the weather data, holidays and seasons, the current notebook only explores a simple linear regression model, as our main objective is to demonstrate the model deployment with Vertex Explainable AI configured on Vertex AI.\n",
         "\n",
-        "## Accessing the data through BigQuery Integration\n",
+        "## Accessing the data through \"BigQuery in Notebooks\"\n",
         "\n",
-        "The **BigQuery Integration for Notebooks** feature of Vertex AI Workbench managed notebooks lets you use BigQuery and its features from the notebook itself eliminating the need to switch between tabs everytime. For every cell in the notebook, there is an option for the BigQuery integration at the top right, and selecting it enables you to compose an SQL query that can be executed in BigQuery. \n",
+        "The \"BigQuery in Notebooks\" feature of Vertex AI Workbench managed notebooks lets you use BigQuery and its features from the notebook itself eliminating the need to switch between tabs everytime. For every cell in the notebook, there is an option for the BigQuery integration at the top right, and selecting it enables you to compose an SQL query that can be executed in BigQuery. \n",
         "\n",
         "The chosen dataset consists of the following fields:\n",
         "\n",
@@ -634,9 +275,7 @@
         "- Time taken for the trip > 0.\n",
         "- Distance covered during the trip > 0.\n",
         "- Total trip charges > 0 and\n",
-        "- Pickup and dropoff areas are valid (not empty).\n",
-        "\n",
-        "Note: The below cell is a Bigquery Integration cell and can only execute on Vertex AI Workbench's managed instances. If your notebook environment is different, you can skip it."
+        "- Pickup and dropoff areas are valid (not empty)."
       ]
     },
     {
@@ -686,7 +325,7 @@
         "# Comment out otherwise for speed-up.\n",
         "from google.cloud.bigquery import Client\n",
         "\n",
-        "client = Client(project=PROJECT_ID)\n",
+        "client = Client()\n",
         "\n",
         "query = \"\"\"select \n",
         "taxi_id, trip_start_timestamp, \n",
@@ -698,7 +337,7 @@
         "where \n",
         "trip_start_timestamp >= '2018-05-12' and \n",
         "trip_end_timestamp <= '2018-05-18' and\n",
-        "trip_seconds > 60 and trip_seconds < 6*60*60 and\n",
+        "trip_seconds > 0 and trip_seconds < 6*60*60 and\n",
         "trip_miles > 0 and\n",
         "trip_total > 3 and\n",
         "pickup_community_area is not NULL and \n",
@@ -850,7 +489,7 @@
         "## Analyze numerical data\n",
         "<a name=\"section-5\"></a>\n",
         "\n",
-        "To further anaylyze the data, there are various plots that can be used on numerical and categorical fields. In case of numerical data, you can use histograms and box plots. Bar charts are suited for categorical data to better understand the distribution of the data and the outliers in the data."
+        "To further anaylyze the data, there are various plots that can be used on numerical and categorical fields. In case of numerical data, one can use histograms and box plots while bar charts are suited for categorical data to better understand the distribution of the data and the outliers in the data."
       ]
     },
     {
@@ -885,7 +524,7 @@
         "id": "c3672976d67b"
       },
       "source": [
-        "The field `trip_seconds` describes the time taken for the trip in seconds. For ease of our analysis, let us convert it into hours."
+        "The field `trip_seconds` describes the time taken for the trip in seconds. Optionally, it can be converted into hours."
       ]
     },
     {
@@ -938,7 +577,6 @@
       },
       "outputs": [],
       "source": [
-        "# generate a pairplot for 10K samples\n",
         "sns.pairplot(\n",
         "    data=df[[\"trip_seconds\", \"trip_miles\", \"trip_total\", \"trip_speed\"]].sample(10000)\n",
         ")\n",
@@ -951,7 +589,7 @@
         "id": "b69e8094ba39"
       },
       "source": [
-        "From the box plots and the histograms visualized so far, it is evident that there are some outliers causing skewness in the data which perhaps could be removed. Also, you can see some linear relationships between the independent variables considered in the pair-plot. For example, `trip_seconds` and `trip_miles` and the dependant variable `trip_total`."
+        "From the box plots and the histograms visualized so far, it is evident that there are some outliers causing skewness in the data which perhaps could be removed. Also, you can see some linear relationships between the independent variables considered in the pair-plot, for example, `trip_seconds` and `trip_miles` and the dependant variable `trip_total`."
       ]
     },
     {
@@ -1376,17 +1014,6 @@
         "Configure Vertex Explainable AI before deploying the model. For further details, see [Configuring Vertex Explainable AI in Vertex AI models](https://cloud.google.com/vertex-ai/docs/explainable-ai/configuring-explanations#scikit-learn-and-xgboost-pre-built-containers)."
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "a130721a5375"
-      },
-      "outputs": [],
-      "source": [
-        "MODEL_DISPLAY_NAME = \"[your-model-display-name]\"  # @param {type: \"string\"}"
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -1395,11 +1022,7 @@
       },
       "outputs": [],
       "source": [
-        "# If the model display name is not set, choose the default one\n",
-        "if MODEL_DISPLAY_NAME == \"[your-model-display-name]\":\n",
-        "    MODEL_DISPLAY_NAME = \"taxi_fare_prediction_model\"\n",
-        "\n",
-        "\n",
+        "MODEL_DISPLAY_NAME = \"taxi_fare_prediction_model\"\n",
         "ARTIFACT_GCS_PATH = f\"{BUCKET_URI}/{BLOB_PATH}\"\n",
         "\n",
         "# Feature-name(Inp_feature) and Output-name(Model_output) can be arbitrary\n",
@@ -1429,7 +1052,7 @@
         "\n",
         "# Create a Vertex AI model resource with support for Vertex Explainable AI\n",
         "\n",
-        "aiplatform.init(project=PROJECT_ID, location=REGION)\n",
+        "aiplatform.init(project=PROJECT, location=LOCATION)\n",
         "\n",
         "model = aiplatform.Model.upload(\n",
         "    display_name=MODEL_DISPLAY_NAME,\n",
@@ -1453,20 +1076,7 @@
         "id": "1ed1bd9f0957"
       },
       "source": [
-        "### Create an Endpoint resource for the model\n",
-        "\n",
-        "Set a display name for the endpoint and create the endpoint resource."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "f0e5cea786b4"
-      },
-      "outputs": [],
-      "source": [
-        "ENDPOINT_DISPLAY_NAME = \"[your-endpoint-display-name]\"  # @param {type: \"string\"}"
+        "Create an Endpoint resource for the model."
       ]
     },
     {
@@ -1477,12 +1087,10 @@
       },
       "outputs": [],
       "source": [
-        "# If the display name is not set, choose the default one\n",
-        "if ENDPOINT_DISPLAY_NAME == \"[your-endpoint-display-name]\":\n",
-        "    ENDPOINT_DISPLAY_NAME = \"taxi_fare_prediction_endpoint\"\n",
+        "ENDPOINT_DISPLAY_NAME = \"taxi_fare_prediction_endpoint\"\n",
         "\n",
         "endpoint = aiplatform.Endpoint.create(\n",
-        "    display_name=ENDPOINT_DISPLAY_NAME, project=PROJECT_ID, location=REGION\n",
+        "    display_name=ENDPOINT_DISPLAY_NAME, project=PROJECT, location=LOCATION\n",
         ")\n",
         "\n",
         "print(endpoint.display_name)\n",
@@ -1492,23 +1100,30 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "9eaab1c54d66"
+        "id": "2a1b280dbec6"
       },
       "source": [
-        "### Deploy the model to the created endpoint with the required machine type\n",
-        "\n",
-        "Set a name for the deployment and deploy the model to the created endpoint."
+        "Save the Endpoint Id for inference."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "6fd9517c3403"
+        "id": "6516bfdd5066"
       },
       "outputs": [],
       "source": [
-        "DEPLOYED_MODEL_NAME = \"[your-deployed-model-name]\"  # @param {type: \"string\"}"
+        "ENDPOINT_ID = \"\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9eaab1c54d66"
+      },
+      "source": [
+        "Deploy the model to the created endpoint with the required machine type."
       ]
     },
     {
@@ -1519,14 +1134,10 @@
       },
       "outputs": [],
       "source": [
-        "# If the deployment name is not set, choose the default one\n",
-        "if DEPLOYED_MODEL_NAME == \"[your-deployed-model-name]\":\n",
-        "    DEPLOYED_MODEL_NAME = \"taxi_fare_prediction_deployment\"\n",
-        "\n",
-        "# Set the machine type to n1-standard2\n",
+        "DEPLOYED_MODEL_NAME = \"taxi_fare_prediction_deployment\"\n",
         "MACHINE_TYPE = \"n1-standard-2\"\n",
         "\n",
-        "# Deploy the model to the endpoint\n",
+        "# deploy the model to the endpoint\n",
         "model.deploy(\n",
         "    endpoint=endpoint,\n",
         "    deployed_model_display_name=DEPLOYED_MODEL_NAME,\n",
@@ -1545,18 +1156,18 @@
         "id": "686cfdcbaef8"
       },
       "source": [
-        "To ensure the model is deployed, the ID of the deployed model can be checked using the `endpoint.list_models()` method."
+        "Save the ID of the deployed model. The ID of the deployed model can also checked using the `endpoint.list_models()` method."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "bab07adf5339"
+        "id": "018f0fdb1d60"
       },
       "outputs": [],
       "source": [
-        "endpoint.list_models()"
+        "DEPLOYED_MODEL_ID = \"\""
       ]
     },
     {
@@ -1625,7 +1236,7 @@
         "    \"\"\"\n",
         "    aiplatform.init(project=project, location=location)\n",
         "\n",
-        "    # endpoint = aiplatform.Endpoint(endpoint_id)\n",
+        "    endpoint = aiplatform.Endpoint(endpoint_id)\n",
         "\n",
         "    response = endpoint.explain(instances=instances)\n",
         "    print(\"#\" * 10 + \"Explanations\" + \"#\" * 10)\n",
@@ -1655,7 +1266,7 @@
         "\n",
         "\n",
         "test_json = [X_test.iloc[0].tolist(), X_test.iloc[1].tolist()]\n",
-        "prediction = explain_tabular_sample(PROJECT_ID, REGION, endpoint, test_json)"
+        "prediction = explain_tabular_sample(PROJECT, LOCATION, ENDPOINT_ID, test_json)"
       ]
     },
     {
@@ -1666,7 +1277,7 @@
       "source": [
         "## Next steps\n",
         "\n",
-        "Since the Chicago Taxi Trips dataset is continuously updating, one can preform the same kind of analysis and model training every time a new set of data is available. The date range can also be increased from a week to a month or more depending on the quality of the data. Most of the steps followed in this notebook would still be valid and can be applied over the new data unless the data is too noisy. In fact, the notebook itself can be scheduled to run at the specified times to retrain the model using the scheduling option of [Vertex AI Workbench's executor](https://console.cloud.google.com/vertex-ai/workbench/list/executions). "
+        "Since the Chicago Taxi Trips dataset is continuously updating, one can preform the same kind of analysis and model training every time a new set of data is available. The date range can also be increased from a week to a month or more depending on the quality of the data. Most of the steps followed in this notebook would still be valid and can be applied over the new data unless the data is too noisy. Perhaps, the notebook itself can be scheduled to run at the specified times to retrain the model using the scheduling option of [Vertex AI Workbench's executor](https://console.cloud.google.com/vertex-ai/workbench/list/executions). "
       ]
     },
     {
@@ -1676,24 +1287,11 @@
       },
       "source": [
         "## Clean up\n",
+        "<a name=\"section-10\"></a>\n",
         "\n",
-        "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
-        "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
+        "Delete the resources created in this notebook.\n",
         "\n",
-        "Otherwise, you can delete the individual resources you created in this tutorial:\n",
-        "\n",
-        "- Model\n",
-        "- Endpoint\n",
-        "- Cloud Storage Bucket"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "f28a9843a13e"
-      },
-      "source": [
-        "Undeploy the model"
+        "Undeploy the model by specifying the `DEPLOYED_MODEL_ID`."
       ]
     },
     {
@@ -1704,7 +1302,7 @@
       },
       "outputs": [],
       "source": [
-        "endpoint.undeploy_all()"
+        "endpoint.undeploy(deployed_model_id=DEPLOYED_MODEL_ID)"
       ]
     },
     {
@@ -1764,11 +1362,7 @@
       },
       "outputs": [],
       "source": [
-        "# Set this to true only if you'd like to delete your bucket\n",
-        "delete_bucket = False\n",
-        "\n",
-        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
-        "    ! gsutil -m rm -r $BUCKET_URI"
+        "! gsutil -m rm -r $BUCKET_URI"
       ]
     }
   ],

notebooks/community/managed_notebooks/fraud_detection/fraud-detection-model.ipynb

@@ -9,57 +9,6 @@
         "# Build a fraud detection model on Vertex AI"
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "5fcd3e4da897"
-      },
-      "outputs": [],
-      "source": [
-        "# Copyright 2022 Google LLC\n",
-        "#\n",
-        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
-        "# you may not use this file except in compliance with the License.\n",
-        "# You may obtain a copy of the License at\n",
-        "#\n",
-        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
-        "#\n",
-        "# Unless required by applicable law or agreed to in writing, software\n",
-        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
-        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
-        "# See the License for the specific language governing permissions and\n",
-        "# limitations under the License."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "05c670d35496"
-      },
-      "source": [
-        "<table align=\"left\">\n",
-        "\n",
-        "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/fraud_detection/fraud-detection-model.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
-        "    </a>\n",
-        "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/fraud_detection/fraud-detection-model.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
-        "      View on GitHub\n",
-        "    </a>\n",
-        "  </td>\n",
-        "   <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/fraud_detection/fraud-detection-model.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
-        "    </a>\n",
-        "  </td>\n",
-        "</table>"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -91,7 +40,9 @@
         "## Overview\n",
         "<a name=\"section-1\"></a>\n",
         "\n",
-        "This tutorial shows you how to build, deploy, and analyze predictions from a simple [random forest](https://en.wikipedia.org/wiki/Random_forest) model using tools like scikit-learn, Vertex AI, and the [What-IF Tool (WIT)](https://cloud.google.com/ai-platform/prediction/docs/using-what-if-tool) on a synthetic fraud transaction dataset to solve a financial fraud detection problem.\n"
+        "This tutorial shows you how to build, deploy, and analyze predictions from a simple [random forest](https://en.wikipedia.org/wiki/Random_forest) model using tools like scikit-learn, Vertex AI, and the [What-IF Tool (WIT)](https://cloud.google.com/ai-platform/prediction/docs/using-what-if-tool) on a synthetic fraud transaction dataset to solve a financial fraud detection problem.\n",
+        "\n",
+        "*Note: This notebook file was designed to run in a [Vertex AI Workbench managed notebooks](https://cloud.google.com/vertex-ai/docs/workbench/managed/create-instance) instance using the `TensorFlow 2 (Local)` kernel. Some components of this notebook may not work in other notebook environments.*"
       ]
     },
     {
@@ -151,62 +102,13 @@
         "to generate a cost estimate based on your projected usage. "
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "1ba37fa1511f"
-      },
-      "source": [
-        "### Set up your local development environment\n",
-        "\n",
-        "**If you are using Colab or Google Cloud Notebooks**, your environment already meets\n",
-        "all the requirements to run this notebook. You can skip this step."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "cd1bc75a1cb2"
-      },
-      "source": [
-        "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
-        "You need the following:\n",
-        "\n",
-        "* The Google Cloud SDK\n",
-        "* Git\n",
-        "* Python 3\n",
-        "* virtualenv\n",
-        "* Jupyter notebook running in a virtual environment with Python 3\n",
-        "\n",
-        "The Google Cloud guide to [Setting up a Python development\n",
-        "environment](https://cloud.google.com/python/setup) and the [Jupyter\n",
-        "installation guide](https://jupyter.org/install) provide detailed instructions\n",
-        "for meeting these requirements. The following steps provide a condensed set of\n",
-        "instructions:\n",
-        "\n",
-        "1. [Install and initialize the Cloud SDK.](https://cloud.google.com/sdk/docs/)\n",
-        "\n",
-        "1. [Install Python 3.](https://cloud.google.com/python/setup#installing_python)\n",
-        "\n",
-        "1. [Install\n",
-        "   virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv)\n",
-        "   and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
-        "\n",
-        "1. To install Jupyter, run `pip3 install jupyter` on the\n",
-        "command-line in a terminal shell.\n",
-        "\n",
-        "1. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
-        "\n",
-        "1. Open this notebook in the Jupyter Notebook Dashboard."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "611991f03b38"
       },
       "source": [
-        "## Install additional packages"
+        "## Installation"
       ]
     },
     {
@@ -235,7 +137,7 @@
       "source": [
         "Install the latest version of the Vertex AI client library.\n",
         "\n",
-        "Run the following command in your notebook environment to install the Vertex SDK for Python:"
+        "Run the following command in your virtual environment to install the Vertex SDK for Python:"
       ]
     },
     {
@@ -249,168 +151,6 @@
         "! pip install {USER_FLAG} --upgrade google-cloud-aiplatform"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "1969a1cc46cf"
-      },
-      "source": [
-        "Run the following command in your notebook environment to install witwidget:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "8b10e59b0911"
-      },
-      "outputs": [],
-      "source": [
-        "! pip install {USER_FLAG} witwidget"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "4099ce79705a"
-      },
-      "source": [
-        "Run the following command in your notebook environment to install joblib:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "1e56d524753a"
-      },
-      "outputs": [],
-      "source": [
-        "! pip install {USER_FLAG} joblib"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "b87ee3041f7d"
-      },
-      "source": [
-        "Run the following command in your notebook environment to install scikit-learn:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "c3ebecd9bd72"
-      },
-      "outputs": [],
-      "source": [
-        "! pip install {USER_FLAG} scikit-learn"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "b624b5163531"
-      },
-      "source": [
-        "Run the following command in your notebook environment to install fsspec:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "79c7a64b04de"
-      },
-      "outputs": [],
-      "source": [
-        "! pip install {USER_FLAG} fsspec"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "5593090dcf0a"
-      },
-      "source": [
-        "Run the following command in your notebook environment to install gcsfs:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "7bf981bc5bf6"
-      },
-      "outputs": [],
-      "source": [
-        "! pip install {USER_FLAG} gcsfs"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "1c7b2a25df27"
-      },
-      "source": [
-        "### Restart the kernel\n",
-        "\n",
-        "After you install the additional packages, you need to restart the notebook kernel so it can find the packages."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "2117d92e6766"
-      },
-      "outputs": [],
-      "source": [
-        "# Automatically restart kernel after installs\n",
-        "\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    # Automatically restart kernel after installs\n",
-        "    import IPython\n",
-        "\n",
-        "    app = IPython.Application.instance()\n",
-        "    app.kernel.do_shutdown(True)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "2d9b3731b3e0"
-      },
-      "source": [
-        "## Before you begin"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "7a5cb1df1ef7"
-      },
-      "source": [
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com). {TODO: Update the APIs needed for your tutorial. Edit the API names, and update the link to append the API IDs, separating each one with a comma. For example, container.googleapis.com,cloudbuild.googleapis.com}\n",
-        "\n",
-        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -430,8 +170,6 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
         "PROJECT_ID = \"\"\n",
         "\n",
         "# Get your Google Cloud project ID from gcloud\n",
@@ -462,17 +200,6 @@
         "    PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "b11114d77c5f"
-      },
-      "outputs": [],
-      "source": [
-        "! gcloud config set project $PROJECT_ID"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -497,81 +224,6 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "0c7f603fcdcf"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Google Cloud Notebooks**, your environment is already\n",
-        "authenticated. Skip this step."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "72bf8f7c9ab3"
-      },
-      "source": [
-        "**If you are using Colab**, run the cell below and follow the instructions\n",
-        "when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "1. In the Cloud Console, go to the [**Create service account key**\n",
-        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
-        "\n",
-        "2. Click **Create service account**.\n",
-        "\n",
-        "3. In the **Service account name** field, enter a name, and\n",
-        "   click **Create**.\n",
-        "\n",
-        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
-        "into the filter box, and select\n",
-        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
-        "local environment.\n",
-        "\n",
-        "6. Enter the path to your service account key as the\n",
-        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "da63f3587ef9"
-      },
-      "outputs": [],
-      "source": [
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "# The Google Cloud Notebook product has specific requirements\n",
-        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
-        "\n",
-        "# If on Google Cloud Notebooks, then don't execute this code\n",
-        "if not IS_GOOGLE_CLOUD_NOTEBOOK:\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -606,7 +258,7 @@
       "outputs": [],
       "source": [
         "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "REGION = \"[your-region]\"  # @param {type:\"string\"}"
+        "REGION = \"us-central1\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -619,9 +271,7 @@
       "source": [
         "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
         "    BUCKET_NAME = PROJECT_ID + \"-vertex-ai-\" + TIMESTAMP\n",
-        "BUCKET_URI = f\"gs://{BUCKET_NAME}\"\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
     {
@@ -696,7 +346,7 @@
         "import matplotlib.pyplot as plt\n",
         "import numpy as np\n",
         "import pandas as pd\n",
-        "from google.cloud import aiplatform, storage\n",
+        "from google.cloud import storage\n",
         "from sklearn.ensemble import RandomForestClassifier\n",
         "from sklearn.metrics import (average_precision_score, classification_report,\n",
         "                             confusion_matrix, f1_score)\n",
@@ -937,11 +587,9 @@
       },
       "outputs": [],
       "source": [
-        "print(\"before initiating\")\n",
-        "forest = RandomForestClassifier(verbose=1)\n",
-        "print(\"after initiating\")\n",
-        "forest.fit(X_train, y_train)\n",
-        "print(\"after fitting\")"
+        "%%time\n",
+        "forest = RandomForestClassifier()\n",
+        "forest.fit(X_train, y_train)"
       ]
     },
     {
@@ -964,9 +612,7 @@
       },
       "outputs": [],
       "source": [
-        "print(\"before predicting\")\n",
         "y_prob = forest.predict_proba(X_test)\n",
-        "print(\"after predicting y_prob\")\n",
         "y_pred = forest.predict(X_test)\n",
         "\n",
         "print(\"AUPRC :\", (average_precision_score(y_test, y_prob[:, 1])))\n",
@@ -976,8 +622,7 @@
         "print(confusion_matrix(y_test, y_pred))\n",
         "\n",
         "print(\"classification_report\")\n",
-        "print(classification_report(y_test, y_pred))\n",
-        "print(\"after printing classification_report\")"
+        "print(classification_report(y_test, y_pred))"
       ]
     },
     {
@@ -1033,7 +678,7 @@
         "BLOB_PATH = \"[your-blob-path]\"\n",
         "BLOB_NAME = os.path.join(BLOB_PATH, FILE_NAME)\n",
         "\n",
-        "bucket = storage.Client(PROJECT_ID).bucket(BUCKET_NAME)\n",
+        "bucket = storage.Client().bucket(BUCKET_NAME)\n",
         "blob = bucket.blob(BLOB_NAME)\n",
         "blob.upload_from_filename(FILE_NAME)"
       ]
@@ -1057,10 +702,7 @@
       "outputs": [],
       "source": [
         "MODEL_DISPLAY_NAME = \"[your-model-display-name]\"\n",
-        "ARTIFACT_GCS_PATH = f\"{BUCKET_URI}/{BLOB_PATH}\"\n",
-        "SERVING_CONTAINER_IMAGE_URI = (\n",
-        "    \"us-docker.pkg.dev/vertex-ai/prediction/sklearn-cpu.1-0:latest\"\n",
-        ")"
+        "ARTIFACT_GCS_PATH = f\"{BUCKET_URI}/{BLOB_PATH}\""
       ]
     },
     {
@@ -1072,13 +714,14 @@
       "outputs": [],
       "source": [
         "# Create a Vertex AI model resource\n",
+        "from google.cloud import aiplatform\n",
         "\n",
         "aiplatform.init(project=PROJECT_ID, location=REGION)\n",
         "\n",
         "model = aiplatform.Model.upload(\n",
         "    display_name=MODEL_DISPLAY_NAME,\n",
         "    artifact_uri=ARTIFACT_GCS_PATH,\n",
-        "    serving_container_image_uri=SERVING_CONTAINER_IMAGE_URI,\n",
+        "    serving_container_image_uri=\"us-docker.pkg.dev/vertex-ai/prediction/sklearn-cpu.0-24:latest\",\n",
         ")\n",
         "\n",
         "model.wait()\n",
@@ -1154,6 +797,10 @@
       },
       "outputs": [],
       "source": [
+        "# Uncomment if starting over without model and endpoint references\n",
+        "# model = aiplatform.Model('[your-model-resource-name]')\n",
+        "# endpoint = aiplatform.Endpoint('[your-endpoint-resource-name]')\n",
+        "\n",
         "# deploy the model to the endpoint\n",
         "model.deploy(\n",
         "    endpoint=endpoint,\n",
@@ -1167,6 +814,26 @@
         "print(model.resource_name)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "602a1a615bb0"
+      },
+      "source": [
+        "Save the ID of the deployed model. The ID of the deployed model can also be checked by using the `endpoint.list_models()` method."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "84a1da5b5e93"
+      },
+      "outputs": [],
+      "source": [
+        "DEPLOYED_MODEL_ID = \"[your-deployed-model-id]\""
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -1176,7 +843,7 @@
         "## What-If Tool \n",
         "<a name=\"section-11\"></a>\n",
         "\n",
-        "The What-If Tool can be used to analyze the model predictions on a test data. See a [brief introduction to the What-If Tool](https://pair-code.github.io/what-if-tool/). In this tutorial, the What-If Tool will be configured and run on the model trained locally, and on the model deployed on Vertex AI Endpoint in the previous steps.\n",
+        "The What-If Tool can be used to analyze the model predictions on a test data. See a [brief introduction to the What-If Tool](https://pair-code.github.io/what-if-tool/). In this tutorial, the What-If Tool will be configured and run on the model trained locally, and on the model deployed on Vertex AI Endpoints in the previous steps.\n",
         "\n",
         "[WitConfigBuilder](https://github.com/PAIR-code/what-if-tool/blob/master/witwidget/notebook/visualization.py#L30) provides the  `set_ai_platform_model()` method to configure the What-If Tool with a model deployed as a version on Ai Platform models. This feature currently supports Ai Platform only but not Vertex AI models. Fortunately, there is also an option to pass a custom function for generating predictions through the `set_custom_predict_fn()` method where either the locally trained model or a function that returns predictions from a Vertex AI model can be passed."
       ]
@@ -1303,27 +970,6 @@
         "WitWidget(config_builder, height=400)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "c446b1263b34"
-      },
-      "source": [
-        "## Undeploy the model\n",
-        "When you are done doing predictions, you undeploy the model from the Endpoint resouce. This deprovisions all compute resources and ends billing for the deployed model."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "72eb599403d4"
-      },
-      "outputs": [],
-      "source": [
-        "endpoint.undeploy_all()"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -1340,6 +986,18 @@
         "Otherwise, you can delete the individual resources you created in this tutorial:"
       ]
     },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "46061cbb656d"
+      },
+      "outputs": [],
+      "source": [
+        "# undeploy the model\n",
+        "endpoint.undeploy(deployed_model_id=DEPLOYED_MODEL_ID)"
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -1372,9 +1030,8 @@
       },
       "outputs": [],
       "source": [
-        "delete_bucket = True\n",
-        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
-        "    ! gsutil rm -r $BUCKET_URI"
+        "# uncomment to remove the contents of the Cloud Storage bucket\n",
+        "# ! gsutil -m rm -r $BUCKET_NAME"
       ]
     }
   ],

notebooks/community/managed_notebooks/subscriber_churn_prediction/telecom-subscriber-churn-prediction.ipynb

@@ -1,56 +1,12 @@
 {
   "cells": [
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "18ebbd838e32"
-      },
-      "outputs": [],
-      "source": [
-        "# Copyright 2022 Google LLC\n",
-        "#\n",
-        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
-        "# you may not use this file except in compliance with the License.\n",
-        "# You may obtain a copy of the License at\n",
-        "#\n",
-        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
-        "#\n",
-        "# Unless required by applicable law or agreed to in writing, software\n",
-        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
-        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
-        "# See the License for the specific language governing permissions and\n",
-        "# limitations under the License."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "64f7165bd1ac"
       },
       "source": [
-        "# Telecom subscriber churn prediction on Vertex AI\n",
-        "\n",
-        "<table align=\"left\">\n",
-        "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/subscriber_churn_prediction/telecom-subscriber-churn-prediction.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
-        "      View on GitHub\n",
-        "    </a>\n",
-        "  </td>\n",
-        "    <td>\n",
-        "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/subscriber_churn_prediction/telecom-subscriber-churn-prediction.ipynb\">\n",
-        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\\\" alt=\"Colab logo\"> Run in Colab\n",
-        "        </a>\n",
-        "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/subscriber_churn_prediction/telecom-subscriber-churn-prediction.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
-        "    </a>\n",
-        "  </td>\n",
-        "</table>\n",
-        "<br/><br/><br/>"
+        "# Telecom subscriber churn prediction on Vertex AI"
       ]
     },
     {
@@ -83,7 +39,9 @@
         "## Overview\n",
         "<a name=\"section-1\"></a>\n",
         "\n",
-        "This example demonstrates building a subscriber churn prediction model on a [telecom customer churn dataset](https://www.kaggle.com/c/customer-churn-prediction-2020/overview). The generated churn model is further deployed to Vertex AI Endpoints and explanations are generated using the Explainable AI feature of Vertex AI. "
+        "This example demonstrates building a subscriber churn prediction model on a [telecom customer churn dataset](https://www.kaggle.com/c/customer-churn-prediction-2020/overview). The generated churn model is further deployed to Vertex AI Endpoints and explanations are generated using the Explainable AI feature of Vertex AI. \n",
+        "\n",
+        "*Note: This notebook file was designed to run in a [Vertex AI Workbench managed notebooks](https://cloud.google.com/vertex-ai/docs/workbench/managed/create-instance) instance using the `Python (Local)` kernel. Some components of this notebook may not work in other notebook environments.*"
       ]
     },
     {
@@ -95,7 +53,7 @@
         "## Dataset\n",
         "<a name=\"section-2\"></a>\n",
         "\n",
-        "The dataset used in this tutorial is Telecom-Customer Churn dataset publicly available on Kaggle. See [Customer Churn Prediction 2020](https://www.kaggle.com/c/customer-churn-prediction-2020/data). This dataset is used to build and deploy a churn prediction model using Vertex AI in this notebook."
+        "The dataset used in this tutorial is publicly available at Kaggle. See [Customer Churn Prediction 2020](https://www.kaggle.com/c/customer-churn-prediction-2020/data). "
       ]
     },
     {
@@ -107,7 +65,7 @@
         "## Objective\n",
         "<a name=\"section-3\"></a>\n",
         "\n",
-        "This tutorial shows you how to do exploratory data analysis, preprocess data, train, deploy and get predictions from a churn prediction model on a tabular churn dataset. The objectives of this tutorial are as follows:\n",
+        "This tutorial shows you how to do exploratory data analysis, preprocess data, and train a churn prediction model on a tabular churn dataset. The steps include the following:\n",
         "\n",
         "- Load data from a Cloud Storage path\n",
         "- Perform exploratory data analysis (EDA)\n",
@@ -149,9 +107,7 @@
         "id": "44b8ae8e2d19"
       },
       "source": [
-        "## Installation\n",
-        "\n",
-        "Install the following packages to run this notebook."
+        "## Installation"
       ]
     },
     {
@@ -173,6 +129,17 @@
         "    USER_FLAG = \"--user\""
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "606337930991"
+      },
+      "source": [
+        "Install the latest version of the Vertex AI client library.\n",
+        "\n",
+        "Run the following command in your virtual environment to install the Vertex SDK for Python:"
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -181,43 +148,67 @@
       },
       "outputs": [],
       "source": [
-        "! pip install {USER_FLAG} --upgrade google-cloud-aiplatform \\\n",
-        "                                    google-cloud-storage \\\n",
-        "                                    category_encoders \\\n",
-        "                                    seaborn \\\n",
-        "                                    sklearn \\\n",
-        "                                    pandas \\\n",
-        "                                    fsspec \\\n",
-        "                                    gcsfs -q"
+        "! pip install {USER_FLAG} --upgrade google-cloud-aiplatform"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "b24902cde81b"
+        "id": "e67139e68463"
       },
       "source": [
-        "### Restart the kernel\n",
-        "\n",
-        "Once you've installed the additional packages, you need to restart the notebook kernel so it can find the packages."
+        "Install the Cloud Storage library:"
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "c61d171395d7"
+        "id": "2ad918f94f5d"
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    # Automatically restart kernel after installs\n",
-        "    import IPython\n",
-        "\n",
-        "    app = IPython.Application.instance()\n",
-        "    app.kernel.do_shutdown(True)"
+        "! pip install {USER_FLAG} --upgrade google-cloud-storage"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "eb0c1e24a8f0"
+      },
+      "source": [
+        "Install the `category_encoders` library:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "deb95a7f2104"
+      },
+      "outputs": [],
+      "source": [
+        "! pip install --upgrade category_encoders"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "184560c1b742"
+      },
+      "source": [
+        "Install the `seaborn` library for the EDA step. If a Vertex AI Workbench managed notebooks instance is being used, this step is optional as the library is already available in the `Python (Local)` kernel."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "0d99cdcdc470"
+      },
+      "outputs": [],
+      "source": [
+        "! pip install --upgrade seaborn"
       ]
     },
     {
@@ -252,7 +243,7 @@
         "id": "96ff17f75e21"
       },
       "source": [
-        "### Set your project ID\n",
+        "#### Set your project ID\n",
         "\n",
         "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
       ]
@@ -265,13 +256,11 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
         "PROJECT_ID = \"\"\n",
         "\n",
         "# Get your Google Cloud project ID from gcloud\n",
         "if not os.getenv(\"IS_TESTING\"):\n",
-        "    shell_output = !gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "    shell_output=!gcloud config list --format 'value(core.project)' 2>/dev/null\n",
         "    PROJECT_ID = shell_output[0]\n",
         "    print(\"Project ID: \", PROJECT_ID)"
       ]
@@ -297,58 +286,13 @@
         "    PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "f2e3c0f2cbfb"
-      },
-      "outputs": [],
-      "source": [
-        "! gcloud config set project $PROJECT_ID"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "60d535f443ac"
-      },
-      "source": [
-        "### Region\n",
-        "\n",
-        "You can also change the `REGION` variable, which is used for operations\n",
-        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
-        "\n",
-        "- Americas: `us-central1`\n",
-        "- Europe: `europe-west4`\n",
-        "- Asia Pacific: `asia-east1`\n",
-        "\n",
-        "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
-        "\n",
-        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "3aaadaaf9b30"
-      },
-      "outputs": [],
-      "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "e663bd062c6f"
       },
       "source": [
-        "### Timestamp\n",
+        "#### Timestamp\n",
         "\n",
         "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append it onto the name of resources you create in this tutorial."
       ]
@@ -366,63 +310,6 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "3ffa6b6c7cdb"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Google Cloud Notebooks**, your environment is already authenticated. Skip this step.\n",
-        "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
-        "\n",
-        "1. **Click Create service account**.\n",
-        "\n",
-        "2. In the **Service account name** field, enter a name, and click **Create**.\n",
-        "\n",
-        "3. In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex AI\" into the filter box, and select **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "4. Click Create. A JSON file that contains your key downloads to your local environment.\n",
-        "\n",
-        "5. Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "2b72272258fc"
-      },
-      "outputs": [],
-      "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Google Cloud Notebook, then don't execute this code\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -440,7 +327,12 @@
         "online predictions.\n",
         "\n",
         "Set the name of your Cloud Storage bucket below. It must be unique across all\n",
-        "Cloud Storage buckets."
+        "Cloud Storage buckets.\n",
+        "\n",
+        "You may also change the `REGION` variable, which is used for operations\n",
+        "throughout the rest of this notebook. Make sure to [choose a region where Vertex AI services are\n",
+        "available](https://cloud.google.com/vertex-ai/docs/general/locations#available_regions). You may\n",
+        "not use a Multi-Regional Storage bucket for training with Vertex AI."
       ]
     },
     {
@@ -451,8 +343,8 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "REGION = \"[your-region]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -463,9 +355,8 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
-        "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
       ]
     },
     {
@@ -485,7 +376,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_URI"
+        "! gsutil mb -l $REGION $BUCKET_NAME"
       ]
     },
     {
@@ -505,7 +396,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_URI"
+        "! gsutil ls -al $BUCKET_NAME"
       ]
     },
     {
@@ -559,13 +450,7 @@
         "id": "e37354341588"
       },
       "source": [
-        "### Load data from Cloud Storage using Pandas\n",
-        "\n",
-        "The Telecom-Customer Churn dataset from [Kaggle](https://www.kaggle.com/c/customer-churn-prediction-2020/overview) is made available on a public Cloud Storage bucket at: \n",
-        "\n",
-        "```gs://cloud-samples-data/vertex-ai/managed_notebooks/telecom_churn_prediction/train.csv```\n",
-        "\n",
-        "Use Pandas to read data directly from the URI."
+        "### Load data from Cloud Storage path using Pandas"
       ]
     },
     {
@@ -1231,8 +1116,6 @@
         "    \"[your-blob-path]\"  # leave blank if no folders inside the bucket are needed.\n",
         ")\n",
         "\n",
-        "if BLOB_PATH == (\"[your-blob-path]\"):\n",
-        "    BLOB_PATH = \"\"\n",
         "\n",
         "BLOB_NAME = BLOB_PATH + FILE_NAME\n",
         "\n",
@@ -1250,9 +1133,7 @@
         "## Create a model with Explainable AI support in Vertex AI\n",
         "<a name=\"section-9\"></a>\n",
         "\n",
-        "Before creating a model, configure the explanations for the model. For further details, see [Configuring explanations in Vertex AI](https://cloud.google.com/vertex-ai/docs/explainable-ai/configuring-explanations#scikit-learn-and-xgboost-pre-built-containers).\n",
-        "\n",
-        "Set a display name for the model resource."
+        "Before creating a model, configure the explanations for the model. For further details, see [Configuring explanations in Vertex AI](https://cloud.google.com/vertex-ai/docs/explainable-ai/configuring-explanations#scikit-learn-and-xgboost-pre-built-containers)."
       ]
     },
     {
@@ -1263,13 +1144,10 @@
       },
       "outputs": [],
       "source": [
-        "# Set the model display name\n",
-        "MODEL_DISPLAY_NAME = \"[your-model-display-name]\"  # @param {type:\"string\"}\n",
-        "\n",
-        "if MODEL_DISPLAY_NAME == \"[your-model-display-name]\":\n",
-        "    MODEL_DISPLAY_NAME = \"subscriber_churn_model\"\n",
-        "\n",
+        "MODEL_DISPLAY_NAME = \"[your-model-display-name]\"\n",
         "ARTIFACT_GCS_PATH = f\"gs://{BUCKET_NAME}/{BLOB_PATH}\"\n",
+        "PROJECT = \"[your-project-id]\"\n",
+        "LOCATION = REGION\n",
         "\n",
         "# Feature-name(Inp_feature) and Output-name(Model_output) can be arbitrary\n",
         "exp_metadata = {\"inputs\": {\"Inp_feature\": {}}, \"outputs\": {\"Model_output\": {}}}"
@@ -1283,20 +1161,17 @@
       },
       "outputs": [],
       "source": [
-        "from google.cloud.aiplatform_v1.types import SampledShapleyAttribution\n",
         "# Create a Vertex AI model resource with support for explanations\n",
-        "from google.cloud.aiplatform_v1.types.explanation import ExplanationParameters\n",
         "\n",
-        "aiplatform.init(project=PROJECT_ID, location=REGION)\n",
+        "aiplatform.init(project=PROJECT, location=LOCATION)\n",
+        "explanation_parameters = {\"sampledShapleyAttribution\": {\"pathCount\": 25}}\n",
         "\n",
         "model = aiplatform.Model.upload(\n",
         "    display_name=MODEL_DISPLAY_NAME,\n",
         "    artifact_uri=ARTIFACT_GCS_PATH,\n",
-        "    serving_container_image_uri=\"us-docker.pkg.dev/vertex-ai/prediction/sklearn-cpu.1-0:latest\",\n",
+        "    serving_container_image_uri=\"us-docker.pkg.dev/vertex-ai/prediction/sklearn-cpu.0-24:latest\",\n",
         "    explanation_metadata=exp_metadata,\n",
-        "    explanation_parameters=ExplanationParameters(\n",
-        "        sampled_shapley_attribution=SampledShapleyAttribution(path_count=25)\n",
-        "    ),\n",
+        "    explanation_parameters=explanation_parameters,\n",
         ")\n",
         "\n",
         "model.wait()\n",
@@ -1317,7 +1192,7 @@
         "gcloud beta ai models upload \\\n",
         "  --region=$REGION \\\n",
         "  --display-name=$MODEL_DISPLAY_NAME \\\n",
-        "  --container-image-uri=\"us-docker.pkg.dev/vertex-ai/prediction/sklearn-cpu.1-0:latest\" \\\n",
+        "  --container-image-uri=\"us-docker.pkg.dev/vertex-ai/prediction/sklearn-cpu.0-24:latest\" \\\n",
         "  --artifact-uri=$ARTIFACT_GCS_PATH \\\n",
         "  --explanation-method=sampled-shapley \\\n",
         "  --explanation-path-count=25 \\\n",
@@ -1342,9 +1217,7 @@
       },
       "outputs": [],
       "source": [
-        "ENDPOINT_DISPLAY_NAME = \"[your-endpoint-display-name]\"  # @param {type:\"string\"}\n",
-        "if ENDPOINT_DISPLAY_NAME == \"[your-endpoint-display-name]\":\n",
-        "    ENDPOINT_DISPLAY_NAME = \"subsc_churn_endpoint\""
+        "ENDPOINT_DISPLAY_NAME = \"[your-endpoint-display-name]\""
       ]
     },
     {
@@ -1356,13 +1229,33 @@
       "outputs": [],
       "source": [
         "endpoint = aiplatform.Endpoint.create(\n",
-        "    display_name=ENDPOINT_DISPLAY_NAME, project=PROJECT_ID, location=REGION\n",
+        "    display_name=ENDPOINT_DISPLAY_NAME, project=PROJECT, location=LOCATION\n",
         ")\n",
         "\n",
         "print(endpoint.display_name)\n",
         "print(endpoint.resource_name)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ae4c69ef8a8c"
+      },
+      "source": [
+        "Save the endpoint ID after the endpoint is created."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6aa73d9a88d3"
+      },
+      "outputs": [],
+      "source": [
+        "ENDPOINT_ID = \"[your-endpoint-id]\""
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -1382,11 +1275,8 @@
       },
       "outputs": [],
       "source": [
-        "DEPLOYED_MODEL_NAME = \"[deployment-model-name]\"  # @param {type:\"string\"}\n",
-        "MACHINE_TYPE = \"n1-standard-4\"\n",
-        "\n",
-        "if DEPLOYED_MODEL_NAME == \"[deployment-model-name]\":\n",
-        "    DEPLOYED_MODEL_NAME = \"subsc_churn_deployment\""
+        "DEPLOYED_MODEL_NAME = \"[deployment-model-name]\"\n",
+        "MACHINE_TYPE = \"n1-standard-4\""
       ]
     },
     {
@@ -1416,7 +1306,7 @@
         "id": "359c43e630cb"
       },
       "source": [
-        "To ensure the model is deployed, the ID of the deployed model can be checked using the `endpoint.list_models()` method."
+        "Save the ID of the deployed model. The ID of the deployed model can also checked using the `endpoint.list_models()` method."
       ]
     },
     {
@@ -1427,7 +1317,7 @@
       },
       "outputs": [],
       "source": [
-        "endpoint.list_models()"
+        "DEPLOYED_MODEL_ID = \"[your-deployed-model-id]\""
       ]
     },
     {
@@ -1446,7 +1336,7 @@
         "id": "7b50c31e0552"
       },
       "source": [
-        "Get explanations for a test instance from the hosted model."
+        "Get explanations for some test instances from the hosted model."
       ]
     },
     {
@@ -1457,8 +1347,8 @@
       },
       "outputs": [],
       "source": [
-        "# format a test instance as the request's payload\n",
-        "test_json = [X_test.iloc[0].tolist()]"
+        "# format the top 2 test instances as the request's payload\n",
+        "test_json = {\"instances\": [X_test.iloc[0].tolist(), X_test.iloc[1].tolist()]}"
       ]
     },
     {
@@ -1495,13 +1385,15 @@
         "    return\n",
         "\n",
         "\n",
-        "def explain_tabular_sample(project: str, location: str, endpoint, instances: list):\n",
+        "def explain_tabular_sample(\n",
+        "    project: str, location: str, endpoint_id: str, instances: list\n",
+        "):\n",
         "    \"\"\"\n",
         "    Function to make an explanation request for the specified payload and generate feature attribution plots\n",
         "    \"\"\"\n",
         "    aiplatform.init(project=project, location=location)\n",
         "\n",
-        "    # endpoint = aiplatform.Endpoint(endpoint_id)\n",
+        "    endpoint = aiplatform.Endpoint(endpoint_id)\n",
         "\n",
         "    response = endpoint.explain(instances=instances)\n",
         "    print(\"#\" * 10 + \"Explanations\" + \"#\" * 10)\n",
@@ -1530,8 +1422,8 @@
         "    return response\n",
         "\n",
         "\n",
-        "# Get explanations for the test instance\n",
-        "prediction = explain_tabular_sample(PROJECT_ID, REGION, endpoint, test_json)"
+        "test_json = [X_test.iloc[0].tolist(), X_test.iloc[1].tolist()]\n",
+        "prediction = explain_tabular_sample(PROJECT, LOCATION, ENDPOINT_ID, test_json)"
       ]
     },
     {
@@ -1546,12 +1438,7 @@
         "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
         "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
         "\n",
-        "Otherwise, you can delete the individual resources you created in this tutorial:\n",
-        "* Vertex AI Model\n",
-        "* Vertex AI Endpoint\n",
-        "* Cloud Storage bucket\n",
-        "\n",
-        "Set `delete_bucket` to *True* to delete the Cloud Storage bucket."
+        "Otherwise, you can delete the individual resources you created in this tutorial:"
       ]
     },
     {
@@ -1562,8 +1449,8 @@
       },
       "outputs": [],
       "source": [
-        "# Undeploy model\n",
-        "endpoint.undeploy_all()"
+        "# undeploy the model\n",
+        "endpoint.undeploy(deployed_model_id=DEPLOYED_MODEL_ID)"
       ]
     },
     {
@@ -1574,7 +1461,7 @@
       },
       "outputs": [],
       "source": [
-        "# Delete the endpoint\n",
+        "# delete the endpoint\n",
         "endpoint.delete()"
       ]
     },
@@ -1586,7 +1473,7 @@
       },
       "outputs": [],
       "source": [
-        "# Delete the model\n",
+        "# delete the model\n",
         "model.delete()"
       ]
     },
@@ -1598,10 +1485,8 @@
       },
       "outputs": [],
       "source": [
-        "# Delete the Cloud Storage bucket\n",
-        "delete_bucket = True\n",
-        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
-        "    ! gsutil -m rm -r $BUCKET_URI"
+        "# remove the contents of the Cloud Storage bucket\n",
+        "! gsutil -m rm -r $BUCKET_NAME"
       ]
     }
   ],

notebooks/community/matching_engine/sdk_matching_engine_for_indexing.ipynb

@@ -97,9 +97,11 @@
       "source": [
         "## Before you begin\n",
         "\n",
-        "* **Prepare a VPC network**.  To reduce any network overhead that might lead to unnecessary increase in overhead latency, it is best to call the ANN endpoints from your VPC via a direct [VPC Peering](https://cloud.google.com/vertex-ai/docs/general/vpc-peering) connection. \n",
-        "  * The following section describes how to setup a VPC Peering connection if you don't have one. \n",
-        "  * This is a one-time initial setup task. You can also reuse existing VPC network and skip this section."
+        "* **Prepare a VPC network**.  To reduce any network overhead that might lead to unnecessary increase in overhead latency, it is best to call the ANN endpoints from your VPC via a direct [VPC Peering](https://cloud.google.com/vertex-ai/docs/general/vpc-peering) connection. The following section describes how to setup a VPC Peering connection if you don't have one. This is a one-time initial setup task. You can also reuse existing VPC network and skip this section.\n",
+        "* **WARNING:** The match service gRPC API (to create online queries against your deployed index) has to be executed in a Google Cloud Notebook instance that is created with the following requirements:\n",
+        "  * **In the same region as where your ANN service is deployed** (for example, if you set `REGION = \"us-central1\"` as same as the tutorial, the notebook instance has to be in `us-central1`).\n",
+        "  * **Make sure you select the VPC network you created for ANN service** (instead of using the \"default\" one). That is, you will have to create the VPC network below and then create a new notebook instance that uses that VPC.  \n",
+        "  * If you run it in the colab or a Google Cloud Notebook instance in a different VPC network or region, the gRPC API will fail to peer the network (InactiveRPCError)."
       ]
     },
     {
@@ -110,11 +112,11 @@
       },
       "outputs": [],
       "source": [
-        "PROJECT_ID = \"python-docs-samples-tests\"  # @param {type:\"string\"}\n",
+        "PROJECT_ID = \"<your_project_id>\"  # @param {type:\"string\"}\n",
         "\n",
-        "NETWORK_NAME = \"ann-vpc-network\"  # @param {type:\"string\"}\n",
+        "NETWORK_NAME = \"ucaip-haystack-vpc-network\"  # @param {type:\"string\"}\n",
         "\n",
-        "PEERING_RANGE_NAME = \"ann-haystack-range\""
+        "PEERING_RANGE_NAME = \"ucaip-haystack-range\""
       ]
     },
     {
@@ -138,10 +140,9 @@
         "! gcloud compute firewall-rules create {NETWORK_NAME}-allow-ssh --network {NETWORK_NAME} --priority 65534 --project {PROJECT_ID} --allow tcp:22\n",
         "\n",
         "# Reserve IP range\n",
-        "! gcloud compute addresses create {PEERING_RANGE_NAME} --global --prefix-length=16 --network={NETWORK_NAME} --purpose=VPC_PEERING --project={PROJECT_ID} --description=\"peering range\"\n",
+        "! gcloud compute addresses create {PEERING_RANGE_NAME} --global --prefix-length=16 --network={NETWORK_NAME} --purpose=VPC_PEERING --project={PROJECT_ID} --description=\"peering range for uCAIP Haystack.\"\n",
         "\n",
         "# Set up peering with service networking\n",
-        "# Your account must have the \"Compute Network Admin\" role to run the following.\n",
         "! gcloud services vpc-peerings connect --service=servicenetworking.googleapis.com --network={NETWORK_NAME} --ranges={PEERING_RANGE_NAME} --project={PROJECT_ID}"
       ]
     },
@@ -151,21 +152,7 @@
         "id": "d3uj8x73nDX_"
       },
       "source": [
-        "* Authentication: Rerun the `gcloud auth login` command in the Vertex AI Workbench notebook terminal when you are logged out and need the credential again."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "d5de53b31bf1"
-      },
-      "source": [
-        "## Make sure the following cells are run from inside the VPC network that you created in the previous step.\n",
-        "\n",
-        "* **WARNING:** The MatchingIndexEndpoint.match method (to create online queries against your deployed index) has to be executed in a Vertex AI Workbench notebook instance that is created with the following requirements:\n",
-        "  * **In the same region as where your ANN service is deployed** (for example, if you set `REGION = \"us-central1\"` as same as the tutorial, the notebook instance has to be in `us-central1`).\n",
-        "  * **Make sure you select the VPC network you created for ANN service** (instead of using the \"default\" one). That is, you will have to create the VPC network below and then create a new notebook instance that uses that VPC.  \n",
-        "  * If you run it in the colab or a Vertex AI Workbench notebook instance in a different VPC network or region, \"Create Online Queries\" section will fail."
+        "* Authentication: `$ gcloud auth login` rerun this in Google Cloud Notebook terminal when you are logged out and need the credential again."
       ]
     },
     {
@@ -176,7 +163,7 @@
       "source": [
         "### Installation\n",
         "\n",
-        "Download and install the latest version of the Vertex SDK for Python."
+        "Download and install the latest (preview) version of the Vertex SDK for Python."
       ]
     },
     {
@@ -187,7 +174,7 @@
       },
       "outputs": [],
       "source": [
-        "! pip install -U google-cloud-aiplatform"
+        "! pip install -U git+https://github.com/ivanmkc/python-aiplatform.git@imkc--matching-engine"
       ]
     },
     {
@@ -284,7 +271,7 @@
       "source": [
         "import os\n",
         "\n",
-        "PROJECT_ID = \"python-docs-samples-tests\"\n",
+        "PROJECT_ID = \"\"\n",
         "\n",
         "# Get your Google Cloud project ID from gcloud\n",
         "if not os.getenv(\"IS_TESTING\"):\n",
@@ -346,7 +333,7 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using a Vertex AI Workbench notebook**, your environment is already\n",
+        "**If you are using Google Cloud Notebooks**, your environment is already\n",
         "authenticated. Skip this step."
       ]
     },
@@ -396,13 +383,11 @@
         "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
         "# requests.\n",
         "\n",
-        "# The Vertex AI Workbench notebook product has specific requirements\n",
-        "IS_VERTEX_AI_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
+        "# The Google Cloud Notebook product has specific requirements\n",
+        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
         "\n",
-        "# If on a Vertex AI Workbench notebook, then don't execute this code\n",
-        "if not IS_VERTEX_AI_WORKBENCH_NOTEBOOK:\n",
+        "# If on Google Cloud Notebooks, then don't execute this code\n",
+        "if not IS_GOOGLE_CLOUD_NOTEBOOK:\n",
         "    if \"google.colab\" in sys.modules:\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
@@ -554,12 +539,12 @@
         "id": "lR6Wwv-hCCN-"
       },
       "source": [
-        "## Prepare the data\n",
+        "## Prepare the Data\n",
         "\n",
         "The GloVe dataset consists of a set of pre-trained embeddings. The embeddings are split into a \"train\" split, and a \"test\" split.\n",
         "We will create a vector search index from the \"train\" split, and use the embedding vectors in the \"test\" split as query vectors to test the vector search index.\n",
         "\n",
-        "**Note:** While the data split uses the term \"train\", these are pre-trained embeddings and therefore are ready to be indexed for search. The terms \"train\" and \"test\" split are used just to be consistent with machine learning terminology.\n",
+        "NOTE: While the data split uses the term \"train\", these are pre-trained embeddings and thus are ready to be indexed for search. The terms \"train\" and \"test\" split are used just to be consistent with usual machine learning terminology.\n",
         "\n",
         "Download the GloVe dataset.\n"
       ]
@@ -747,28 +732,6 @@
         "INDEX_RESOURCE_NAME"
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "0f1a9fbecabb"
-      },
-      "outputs": [],
-      "source": [
-        "Using the resource name, you can retrieve an existing MatchingEngineIndex."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "1ddb70647d98"
-      },
-      "outputs": [],
-      "source": [
-        "tree_ah_index = aiplatform.MatchingEngineIndex(INDEX_RESOURCE_NAME)"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -777,7 +740,7 @@
       "source": [
         "### Create Brute Force Index (for Ground Truth)\n",
         "\n",
-        "The brute force index uses a naive brute force method to find the nearest neighbors. This method is not fast or efficient. Hence brute force indices are not recommended for production usage. They are to be used to find the \"ground truth\" set of neighbors, so that the \"ground truth\" set can be used to measure recall of the indices being tuned for production usage. To ensure an apples to apples comparison, the `distanceMeasureType` and `dimensions` of the brute force index should match those of the production indices being tuned.\n",
+        "The brute force index uses a naive brute force method to find the nearest neighbors. This method is not fast or efficient. Hence brute force indices are not recommended for production usage. They are to be used to find the \"ground truth\" set of neighbors, so that the \"ground truth\" set can be used to measure recall of the indices being tuned for production usage. To ensure an apples to apples comparison, the `distanceMeasureType` and `featureNormType`, `dimensions` of the brute force index should match those of the production indices being tuned.\n",
         "\n",
         "Create the brute force index configuration:"
       ]
@@ -812,19 +775,6 @@
         "INDEX_BRUTE_FORCE_RESOURCE_NAME"
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "865fcad494d7"
-      },
-      "outputs": [],
-      "source": [
-        "brute_force_index = aiplatform.MatchingEngineIndex(\n",
-        "    \"projects/1012616486416/locations/us-central1/indexes/6738176690918260736\"\n",
-        ")"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -989,7 +939,7 @@
       },
       "outputs": [],
       "source": [
-        "DEPLOYED_INDEX_ID = f\"tree_ah_glove_deployed_{TIMESTAMP}\""
+        "DEPLOYED_INDEX_ID = \"tree_ah_glove_deployed\""
       ]
     },
     {
@@ -1024,7 +974,7 @@
       },
       "outputs": [],
       "source": [
-        "DEPLOYED_BRUTE_FORCE_INDEX_ID = f\"glove_brute_force_deployed_{TIMESTAMP}\""
+        "DEPLOYED_BRUTE_FORCE_INDEX_ID = \"glove_brute_force_deployed\""
       ]
     },
     {
@@ -1071,13 +1021,344 @@
       "outputs": [],
       "source": [
         "# Test query\n",
+        "query = [\n",
+        "    -0.11333,\n",
+        "    0.48402,\n",
+        "    0.090771,\n",
+        "    -0.22439,\n",
+        "    0.034206,\n",
+        "    -0.55831,\n",
+        "    0.041849,\n",
+        "    -0.53573,\n",
+        "    0.18809,\n",
+        "    -0.58722,\n",
+        "    0.015313,\n",
+        "    -0.014555,\n",
+        "    0.80842,\n",
+        "    -0.038519,\n",
+        "    0.75348,\n",
+        "    0.70502,\n",
+        "    -0.17863,\n",
+        "    0.3222,\n",
+        "    0.67575,\n",
+        "    0.67198,\n",
+        "    0.26044,\n",
+        "    0.4187,\n",
+        "    -0.34122,\n",
+        "    0.2286,\n",
+        "    -0.53529,\n",
+        "    1.2582,\n",
+        "    -0.091543,\n",
+        "    0.19716,\n",
+        "    -0.037454,\n",
+        "    -0.3336,\n",
+        "    0.31399,\n",
+        "    0.36488,\n",
+        "    0.71263,\n",
+        "    0.1307,\n",
+        "    -0.24654,\n",
+        "    -0.52445,\n",
+        "    -0.036091,\n",
+        "    0.55068,\n",
+        "    0.10017,\n",
+        "    0.48095,\n",
+        "    0.71104,\n",
+        "    -0.053462,\n",
+        "    0.22325,\n",
+        "    0.30917,\n",
+        "    -0.39926,\n",
+        "    0.036634,\n",
+        "    -0.35431,\n",
+        "    -0.42795,\n",
+        "    0.46444,\n",
+        "    0.25586,\n",
+        "    0.68257,\n",
+        "    -0.20821,\n",
+        "    0.38433,\n",
+        "    0.055773,\n",
+        "    -0.2539,\n",
+        "    -0.20804,\n",
+        "    0.52522,\n",
+        "    -0.11399,\n",
+        "    -0.3253,\n",
+        "    -0.44104,\n",
+        "    0.17528,\n",
+        "    0.62255,\n",
+        "    0.50237,\n",
+        "    -0.7607,\n",
+        "    -0.071786,\n",
+        "    0.0080131,\n",
+        "    -0.13286,\n",
+        "    0.50097,\n",
+        "    0.18824,\n",
+        "    -0.54722,\n",
+        "    -0.42664,\n",
+        "    0.4292,\n",
+        "    0.14877,\n",
+        "    -0.0072514,\n",
+        "    -0.16484,\n",
+        "    -0.059798,\n",
+        "    0.9895,\n",
+        "    -0.61738,\n",
+        "    0.054169,\n",
+        "    0.48424,\n",
+        "    -0.35084,\n",
+        "    -0.27053,\n",
+        "    0.37829,\n",
+        "    0.11503,\n",
+        "    -0.39613,\n",
+        "    0.24266,\n",
+        "    0.39147,\n",
+        "    -0.075256,\n",
+        "    0.65093,\n",
+        "    -0.20822,\n",
+        "    -0.17456,\n",
+        "    0.53571,\n",
+        "    -0.16537,\n",
+        "    0.13582,\n",
+        "    -0.56016,\n",
+        "    0.016964,\n",
+        "    0.1277,\n",
+        "    0.94071,\n",
+        "    -0.22608,\n",
+        "    -0.021106,\n",
+        "]\n",
+        "\n",
         "response = my_index_endpoint.match(\n",
-        "    deployed_index_id=DEPLOYED_INDEX_ID, queries=test[:1], num_neighbors=NUM_NEIGHBOURS\n",
+        "    deployed_index_id=DEPLOYED_INDEX_ID, queries=[query], num_neighbors=NUM_NEIGHBOURS\n",
         ")\n",
         "\n",
         "response"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "_mNwdU9_B_Ez"
+      },
+      "source": [
+        "### Batch Query\n",
+        "\n",
+        "You can run multiple queries in a single match call:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "A0XL0PJ1GoM9"
+      },
+      "outputs": [],
+      "source": [
+        "# Test query\n",
+        "queries = [\n",
+        "    [\n",
+        "        -0.11333,\n",
+        "        0.48402,\n",
+        "        0.090771,\n",
+        "        -0.22439,\n",
+        "        0.034206,\n",
+        "        -0.55831,\n",
+        "        0.041849,\n",
+        "        -0.53573,\n",
+        "        0.18809,\n",
+        "        -0.58722,\n",
+        "        0.015313,\n",
+        "        -0.014555,\n",
+        "        0.80842,\n",
+        "        -0.038519,\n",
+        "        0.75348,\n",
+        "        0.70502,\n",
+        "        -0.17863,\n",
+        "        0.3222,\n",
+        "        0.67575,\n",
+        "        0.67198,\n",
+        "        0.26044,\n",
+        "        0.4187,\n",
+        "        -0.34122,\n",
+        "        0.2286,\n",
+        "        -0.53529,\n",
+        "        1.2582,\n",
+        "        -0.091543,\n",
+        "        0.19716,\n",
+        "        -0.037454,\n",
+        "        -0.3336,\n",
+        "        0.31399,\n",
+        "        0.36488,\n",
+        "        0.71263,\n",
+        "        0.1307,\n",
+        "        -0.24654,\n",
+        "        -0.52445,\n",
+        "        -0.036091,\n",
+        "        0.55068,\n",
+        "        0.10017,\n",
+        "        0.48095,\n",
+        "        0.71104,\n",
+        "        -0.053462,\n",
+        "        0.22325,\n",
+        "        0.30917,\n",
+        "        -0.39926,\n",
+        "        0.036634,\n",
+        "        -0.35431,\n",
+        "        -0.42795,\n",
+        "        0.46444,\n",
+        "        0.25586,\n",
+        "        0.68257,\n",
+        "        -0.20821,\n",
+        "        0.38433,\n",
+        "        0.055773,\n",
+        "        -0.2539,\n",
+        "        -0.20804,\n",
+        "        0.52522,\n",
+        "        -0.11399,\n",
+        "        -0.3253,\n",
+        "        -0.44104,\n",
+        "        0.17528,\n",
+        "        0.62255,\n",
+        "        0.50237,\n",
+        "        -0.7607,\n",
+        "        -0.071786,\n",
+        "        0.0080131,\n",
+        "        -0.13286,\n",
+        "        0.50097,\n",
+        "        0.18824,\n",
+        "        -0.54722,\n",
+        "        -0.42664,\n",
+        "        0.4292,\n",
+        "        0.14877,\n",
+        "        -0.0072514,\n",
+        "        -0.16484,\n",
+        "        -0.059798,\n",
+        "        0.9895,\n",
+        "        -0.61738,\n",
+        "        0.054169,\n",
+        "        0.48424,\n",
+        "        -0.35084,\n",
+        "        -0.27053,\n",
+        "        0.37829,\n",
+        "        0.11503,\n",
+        "        -0.39613,\n",
+        "        0.24266,\n",
+        "        0.39147,\n",
+        "        -0.075256,\n",
+        "        0.65093,\n",
+        "        -0.20822,\n",
+        "        -0.17456,\n",
+        "        0.53571,\n",
+        "        -0.16537,\n",
+        "        0.13582,\n",
+        "        -0.56016,\n",
+        "        0.016964,\n",
+        "        0.1277,\n",
+        "        0.94071,\n",
+        "        -0.22608,\n",
+        "        -0.021106,\n",
+        "    ],\n",
+        "    [\n",
+        "        -0.99544,\n",
+        "        -2.3651,\n",
+        "        -0.24332,\n",
+        "        -1.0321,\n",
+        "        0.42052,\n",
+        "        -1.1817,\n",
+        "        -0.16451,\n",
+        "        -1.683,\n",
+        "        0.49673,\n",
+        "        -0.27258,\n",
+        "        -0.025397,\n",
+        "        0.34188,\n",
+        "        1.5523,\n",
+        "        1.3532,\n",
+        "        0.33297,\n",
+        "        -0.0056677,\n",
+        "        -0.76525,\n",
+        "        0.49587,\n",
+        "        1.2211,\n",
+        "        0.83394,\n",
+        "        -0.20031,\n",
+        "        -0.59657,\n",
+        "        0.38485,\n",
+        "        -0.23487,\n",
+        "        -1.0725,\n",
+        "        0.95856,\n",
+        "        0.16161,\n",
+        "        -1.2496,\n",
+        "        1.6751,\n",
+        "        0.73899,\n",
+        "        0.051347,\n",
+        "        -0.42702,\n",
+        "        0.16257,\n",
+        "        -0.16772,\n",
+        "        0.40146,\n",
+        "        0.29837,\n",
+        "        0.96204,\n",
+        "        -0.36232,\n",
+        "        -0.47848,\n",
+        "        0.78278,\n",
+        "        0.14834,\n",
+        "        1.3407,\n",
+        "        0.47834,\n",
+        "        -0.39083,\n",
+        "        -1.037,\n",
+        "        -0.24643,\n",
+        "        -0.75841,\n",
+        "        0.7669,\n",
+        "        -0.37363,\n",
+        "        0.52741,\n",
+        "        0.018563,\n",
+        "        -0.51301,\n",
+        "        0.97674,\n",
+        "        0.55232,\n",
+        "        1.1584,\n",
+        "        0.73715,\n",
+        "        1.3055,\n",
+        "        -0.44743,\n",
+        "        -0.15961,\n",
+        "        0.85006,\n",
+        "        -0.34092,\n",
+        "        -0.67667,\n",
+        "        0.2317,\n",
+        "        1.5582,\n",
+        "        1.2308,\n",
+        "        -0.62213,\n",
+        "        -0.032801,\n",
+        "        0.1206,\n",
+        "        -0.25899,\n",
+        "        -0.02756,\n",
+        "        -0.52814,\n",
+        "        -0.93523,\n",
+        "        0.58434,\n",
+        "        -0.24799,\n",
+        "        0.37692,\n",
+        "        0.86527,\n",
+        "        0.069626,\n",
+        "        1.3096,\n",
+        "        0.29975,\n",
+        "        -1.3651,\n",
+        "        -0.32048,\n",
+        "        -0.13741,\n",
+        "        0.33329,\n",
+        "        -1.9113,\n",
+        "        -0.60222,\n",
+        "        -0.23921,\n",
+        "        0.12664,\n",
+        "        -0.47961,\n",
+        "        -0.89531,\n",
+        "        0.62054,\n",
+        "        0.40869,\n",
+        "        -0.08503,\n",
+        "        0.6413,\n",
+        "        -0.84044,\n",
+        "        -0.74325,\n",
+        "        -0.19426,\n",
+        "        0.098722,\n",
+        "        0.32648,\n",
+        "        -0.67621,\n",
+        "        -0.62692,\n",
+        "    ],\n",
+        "]"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -1086,7 +1367,7 @@
       "source": [
         "### Compute Recall\n",
         "\n",
-        "Use the deployed brute force Index as the ground truth to calculate the recall of ANN Index. Note that you can run multiple queries in a single match call."
+        "Use deployed brute force Index as the ground truth to calculate the recall of ANN Index:"
       ]
     },
     {
@@ -1119,20 +1400,18 @@
       "outputs": [],
       "source": [
         "# Calculate recall by determining how many neighbors were correctly retrieved as compared to the brute-force option.\n",
-        "recalled_neighbors = 0\n",
+        "correct_neighbors = 0\n",
         "for tree_ah_neighbors, brute_force_neighbors in zip(\n",
         "    tree_ah_response_test, brute_force_response_test\n",
         "):\n",
         "    tree_ah_neighbor_ids = [neighbor.id for neighbor in tree_ah_neighbors]\n",
         "    brute_force_neighbor_ids = [neighbor.id for neighbor in brute_force_neighbors]\n",
         "\n",
-        "    recalled_neighbors += len(\n",
+        "    correct_neighbors += len(\n",
         "        set(tree_ah_neighbor_ids).intersection(brute_force_neighbor_ids)\n",
         "    )\n",
         "\n",
-        "recall = recalled_neighbors / len(\n",
-        "    [neighbor for neighbors in brute_force_response_test for neighbor in neighbors]\n",
-        ")\n",
+        "recall = correct_neighbors / (len(test) * NUM_NEIGHBOURS)\n",
         "\n",
         "print(\"Recall: {}\".format(recall))"
       ]

notebooks/community/ml_metadata/sdk-metric-parameter-tracking-for-locally-trained-models.ipynb

@@ -8,7 +8,7 @@
       },
       "outputs": [],
       "source": [
-        "# Copyright 2022 Google LLC\n",
+        "# Copyright 2021 Google LLC\n",
         "#\n",
         "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
         "# you may not use this file except in compliance with the License.\n",
@@ -32,25 +32,28 @@
         "<table align=\"left\">\n",
         "\n",
         "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/official/tabnet/ai-explanations-tabnet-algorithm.ipynb\">\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/ml_metadata/sdk-metric-parameter-tracking-for-locally-trained-models.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samplestree/master/notebooks/official/tabnet/ai-explanations-tabnet-algorithm.ipynb\">\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/ml_metadata/sdk-metric-parameter-tracking-for-locally-trained-models.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/tabnet/ai-explanations-tabnet-algorithm.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
-        "    </a>\n",
-        "  </td>                                                                                               \n",
         "</table>"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WBFL9LagqmwT"
+      },
+      "source": [
+        "#Vertex AI: Track parameters and metrics for locally trained models"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -59,33 +62,28 @@
       "source": [
         "## Overview\n",
         "\n",
-        "Vertex AI provides a algorithm called on [TabNet] (https://arxiv.org/abs/1908.07442). TabNet is an interpretable deep learning architecture for tabular (structured) data, the most common data type among enterprises. TabNet combines the best of two worlds: it is explainable, like simpler tree-based models, and can achieve the high accuracy of complex black-box models and ensembles, meaning it is precise without obscuring how the model works. This makes TabNet well-suited for a wide range of tabular data tasks where model explainability is just as important as accuracy.\n",
-        "\n",
-        "The goal of the tutorial is to provide a sample plotting tool to visualize the output of TabNet, which is helpful in explaining the algorithm.\n",
+        "This notebook demonstrates how to track metrics and parameters for ML training jobs and analyze this metadata using Vertex SDK for Python.\n",
         "\n",
         "### Dataset\n",
         "\n",
-        "This tutorial uses Synthetic_2 (Syn2) data, described in Section 4.1 of the [Learning to Explain](https://arxiv.org/pdf/1802.07814.pdf) paper. The input feature X is generated from a 10-dimensional standard Gaussian. The response variable Y is generated from feature X[3:6] only. The data has been split into training and prediction sets and has been uploaded to Google Cloud Storage:\n",
-        "* Training data: gs://cloud-samples-data/ai-platform-unified/datasets/tabnet/tab_net_input/syn2_train.csv.\n",
-        "* Prediction output data: gs://cloud-samples-data/ai-platform-unified/datasets/tabnet/tab_net_output/syn2\n",
-        "\n",
-        "At this time, the TabNet pre-trained model file is not publicly available.\n",
+        "In this notebook, we will train a simple distributed neural network (DNN) model to predict automobile's miles per gallon (MPG) based on automobile information in the [auto-mpg dataset](https://www.kaggle.com/devanshbesain/exploration-and-analysis-auto-mpg).\n",
         "\n",
         "### Objective\n",
         "\n",
-        "The goal is to provide a sample plotting tool to visualize the output of TabNet, which is helpful in explaining the algorithm. It includes the following steps:\n",
-        "* Setup the the project.\n",
-        "* Download the prediction data of pretrain model onf Syn2 data.\n",
-        "* Visualize and understand the feature importance based on the masks output.\n",
-        "* Clean up the resource created by this tutorial.\n",
+        "In this notebook, you will learn how to use Vertex SDK for Python to:\n",
+        "\n",
+        "    * Track parameters and metrics for a locally trainined model.\n",
+        "    * Extract and perform analysis for all parameters and metrics within an Experiment.\n",
         "\n",
         "### Costs \n",
         "\n",
+        "\n",
         "This tutorial uses billable components of Google Cloud:\n",
         "\n",
         "* Vertex AI\n",
         "* Cloud Storage\n",
         "\n",
+        "\n",
         "Learn about [Vertex AI\n",
         "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
         "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
@@ -134,7 +132,7 @@
         "   virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv)\n",
         "   and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
         "\n",
-        "1. To install Jupyter, run `pip3 install jupyter` on the\n",
+        "1. To install Jupyter, run `pip install jupyter` on the\n",
         "command-line in a terminal shell.\n",
         "\n",
         "1. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
@@ -142,23 +140,72 @@
         "1. Open this notebook in the Jupyter Notebook Dashboard."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "i7EUnXsZhAGF"
+      },
+      "source": [
+        "### Install additional packages\n",
+        "\n",
+        "Run the following commands to install the Vertex SDK for Python."
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "2b4ef9b72d43"
+        "id": "IaYsrh0Tc17L"
       },
       "outputs": [],
       "source": [
+        "import sys\n",
+        "\n",
+        "if \"google.colab\" in sys.modules:\n",
+        "    USER_FLAG = \"\"\n",
+        "else:\n",
+        "    USER_FLAG = \"--user\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "wyy5Lbnzg5fi"
+      },
+      "outputs": [],
+      "source": [
+        "!python3 -m pip install {USER_FLAG} google-cloud-aiplatform --upgrade"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "hhq5zEbGg0XX"
+      },
+      "source": [
+        "### Restart the kernel\n",
+        "\n",
+        "After you install the additional packages, you need to restart the notebook kernel so it can find the packages."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "EzrelQZ22IZj"
+      },
+      "outputs": [],
+      "source": [
+        "# Automatically restart kernel after installs\n",
         "import os\n",
         "\n",
-        "# The Google Cloud Notebook product has specific requirements\n",
-        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
+        "if not os.getenv(\"IS_TESTING\"):\n",
+        "    # Automatically restart kernel after installs\n",
+        "    import IPython\n",
         "\n",
-        "# Google Cloud Notebook requires dependencies to be installed with '--user'\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_GOOGLE_CLOUD_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\""
+        "    app = IPython.Application.instance()\n",
+        "    app.kernel.do_shutdown(True)"
       ]
     },
     {
@@ -167,7 +214,11 @@
         "id": "lWEdiXsJg0XY"
       },
       "source": [
-        "## Before you begin"
+        "## Before you begin\n",
+        "\n",
+        "### Select a GPU runtime\n",
+        "\n",
+        "**Make sure you're running this notebook in a GPU runtime if you have that option. In Colab, select \"Runtime --> Change runtime type > GPU\"**"
       ]
     },
     {
@@ -184,7 +235,7 @@
         "\n",
         "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
         "\n",
-        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com). {TODO: Update the APIs needed for your tutorial. Edit the API names, and update the link to append the API IDs, separating each one with a comma. For example, container.googleapis.com,cloudbuild.googleapis.com}\n",
+        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com).\n",
         "\n",
         "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
         "\n",
@@ -213,11 +264,13 @@
       },
       "outputs": [],
       "source": [
+        "import os\n",
+        "\n",
         "PROJECT_ID = \"\"\n",
         "\n",
         "# Get your Google Cloud project ID from gcloud\n",
         "if not os.getenv(\"IS_TESTING\"):\n",
-        "    shell_output = !gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "    shell_output=!gcloud config list --format 'value(core.project)' 2>/dev/null\n",
         "    PROJECT_ID = shell_output[0]\n",
         "    print(\"Project ID: \", PROJECT_ID)"
       ]
@@ -243,6 +296,30 @@
         "    PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "06571eb4063b"
+      },
+      "source": [
+        "#### Timestamp\n",
+        "\n",
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append it onto the name of resources you create in this tutorial."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "697568e92bd6"
+      },
+      "outputs": [],
+      "source": [
+        "from datetime import datetime\n",
+        "\n",
+        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -301,11 +378,8 @@
         "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
         "# requests.\n",
         "\n",
-        "# The Google Cloud Notebook product has specific requirements\n",
-        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
-        "\n",
         "# If on Google Cloud Notebooks, then don't execute this code\n",
-        "if not IS_GOOGLE_CLOUD_NOTEBOOK:\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
         "    if \"google.colab\" in sys.modules:\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
@@ -318,115 +392,6 @@
         "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "zgPO1eR3CYjk"
-      },
-      "source": [
-        "### Create a Cloud Storage bucket\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "Set the name of your Cloud Storage bucket below. It must be unique across all\n",
-        "Cloud Storage buckets.\n",
-        "\n",
-        "You may also change the `REGION` variable, which is used for operations\n",
-        "throughout the rest of this notebook. We suggest that you [choose a region where Vertex AI services are\n",
-        "available](https://cloud.google.com/vertex-ai/docs/general/locations#available_regions)."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "e663bd062c6f"
-      },
-      "source": [
-        "#### Timestamp\n",
-        "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append it onto the name of resources you create in this tutorial."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "953fa6e5ddda"
-      },
-      "outputs": [],
-      "source": [
-        "from datetime import datetime\n",
-        "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "MzGDU7TWdts_"
-      },
-      "outputs": [],
-      "source": [
-        "BUCKET_URI = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "REGION = \"[your-region]\"  # @param {type:\"string\"}"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "cf221059d072"
-      },
-      "outputs": [],
-      "source": [
-        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "-EcIXiGsCePi"
-      },
-      "source": [
-        "**Only if your bucket doesn't already exist**: Run the following cell to create your Cloud Storage bucket."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "NIq7R4HZCfIc"
-      },
-      "outputs": [],
-      "source": [
-        "! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "ucvCsknMCims"
-      },
-      "source": [
-        "Finally, validate access to your Cloud Storage bucket by examining its contents:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "vhOb7YnwClBb"
-      },
-      "outputs": [],
-      "source": [
-        "! gsutil ls -al $BUCKET_URI"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -436,6 +401,15 @@
         "### Import libraries and define constants"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Y9Uo3tifg1kx"
+      },
+      "source": [
+        "Import required libraries."
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -444,133 +418,434 @@
       },
       "outputs": [],
       "source": [
-        "import json\n",
-        "\n",
-        "import matplotlib.cm as cm\n",
         "import matplotlib.pyplot as plt\n",
-        "import numpy as np\n",
-        "from google.cloud import storage\n",
-        "\n",
-        "%matplotlib inline"
+        "import pandas as pd\n",
+        "from google.cloud import aiplatform\n",
+        "from tensorflow.python.keras import Sequential, layers\n",
+        "from tensorflow.python.keras.utils import data_utils"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "06f6abd8c40e"
+        "id": "xtXZWmYqJ1bh"
       },
       "source": [
-        "## Reading a sample TabNet prediction on syn2 data\n",
-        "\n",
-        "After training and serving your model, you upload the output to Google Cloud Storage. \n",
-        "\n",
-        "Sample prediction data is stored on Google Cloud at gs://cloud-samples-data/ai-platform-unified/datasets/tabnet/tab_net_output/syn2. You can use your own set of prediction data, but you must ensure that the format of the prediction data is the same as the format of the training data.\n",
-        "\n",
-        "Each prediction in TabNet contains a mask that is used to explain the predictions. The mask is stored in an **aggregated_mask_values** field.\n",
-        "\n",
-        "Information about the training set and the model are better suited for the Dataset section at the previous section of the notebook.\n"
+        "Define some constants"
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "0f55ca32f9b3"
+        "id": "JIOrI-hoJ46P"
       },
       "outputs": [],
       "source": [
-        "!gsutil cp gs://cloud-samples-data/ai-platform-unified/datasets/tabnet/tab_net_output/syn2 $BUCKET_URI\n",
-        "\n",
-        "# Replace your the BUCKET_URI and PREDICTION_FILE\n",
-        "# BUCKET_NAME = \"[<your-bucket-name>]\"\n",
-        "# PREDICTION_FILE = \"[<your-prediction-file>]\"\n",
-        "\n",
-        "BUCKET_NAME = BUCKET_URI[5:]\n",
-        "PREDICTION_FILE = \"syn2\"\n",
-        "\n",
-        "MASK_KEY = \"aggregated_mask_values\"\n",
-        "\n",
-        "HEADER = [(\"feat_\" + str(i)) for i in range(1, 12)]\n",
-        "HEADER"
+        "EXPERIMENT_NAME = \"\"  # @param {type:\"string\"}\n",
+        "REGION = \"[your-region]\"  # @param {type:\"string\"}"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "c4782ae3d628"
+        "id": "jWQLXXNVN4Lv"
       },
       "source": [
-        "### Download and preprocess the predictions."
+        "If EXEPERIMENT_NAME is not set, set a default one below:"
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "4aeaa7143830"
+        "id": "Q1QInYWOKsmo"
       },
       "outputs": [],
       "source": [
-        "storage_client = storage.Client()\n",
-        "bucket = storage_client.get_bucket(BUCKET_NAME)\n",
-        "blob = bucket.blob(PREDICTION_FILE)\n",
-        "f = blob.download_as_string(client=None).decode(\"utf-8\").strip()\n",
-        "predictions = f.split(\"\\n\")\n",
-        "predictions[:1]"
+        "if EXPERIMENT_NAME == \"\" or EXPERIMENT_NAME is None:\n",
+        "    EXPERIMENT_NAME = \"my-experiment-\" + TIMESTAMP"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "3f9f6b27280a"
+        "id": "Xuny18aMcWDb"
       },
       "source": [
-        "## Parse the mask values in prediction. Then, concatenate the mask values.\n",
-        "The output is a matrix having Nxk (N is the number of outputs, k is the size of each mask). Concatenating mask values are used to visualize the feature importance."
+        "## Concepts\n",
+        "\n",
+        "To better understanding how parameters and metrics are stored and organized, we'd like to introduce the following concepts:\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "NThDci5bp0Uw"
+      },
+      "source": [
+        "### Experiment\n",
+        "Experiments describe a context that groups your runs and the artifacts you create into a logical session. For example, in this notebook you create an Experiment and log data to that experiment."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "SAyRR3Ydp4X5"
+      },
+      "source": [
+        "### Run\n",
+        "A run represents a single path/avenue that you executed while performing an experiment. A run includes artifacts that you used as inputs or outputs, and parameters that you used in this execution. An Experiment can contain multiple runs. "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "l1YW2pgyegFP"
+      },
+      "source": [
+        "## Getting started tracking parameters and metrics\n",
+        "\n",
+        "You can use the Vertex SDK for Python to track metrics and parameters for models trained locally. \n",
+        "\n",
+        "In the following example, you train a simple distributed neural network (DNN) model to predict automobile's miles per gallon (MPG) based on automobile information in the [auto-mpg dataset](https://www.kaggle.com/devanshbesain/exploration-and-analysis-auto-mpg)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "KPY41M9_AhZU"
+      },
+      "source": [
+        "### Load and process the training dataset"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "bfMQSmRuUuX-"
+      },
+      "source": [
+        "Download and process the dataset."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "7f5b5c11f3dc"
+        "id": "RiQuMv4bmpuV"
       },
       "outputs": [],
       "source": [
-        "masks = []\n",
-        "for prediction in predictions:\n",
-        "    prediction = json.loads(prediction)\n",
-        "    masks.append(prediction[MASK_KEY])\n",
-        "masks = np.matrix(masks)\n",
-        "masks.shape"
+        "def read_data(uri):\n",
+        "    dataset_path = data_utils.get_file(\"auto-mpg.data\", uri)\n",
+        "    column_names = [\n",
+        "        \"MPG\",\n",
+        "        \"Cylinders\",\n",
+        "        \"Displacement\",\n",
+        "        \"Horsepower\",\n",
+        "        \"Weight\",\n",
+        "        \"Acceleration\",\n",
+        "        \"Model Year\",\n",
+        "        \"Origin\",\n",
+        "    ]\n",
+        "    raw_dataset = pd.read_csv(\n",
+        "        dataset_path,\n",
+        "        names=column_names,\n",
+        "        na_values=\"?\",\n",
+        "        comment=\"\\t\",\n",
+        "        sep=\" \",\n",
+        "        skipinitialspace=True,\n",
+        "    )\n",
+        "    dataset = raw_dataset.dropna()\n",
+        "    dataset[\"Origin\"] = dataset[\"Origin\"].map(\n",
+        "        lambda x: {1: \"USA\", 2: \"Europe\", 3: \"Japan\"}.get(x)\n",
+        "    )\n",
+        "    dataset = pd.get_dummies(dataset, prefix=\"\", prefix_sep=\"\")\n",
+        "    return dataset\n",
+        "\n",
+        "\n",
+        "dataset = read_data(\n",
+        "    \"http://archive.ics.uci.edu/ml/machine-learning-databases/auto-mpg/auto-mpg.data\"\n",
+        ")"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "8455a22d412c"
+        "id": "Y06J7A7yU21t"
       },
       "source": [
-        "## Visualize the mask value matrix.\n",
-        "The lighter color indicates more important feature. For example, only features 3-6 are meaningful in prediction output in Syn2 data. In the plot, the column 3-6 have light color."
+        "Split dataset for training and testing."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "ab63558c2593"
+        "id": "p5JBCBKyH-NC"
       },
       "outputs": [],
       "source": [
-        "fig = plt.figure(figsize=(20, 10))\n",
-        "ax = fig.add_subplot(121)\n",
-        "ax.imshow(masks[:50, :], interpolation=\"bilinear\", cmap=cm.Greys_r)\n",
-        "ax.set_xlabel(\"Features\")\n",
-        "ax.set_ylabel(\"Sample index\")\n",
-        "ax.xaxis.set_ticks(np.arange(len(HEADER)))\n",
-        "ax.set_xticklabels(HEADER, rotation=\"vertical\")\n",
-        "plt.show()"
+        "def train_test_split(dataset, split_frac=0.8, random_state=0):\n",
+        "    train_dataset = dataset.sample(frac=split_frac, random_state=random_state)\n",
+        "    test_dataset = dataset.drop(train_dataset.index)\n",
+        "    train_labels = train_dataset.pop(\"MPG\")\n",
+        "    test_labels = test_dataset.pop(\"MPG\")\n",
+        "\n",
+        "    return train_dataset, test_dataset, train_labels, test_labels\n",
+        "\n",
+        "\n",
+        "train_dataset, test_dataset, train_labels, test_labels = train_test_split(dataset)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gaNNTFPaU7KT"
+      },
+      "source": [
+        "Normalize the features in the dataset for better model performance."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "VGq5QCoyIEWJ"
+      },
+      "outputs": [],
+      "source": [
+        "def normalize_dataset(train_dataset, test_dataset):\n",
+        "    train_stats = train_dataset.describe()\n",
+        "    train_stats = train_stats.transpose()\n",
+        "\n",
+        "    def norm(x):\n",
+        "        return (x - train_stats[\"mean\"]) / train_stats[\"std\"]\n",
+        "\n",
+        "    normed_train_data = norm(train_dataset)\n",
+        "    normed_test_data = norm(test_dataset)\n",
+        "\n",
+        "    return normed_train_data, normed_test_data\n",
+        "\n",
+        "\n",
+        "normed_train_data, normed_test_data = normalize_dataset(train_dataset, test_dataset)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "UBXUgxgqA_GB"
+      },
+      "source": [
+        "### Define ML model and training function"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "66odBYKrIN4q"
+      },
+      "outputs": [],
+      "source": [
+        "def train(\n",
+        "    train_data,\n",
+        "    train_labels,\n",
+        "    num_units=64,\n",
+        "    activation=\"relu\",\n",
+        "    dropout_rate=0.0,\n",
+        "    validation_split=0.2,\n",
+        "    epochs=1000,\n",
+        "):\n",
+        "\n",
+        "    model = Sequential(\n",
+        "        [\n",
+        "            layers.Dense(\n",
+        "                num_units,\n",
+        "                activation=activation,\n",
+        "                input_shape=[len(train_dataset.keys())],\n",
+        "            ),\n",
+        "            layers.Dropout(rate=dropout_rate),\n",
+        "            layers.Dense(num_units, activation=activation),\n",
+        "            layers.Dense(1),\n",
+        "        ]\n",
+        "    )\n",
+        "\n",
+        "    model.compile(loss=\"mse\", optimizer=\"adam\", metrics=[\"mae\", \"mse\"])\n",
+        "    print(model.summary())\n",
+        "\n",
+        "    history = model.fit(\n",
+        "        train_data, train_labels, epochs=epochs, validation_split=validation_split\n",
+        "    )\n",
+        "\n",
+        "    return model, history"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "O8XJZB3gR8eL"
+      },
+      "source": [
+        "### Initialize the Vertex AI SDK for Python and create an Experiment\n",
+        "\n",
+        "Initialize the *client* for Vertex AI and create an experiment."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "o_wnT10RJ7-W"
+      },
+      "outputs": [],
+      "source": [
+        "aiplatform.init(project=PROJECT_ID, location=REGION, experiment=EXPERIMENT_NAME)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "u-iTnzt3B6Z_"
+      },
+      "source": [
+        "### Start several model training runs\n",
+        "\n",
+        "Training parameters and metrics are logged for each run."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "i2wnpu8_7JfV"
+      },
+      "outputs": [],
+      "source": [
+        "parameters = [\n",
+        "    {\"num_units\": 16, \"epochs\": 3, \"dropout_rate\": 0.1},\n",
+        "    {\"num_units\": 16, \"epochs\": 10, \"dropout_rate\": 0.1},\n",
+        "    {\"num_units\": 16, \"epochs\": 10, \"dropout_rate\": 0.2},\n",
+        "    {\"num_units\": 32, \"epochs\": 10, \"dropout_rate\": 0.1},\n",
+        "    {\"num_units\": 32, \"epochs\": 10, \"dropout_rate\": 0.2},\n",
+        "]\n",
+        "\n",
+        "for i, params in enumerate(parameters):\n",
+        "    aiplatform.start_run(run=f\"auto-mpg-local-run-{i}\")\n",
+        "    aiplatform.log_params(params)\n",
+        "    model, history = train(\n",
+        "        normed_train_data,\n",
+        "        train_labels,\n",
+        "        num_units=params[\"num_units\"],\n",
+        "        activation=\"relu\",\n",
+        "        epochs=params[\"epochs\"],\n",
+        "        dropout_rate=params[\"dropout_rate\"],\n",
+        "    )\n",
+        "    aiplatform.log_metrics(\n",
+        "        {metric: values[-1] for metric, values in history.history.items()}\n",
+        "    )\n",
+        "\n",
+        "    loss, mae, mse = model.evaluate(normed_test_data, test_labels, verbose=2)\n",
+        "    aiplatform.log_metrics({\"eval_loss\": loss, \"eval_mae\": mae, \"eval_mse\": mse})"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "jZLrJZTfL7tE"
+      },
+      "source": [
+        "### Extract parameters and metrics into a dataframe for analysis"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "A1PqKxlpOZa2"
+      },
+      "source": [
+        "We can also extract all parameters and metrics associated with any Experiment into a dataframe for further analysis."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "jbRf1WoH_vbY"
+      },
+      "outputs": [],
+      "source": [
+        "experiment_df = aiplatform.get_experiment_df()\n",
+        "experiment_df"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "EYuYgqVCMKU1"
+      },
+      "source": [
+        "### Visualizing an experiment's parameters and metrics"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "r8orCj8iJuO1"
+      },
+      "outputs": [],
+      "source": [
+        "plt.rcParams[\"figure.figsize\"] = [15, 5]\n",
+        "\n",
+        "ax = pd.plotting.parallel_coordinates(\n",
+        "    experiment_df.reset_index(level=0),\n",
+        "    \"run_name\",\n",
+        "    cols=[\n",
+        "        \"param.num_units\",\n",
+        "        \"param.dropout_rate\",\n",
+        "        \"param.epochs\",\n",
+        "        \"metric.loss\",\n",
+        "        \"metric.val_loss\",\n",
+        "        \"metric.eval_loss\",\n",
+        "    ],\n",
+        "    color=[\"blue\", \"green\", \"pink\", \"red\"],\n",
+        ")\n",
+        "ax.set_yscale(\"symlog\")\n",
+        "ax.legend(bbox_to_anchor=(1.0, 0.5))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WTHvPMweMlP1"
+      },
+      "source": [
+        "## Visualizing experiments in Cloud Console"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "F19_5lw0MqXv"
+      },
+      "source": [
+        "Run the following to get the URL of Vertex AI Experiments for your project.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "GmN9vE9pqqzt"
+      },
+      "outputs": [],
+      "source": [
+        "print(\"Vertex AI Experiments:\")\n",
+        "print(\n",
+        "    f\"https://console.cloud.google.com/ai/platform/experiments/experiments?folder=&organizationId=&project={PROJECT_ID}\"\n",
+        ")"
       ]
     },
     {
@@ -582,42 +857,14 @@
         "## Cleaning up\n",
         "\n",
         "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
-        "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
-        "\n",
-        "Otherwise, you can delete the individual resources you created in this tutorial:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "sx_vKniMq9ZX"
-      },
-      "outputs": [],
-      "source": [
-        "# Delete Cloud Storage that were created\n",
-        "if os.getenv(\"IS_TESTING\"):\n",
-        "    ! gsutil -m rm -r $BUCKET_URI"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "cb0f06c7ba0d"
-      },
-      "source": [
-        "## What's next?\n",
-        "\n",
-        "To learn more about TabNet, check out the resources here.\n",
-        "\n",
-        "* [TabNet: Attentive Interpretable Tabular Learning](https://arxiv.org/abs/1908.07442)"
+        "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial."
       ]
     }
   ],
   "metadata": {
     "colab": {
       "collapsed_sections": [],
-      "name": "ai-explanations-tabnet-algorithm.ipynb",
+      "name": "sdk-metric-parameter-tracking-for-locally-trained-models.ipynb",
       "toc_visible": true
     },
     "kernelspec": {

notebooks/community/ml_ops/README.md

@@ -12,7 +12,7 @@ The purpose of this set of notebooks and markdown files is to demonstrate Google
 2. [Experimentation](stage2)
 3. [Formalization](stage3)
 4. [Evaluation](stage4)
-5. [Deployment](stage5)
+5. Deployment
 6. [Serving](stage6)
 7. Monitoring
 8. Continuous Training

notebooks/community/ml_ops/stage1/README.md

@@ -22,56 +22,17 @@ The first stage in MLOps is the collection and preparation for the purpose of de
 - Data is preprocessed for training and evaluation using Dataflow.
 - Data augmentation is performed on-the-fly and is coupled with model feeding.
 
-<img src='stage1v2.png'>
+<img src='stage1.jpg'>
 
 ## Notebooks
 
 ### Get Started
 
-[Get started with Vertex AI datasets](get_started_vertex_datasets.ipynb)
+[Get Started with BQ datasets](get_started_bq_datasets.ipynb)
 
 ```
 The steps performed include:
-- Create a Vertex AI `Dataset` resource for:
-    - image data
-    - text data
-    - video data
-    - tabular data
-    - forecasting data
-- Search `Dataset` resources using a filter.
-- Read a sample of a `BigQuery` dataset into a dataframe.
-- Generate statistics and data schema using TensorFlow Data Validation from the samples in the dataframe.
-- Detect anomalies in new data using TensorFlow Data Validation.
-- Generate a TFRecord feature specification using TensorFlow Transform from the data schema.
-- Export a dataset and convert to TFRecords.
-```
 
-[Get started with Dataflow](get_started_dataflow.ipynb)
-
-```
-The steps performed include:
-- Offline preprocessing of data:
-    - Serially - w/o dataflow
-    - Parallel - with dataflow
-- Upstream preprocessing of data:
-    - tabular data
-    - image data
-```
-
-[Create an unlabelled Vertex AI AutoML text entity extraction dataset from pdfs using Vision API](get_started_with_visionapi_and_vertex_datasets.ipynb)
-
-```
-The steps performed include:
-1. Using `Vision API` to perform Optical Character Recognition (OCR) to extract text from PDF files.
-2. Processing the results and saving them to text files.
-3. Generating a `Vertex AI Dataset` import file.
-4. Creating a new unlabelled text entity extraction `Vertex AI Dataset` resource in `Vertex AI`.
-```
-
-[Get started with BigQuery datasets](get_started_bq_datasets.ipynb)
-
-```
-The steps performed include:
 - Create a Vertex AI `Dataset` resource from `BigQuery` table -- compatible for `AutoML` training.
 - Extract a copy of the dataset from `BigQuery` to a CSV file in Cloud Storage -- compatible for `AutoML` or custom training.
 - Select rows from a `BigQuery` dataset into a `pandas` dataframe -- compatible for custom training.
@@ -81,25 +42,47 @@ The steps performed include:
 - Extract data from `BigQuery` table into a `DMatrix` -- compatible for custom training `XGBoost` models.
 ```
 
-[Get started with Vertex AI data labeling](get_started_with_data_labeling.ipynb)
+[Get Started with Vertex datasets](get_started_vertex_datasets.ipynb)
 
 ```
 The steps performed include:
-- Create a Specialist Pool for data labelers.
-- Create a data labeling job.
-- Submit the data labeling job.
-- List data labeling jobs.
-- Cancel a data labeling job.
+  
+- Create a Vertex AI `Dataset` resource for:
+    - image data
+    - text data
+    - video data
+    - tabular data
+    - forecasting data
 
+
+- Search `Dataset` resources using a filter.
+- Read a sample of a `BigQuery` dataset into a dataframe.
+- Generate statistics and data schema using TensorFlow Data Validation from the samples in the dataframe.
+- Detect anomalies in new data using TensorFlow Data Validation.
+- Generate a TFRecord feature specification using TensorFlow Transform from the data schema.
+- Export a dataset and convert to TFRecords.
+```
+
+[Get Started with Dataflow](get_started_dataflow.ipynb)
+
+```
+The steps performed include:
+
+- Offline preprocessing of data:
+    - Serially - w/o dataflow
+    - Parallel - with dataflow
+- Upstream preprocessing of data:
+    - tabular data
+    - image data
 ```
 
 ### E2E Stage Example
 
 [Stage 1: Data Management](mlops_data_management.ipynb)
-
-
+  
 ```
 The steps performed include:
+
 - Explore and visualize the data.
 - Create a Vertex AI `Dataset` resource from `BigQuery` table -- for AutoML training.
 - Extract a copy of the dataset to a CSV file in Cloud Storage.
@@ -109,3 +92,4 @@ The steps performed include:
 - Generate a TFRecord feature specification using TensorFlow Data Validation from the data schema.
 - Preprocess a portion of the BigQuery data using `Dataflow` -- for custom training.
 ```
+

notebooks/community/ml_ops/stage1/get_started_bq_datasets.ipynb

@@ -8,7 +8,7 @@
       },
       "outputs": [],
       "source": [
-        "# Copyright 2022 Google LLC\n",
+        "# Copyright 2021 Google LLC\n",
         "#\n",
         "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
         "# you may not use this file except in compliance with the License.\n",
@@ -34,18 +34,13 @@
         "<table align=\"left\">\n",
         "  <td>\n",
         "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_bq_datasets.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
-        "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_bq_datasets.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
-        "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage1/get_started_bq_datasets.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
+        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_bq_datasets.ipynb\">\n",
+        "      Open in Google Cloud Notebooks\n",
         "    </a>\n",
         "  </td>\n",
         "</table>\n",
@@ -64,6 +59,17 @@
         "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 1 : data management: get started with BigQuery datasets."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dataset:gsod,lrg"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the GSOD dataset from [BigQuery public datasets](https://cloud.google.com/bigquery/public-data). The version of the dataset you use only the fields year, month and day to predict the value of mean daily temperature (mean_temp)."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -98,7 +104,7 @@
       "source": [
         "### Recommendations\n",
         "\n",
-        "When doing E2E MLOps on Google Cloud, following are the best practices when dealing with structured (tabular) data in BigQuery:\n",
+        "When doing E2E MLOps on Google Cloud, the following best practices with structured (tabular) data in BigQuery:\n",
         "\n",
         "- For AutoML training:\n",
         "  - Create a managed dataset with Vertex AI `TabularDataset`.\n",
@@ -118,7 +124,7 @@
         "        - Within the generator (upstream)\n",
         "        - Within the model (downstream)\n",
         "    - XGBoost model training:\n",
-        "      - Use BigQuery ML built-in XGBoost training.\n",
+        "      - Use BigQuery ML builtin XGBoost training.\n",
         "      - Alternatively, create a DMatrix generator from CSV files extracted from BigQuery table.\n",
         "    - Pytorch model training:\n",
         "        - Extract the BigQuery to a pandas dataframe.\n",
@@ -126,39 +132,12 @@
         "        - Create a DataLoader generator from the pandas dataframe.\n",
         "\n",
         "\n",
-        "- Alternatively:\n",
+        "- Alternately:\n",
         "    - Extract the BigQuery table to CSV files.\n",
         "    - Preprocess the CSV files.\n",
         "    - Create a tf.data.Dataset generator from the CSV files."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "dataset:gsod,lrg"
-      },
-      "source": [
-        "### Dataset\n",
-        "\n",
-        "The dataset used for this tutorial is the GSOD dataset from [BigQuery public datasets](https://cloud.google.com/bigquery/public-data). In this version of the dataset you consider the fields year, month and day to predict the value of mean daily temperature (mean_temp)."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "9e483012a752"
-      },
-      "source": [
-        "### Costs\n",
-        "This tutorial uses billable components of Google Cloud:\n",
-        "\n",
-        "- Vertex AI\n",
-        "- Cloud Storage\n",
-        "- BigQuery\n",
-        "\n",
-        "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing), [Cloud Storage pricing](https://cloud.google.com/storage/pricing) and [BigQuery pricing](https://cloud.google.com/bigquery/pricing) and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -167,7 +146,7 @@
       "source": [
         "## Installations\n",
         "\n",
-        "Install the following packages to execute this notebook."
+        "Install *one time* the packages for executing the MLOps notebooks."
       ]
     },
     {
@@ -178,26 +157,40 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
-        "# Install the packages\n",
-        "! pip3 install --upgrade pyarrow $USER_FLAG -q\n",
-        "! pip3 install --upgrade google-cloud-bigquery $USER_FLAG -q\n",
-        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG -q\n",
-        "! pip3 install -U xgboost $USER_FLAG -q\n",
-        "! pip3 install -U tensorflow $USER_FLAG -q\n",
-        "! pip3 install -U tensorflow-io==0.18 $USER_FLAG -q"
+        "ONCE_ONLY = False\n",
+        "if ONCE_ONLY:\n",
+        "    ! pip3 install -U tensorflow==2.5 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-transform==1.2 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-io==0.18 $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-bigquery $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-logging $USER_FLAG\n",
+        "    ! pip3 install --upgrade apache-beam[gcp] $USER_FLAG\n",
+        "    ! pip3 install --upgrade pyarrow $USER_FLAG\n",
+        "    ! pip3 install --upgrade cloudml-hypertune $USER_FLAG\n",
+        "    ! pip3 install --upgrade kfp $USER_FLAG"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "install_xgboost"
+      },
+      "source": [
+        "Install the latest GA version of *XGBoost* library as well."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "install_xgboost"
+      },
+      "outputs": [],
+      "source": [
+        "! pip3 install -U xgboost $USER_FLAG"
       ]
     },
     {
@@ -229,32 +222,6 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "84cd83853240"
-      },
-      "source": [
-        "## Before you begin\n",
-        "\n",
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI, BigQuery, Compute Engine and Cloud Storage APIs](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,bigquery,compute_component,storage_component).\n",
-        "\n",
-        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -331,10 +298,7 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
       ]
     },
     {
@@ -361,67 +325,6 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "77c385f0db59"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
-        "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
-        "\n",
-        "1. **Click Create service account**.\n",
-        "\n",
-        "2. In the **Service account name** field, enter a name, and click **Create**.\n",
-        "\n",
-        "3. In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex AI\" into the filter box, and select **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "4. Click Create. A JSON file that contains your key downloads to your local environment.\n",
-        "\n",
-        "5. Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "535223fa4b84"
-      },
-      "outputs": [],
-      "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = False\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        IS_COLAB = True\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -432,7 +335,12 @@
         "\n",
         "**The following steps are required, regardless of your notebook environment.**\n",
         "\n",
-        "When you create a dataset resource using the Vertex SDK, you can provide a Cloud Storage bucket that contains the data. Vertex AI creates the dataset resource from the data. In this tutorial, Vertex AI also creates a dataset resource from your data in the Cloud Storage bucket.\n",
+        "When you submit a custom training job using the Vertex SDK, you upload a Python package\n",
+        "containing your training code to a Cloud Storage bucket. Vertex AI runs\n",
+        "the code from this package. In this tutorial, Vertex AI also saves the\n",
+        "trained model that results from your job in the same bucket. You can then\n",
+        "create an `Endpoint` resource based on this output in order to serve\n",
+        "online predictions.\n",
         "\n",
         "Set the name of your Cloud Storage bucket below. Bucket names must be globally unique across all Google Cloud projects, including those outside of your organization."
       ]
@@ -445,8 +353,7 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -457,9 +364,8 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
-        "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
       ]
     },
     {
@@ -479,7 +385,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_URI"
+        "! gsutil mb -l $REGION $BUCKET_NAME"
       ]
     },
     {
@@ -499,7 +405,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_URI"
+        "! gsutil ls -al $BUCKET_NAME"
       ]
     },
     {
@@ -508,6 +414,9 @@
         "id": "setup_vars"
       },
       "source": [
+        "### Set up variables\n",
+        "\n",
+        "Next, set up some variables used throughout the tutorial.\n",
         "### Import libraries and define constants"
       ]
     },
@@ -519,12 +428,75 @@
       },
       "outputs": [],
       "source": [
-        "import google.cloud.aiplatform as aiplatform\n",
-        "import pandas as pd\n",
-        "import xgboost as xgb\n",
+        "import google.cloud.aiplatform as aip"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "import_bq"
+      },
+      "source": [
+        "#### Import BigQuery\n",
+        "\n",
+        "Import the BigQuery package into your Python environment."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "import_bq"
+      },
+      "outputs": [],
+      "source": [
         "from google.cloud import bigquery"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "import_xgboost"
+      },
+      "source": [
+        "#### Import XGBoost\n",
+        "\n",
+        "Import the XGBoost package into your Python environment."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "import_xgboost"
+      },
+      "outputs": [],
+      "source": [
+        "import xgboost as xgb"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "import_pandas"
+      },
+      "source": [
+        "#### Import pandas\n",
+        "\n",
+        "Import the pandas package into your Python environment."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "import_pandas"
+      },
+      "outputs": [],
+      "source": [
+        "import pandas as pd"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -544,7 +516,7 @@
       },
       "outputs": [],
       "source": [
-        "aiplatform.init(project=PROJECT_ID, location=REGION)"
+        "aip.init(project=PROJECT_ID, location=REGION)"
       ]
     },
     {
@@ -566,7 +538,7 @@
       },
       "outputs": [],
       "source": [
-        "bqclient = bigquery.Client(project=PROJECT_ID)"
+        "bqclient = bigquery.Client()"
       ]
     },
     {
@@ -577,7 +549,7 @@
       "source": [
         "#### Location of BigQuery training data.\n",
         "\n",
-        "Now, set the variable `IMPORT_FILE` to the location of the data table in BigQuery and `BQ_TABLE` with the table id."
+        "Now set the variable `IMPORT_FILE` to the location of the data table in BigQuery."
       ]
     },
     {
@@ -619,10 +591,10 @@
       },
       "outputs": [],
       "source": [
-        "dataset = aiplatform.TabularDataset.create(\n",
+        "dataset = aip.TabularDataset.create(\n",
         "    display_name=\"NOAA historical weather data\" + \"_\" + TIMESTAMP,\n",
         "    bq_source=[IMPORT_FILE],\n",
-        "    labels={\"user_metadata\": BUCKET_URI[5:]},\n",
+        "    labels={\"user_metadata\": BUCKET_NAME[5:]},\n",
         ")\n",
         "\n",
         "label_column = \"mean_temp\"\n",
@@ -638,7 +610,7 @@
       "source": [
         "### Copy the dataset to Cloud Storage\n",
         "\n",
-        "Next, you make a copy of the BigQuery table as a CSV file, to Cloud Storage using the BigQuery extract command.\n",
+        "Next, you make a copy of the BigQuery dataset, as a CSV file, to Cloud Storage using the BigQuery extract command.\n",
         "\n",
         "Learn more about [BigQuery command line interface](https://cloud.google.com/bigquery/docs/reference/bq-cli-reference)."
       ]
@@ -654,9 +626,9 @@
         "comps = BQ_TABLE.split(\".\")\n",
         "BQ_PROJECT_DATASET_TABLE = comps[0] + \":\" + comps[1] + \".\" + comps[2]\n",
         "\n",
-        "! bq --location=us extract --destination_format CSV $BQ_PROJECT_DATASET_TABLE $BUCKET_URI/mydata*.csv\n",
+        "! bq --location=us extract --destination_format CSV $BQ_PROJECT_DATASET_TABLE $BUCKET_NAME/mydata*.csv\n",
         "\n",
-        "IMPORT_FILES = ! gsutil ls $BUCKET_URI/mydata*.csv\n",
+        "IMPORT_FILES = ! gsutil ls $BUCKET_NAME/mydata*.csv\n",
         "\n",
         "print(IMPORT_FILES)\n",
         "\n",
@@ -692,12 +664,15 @@
       },
       "outputs": [],
       "source": [
-        "gcs_source = IMPORT_FILES\n",
+        "if \"IMPORT_FILES\" in globals():\n",
+        "    gcs_source = IMPORT_FILES\n",
+        "else:\n",
+        "    gcs_source = [IMPORT_FILE]\n",
         "\n",
-        "dataset = aiplatform.TabularDataset.create(\n",
+        "dataset = aip.TabularDataset.create(\n",
         "    display_name=\"NOAA historical weather data\" + \"_\" + TIMESTAMP,\n",
         "    gcs_source=gcs_source,\n",
-        "    labels={\"user_metadata\": BUCKET_URI[5:]},\n",
+        "    labels={\"user_metadata\": BUCKET_NAME[5:]},\n",
         ")\n",
         "\n",
         "\n",
@@ -719,30 +694,6 @@
         "Learn more about [Creating BigQuery views](https://cloud.google.com/bigquery/docs/views)."
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "7dc142433e50"
-      },
-      "outputs": [],
-      "source": [
-        "# Set dataset name and view name in BigQuery\n",
-        "BQ_MY_DATASET = \"[your-dataset-name]\"\n",
-        "BQ_MY_TABLE = \"[your-view-name]\"\n",
-        "\n",
-        "# Otherwise, use the default names\n",
-        "if (\n",
-        "    BQ_MY_DATASET == \"\"\n",
-        "    or BQ_MY_DATASET is None\n",
-        "    or BQ_MY_DATASET == \"[your-dataset-name]\"\n",
-        "):\n",
-        "    BQ_MY_DATASET = \"mlops_dataset_\" + TIMESTAMP\n",
-        "\n",
-        "if BQ_MY_TABLE == \"\" or BQ_MY_TABLE is None or BQ_MY_TABLE == \"[your-view-name]\":\n",
-        "    BQ_MY_TABLE = \"mlops_view_\" + TIMESTAMP"
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -751,7 +702,8 @@
       },
       "outputs": [],
       "source": [
-        "# Create the resources\n",
+        "BQ_MY_DATASET = 'mydataset'\n",
+        "BQ_MY_TABLE = 'myview'\n",
         "! bq --location=US mk -d \\\n",
         "$PROJECT_ID:$BQ_MY_DATASET\n",
         "\n",
@@ -792,8 +744,8 @@
       },
       "outputs": [],
       "source": [
-        "# Download the table.\n",
-        "table = bigquery.TableReference.from_string(BQ_TABLE)\n",
+        "# Download a table.\n",
+        "table = bigquery.TableReference.from_string(\"bigquery-public-data.samples.gsod\")\n",
         "\n",
         "rows = bqclient.list_rows(\n",
         "    table,\n",
@@ -1079,6 +1031,22 @@
         "TABLE_ID = \"gsod\"\n",
         "\n",
         "\n",
+        "def create_bigquery_dataset(dataset_id):\n",
+        "    dataset = bigquery.Dataset(\n",
+        "        bigquery.dataset.DatasetReference(PROJECT_ID, dataset_id)\n",
+        "    )\n",
+        "    dataset.location = \"us\"\n",
+        "\n",
+        "    try:\n",
+        "        dataset = bqclient.create_dataset(dataset)  # API request\n",
+        "        return True\n",
+        "    except Exception as err:\n",
+        "        print(err)\n",
+        "        if err.code != 409:  # http_client.CONFLICT\n",
+        "            raise\n",
+        "    return False\n",
+        "\n",
+        "\n",
         "def load_data_into_bigquery(url, dataset_id, table_id):\n",
         "    create_bigquery_dataset(dataset_id)\n",
         "    dataset = bqclient.dataset(dataset_id)\n",
@@ -1111,11 +1079,13 @@
       "source": [
         "### Read BigQuery table into XGboost DMatrix\n",
         "\n",
-        "Currently, there is no direct data feeding connector between BigQuery and the open source XGBoost. The BigQuery ML service has a built-in XGBoost training module.\n",
+        "Currently, there is no direct data feeding connector between BigQuery and the open source XGBoost.\n",
         "\n",
-        "Alernatively, you extract the data either as a pandas dataframe or as CSV files. The extracted data is then given as an input to a `DMatrix` object when training the model.\n",
+        "The BigQuery ML service has XGBoost training builtin.\n",
         "\n",
-        "Learn more about [Getting started with built-in XGBoost](https://cloud.google.com/ai-platform/training/docs/algorithms/xgboost-start)."
+        "Alernatively, you extract the data either as a pandas dataframe or as CSV files. The extracted data is then inputted to a `DMatrix` object when training the model.\n",
+        "\n",
+        "Learn more about [Getting started with builtin XGBoost](https://cloud.google.com/ai-platform/training/docs/algorithms/xgboost-start)"
       ]
     },
     {
@@ -1126,7 +1096,7 @@
       "source": [
         "### Read pandas table into XGboost DMatrix\n",
         "\n",
-        "Next, you load the pandas dataframe into a `DMatrix` object. XGBoost does not support non-numeric inputs. Any column that is categorical need to be one-hot encoded prior to loading the dataframe."
+        "Next, you load the pandas dataframe into a `DMatrix` object. XGBoost does not support non-numeric inputs. Any column that is categorical will need to be one-hot encoded prior to loading the dataframe."
       ]
     },
     {
@@ -1139,7 +1109,7 @@
       "source": [
         "dataframe[\"station_number\"] = pd.to_numeric(dataframe[\"station_number\"])\n",
         "labels = dataframe[\"mean_temp\"]\n",
-        "data = dataframe.drop([\"mean_temp\"], axis=1)\n",
+        "data = dataframe.drop(4)\n",
         "\n",
         "dtrain = xgb.DMatrix(data, label=labels)"
       ]
@@ -1152,7 +1122,7 @@
       "source": [
         "### Read CSV files into XGboost DMatrix\n",
         "\n",
-        "Currently, there is no Cloud Storage support in XGBoost. If you use CSV files for input, you need to download them locally."
+        "Currently, there is no Cloud Storage support in XGBoost. If you use CSV files for input, you will need to download them locally."
       ]
     },
     {
@@ -1174,42 +1144,87 @@
         "id": "cleanup:mbsdk"
       },
       "source": [
-        "# Clean up\n",
+        "# Cleaning up\n",
         "\n",
         "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
         "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
         "\n",
         "Otherwise, you can delete the individual resources you created in this tutorial:\n",
         "\n",
-        "- Vertex AI Dataset resource\n",
-        "- Cloud Storage Bucket\n",
-        "- BigQuery Dataset\n",
-        "\n",
-        "Set `delete_storage` to _True_ to delete the storage resources used in this notebook."
+        "- Dataset\n",
+        "- Pipeline\n",
+        "- Model\n",
+        "- Endpoint\n",
+        "- AutoML Training Job\n",
+        "- Batch Job\n",
+        "- Custom Job\n",
+        "- Hyperparameter Tuning Job\n",
+        "- Cloud Storage Bucket"
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "47ad926d84e8"
+        "id": "cleanup:mbsdk"
       },
       "outputs": [],
       "source": [
-        "import os\n",
+        "delete_all = True\n",
         "\n",
-        "# Delete the dataset using the Vertex dataset object\n",
-        "dataset.delete()\n",
+        "if delete_all:\n",
+        "    # Delete the dataset using the Vertex dataset object\n",
+        "    try:\n",
+        "        if \"dataset\" in globals():\n",
+        "            dataset.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
         "\n",
-        "# Delete the temporary BigQuery dataset\n",
-        "! bq rm -r -f $PROJECT_ID:$DATASET_ID\n",
+        "    # Delete the model using the Vertex model object\n",
+        "    try:\n",
+        "        if \"model\" in globals():\n",
+        "            model.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
         "\n",
-        "delete_storage = False\n",
-        "if delete_storage or os.getenv(\"IS_TESTING\"):\n",
-        "    # Delete the created GCS bucket\n",
-        "    ! gsutil rm -r $BUCKET_URI\n",
-        "    # Delete the created BigQuery datasets\n",
-        "    ! bq rm -r -f $PROJECT_ID:$BQ_MY_DATASET"
+        "    # Delete the endpoint using the Vertex endpoint object\n",
+        "    try:\n",
+        "        if \"endpoint\" in globals():\n",
+        "            endpoint.undeploy_all()\n",
+        "            endpoint.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the AutoML or Pipeline training job\n",
+        "    try:\n",
+        "        if \"dag\" in globals():\n",
+        "            dag.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the custom training job\n",
+        "    try:\n",
+        "        if \"job\" in globals():\n",
+        "            job.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the batch prediction job using the Vertex batch prediction object\n",
+        "    try:\n",
+        "        if \"batch_predict_job\" in globals():\n",
+        "            batch_predict_job.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the hyperparameter tuning job using the Vertex hyperparameter tuning object\n",
+        "    try:\n",
+        "        if \"hpt_job\" in globals():\n",
+        "            hpt_job.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    if \"BUCKET_NAME\" in globals():\n",
+        "        ! gsutil rm -r $BUCKET_NAME"
       ]
     }
   ],

notebooks/community/ml_ops/stage1/get_started_dataflow.ipynb

@@ -39,14 +39,8 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_dataflow.ipynb\">\n",
-        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\\\" alt=\"Colab logo\"> Run in Colab\n",
-        "        </a>\n",
-        "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_dataflow.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
+        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_dataflow.ipynb\">\n",
+        "      Open in Google Cloud Notebooks\n",
         "    </a>\n",
         "  </td>\n",
         "</table>\n",
@@ -65,6 +59,17 @@
         "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 1 : data management: get started with Dataflow."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dataset:gsod,lrg"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the GSOD dataset from [BigQuery public datasets](https://cloud.google.com/bigquery/public-data). The version of the dataset you use only the fields year, month and day to predict the value of mean daily temperature (mean_temp)."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -126,34 +131,6 @@
         "Alternately for AutoML tabular model training, you can reconfigure the otherwise default preprocessing."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "dataset:gsod,lrg"
-      },
-      "source": [
-        "### Dataset\n",
-        "\n",
-        "The dataset used for this tutorial is the GSOD dataset from [BigQuery public datasets](https://cloud.google.com/bigquery/public-data). The version of the dataset you use only the fields year, month and day to predict the value of mean daily temperature (mean_temp)."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "9e483012a752"
-      },
-      "source": [
-        "### Costs\n",
-        "This tutorial uses billable components of Google Cloud:\n",
-        "\n",
-        "- Vertex AI\n",
-        "- Cloud Storage\n",
-        "- BigQuery\n",
-        "- Dataflow\n",
-        "\n",
-        "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing), [Cloud Storage pricing](https://cloud.google.com/storage/pricing), [BigQuery pricing](https://cloud.google.com/bigquery/pricing), and [Dataflow pricing](https://cloud.google.com/dataflow/pricing) and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -162,7 +139,7 @@
       "source": [
         "## Installations\n",
         "\n",
-        "Install the following packages to execute this notebook."
+        "Install *one time* the packages for executing the MLOps notebooks."
       ]
     },
     {
@@ -173,26 +150,20 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
-        "! pip3 install -U tensorflow==2.5 $USER_FLAG -q\n",
-        "! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG -q\n",
-        "! pip3 install -U tensorflow-transform==1.2 $USER_FLAG -q\n",
-        "! pip3 install -U tensorflow-io==0.18 $USER_FLAG -q\n",
-        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG -q\n",
-        "! pip3 install --upgrade google-cloud-bigquery $USER_FLAG -q\n",
-        "! pip3 install --upgrade apache-beam[gcp] $USER_FLAG -q"
+        "ONCE_ONLY = False\n",
+        "if ONCE_ONLY:\n",
+        "    ! pip3 install -U tensorflow==2.5 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-transform==1.2 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-io==0.18 $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-bigquery $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-logging $USER_FLAG\n",
+        "    ! pip3 install --upgrade apache-beam[gcp] $USER_FLAG\n",
+        "    ! pip3 install --upgrade pyarrow $USER_FLAG\n",
+        "    ! pip3 install --upgrade cloudml-hypertune $USER_FLAG\n",
+        "    ! pip3 install --upgrade kfp $USER_FLAG"
       ]
     },
     {
@@ -224,32 +195,6 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "84cd83853240"
-      },
-      "source": [
-        "## Before you begin\n",
-        "\n",
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI, BigQuery, Compute Engine and Cloud Storage APIs](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,bigquery,compute_component,storage_component).\n",
-        "\n",
-        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -326,10 +271,7 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
       ]
     },
     {
@@ -356,67 +298,6 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "77c385f0db59"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
-        "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
-        "\n",
-        "1. **Click Create service account**.\n",
-        "\n",
-        "2. In the **Service account name** field, enter a name, and click **Create**.\n",
-        "\n",
-        "3. In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex AI\" into the filter box, and select **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "4. Click Create. A JSON file that contains your key downloads to your local environment.\n",
-        "\n",
-        "5. Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "535223fa4b84"
-      },
-      "outputs": [],
-      "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = False\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        IS_COLAB = True\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -445,8 +326,7 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -457,9 +337,8 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
-        "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
       ]
     },
     {
@@ -479,7 +358,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_URI"
+        "! gsutil mb -l $REGION $BUCKET_NAME"
       ]
     },
     {
@@ -499,7 +378,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_URI"
+        "! gsutil ls -al $BUCKET_NAME"
       ]
     },
     {
@@ -522,7 +401,7 @@
       },
       "outputs": [],
       "source": [
-        "import google.cloud.aiplatform as aiplatform"
+        "import google.cloud.aiplatform as aip"
       ]
     },
     {
@@ -676,7 +555,7 @@
       },
       "outputs": [],
       "source": [
-        "aiplatform.init(project=PROJECT_ID, location=REGION)"
+        "aip.init(project=PROJECT_ID, location=REGION)"
       ]
     },
     {
@@ -1125,7 +1004,7 @@
       },
       "outputs": [],
       "source": [
-        "SCHEMA_LOCATION = BUCKET_URI + \"/schema.txt\"\n",
+        "SCHEMA_LOCATION = BUCKET_NAME + \"/schema.txt\"\n",
         "\n",
         "# When running Apache Beam directly (file is directly accessed)\n",
         "tfdv.write_schema_text(output_path=SCHEMA_LOCATION, schema=schema)\n",
@@ -1270,7 +1149,7 @@
         "            )\n",
         "\n",
         "\n",
-        "EXPORTED_DATA_PREFIX = os.path.join(BUCKET_URI, \"exported_data\")\n",
+        "EXPORTED_DATA_PREFIX = os.path.join(BUCKET_NAME, \"exported_data\")\n",
         "\n",
         "QUERY_STRING = \"SELECT {},{} FROM {} LIMIT 500\".format(\n",
         "    \"CAST(station_number as STRING) AS station_number,year,month,day\",\n",
@@ -1283,7 +1162,7 @@
         "    \"runner\": RUNNER,\n",
         "    \"raw_data_query\": QUERY_STRING,\n",
         "    \"exported_data_prefix\": EXPORTED_DATA_PREFIX,\n",
-        "    \"temp_location\": os.path.join(BUCKET_URI, \"temp\"),\n",
+        "    \"temp_location\": os.path.join(BUCKET_NAME, \"temp\"),\n",
         "    \"project\": PROJECT_ID,\n",
         "    \"region\": REGION,\n",
         "    \"setup_file\": \"./setup.py\",\n",
@@ -1308,7 +1187,17 @@
         "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
         "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
         "\n",
-        "Otherwise, you can delete the individual resources you created in this tutorial."
+        "Otherwise, you can delete the individual resources you created in this tutorial:\n",
+        "\n",
+        "- Dataset\n",
+        "- Pipeline\n",
+        "- Model\n",
+        "- Endpoint\n",
+        "- AutoML Training Job\n",
+        "- Batch Job\n",
+        "- Custom Job\n",
+        "- Hyperparameter Tuning Job\n",
+        "- Cloud Storage Bucket"
       ]
     },
     {
@@ -1319,11 +1208,61 @@
       },
       "outputs": [],
       "source": [
-        "delete_storage = True\n",
+        "delete_all = True\n",
         "\n",
-        "if delete_storage or os.getenv(\"IS_TESTING\"):\n",
-        "    if \"BUCKET_URI\" in globals():\n",
-        "        ! gsutil rm -r $BUCKET_URI"
+        "if delete_all:\n",
+        "    # Delete the dataset using the Vertex dataset object\n",
+        "    try:\n",
+        "        if \"dataset\" in globals():\n",
+        "            dataset.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the model using the Vertex model object\n",
+        "    try:\n",
+        "        if \"model\" in globals():\n",
+        "            model.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the endpoint using the Vertex endpoint object\n",
+        "    try:\n",
+        "        if \"endpoint\" in globals():\n",
+        "            endpoint.undeploy_all()\n",
+        "            endpoint.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the AutoML or Pipeline training job\n",
+        "    try:\n",
+        "        if \"dag\" in globals():\n",
+        "            dag.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the custom training job\n",
+        "    try:\n",
+        "        if \"job\" in globals():\n",
+        "            job.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the batch prediction job using the Vertex batch prediction object\n",
+        "    try:\n",
+        "        if \"batch_predict_job\" in globals():\n",
+        "            batch_predict_job.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the hyperparameter tuning job using the Vertex hyperparameter tuning object\n",
+        "    try:\n",
+        "        if \"hpt_job\" in globals():\n",
+        "            hpt_job.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    if \"BUCKET_NAME\" in globals():\n",
+        "        ! gsutil rm -r $BUCKET_NAME"
       ]
     }
   ],

notebooks/community/ml_ops/stage1/get_started_vertex_datasets.ipynb

@@ -8,7 +8,7 @@
       },
       "outputs": [],
       "source": [
-        "# Copyright 2022 Google LLC\n",
+        "# Copyright 2021 Google LLC\n",
         "#\n",
         "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
         "# you may not use this file except in compliance with the License.\n",
@@ -29,26 +29,20 @@
         "id": "title:generic,gcp"
       },
       "source": [
-        "# E2E ML on GCP: MLOps stage 1 : data management: get started with Vertex AI datasets\n",
+        "# E2E ML on GCP: MLOps stage 1 : data management: get started with Vertex datasets\n",
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
         "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_vertex_datasets.ipynb\">\n",
-        "     <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
         "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_vertex_datasets.ipynb\">\n",
-        "<img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
-        "      View on GitHub\n",
+        "      Open in Google Cloud Notebooks\n",
         "    </a>\n",
         "  </td>\n",
-        "   <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage1/get_started_vertex_datasets.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
-        "    </a>\n",
-        "  </td>   \n",
         "</table>\n",
         "<br/><br/><br/>"
       ]
@@ -139,33 +133,6 @@
         "            - Create a tf.data.Dataset from the TFRecords."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "533dd6fe83c8"
-      },
-      "source": [
-        "### Datasets\n",
-        "\n",
-        "This tutorial uses a variety of public datasets to demonstrate using a `Vertex AI` managed dataset."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "9e483012a752"
-      },
-      "source": [
-        "### Costs\n",
-        "This tutorial uses billable components of Google Cloud:\n",
-        "\n",
-        "- Vertex AI\n",
-        "- Cloud Storage\n",
-        "- BigQuery\n",
-        "\n",
-        "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing), [Cloud Storage pricing](https://cloud.google.com/storage/pricing) and [BigQuery pricing](https://cloud.google.com/bigquery/pricing) and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -174,7 +141,7 @@
       "source": [
         "## Installations\n",
         "\n",
-        "Install the packages required for executing this notebook."
+        "Install *one time* the packages for executing the MLOps notebooks."
       ]
     },
     {
@@ -185,27 +152,20 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
-        "! pip3 install -U tensorflow $USER_FLAG -q\n",
-        "! pip3 install -U tensorflow-data-validation $USER_FLAG -q\n",
-        "! pip3 install -U tensorflow-transform $USER_FLAG -q\n",
-        "! pip3 install -U tensorflow-io $USER_FLAG -q\n",
-        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG -q\n",
-        "! pip3 install --upgrade google-cloud-bigquery $USER_FLAG -q\n",
-        "! pip3 install -U tensorflow-io==0.18 $USER_FLAG -q\n",
-        "! pip3 install --upgrade db-dtypes $USER_FLAG -q! pip3 install --upgrade future $USER_FLAG -q"
+        "ONCE_ONLY = False\n",
+        "if ONCE_ONLY:\n",
+        "    ! pip3 install -U tensorflow==2.5 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-transform==1.2 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-io==0.18 $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-bigquery $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-logging $USER_FLAG\n",
+        "    ! pip3 install --upgrade apache-beam[gcp] $USER_FLAG\n",
+        "    ! pip3 install --upgrade pyarrow $USER_FLAG\n",
+        "    ! pip3 install --upgrade cloudml-hypertune $USER_FLAG\n",
+        "    ! pip3 install --upgrade kfp $USER_FLAG"
       ]
     },
     {
@@ -237,32 +197,6 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "cb082379ed5b"
-      },
-      "source": [
-        "## Before you begin\n",
-        "\n",
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com). \n",
-        "\n",
-        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -289,7 +223,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "nWlzLu5ELxWd"
+        "id": "autoset_project_id"
       },
       "outputs": [],
       "source": [
@@ -304,7 +238,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "c021ca495967"
+        "id": "set_gcloud_project_id"
       },
       "outputs": [],
       "source": [
@@ -339,10 +273,7 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
       ]
     },
     {
@@ -369,66 +300,6 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "927085b84a07"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
-        "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
-        "\n",
-        "**Click Create service account**.\n",
-        "\n",
-        "In the **Service account name** field, enter a name, and click **Create**.\n",
-        "\n",
-        "In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex\" into the filter box, and select **Vertex Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "Click Create. A JSON file that contains your key downloads to your local environment.\n",
-        "\n",
-        "Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "89788a802687"
-      },
-      "outputs": [],
-      "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = \"google.colab\" in sys.modules\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -457,7 +328,7 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_URI = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
+        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -468,8 +339,8 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
       ]
     },
     {
@@ -489,7 +360,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_URI"
+        "! gsutil mb -l $REGION $BUCKET_NAME"
       ]
     },
     {
@@ -509,7 +380,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_URI"
+        "! gsutil ls -al $BUCKET_NAME"
       ]
     },
     {
@@ -521,11 +392,7 @@
         "### Set up variables\n",
         "\n",
         "Next, set up some variables used throughout the tutorial.\n",
-        "### Import libraries and define constants\n",
-        "\n",
-        "Import the BigQuery package, TensorFlow Data Validation (TFDV) package and TensorFlow Data Validation package into your Python environment. \n",
-        "\n",
-        "Import TensorFlow Transform (TFT) package and pandas into your Python environment."
+        "### Import libraries and define constants"
       ]
     },
     {
@@ -536,13 +403,97 @@
       },
       "outputs": [],
       "source": [
-        "import google.cloud.aiplatform as aip\n",
-        "import pandas as pd\n",
-        "import tensorflow_data_validation as tfdv\n",
-        "import tensorflow_transform as tft\n",
+        "import google.cloud.aiplatform as aip"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "import_bq"
+      },
+      "source": [
+        "#### Import BigQuery\n",
+        "\n",
+        "Import the BigQuery package into your Python environment."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "import_bq"
+      },
+      "outputs": [],
+      "source": [
         "from google.cloud import bigquery"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "import_tfdv"
+      },
+      "source": [
+        "#### Import TensorFlow Data Validation\n",
+        "\n",
+        "Import the TensorFlow Data Validation (TFDV) package into your Python environment."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "import_tfdv"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow_data_validation as tfdv"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "import_tft"
+      },
+      "source": [
+        "#### Import TensorFlow Transform\n",
+        "\n",
+        "Import the TensorFlow Transform (TFT) package into your Python environment."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "import_tft"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow_transform as tft"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "import_pandas"
+      },
+      "source": [
+        "#### Import pandas\n",
+        "\n",
+        "Import the pandas package into your Python environment."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "import_pandas"
+      },
+      "outputs": [],
+      "source": [
+        "import pandas as pd"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -562,7 +513,7 @@
       },
       "outputs": [],
       "source": [
-        "aip.init(project=PROJECT_ID, location=REGION, staging_bucket=BUCKET_URI)"
+        "aip.init(project=PROJECT_ID, location=REGION)"
       ]
     },
     {
@@ -616,13 +567,26 @@
         "Learn more about [All dataset documentation](https://cloud.google.com/vertex-ai/docs/datasets/datasets)"
       ]
     },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "import_file:flowers,csv,icn"
+      },
+      "outputs": [],
+      "source": [
+        "IMPORT_FILE = (\n",
+        "    \"gs://cloud-samples-data/vision/automl_classification/flowers/all_data_v2.csv\"\n",
+        ")"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "create_dataset:image,icn"
       },
       "source": [
-        "### Create an Image Dataset\n",
+        "### Create the Dataset\n",
         "\n",
         "Next, create the `Dataset` resource using the `create` method for the `ImageDataset` class, which takes the following parameters:\n",
         "\n",
@@ -637,19 +601,6 @@
         "Learn more about [ImageDataset](https://cloud.google.com/vertex-ai/docs/datasets/prepare-image)."
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "import_file:flowers,csv,icn"
-      },
-      "outputs": [],
-      "source": [
-        "IMPORT_FILE = (\n",
-        "    \"gs://cloud-samples-data/vision/automl_classification/flowers/all_data_v2.csv\"\n",
-        ")"
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -667,13 +618,24 @@
         "print(dataset.resource_name)"
       ]
     },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "import_file:hmdb,csv,vcn"
+      },
+      "outputs": [],
+      "source": [
+        "IMPORT_FILE = \"gs://automl-video-demo-data/hmdb_split1_5classes_train_inf.csv\""
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "create_dataset:video,vcn"
       },
       "source": [
-        "### Create a Video Dataset\n",
+        "### Create the Dataset\n",
         "\n",
         "Next, create the `Dataset` resource using the `create` method for the `VideoDataset` class, which takes the following parameters:\n",
         "\n",
@@ -687,17 +649,6 @@
         "Learn more about [VideoDataset](https://cloud.google.com/vertex-ai/docs/datasets/prepare-video)."
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "import_file:hmdb,csv,vcn"
-      },
-      "outputs": [],
-      "source": [
-        "IMPORT_FILE = \"gs://automl-video-demo-data/hmdb_split1_5classes_train_inf.csv\""
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -715,13 +666,24 @@
         "print(dataset.resource_name)"
       ]
     },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "import_file:happydb,csv,tcn"
+      },
+      "outputs": [],
+      "source": [
+        "IMPORT_FILE = \"gs://cloud-ml-data/NL-classification/happiness.csv\""
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "create_dataset:text,tcn"
       },
       "source": [
-        "### Create a Text Dataset\n",
+        "### Create the Dataset\n",
         "\n",
         "Next, create the `Dataset` resource using the `create` method for the `TextDataset` class, which takes the following parameters:\n",
         "\n",
@@ -736,17 +698,6 @@
         "Learn more about [TextDataset](https://cloud.google.com/vertex-ai/docs/datasets/prepare-text)."
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "import_file:happydb,csv,tcn"
-      },
-      "outputs": [],
-      "source": [
-        "IMPORT_FILE = \"gs://cloud-ml-data/NL-classification/happiness.csv\""
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -764,24 +715,6 @@
         "print(dataset.resource_name)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "create_dataset:tabular,bq,lrg,v2"
-      },
-      "source": [
-        "### Create a Tabular Dataset\n",
-        "\n",
-        "#### CSV input data\n",
-        "\n",
-        "Next, create the `Dataset` resource using the `create` method for the `TabularDataset` class for CSV input data, which takes the following parameters:\n",
-        "\n",
-        "- `display_name`: The human readable name for the `Dataset` resource.\n",
-        "- `gcs_source`: A list of one or more dataset index files to import the data items into the `Dataset` resource.\n",
-        "\n",
-        "Learn more about [TabularDataset from CSV files](https://cloud.google.com/vertex-ai/docs/datasets/create-dataset-api#aiplatform_create_dataset_tabular_gcs_sample-python)"
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -790,50 +723,27 @@
       },
       "outputs": [],
       "source": [
-        "IMPORT_FILE = \"gs://cloud-samples-data/tables/iris_1000.csv\""
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "create_dataset:tabular,bq,lrg,v2"
-      },
-      "outputs": [],
-      "source": [
-        "dataset = aip.TabularDataset.create(\n",
-        "    display_name=\"example\" + \"_\" + TIMESTAMP, gcs_source=[IMPORT_FILE]\n",
-        ")\n",
-        "\n",
-        "print(dataset.resource_name)"
+        "IMPORT_FILE = \"bq://bigquery-public-data.samples.gsod\"\n",
+        "BQ_TABLE = \"bigquery-public-data.samples.gsod\""
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "854dd1e0195c"
+        "id": "create_dataset:tabular,bq,lrg,v2"
       },
       "source": [
-        "#### BigQuery input data\n",
+        "### Create the Dataset\n",
         "\n",
-        "Next, create the `Dataset` resource using the `create` method for the `TabularDataset` class for BigQuery table input, which takes the following parameters:\n",
+        "#### CSV input data\n",
+        "\n",
+        "Next, create the `Dataset` resource using the `create` method for the `TabularDataset` class, which takes the following parameters:\n",
         "\n",
         "- `display_name`: The human readable name for the `Dataset` resource.\n",
-        "- `bq_source`: A list of one or more BigQuery tables to import the data items into the `Dataset` resource.\n",
+        "- `gcs_source`: A list of one or more dataset index files to import the data items into the `Dataset` resource.\n",
+        "- `labels`: User defined metadata. In this example, you store the location of the Cloud Storage bucket containing the user defined data.\n",
         "\n",
-        "Learn more about [TabularDataset from BigQuery table](https://cloud.google.com/vertex-ai/docs/datasets/create-dataset-api#aiplatform_create_dataset_tabular_bigquery_sample-pythonn)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "86343c146300"
-      },
-      "outputs": [],
-      "source": [
-        "IMPORT_FILE = \"bq://bigquery-public-data.samples.gsod\"\n",
-        "BQ_TABLE = \"bigquery-public-data.samples.gsod\""
+        "Learn more about [TabularDataset from CSV files](https://cloud.google.com/vertex-ai/docs/datasets/create-dataset-api#aiplatform_create_dataset_tabular_gcs_sample-python)"
       ]
     },
     {
@@ -851,82 +761,6 @@
         "print(dataset.resource_name)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "82e9fe20ce71"
-      },
-      "source": [
-        "#### Dataframe input data\n",
-        "\n",
-        "Next, create the `Dataset` resource using the `create_from_dataframe` method for the `TabularDataset` class for pandas dataframe input, which takes the following parameters:\n",
-        "\n",
-        "- `display_name`: The human readable name for the `Dataset` resource.\n",
-        "- `df_source`: The pandas dataframe to import the data items into the `Dataset` resource.\n",
-        "- `staging_path`: The BigQuery table to store the imported data."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "3805f945ffdd"
-      },
-      "outputs": [],
-      "source": [
-        "# Download the table.\n",
-        "table = bigquery.TableReference.from_string(BQ_TABLE)\n",
-        "\n",
-        "rows = bqclient.list_rows(\n",
-        "    table,\n",
-        "    max_results=10000,\n",
-        "    selected_fields=[\n",
-        "        bigquery.SchemaField(\"station_number\", \"STRING\"),\n",
-        "        bigquery.SchemaField(\"year\", \"INTEGER\"),\n",
-        "        bigquery.SchemaField(\"month\", \"INTEGER\"),\n",
-        "        bigquery.SchemaField(\"day\", \"INTEGER\"),\n",
-        "        bigquery.SchemaField(\"mean_temp\", \"FLOAT\"),\n",
-        "    ],\n",
-        ")\n",
-        "\n",
-        "dataframe = rows.to_dataframe()\n",
-        "print(dataframe.head())"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "create_dataset:tabular,bq,lrg,v2"
-      },
-      "outputs": [],
-      "source": [
-        "dataset = aip.TabularDataset.create_from_dataframe(\n",
-        "    display_name=\"example\" + \"_\" + TIMESTAMP,\n",
-        "    df_source=dataframe,\n",
-        "    staging_path=f\"bq://{PROJECT_ID}.samples.gsod\",\n",
-        ")\n",
-        "\n",
-        "print(dataset.resource_name)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "create_dataset:tabular,forecast,v2"
-      },
-      "source": [
-        "### Create a Time Series Dataset\n",
-        "\n",
-        "Next, create the `Dataset` resource using the `create` method for the `TimeSeriesDataset` class, which takes the following parameters:\n",
-        "\n",
-        "- `display_name`: The human readable name for the `Dataset` resource.\n",
-        "- `gcs_source`: A list of one or more dataset index files to import the data items into the `Dataset` resource.\n",
-        "- `bq_source`: Alternatively, import data items from a BigQuery table into the `Dataset` resource.\n",
-        "\n",
-        "Learn more about [TimeSeriesDataset](https://cloud.google.com/vertex-ai/docs/datasets/prepare-tabular)."
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -938,6 +772,23 @@
         "IMPORT_FILE = \"gs://cloud-samples-data/ai-platform/covid/bigquery-public-covid-nyt-us-counties-train.csv\""
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "create_dataset:tabular,forecast,v2"
+      },
+      "source": [
+        "### Create the Dataset\n",
+        "\n",
+        "Next, create the `Dataset` resource using the `create` method for the `TimeSeriesDataset` class, which takes the following parameters:\n",
+        "\n",
+        "- `display_name`: The human readable name for the `Dataset` resource.\n",
+        "- `gcs_source`: A list of one or more dataset index files to import the data items into the `Dataset` resource.\n",
+        "- `bq_source`: Alternatively, import data items from a BigQuery table into the `Dataset` resource.\n",
+        "\n",
+        "Learn more about [TimeSeriesDataset](https://cloud.google.com/vertex-ai/docs/datasets/prepare-tabular)."
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -1298,9 +1149,9 @@
         "comps = BQ_TABLE.split(\".\")\n",
         "BQ_PROJECT_DATASET_TABLE = comps[0] + \":\" + comps[1] + \".\" + comps[2]\n",
         "\n",
-        "! bq --location=us extract --destination_format CSV $BQ_PROJECT_DATASET_TABLE $BUCKET_URI/mydata*.csv\n",
+        "! bq --location=us extract --destination_format CSV $BQ_PROJECT_DATASET_TABLE $BUCKET_NAME/mydata*.csv\n",
         "\n",
-        "IMPORT_FILES = ! gsutil ls $BUCKET_URI/mydata*.csv\n",
+        "IMPORT_FILES = ! gsutil ls $BUCKET_NAME/mydata*.csv\n",
         "\n",
         "print(IMPORT_FILES)\n",
         "\n",
@@ -1358,38 +1209,6 @@
         "To create a dataframe from multiple CSV sources, you read each CSV file and concatenate the dataframes together."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "bcd2e4e0703b"
-      },
-      "source": [
-        "If you are running this notebook on Colab, run the following cell to install packages fsspec and gcsfs."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "927bd3f92268"
-      },
-      "outputs": [],
-      "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Workbench AI Notebook, then don't execute this code\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        ! pip3 install fsspec\n",
-        "        ! pip3 install gcsfs"
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -1450,7 +1269,7 @@
       },
       "outputs": [],
       "source": [
-        "EXPORTED_DIR = f\"{BUCKET_URI}/exported\"\n",
+        "EXPORTED_DIR = f\"{BUCKET_NAME}/exported\"\n",
         "exported_files = dataset.export_data(output_dir=EXPORTED_DIR)\n",
         "\n",
         "! gsutil ls $EXPORTED_DIR"
@@ -1679,7 +1498,7 @@
         "    data = f.readlines()\n",
         "\n",
         "# The path to the TFRecord cached file.\n",
-        "GCS_TFRECORD_URI = BUCKET_URI + \"/flowers.tfrecord\"\n",
+        "GCS_TFRECORD_URI = BUCKET_NAME + \"/flowers.tfrecord\"\n",
         "\n",
         "# Create the TFRecord cached file\n",
         "with tf.io.TFRecordWriter(GCS_TFRECORD_URI) as writer:\n",
@@ -1713,7 +1532,14 @@
         "Otherwise, you can delete the individual resources you created in this tutorial:\n",
         "\n",
         "- Dataset\n",
-        "- Bucket"
+        "- Pipeline\n",
+        "- Model\n",
+        "- Endpoint\n",
+        "- AutoML Training Job\n",
+        "- Batch Job\n",
+        "- Custom Job\n",
+        "- Hyperparameter Tuning Job\n",
+        "- Cloud Storage Bucket"
       ]
     },
     {
@@ -1724,16 +1550,61 @@
       },
       "outputs": [],
       "source": [
-        "delete_bucket = False\n",
+        "delete_all = True\n",
         "\n",
-        "# Delete the dataset using the Vertex dataset object\n",
-        "datasets = aip.TabularDataset.list(filter=f'display_name=\"example_{TIMESTAMP}\"')\n",
-        "for dataset in datasets:\n",
-        "    dataset.delete()\n",
+        "if delete_all:\n",
+        "    # Delete the dataset using the Vertex dataset object\n",
+        "    try:\n",
+        "        if \"dataset\" in globals():\n",
+        "            dataset.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
         "\n",
-        "# Delete the bucket\n",
-        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
-        "    ! gsutil rm -r $BUCKET_URI"
+        "    # Delete the model using the Vertex model object\n",
+        "    try:\n",
+        "        if \"model\" in globals():\n",
+        "            model.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the endpoint using the Vertex endpoint object\n",
+        "    try:\n",
+        "        if \"endpoint\" in globals():\n",
+        "            endpoint.undeploy_all()\n",
+        "            endpoint.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the AutoML or Pipeline training job\n",
+        "    try:\n",
+        "        if \"dag\" in globals():\n",
+        "            dag.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the custom training job\n",
+        "    try:\n",
+        "        if \"job\" in globals():\n",
+        "            job.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the batch prediction job using the Vertex batch prediction object\n",
+        "    try:\n",
+        "        if \"batch_predict_job\" in globals():\n",
+        "            batch_predict_job.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the hyperparameter tuning job using the Vertex hyperparameter tuning object\n",
+        "    try:\n",
+        "        if \"hpt_job\" in globals():\n",
+        "            hpt_job.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    if \"BUCKET_NAME\" in globals():\n",
+        "        ! gsutil rm -r $BUCKET_NAME"
       ]
     }
   ],

notebooks/community/ml_ops/stage1/mlops_data_management.ipynb

@@ -39,14 +39,8 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/mlops_data_management.ipynb\">\n",
-        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\\\" alt=\"Colab logo\"> Run in Colab\n",
-        "        </a>\n",
-        "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage1/mlops_data_management.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
+        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/mlops_data_management.ipynb\">\n",
+        "      Open in Google Cloud Notebooks\n",
         "    </a>\n",
         "  </td>\n",
         "</table>\n",
@@ -65,6 +59,17 @@
         "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 1 : data management."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dataset:bq,chicago,lbn"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the [Chicago Taxi](https://www.kaggle.com/chicago/chicago-taxi-trips-bq). The version of the dataset you will use in this tutorial is stored in a public BigQuery table. The trained model predicts whether someone would leave a tip for a taxi fare."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -109,34 +114,6 @@
         " - Preprocess the data with `Dataflow`"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "dataset:bq,chicago,lbn"
-      },
-      "source": [
-        "### Dataset\n",
-        "\n",
-        "The dataset used for this tutorial is the [Chicago Taxi](https://www.kaggle.com/chicago/chicago-taxi-trips-bq). The version of the dataset used in this tutorial is stored in a public BigQuery table. The trained model predicts whether someone leaves a tip for a taxi fare."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "9e483012a752"
-      },
-      "source": [
-        "### Costs\n",
-        "This tutorial uses billable components of Google Cloud:\n",
-        "\n",
-        "- Vertex AI\n",
-        "- Cloud Storage\n",
-        "- BigQuery\n",
-        "- Dataflow\n",
-        "\n",
-        "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing), [Cloud Storage pricing](https://cloud.google.com/storage/pricing), [BigQuery pricing](https://cloud.google.com/bigquery/pricing), and [Dataflow pricing](https://cloud.google.com/dataflow/pricing) and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -156,34 +133,20 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
-        "ONCE_ONLY = True\n",
+        "ONCE_ONLY = False\n",
         "if ONCE_ONLY:\n",
-        "    ! pip3 install -U tensorflow==2.5 $USER_FLAG -q\n",
-        "    ! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG -q\n",
-        "    ! pip3 install -U tensorflow-transform==1.2 $USER_FLAG -q\n",
-        "    ! pip3 install -U tensorflow-io==0.18 $USER_FLAG -q\n",
-        "    ! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG -q\n",
-        "    ! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG -q\n",
-        "    ! pip3 install --upgrade google-cloud-bigquery $USER_FLAG -q\n",
-        "    ! pip3 install --upgrade google-cloud-logging $USER_FLAG -q\n",
-        "    ! pip3 install --upgrade apache-beam[gcp]==2.33.0 $USER_FLAG -q\n",
-        "    ! pip3 install --upgrade pyarrow $USER_FLAG -q\n",
-        "    ! pip3 install --upgrade cloudml-hypertune $USER_FLAG -q\n",
-        "    ! pip3 install --upgrade kfp $USER_FLAG -q\n",
-        "    ! pip3 install future $USER_FLAG -q"
+        "    ! pip3 install -U tensorflow==2.5 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-transform==1.2 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-io==0.18 $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-bigquery $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-logging $USER_FLAG\n",
+        "    ! pip3 install --upgrade apache-beam[gcp] $USER_FLAG\n",
+        "    ! pip3 install --upgrade pyarrow $USER_FLAG\n",
+        "    ! pip3 install --upgrade cloudml-hypertune $USER_FLAG\n",
+        "    ! pip3 install --upgrade kfp $USER_FLAG"
       ]
     },
     {
@@ -215,32 +178,6 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "84cd83853240"
-      },
-      "source": [
-        "## Before you begin\n",
-        "\n",
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI, BigQuery, Compute Engine and Cloud Storage APIs](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,bigquery,compute_component,storage_component).\n",
-        "\n",
-        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -317,10 +254,7 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
       ]
     },
     {
@@ -347,66 +281,6 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "77c385f0db59"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
-        "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
-        "\n",
-        "1. **Click Create service account**.\n",
-        "\n",
-        "2. In the **Service account name** field, enter a name, and click **Create**.\n",
-        "\n",
-        "3. In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex AI\" into the filter box, and select **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "4. Click Create. A JSON file that contains your key downloads to your local environment.\n",
-        "\n",
-        "5. Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "535223fa4b84"
-      },
-      "outputs": [],
-      "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = \"google.colab\" in sys.modules\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -435,8 +309,7 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -447,9 +320,8 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
-        "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
       ]
     },
     {
@@ -469,7 +341,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_URI"
+        "! gsutil mb -l $REGION $BUCKET_NAME"
       ]
     },
     {
@@ -489,7 +361,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_URI"
+        "! gsutil ls -al $BUCKET_NAME"
       ]
     },
     {
@@ -644,7 +516,7 @@
       },
       "outputs": [],
       "source": [
-        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
+        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_NAME)"
       ]
     },
     {
@@ -666,7 +538,7 @@
       },
       "outputs": [],
       "source": [
-        "bqclient = bigquery.Client(project=PROJECT_ID)"
+        "bqclient = bigquery.Client()"
       ]
     },
     {
@@ -789,11 +661,6 @@
         "LIMIT = 300000\n",
         "YEAR = 2020\n",
         "\n",
-        "# First, create the dataset entry\n",
-        "dataset = bigquery.Dataset(f\"{PROJECT_ID}.{BQ_DATASET}\")\n",
-        "dataset.location = \"US\"\n",
-        "dataset = bqclient.create_dataset(dataset, timeout=30)\n",
-        "\n",
         "query = f\"\"\"\n",
         "CREATE OR REPLACE TABLE `{BQ_TABLE_COPY}`\n",
         "AS (\n",
@@ -890,7 +757,7 @@
         "dataset = aip.TabularDataset.create(\n",
         "    display_name=\"Chicago Taxi\" + \"_\" + TIMESTAMP,\n",
         "    bq_source=[IMPORT_FILE],\n",
-        "    labels={\"user_metadata\": BUCKET_NAME},\n",
+        "    labels={\"user_metadata\": BUCKET_NAME[5:]},\n",
         ")\n",
         "\n",
         "label_column = \"tip_bin\"\n",
@@ -1082,9 +949,9 @@
       },
       "outputs": [],
       "source": [
-        "STATISTICS_SCHEMA = BUCKET_URI + \"/statistics.jsonl\"\n",
+        "STATISTICS_SCHEMA = BUCKET_NAME + \"/statistics.jsonl\"\n",
         "\n",
-        "tfdv.write_stats_text(stats, BUCKET_URI + \"/statistics.jsonl\")\n",
+        "tfdv.write_stats_text(stats, BUCKET_NAME + \"/statistics.jsonl\")\n",
         "\n",
         "with tf.io.gfile.GFile(\n",
         "    \"gs://\" + dataset.labels[\"user_metadata\"] + \"/metadata.jsonl\", \"r\"\n",
@@ -1097,7 +964,7 @@
         ") as f:\n",
         "    json.dump(metadata, f)\n",
         "\n",
-        "! gsutil cat $BUCKET_URI/metadata.jsonl"
+        "!gsutil cat $BUCKET_NAME/metadata.jsonl"
       ]
     },
     {
@@ -1144,7 +1011,7 @@
       },
       "outputs": [],
       "source": [
-        "SCHEMA_LOCATION = BUCKET_URI + \"/schema.txt\"\n",
+        "SCHEMA_LOCATION = BUCKET_NAME + \"/schema.txt\"\n",
         "\n",
         "# When running Apache Beam directly (file is directly accessed)\n",
         "tfdv.write_schema_text(output_path=SCHEMA_LOCATION, schema=schema)\n",
@@ -1182,7 +1049,7 @@
         ") as f:\n",
         "    json.dump(metadata, f)\n",
         "\n",
-        "! gsutil cat $BUCKET_URI/metadata.jsonl"
+        "!gsutil cat $BUCKET_NAME/metadata.jsonl"
       ]
     },
     {
@@ -1234,7 +1101,7 @@
         "import setuptools\n",
         "\n",
         "REQUIRED_PACKAGES = [\n",
-        "    \"google-cloud-aiplatform\",\n",
+        "    \"google-cloud-aiplatform==1.4.2\",\n",
         "    \"tensorflow-transform==1.2.0\",\n",
         "    \"tensorflow-data-validation==1.2.0\",\n",
         "]\n",
@@ -1505,10 +1372,10 @@
         "            )\n",
         "\n",
         "\n",
-        "EXPORTED_JSONL_PREFIX = os.path.join(BUCKET_URI, \"exported_data/jsonl\")\n",
-        "EXPORTED_TFREC_PREFIX = os.path.join(BUCKET_URI, \"exported_data/tfrec\")\n",
-        "TRANSFORMED_DATA_PREFIX = os.path.join(BUCKET_URI, \"transformed_data\")\n",
-        "TRANSFORM_ARTIFACTS_DIR = os.path.join(BUCKET_URI, \"transformed_artifacts\")\n",
+        "EXPORTED_JSONL_PREFIX = os.path.join(BUCKET_NAME, \"exported_data/jsonl\")\n",
+        "EXPORTED_TFREC_PREFIX = os.path.join(BUCKET_NAME, \"exported_data/tfrec\")\n",
+        "TRANSFORMED_DATA_PREFIX = os.path.join(BUCKET_NAME, \"transformed_data\")\n",
+        "TRANSFORM_ARTIFACTS_DIR = os.path.join(BUCKET_NAME, \"transformed_artifacts\")\n",
         "\n",
         "QUERY_STRING = \"SELECT * FROM {} LIMIT 300000\".format(BQ_TABLE)\n",
         "JOB_NAME = \"chicago\" + TIMESTAMP\n",
@@ -1521,7 +1388,7 @@
         "    \"transform_artifact_dir\": TRANSFORM_ARTIFACTS_DIR,\n",
         "    \"exported_jsonl_prefix\": EXPORTED_JSONL_PREFIX,\n",
         "    \"exported_tfrec_prefix\": EXPORTED_TFREC_PREFIX,\n",
-        "    \"temp_location\": os.path.join(BUCKET_URI, \"temp\"),\n",
+        "    \"temp_location\": os.path.join(BUCKET_NAME, \"temp\"),\n",
         "    \"project\": PROJECT_ID,\n",
         "    \"region\": REGION,\n",
         "    \"setup_file\": \"./setup.py\",\n",
@@ -1592,7 +1459,7 @@
         ") as f:\n",
         "    json.dump(metadata, f)\n",
         "\n",
-        "! gsutil cat $BUCKET_URI/metadata.jsonl"
+        "!gsutil cat $BUCKET_NAME/metadata.jsonl"
       ]
     },
     {
@@ -1606,9 +1473,17 @@
         "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
         "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
         "\n",
-        "Otherwise, you can delete the individual resources you created in this tutorial.\n",
+        "Otherwise, you can delete the individual resources you created in this tutorial:\n",
         "\n",
-        "*Note:* stage2/mlops_experimentation is dependent on the resources created by this stage1 notebook."
+        "- Dataset\n",
+        "- Pipeline\n",
+        "- Model\n",
+        "- Endpoint\n",
+        "- AutoML Training Job\n",
+        "- Batch Job\n",
+        "- Custom Job\n",
+        "- Hyperparameter Tuning Job\n",
+        "- Cloud Storage Bucket"
       ]
     },
     {
@@ -1629,8 +1504,8 @@
         "    except Exception as e:\n",
         "        print(e)\n",
         "\n",
-        "    if \"BUCKET_URI\" in globals():\n",
-        "        ! gsutil rm -r $BUCKET_URI"
+        "    if \"BUCKET_NAME\" in globals():\n",
+        "        ! gsutil rm -r $BUCKET_NAME"
       ]
     }
   ],

notebooks/community/ml_ops/stage2/README.md

@@ -25,154 +25,85 @@ The second stage in MLOps is experimenting in developing one or more baseline mo
 - Use the What-if-Tool (WIT) to explore how the trained model would make predictions in different scenarios.
 
 
-<img src='stage2v3.png'>
-<br/>
-<br/>
-<br/>
-<img src='stage2.2v1.png'>
+<img src='stage2.png'>
 
 ## Notebooks
 
 ### Get Started
 
-
-[Get started with Vertex AI Training for Pytorch](get_started_vertex_training_pytorch.ipynb)
+[Get Started with Vertex Experiments and Vertex ML Metadata](get_started_vertex_experiments.ipynb)
 
 ```
 The steps performed include:
+
+- Use Python logging to log training configuration/results locally.
+- Use Google Cloud Logging to log training configuration/results in cloud storage.
+- Create a Vertex AI `Experiment` resource.
+- Instantiate an experiment run.
+- Log parameters for the run.
+- Log metrics for the run.
+- Display the logged experiment run.
+```
+
+[Get Started with Vertex TensorBoard](get_started_vertex_tensorboard.ipynb)
+
+```
+The steps performed include:
+
+- Create a TensorBoard callback when training a model.
+- Using Tensorboard with locally trained model.
+- Using Vertex AI TensorBoard with Vertex AI Training.
+```
+
+[Get Started with Custom Training Packages (Tensorflow)](get_started_vertex_training.ipynb)
+
+```
+The steps performed include:
+
+- Training using a single Python script.
+- Training using a Python package.
+- Training using a custom training image.
+- Laying out a training package.
+```
+
+[Get Started with Custom Training Packages (Scikit-Learn)](get_started_vertex_training_sklearn.ipynb)
+
+```
+The steps performed include:
+
+- Training using a Python package.
+- Report accuracy when hyperparameter tuning.
+- Save the model artifacts to Cloud Storage using GCSFuse.
+- Create a `Vertex AI Model` resource.
+```
+
+[Get Started with Custom Training Packages (XGBoost)](get_started_vertex_training_xgboost.ipynb)
+
+```
+The steps performed include:
+
+- Training using a Python package.
+- Report accuracy when hyperparameter tuning.
+- Save the model artifacts to Cloud Storage using GCSFuse.
+- Create a `Vertex AI Model` resource.
+```
+
+[Get Started with Custom Training Packages (Pytorch)](get_started_vertex_training_pytorch.ipynb)
+
+```
+The steps performed include:
+
 - Single node training using a Python package.
 - Report accuracy when hyperparameter tuning.
 - Save the model artifacts to Cloud Storage using GCSFuse.
 - Create a `Vertex AI Model` resource.
 ```
 
-[Get started with prebuilt TFHub models](get_started_with_tfhub_models.ipynb)
+[Get Started with Custom Training Packages (R)](get_started_vertex_training_r.ipynb)
 
 ```
 The steps performed include:
-- Download a TensorFlow Hub prebuilt model.
-- Add the task component as a classifier for the CIFAR-10 dataset.
-- Fine tune locally the model with transfer learning training.
-- Construct a custom training script:
-    - Get training data from TensorFlow Datasets
-    - Get model architecture from TensorFlow Hub
-    - Train then model
-    - Save model artifacts and upload as Vertex AI Model resource.
-```
 
-[Get started with Vertex AI TensorBoard](get_started_vertex_tensorboard.ipynb)
-
-```
-The steps performed include:
-- Create a TensorBoard callback when training a model.
-- Using Tensorboard with locally trained model.
-- Using Vertex AI TensorBoard with Vertex AI Training.
-```
-
-[Get started with TabNet builtin algorithm for training tabular models](get_started_with_tabnet.ipynb)
-
-```
-The steps performed include:
-- Get the training data.
-- Configure training parameters for the `Vertex AI TabNet` container.
-- Train the model using `Vertex AI Training` using CSV data.
-- Upload the model as a `Vertex AI Model` resource.
-- Deploy the `Vertex AI Model` resource to a `Vertex AI Endpoint` resource.
-- Make a prediction with the deployed model.
-- Hyperparameter tuning the `Vertex AI TabNet` model.
-- Train the model using `Vertex AI Training` using BigQuery table.
-
-```
-
-[Get started with Vertex AI Vizier](get_started_vertex_vizier.ipynb)
-
-```
-The steps performed include:
-- Hyperparameter tuning with Random algorithm.
-- Hyperparameter tuning with Vizier (Bayesian) algorithm.
-```
-
-[Automl image classfication training with customer managed encryption keys (CMEK)](get_started_with_cmek_training.ipynb)
-
-```
-The steps performed include:
-- Creating a customer managed encryption key.
-- Creating an image dataset with CMEK encryption.
-- Train an AutoML model with CMEK encryption.
-```
-
-[Get started with Vertex AI distributed training](get_started_vertex_distributed_training.ipynb)
-
-```
-The steps performed include:
-- `MirroredStrategy`: Train on a single VM with multiple GPUs.
-- `MultiWorkerMirroredStrategy`: Train on multiple VMs with automatic setup of replicas.
-- `MultiWorkerMirroredStrategy`: Train on multiple VMs with fine grain control of replicas.
-- `ReductionServer`: Train on multiple VMS and sync updates across VMS with `Vertex AI Reduction Server`.
-- `TPUTraining`: Train with multiple Cloud TPUs.
-```
-
-[Get started with Vertex AI Training for scikit-learn](get_started_vertex_training_sklearn.ipynb)
-
-```
-The steps performed include:
-- Training using a Python package.
-- Report accuracy when hyperparameter tuning.
-- Save the model artifacts to Cloud Storage using GCSFuse.
-- Create a `Vertex AI Model` resource.
-```
-
-[Get started with Vertex AI Experiments](get_started_vertex_experiments.ipynb)
-
-```
-The steps performed include:
-- Local (notebook) Training
-    - Create an experiment
-    - Create a first run in the experiment
-    - Log parameters and metrics
-    - Create artifact lineage
-    - Visualize the experiment results
-    - Execute a second run
-    - Compare the two runs in the experiment
-- Cloud (`Vertex AI`) Training
-    - Within the training script:
-        - Create an experiment
-        - Log parameters and metrics
-        - Create artifact lineage
-    - Create a `Vertex AI Training` custom job
-    - Execute the custom job
-    - Visualize the experiment results
-```
-
-[Get started with Vertex AI Hyperparameter Tuning for XGBoost](get_started_vertex_hpt_xgboost.ipynb)
-
-```
-The steps performed include:
-- Training using a Python package.
-- Report accuracy when hyperparameter tuning.
-- Save the model artifacts to Cloud Storage using GCSFuse.
-- Create a `Vertex AI Model` resource.
-
-```
-
-[Get started with Vertex AI Feature Store](get_started_vertex_feature_store.ipynb)
-
-```
-The steps performed include:
-- Creating a Vertex AI `Featurestore` resource.
-    - Creating `EntityType` resources for the `Featurestore` resource.
-    - Creating `Feature` resources for each `EntityType` resource.
-- Import feature values (entity data items) into `Featurestore` resource.
-    - From a Cloud Storage location.
-    - From a pandas DataFrame.
-- Perform online serving from a `Featurestore` resource.
-- Perform batch serving from a `Featurestore` resource.
-```
-
-[Get started with Vertex AI Training for R](get_started_vertex_training_r.ipynb)
-
-```
-The steps performed include:
 - Locally train an R model in a notebook using %%R magic commands
 - Create a deployment image with trained R model and serving functions.
 - Test the deployment image locally.
@@ -184,101 +115,76 @@ The steps performed include:
 - Train a R model using `Vertex AI Trainingh` service with the R-to-Python training package.
 ```
 
-[Get started with logging](get_started_with_logging.ipynb)
+[Get Started with Distributed Training](get_started_vertex_distributed_training.ipynb)
 
 ```
 The steps performed include:
-- Use Python logging to log training configuration/results locally.
-- Use Google Cloud Logging to log training configuration/results in cloud storage.
+
+- `MirroredStrategy`: Train on a single VM with multiple GPUs.
+- `MultiWorkerMirroredStrategy`: Train on multiple VMs with automatic setup of replicas.
+- `MultiWorkerMirroredStrategy`: Train on multiple VMs with fine grain control of replicas.
+- `ReductionServer`: Train on multiple VMS and sync updates across VMS with `Vertex AI Reduction Server`.
+- `TPUTraining`: Train with multiple Cloud TPUs.
 ```
 
-[Get started with Vertex AI Training for R using R Kernel](get_started_vertex_training_r_using_r_kernel.ipynb)
+[Get Started with Vizier Hyperparameter Tuning](get_started_vertex_vizier.ipynb)
 
 ```
 The steps performed include:
-- Create a custom R training script
-- Create a custom R serving script
-- Create a custom R deployment (serving) container.
-- Train the model using `Vertex AI` custom training.
-- Create an `Endpoint` resouce.
-- Deploy the `Model` resource (trained R model) to the `Endpoint` resource.
-- Make an online prediction.
 
+- Hyperparameter tuning with Random algorithm.
+- Hyperparameter tuning with Vizier (Bayesian) algorithm.
 ```
 
-[Get started with BigQuery ML training](get_started_bqml_training.ipynb)
+[Get Started with AutoML Training](get_started_automl_training.ipynb)
 
 ```
 The steps performed include:
-- Create a local BigQuery table in your project
-- Train a BQML model
-- Evaluate the BQML model
-- Export the BQML model as a cloud model
-- Upload the exported model as a `Vertex AI Model` resource
-- Hyperparameter tune a BQML model with `Vertex AI Vizier`
-- Automatically register a BQML model to `Vertex AI Model Registry`
 
+- Train an image model.
+- Export the image model as an edge model.
+- Train a tabular model.
+- Export the tabular model as a cloud model.
+- Train a text model.
 ```
 
-[Get started with AutoML training](get_started_automl_training.ipynb)
+[Get Started with BQML Training](get_started_bqml_training.ipynb)
 
 ```
 The steps performed include:
-- Train an image model
-- Export the image model as an edge model
-- Train a tabular model
-- Export the tabular model as a cloud model
-- Train a text model
-- Train a video model
+
+- Create a local BQ table in your project.
+- Train a BQML model.
+- Evaluate the BQML model.
+- Export the BQML model as a cloud model.
+- Upload the exported model as a Vertex AI Model resource.
+- Hyperparameter tune a BQML model with Vertex AI Vizier.
 ```
 
-[Get started with Vertex AI Training for XGBoost](get_started_vertex_training_xgboost.ipynb)
+[Get Started with Vertex Feature Store](get_started_vertex_feature_store.ipynb)
 
 ```
 The steps performed include:
-- Training using a Python package.
-- Report accuracy when hyperparameter tuning.
-- Save the model artifacts to Cloud Storage using GCSFuse.
-- Create a `Vertex AI Model` resource.
 
+- Creating a Vertex AI `Featurestore` resource.
+    - Creating `EntityType` resources for the `Featurestore` resource.
+    - Creating `Feature` resources for each `EntityType` resource.
+- Import feature values (entity data items) into `Featurestore` resource from Cloud Storage.
+- Import feature values (entity data items) into `Featurestore` resource from pandas DataFrame.
+- Perform online serving from a `Featurestore` resource.
+- Perform batch serving from a `Featurestore` resource.
 ```
 
-[Get started with Vertex AI Training](get_started_vertex_training.ipynb)
+[Get Started with Google CMEK Training](get_started_with_cmek_training.ipynb)
 
 ```
 The steps performed include:
-- Training using a single Python script.
-- Training using a Python package.
-- Training using a custom training image.
-- Laying out a training package.
+
+- Creating a customer managed encryption key.
+- Creating an image dataset with CMEK encryption.
+- Train an AutoML model with CMEK encryption.
 ```
 
-[Get started with Vertex AI Training for LightGBM](get_started_vertex_training_lightgbm.ipynb)
-
-```
-The steps performed include:
-- Training using a Python package.
-- Save the model artifacts to Cloud Storage using GCSFuse.
-- Construct a FastAPI prediction server.
-- Construct a Dockerfile deployment image.
-- Test the deployment image locally.
-- Create a `Vertex AI Model` resource.
-
-```
-
-[Get started Vision API test preprocessing and AutoML text model generation](get_started_with_visionapi_and_automl.ipynb)
-
-```
-The steps performed include:
-- Preprocess training files using `Vision AI` APIs to extract the text from PDF files.
-- Create a custom import file that includes annotation data based on the sample `BigQuery` dataset.
-- Create a `Vertex AI Dataset` resource.
-- Train the model.
-- View the model evaluation.
-- Deploy the `Vertex AI Model` resource to a serving `Endpoint` resource.
-- Make a prediction.
-- Undeploy the `Model`.
-```
 
 ### E2E Stage Example
 
@@ -286,6 +192,7 @@ The steps performed include:
 
 ```
 The steps performed include:
+
 - Review the `Dataset` resource created during stage 1.
 - Train an AutoML tabular binary classifier model in the background.
 - Build the experimental model architecture.

notebooks/community/ml_ops/stage2/get_started_automl_training.ipynb

@@ -8,7 +8,7 @@
       },
       "outputs": [],
       "source": [
-        "# Copyright 2022 Google LLC\n",
+        "# Copyright 2021 Google LLC\n",
         "#\n",
         "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
         "# you may not use this file except in compliance with the License.\n",
@@ -39,14 +39,8 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_automl_training.ipynb\">\n",
-        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
-        "        </a>\n",
-        "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage2/get_started_automl_training.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
+        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_automl_training.ipynb\">\n",
+        "      Open in Google Cloud Notebooks\n",
         "    </a>\n",
         "  </td>\n",
         "</table>\n",
@@ -65,6 +59,39 @@
         "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 2 : experimentation: get started with AutoML Training."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dataset:flowers,icn"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the [Flowers dataset](https://www.tensorflow.org/datasets/catalog/tf_flowers) from [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/overview). The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket. The trained model predicts the type of flower an image is from a class of five flowers: daisy, dandelion, rose, sunflower, or tulip."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dataset:gsod,lrg"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the GSOD dataset from [BigQuery public datasets](https://cloud.google.com/bigquery/public-data). The version of the dataset you use only the fields year, month and day to predict the value of mean daily temperature (mean_temp)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dataset:happydb,tcn"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the [Happy Moments dataset](https://www.kaggle.com/ritresearch/happydb) from [Kaggle Datasets](https://www.kaggle.com/ritresearch/happydb). The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -81,12 +108,11 @@
         "\n",
         "The steps performed include:\n",
         "\n",
-        "- Train an image model\n",
-        "- Export the image model as an edge model\n",
-        "- Train a tabular model\n",
-        "- Export the tabular model as a cloud model\n",
-        "- Train a text model\n",
-        "- Train a video model"
+        "- Train an image model.\n",
+        "- Export the image model as an edge model.\n",
+        "- Train a tabular model.\n",
+        "- Export the tabular model as a cloud model.\n",
+        "- Train a text model."
       ]
     },
     {
@@ -99,49 +125,9 @@
         "\n",
         "When doing E2E MLOps on Google Cloud, the following are best practices for when to use AutoML:\n",
         "\n",
-        "* **You have a limited amount of training data**\n",
+        "**You have a limited amount of training data**\n",
         "\n",
-        "* **You want to establish a baseline metric before experimenting with a custom model**"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "dataset:flowers,icn"
-      },
-      "source": [
-        "### Datasets\n",
-        "\n",
-        "#### Image\n",
-        "\n",
-        "The image dataset used for this tutorial is the [Flowers dataset](https://www.tensorflow.org/datasets/catalog/tf_flowers) from [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/overview). The version of the dataset in this tutorial is stored in a public Cloud Storage bucket. The trained model predicts the type of flower in a given image from a class of five flowers: daisy, dandelion, rose, sunflower, or tulip.\n",
-        "\n",
-        "#### Tabular\n",
-        "\n",
-        "The tabular dataset used for this tutorial is the GSOD dataset from [BigQuery public datasets](https://cloud.google.com/bigquery/public-data). The version of the dataset you use only the fields year, month and day to predict the value of mean daily temperature (mean_temp).\n",
-        "\n",
-        "#### Text\n",
-        "\n",
-        "The text dataset used for this tutorial is the [Happy Moments dataset](https://www.kaggle.com/ritresearch/happydb) from [Kaggle Datasets](https://www.kaggle.com/ritresearch/happydb). The version of the dataset you use in this tutorial is stored in a public Cloud Storage bucket.\n",
-        "\n",
-        "#### Video\n",
-        "\n",
-        "The video dataset used for this tutorial is the golf swing recognition portion of the [Human Motion dataset](https://todo) from [MIT](http://cbcl.mit.edu/publications/ps/Kuehne_etal_iccv11.pdf). The version of the dataset you use in this tutorial is stored in a public Cloud Storage bucket. The trained model predicts the start frame where a golf swing begins."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "fb3451ce8e47"
-      },
-      "source": [
-        "### Costs\n",
-        "This tutorial uses billable components of Google Cloud:\n",
-        "\n",
-        "- Vertex AI\n",
-        "- Cloud Storage\n",
-        "\n",
-        "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage pricing](https://cloud.google.com/storage/pricing) and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
+        "**You want to establish a baseline metric before experimenting with a custom model**"
       ]
     },
     {
@@ -152,7 +138,7 @@
       "source": [
         "## Installations\n",
         "\n",
-        "Install the packages required for executing this notebook."
+        "Install *one time* the packages for executing the MLOps notebooks."
       ]
     },
     {
@@ -163,23 +149,20 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
-        "# Install the packages\n",
-        "\n",
-        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG -q\n",
-        "! pip3 install --upgrade google-cloud-storage $USER_FLAG -q"
+        "ONCE_ONLY = False\n",
+        "if ONCE_ONLY:\n",
+        "    ! pip3 install -U tensorflow==2.5 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-transform==1.2 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-io==0.18 $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-bigquery $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-logging $USER_FLAG\n",
+        "    ! pip3 install --upgrade apache-beam[gcp] $USER_FLAG\n",
+        "    ! pip3 install --upgrade pyarrow $USER_FLAG\n",
+        "    ! pip3 install --upgrade cloudml-hypertune $USER_FLAG\n",
+        "    ! pip3 install --upgrade kfp $USER_FLAG"
       ]
     },
     {
@@ -216,32 +199,6 @@
       "metadata": {
         "id": "project_id"
       },
-      "source": [
-        "## Before you begin\n",
-        "\n",
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI, Compute Engine and Cloud Storage APIs](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component,storage_component).\n",
-        "\n",
-        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "5aee4379e8e5"
-      },
       "source": [
         "#### Set your project ID\n",
         "\n",
@@ -313,10 +270,7 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
       ]
     },
     {
@@ -343,67 +297,6 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "3ffa6b6c7cdb"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
-        "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
-        "\n",
-        "1. **Click Create service account**.\n",
-        "\n",
-        "2. In the **Service account name** field, enter a name, and click **Create**.\n",
-        "\n",
-        "3. In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex AI\" into the filter box, and select **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "4. Click Create. A JSON file that contains your key downloads to your local environment.\n",
-        "\n",
-        "5. Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "2b72272258fc"
-      },
-      "outputs": [],
-      "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = False\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        IS_COLAB = True\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -427,8 +320,7 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -439,9 +331,8 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
-        "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
       ]
     },
     {
@@ -461,7 +352,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_URI"
+        "! gsutil mb -l $REGION $BUCKET_NAME"
       ]
     },
     {
@@ -481,7 +372,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_URI"
+        "! gsutil ls -al $BUCKET_NAME"
       ]
     },
     {
@@ -504,7 +395,7 @@
       },
       "outputs": [],
       "source": [
-        "import google.cloud.aiplatform as aiplatform"
+        "import google.cloud.aiplatform as aip"
       ]
     },
     {
@@ -526,7 +417,7 @@
       },
       "outputs": [],
       "source": [
-        "aiplatform.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
+        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_NAME)"
       ]
     },
     {
@@ -557,7 +448,7 @@
       "source": [
         "## AutoML image models\n",
         "\n",
-        "AutoML can train the following types of image models:\n",
+        "AutoML can train the following types of models:\n",
         "\n",
         "- classification\n",
         "- objection detection\n",
@@ -613,7 +504,10 @@
       },
       "outputs": [],
       "source": [
-        "FILE = IMPORT_FILE\n",
+        "if \"IMPORT_FILES\" in globals():\n",
+        "    FILE = IMPORT_FILES[0]\n",
+        "else:\n",
+        "    FILE = IMPORT_FILE\n",
         "\n",
         "count = ! gsutil cat $FILE | wc -l\n",
         "print(\"Number of Examples\", int(count[0]))\n",
@@ -651,10 +545,10 @@
       },
       "outputs": [],
       "source": [
-        "dataset = aiplatform.ImageDataset.create(\n",
-        "    display_name=\"flowers_\" + TIMESTAMP,\n",
+        "dataset = aip.ImageDataset.create(\n",
+        "    display_name=\"Happy Moments\" + \"_\" + TIMESTAMP,\n",
         "    gcs_source=[IMPORT_FILE],\n",
-        "    import_schema_uri=aiplatform.schema.dataset.ioformat.image.single_label_classification,\n",
+        "    import_schema_uri=aip.schema.dataset.ioformat.image.single_label_classification,\n",
         ")\n",
         "\n",
         "print(dataset.resource_name)"
@@ -699,8 +593,8 @@
       },
       "outputs": [],
       "source": [
-        "dag = aiplatform.AutoMLImageTrainingJob(\n",
-        "    display_name=\"flowers_\" + TIMESTAMP,\n",
+        "dag = aip.AutoMLImageTrainingJob(\n",
+        "    display_name=\"happydb_\" + TIMESTAMP,\n",
         "    prediction_type=\"classification\",\n",
         "    multi_label=False,\n",
         "    model_type=\"MOBILE_TF_LOW_LATENCY_1\",\n",
@@ -718,14 +612,14 @@
       "source": [
         "#### Run the training pipeline\n",
         "\n",
-        "Next, you run the created DAG to start the training job by invoking the method `run`, with the following parameters:\n",
+        "Next, you run the DAG to start the training job by invoking the method `run`, with the following parameters:\n",
         "\n",
         "- `dataset`: The `Dataset` resource to train the model.\n",
         "- `model_display_name`: The human readable name for the trained model.\n",
         "- `training_fraction_split`: The percentage of the dataset to use for training.\n",
         "- `test_fraction_split`: The percentage of the dataset to use for test (holdout data).\n",
         "- `validation_fraction_split`: The percentage of the dataset to use for validation.\n",
-        "- `budget_milli_node_hours`: (optional) Maximum training time specified in unit of milli node-hours (1000 = node-hour).\n",
+        "- `budget_milli_node_hours`: (optional) Maximum training time specified in unit of millihours (1000 = hour).\n",
         "- `disable_early_stopping`: If `True`, training maybe completed before using the entire budget if the service believes it cannot further improve on the model objective measurements.\n",
         "\n",
         "The `run` method when completed returns the `Model` resource.\n",
@@ -743,7 +637,7 @@
       "source": [
         "model = dag.run(\n",
         "    dataset=dataset,\n",
-        "    model_display_name=\"flowers_\" + TIMESTAMP,\n",
+        "    model_display_name=\"happydb_\" + TIMESTAMP,\n",
         "    training_fraction_split=0.8,\n",
         "    validation_fraction_split=0.1,\n",
         "    test_fraction_split=0.1,\n",
@@ -759,8 +653,9 @@
       },
       "source": [
         "## Review model evaluation scores\n",
+        "After your model has finished training, you can review the evaluation scores for it.\n",
         "\n",
-        "After your model training has finished, you can review the evaluation scores for it using the `list_model_evaluations()` method. This method will return an iterator for each evaluation slice."
+        "First, you need to get a reference to the new model. As with datasets, you can either use the reference to the model variable you created when you deployed the model or you can list all of the models in your project."
       ]
     },
     {
@@ -771,10 +666,18 @@
       },
       "outputs": [],
       "source": [
-        "model_evaluations = model.list_model_evaluations()\n",
+        "# Get model resource ID\n",
+        "models = aip.Model.list(filter=\"display_name=happydb_\" + TIMESTAMP)\n",
         "\n",
-        "for model_evaluation in model_evaluations:\n",
-        "    print(model_evaluation.to_dict())"
+        "# Get a reference to the Model Service client\n",
+        "client_options = {\"api_endpoint\": f\"{REGION}-aiplatform.googleapis.com\"}\n",
+        "model_service_client = aip.gapic.ModelServiceClient(client_options=client_options)\n",
+        "\n",
+        "model_evaluations = model_service_client.list_model_evaluations(\n",
+        "    parent=models[0].resource_name\n",
+        ")\n",
+        "model_evaluation = list(model_evaluations)[0]\n",
+        "print(model_evaluation)"
       ]
     },
     {
@@ -818,7 +721,7 @@
       "source": [
         "### Get test item\n",
         "\n",
-        "You will use an arbitrary example out of the dataset as a test item. Don't be concerned that the example was likely used in training the model. You are just looking at how to make a prediction."
+        "You will use an arbitrary example out of the dataset as a test item. Don't be concerned that the example was likely used in training the model -- we just want to demonstrate how to make a prediction."
       ]
     },
     {
@@ -850,7 +753,7 @@
         "\n",
         "#### Request\n",
         "\n",
-        "Since your test item is in a public Cloud Storage bucket in this example, you copy it to your bucket and read the contents of the image using `Cloud Storage SDK`. To pass the test data to the prediction service, you encode the bytes into base64 which makes the content safe from modification while transmitting binary data over the network.\n",
+        "Since in this example your test item is in a Cloud Storage bucket, you open and read the contents of the image using `tf.io.gfile.Gfile()`. To pass the test data to the prediction service, you encode the bytes into base64 -- which makes the content safe from modification while transmitting binary data over the network.\n",
         "\n",
         "The format of each instance is:\n",
         "\n",
@@ -872,66 +775,32 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "1c1d53e89beb"
+        "id": "predict_request:mbsdk,icn"
       },
       "outputs": [],
       "source": [
         "import base64\n",
         "\n",
-        "from google.cloud import storage\n",
+        "import tensorflow as tf\n",
         "\n",
-        "# Copy the test image to the Cloud storage bucket as \"test.jpg\"\n",
-        "test_image_local = \"{}/test.jpg\".format(BUCKET_URI)\n",
-        "! gsutil cp $test_item $test_image_local\n",
-        "\n",
-        "# Download the test image in bytes format\n",
-        "storage_client = storage.Client(project=PROJECT_ID)\n",
-        "bucket = storage_client.bucket(bucket_name=BUCKET_NAME)\n",
-        "test_content = bucket.get_blob(\"test.jpg\").download_as_bytes()\n",
+        "with tf.io.gfile.GFile(test_item, \"rb\") as f:\n",
+        "    content = f.read()\n",
         "\n",
         "# The format of each instance should conform to the deployed model's prediction input schema.\n",
-        "instances = [{\"content\": base64.b64encode(test_content).decode(\"utf-8\")}]\n",
+        "instances = [{\"content\": base64.b64encode(content).decode(\"utf-8\")}]\n",
         "\n",
         "prediction = endpoint.predict(instances=instances)\n",
         "\n",
         "print(prediction)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "3b1b67898533"
-      },
-      "source": [
-        "#### Alternate method using [GFile](https://www.tensorflow.org/api_docs/python/tf/io/gfile/GFile)\n",
-        "\n",
-        "Alternatively, [GFile](https://www.tensorflow.org/api_docs/python/tf/io/gfile/GFile) method from tensorflow-io library can be used to read the data from Cloud storage directly. The following code snippet does the same :\n",
-        "\n",
-        "```\n",
-        "import base64\n",
-        "import tensorflow as tf\n",
-        "\n",
-        "# Read the test file using GFile\n",
-        "with tf.io.gfile.GFile(test_item, \"rb\") as f:\n",
-        "    content = f.read()\n",
-        "\n",
-        "# The format of each instance should conform to the deployed model's prediction input schema.\n",
-        "instances = [{\"content\": base64.b64encode(content).decode(\"utf-8\")}]\n",
-        "\n",
-        "prediction = endpoint.predict(instances=instances)\n",
-        "\n",
-        "print(prediction)\n",
-        "```\n",
-        "Nevertheless, `tf.io.gfile.GFile` supports multiple file system implementations, including local files, Google Cloud Storage (using a gs:// prefix), and HDFS (using an hdfs:// prefix)."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "undeploy_model:mbsdk"
       },
       "source": [
-        "#### Undeploy the model\n",
+        "## Undeploy the model\n",
         "\n",
         "When you are done doing predictions, you undeploy the model from the `Endpoint` resouce. This deprovisions all compute resources and ends billing for the deployed model."
       ]
@@ -977,7 +846,7 @@
       "outputs": [],
       "source": [
         "response = model.export_model(\n",
-        "    artifact_destination=BUCKET_URI, export_format_id=\"tflite\", sync=True\n",
+        "    artifact_destination=BUCKET_NAME, export_format_id=\"tflite\", sync=True\n",
         ")\n",
         "\n",
         "model_package = response[\"artifactOutputUri\"]"
@@ -1118,10 +987,10 @@
       },
       "outputs": [],
       "source": [
-        "dataset = aiplatform.TabularDataset.create(\n",
-        "    display_name=\"gsod_\" + TIMESTAMP,\n",
+        "dataset = aip.TabularDataset.create(\n",
+        "    display_name=\"Happy Moments\" + \"_\" + TIMESTAMP,\n",
         "    bq_source=[IMPORT_FILE],\n",
-        "    labels={\"user_metadata\": BUCKET_NAME},\n",
+        "    labels={\"user_metadata\": BUCKET_NAME[5:]},\n",
         ")\n",
         "\n",
         "label_column = \"mean_temp\"\n",
@@ -1177,7 +1046,9 @@
         "  - regression:\n",
         "    - `minimize-rmse`\n",
         "    - `minimize-mae`\n",
-        "    - `minimize-rmsle`"
+        "    - `minimize-rmsle`\n",
+        "\n",
+        "The instantiated object is the DAG (directed acyclic graph) for the training pipeline."
       ]
     },
     {
@@ -1188,8 +1059,8 @@
       },
       "outputs": [],
       "source": [
-        "dag = aiplatform.AutoMLTabularTrainingJob(\n",
-        "    display_name=\"gsod_\" + TIMESTAMP,\n",
+        "dag = aip.AutoMLTabularTrainingJob(\n",
+        "    display_name=\"happydb_\" + TIMESTAMP,\n",
         "    optimization_prediction_type=\"regression\",\n",
         "    optimization_objective=\"minimize-rmse\",\n",
         "    column_transformations=TRANSFORMATIONS,\n",
@@ -1206,7 +1077,7 @@
       "source": [
         "#### Run the training pipeline\n",
         "\n",
-        "Next, you run the created DAG to start the training job by invoking the method `run`, with the following parameters:\n",
+        "Next, you run the DAG to start the training job by invoking the method `run`, with the following parameters:\n",
         "\n",
         "- `dataset`: The `Dataset` resource to train the model.\n",
         "- `model_display_name`: The human readable name for the trained model.\n",
@@ -1232,7 +1103,7 @@
       "source": [
         "model = dag.run(\n",
         "    dataset=dataset,\n",
-        "    model_display_name=\"gsod_\" + TIMESTAMP,\n",
+        "    model_display_name=\"happydb_\" + TIMESTAMP,\n",
         "    training_fraction_split=0.8,\n",
         "    validation_fraction_split=0.1,\n",
         "    test_fraction_split=0.1,\n",
@@ -1249,8 +1120,9 @@
       },
       "source": [
         "## Review model evaluation scores\n",
+        "After your model has finished training, you can review the evaluation scores for it.\n",
         "\n",
-        "After your model training has finished, you can review the evaluation scores for it using the `list_model_evaluations()` method. This method will return an iterator for each evaluation slice."
+        "First, you need to get a reference to the new model. As with datasets, you can either use the reference to the model variable you created when you deployed the model or you can list all of the models in your project."
       ]
     },
     {
@@ -1261,10 +1133,18 @@
       },
       "outputs": [],
       "source": [
-        "model_evaluations = model.list_model_evaluations()\n",
+        "# Get model resource ID\n",
+        "models = aip.Model.list(filter=\"display_name=happydb_\" + TIMESTAMP)\n",
         "\n",
-        "for model_evaluation in model_evaluations:\n",
-        "    print(model_evaluation.to_dict())"
+        "# Get a reference to the Model Service client\n",
+        "client_options = {\"api_endpoint\": f\"{REGION}-aiplatform.googleapis.com\"}\n",
+        "model_service_client = aip.gapic.ModelServiceClient(client_options=client_options)\n",
+        "\n",
+        "model_evaluations = model_service_client.list_model_evaluations(\n",
+        "    parent=models[0].resource_name\n",
+        ")\n",
+        "model_evaluation = list(model_evaluations)[0]\n",
+        "print(model_evaluation)"
       ]
     },
     {
@@ -1297,7 +1177,7 @@
         "id": "undeploy_model:mbsdk"
       },
       "source": [
-        "#### Undeploy the model\n",
+        "## Undeploy the model\n",
         "\n",
         "When you are done doing predictions, you undeploy the model from the `Endpoint` resouce. This deprovisions all compute resources and ends billing for the deployed model."
       ]
@@ -1338,7 +1218,7 @@
       "outputs": [],
       "source": [
         "response = model.export_model(\n",
-        "    artifact_destination=BUCKET_URI, export_format_id=\"tf-saved-model\", sync=True\n",
+        "    artifact_destination=BUCKET_NAME, export_format_id=\"tf-saved-model\", sync=True\n",
         ")\n",
         "\n",
         "model_package = response[\"artifactOutputUri\"]"
@@ -1470,7 +1350,10 @@
       },
       "outputs": [],
       "source": [
-        "FILE = IMPORT_FILE\n",
+        "if \"IMPORT_FILES\" in globals():\n",
+        "    FILE = IMPORT_FILES[0]\n",
+        "else:\n",
+        "    FILE = IMPORT_FILE\n",
         "\n",
         "count = ! gsutil cat $FILE | wc -l\n",
         "print(\"Number of Examples\", int(count[0]))\n",
@@ -1508,10 +1391,10 @@
       },
       "outputs": [],
       "source": [
-        "dataset = aiplatform.TextDataset.create(\n",
-        "    display_name=\"happydb_\" + TIMESTAMP,\n",
+        "dataset = aip.TextDataset.create(\n",
+        "    display_name=\"Happy Moments\" + \"_\" + TIMESTAMP,\n",
         "    gcs_source=[IMPORT_FILE],\n",
-        "    import_schema_uri=aiplatform.schema.dataset.ioformat.text.single_label_classification,\n",
+        "    import_schema_uri=aip.schema.dataset.ioformat.text.single_label_classification,\n",
         ")\n",
         "\n",
         "print(dataset.resource_name)"
@@ -1537,7 +1420,9 @@
         "  - `sentiment`: A text sentiment analysis model.\n",
         "  - `extraction`: A text entity extraction model.\n",
         "- `multi_label`: If a classification task, whether single (False) or multi-labeled (True).\n",
-        "- `sentiment_max`: If a sentiment analysis task, the maximum sentiment value.\n"
+        "- `sentiment_max`: If a sentiment analysis task, the maximum sentiment value.\n",
+        "\n",
+        "The instantiated object is the DAG (directed acyclic graph) for the training pipeline."
       ]
     },
     {
@@ -1548,7 +1433,7 @@
       },
       "outputs": [],
       "source": [
-        "dag = aiplatform.AutoMLTextTrainingJob(\n",
+        "dag = aip.AutoMLTextTrainingJob(\n",
         "    display_name=\"happydb_\" + TIMESTAMP,\n",
         "    prediction_type=\"classification\",\n",
         "    multi_label=False,\n",
@@ -1565,7 +1450,7 @@
       "source": [
         "#### Run the training pipeline\n",
         "\n",
-        "Next, you run the created DAG to start the training job by invoking the method `run`, with the following parameters:\n",
+        "Next, you run the DAG to start the training job by invoking the method `run`, with the following parameters:\n",
         "\n",
         "- `dataset`: The `Dataset` resource to train the model.\n",
         "- `model_display_name`: The human readable name for the trained model.\n",
@@ -1602,8 +1487,9 @@
       },
       "source": [
         "## Review model evaluation scores\n",
+        "After your model has finished training, you can review the evaluation scores for it.\n",
         "\n",
-        "After your model training has finished, you can review the evaluation scores for it using the `list_model_evaluations()` method. This method will return an iterator for each evaluation slice."
+        "First, you need to get a reference to the new model. As with datasets, you can either use the reference to the model variable you created when you deployed the model or you can list all of the models in your project."
       ]
     },
     {
@@ -1614,10 +1500,18 @@
       },
       "outputs": [],
       "source": [
-        "model_evaluations = model.list_model_evaluations()\n",
+        "# Get model resource ID\n",
+        "models = aip.Model.list(filter=\"display_name=happydb_\" + TIMESTAMP)\n",
         "\n",
-        "for model_evaluation in model_evaluations:\n",
-        "    print(model_evaluation.to_dict())"
+        "# Get a reference to the Model Service client\n",
+        "client_options = {\"api_endpoint\": f\"{REGION}-aiplatform.googleapis.com\"}\n",
+        "model_service_client = aip.gapic.ModelServiceClient(client_options=client_options)\n",
+        "\n",
+        "model_evaluations = model_service_client.list_model_evaluations(\n",
+        "    parent=models[0].resource_name\n",
+        ")\n",
+        "model_evaluation = list(model_evaluations)[0]\n",
+        "print(model_evaluation)"
       ]
     },
     {
@@ -1648,7 +1542,7 @@
         "id": "undeploy_model:mbsdk"
       },
       "source": [
-        "#### Undeploy the model\n",
+        "## Undeploy the model\n",
         "\n",
         "When you are done doing predictions, you undeploy the model from the `Endpoint` resouce. This deprovisions all compute resources and ends billing for the deployed model."
       ]
@@ -1792,7 +1686,10 @@
       },
       "outputs": [],
       "source": [
-        "FILE = IMPORT_FILE\n",
+        "if \"IMPORT_FILES\" in globals():\n",
+        "    FILE = IMPORT_FILES[0]\n",
+        "else:\n",
+        "    FILE = IMPORT_FILE\n",
         "\n",
         "count = ! gsutil cat $FILE | wc -l\n",
         "print(\"Number of Examples\", int(count[0]))\n",
@@ -1829,10 +1726,10 @@
       },
       "outputs": [],
       "source": [
-        "dataset = aiplatform.VideoDataset.create(\n",
-        "    display_name=\"human_motion_\" + TIMESTAMP,\n",
+        "dataset = aip.VideoDataset.create(\n",
+        "    display_name=\"Happy Moments\" + \"_\" + TIMESTAMP,\n",
         "    gcs_source=[IMPORT_FILE],\n",
-        "    import_schema_uri=aiplatform.schema.dataset.ioformat.video.classification,\n",
+        "    import_schema_uri=aip.schema.dataset.ioformat.video.classification,\n",
         ")\n",
         "\n",
         "print(dataset.resource_name)"
@@ -1856,7 +1753,9 @@
         "- `prediction_type`: The type task to train the model for.\n",
         "  - `classification`: A video classification model.\n",
         "  - `object_tracking`: A video object tracking model.\n",
-        "  - `action_recognition`: A video action recognition model."
+        "  - `action_recognition`: A video action recognition model.\n",
+        "\n",
+        "The instantiated object is the DAG (directed acyclic graph) for the training pipeline."
       ]
     },
     {
@@ -1867,8 +1766,8 @@
       },
       "outputs": [],
       "source": [
-        "dag = aiplatform.AutoMLVideoTrainingJob(\n",
-        "    display_name=\"human_motion_\" + TIMESTAMP,\n",
+        "dag = aip.AutoMLVideoTrainingJob(\n",
+        "    display_name=\"happydb_\" + TIMESTAMP,\n",
         "    prediction_type=\"classification\",\n",
         ")\n",
         "\n",
@@ -1883,7 +1782,7 @@
       "source": [
         "#### Run the training pipeline\n",
         "\n",
-        "Next, you run the created DAG to start the training job by invoking the method `run`, with the following parameters:\n",
+        "Next, you run the DAG to start the training job by invoking the method `run`, with the following parameters:\n",
         "\n",
         "- `dataset`: The `Dataset` resource to train the model.\n",
         "- `model_display_name`: The human readable name for the trained model.\n",
@@ -1905,7 +1804,7 @@
       "source": [
         "model = dag.run(\n",
         "    dataset=dataset,\n",
-        "    model_display_name=\"human_motion_\" + TIMESTAMP,\n",
+        "    model_display_name=\"happydb_\" + TIMESTAMP,\n",
         "    training_fraction_split=0.8,\n",
         "    test_fraction_split=0.2,\n",
         ")"
@@ -1918,8 +1817,9 @@
       },
       "source": [
         "## Review model evaluation scores\n",
+        "After your model has finished training, you can review the evaluation scores for it.\n",
         "\n",
-        "After your model training has finished, you can review the evaluation scores for it using the `list_model_evaluations()` method. This method will return an iterator for each evaluation slice."
+        "First, you need to get a reference to the new model. As with datasets, you can either use the reference to the model variable you created when you deployed the model or you can list all of the models in your project."
       ]
     },
     {
@@ -1930,10 +1830,18 @@
       },
       "outputs": [],
       "source": [
-        "model_evaluations = model.list_model_evaluations()\n",
+        "# Get model resource ID\n",
+        "models = aip.Model.list(filter=\"display_name=happydb_\" + TIMESTAMP)\n",
         "\n",
-        "for model_evaluation in model_evaluations:\n",
-        "    print(model_evaluation.to_dict())"
+        "# Get a reference to the Model Service client\n",
+        "client_options = {\"api_endpoint\": f\"{REGION}-aiplatform.googleapis.com\"}\n",
+        "model_service_client = aip.gapic.ModelServiceClient(client_options=client_options)\n",
+        "\n",
+        "model_evaluations = model_service_client.list_model_evaluations(\n",
+        "    parent=models[0].resource_name\n",
+        ")\n",
+        "model_evaluation = list(model_evaluations)[0]\n",
+        "print(model_evaluation)"
       ]
     },
     {
@@ -1991,7 +1899,16 @@
         "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
         "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
         "\n",
-        "Otherwise, you can delete the individual resources you created in this tutorial.\n"
+        "Otherwise, you can delete the individual resources you created in this tutorial:\n",
+        "\n",
+        "- Dataset\n",
+        "- Pipeline\n",
+        "- Model\n",
+        "- Endpoint\n",
+        "- Batch Job\n",
+        "- Custom Job\n",
+        "- Hyperparameter Tuning Job\n",
+        "- Cloud Storage Bucket"
       ]
     },
     {
@@ -2002,11 +1919,66 @@
       },
       "outputs": [],
       "source": [
-        "# Set this to true only if you'd like to delete your bucket\n",
-        "delete_bucket = False\n",
+        "delete_dataset = True\n",
+        "delete_pipeline = True\n",
+        "delete_model = True\n",
+        "delete_endpoint = True\n",
+        "delete_batchjob = True\n",
+        "delete_customjob = True\n",
+        "delete_hptjob = True\n",
+        "delete_bucket = True\n",
         "\n",
-        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
-        "    ! gsutil rm -r $BUCKET_URI"
+        "# Delete the dataset using the Vertex fully qualified identifier for the dataset\n",
+        "try:\n",
+        "    if delete_dataset and \"dataset_id\" in globals():\n",
+        "        clients[\"dataset\"].delete_dataset(name=dataset_id)\n",
+        "except Exception as e:\n",
+        "    print(e)\n",
+        "\n",
+        "# Delete the training pipeline using the Vertex fully qualified identifier for the pipeline\n",
+        "try:\n",
+        "    if delete_pipeline and \"pipeline_id\" in globals():\n",
+        "        clients[\"pipeline\"].delete_training_pipeline(name=pipeline_id)\n",
+        "except Exception as e:\n",
+        "    print(e)\n",
+        "\n",
+        "# Delete the model using the Vertex fully qualified identifier for the model\n",
+        "try:\n",
+        "    if delete_model and \"model_to_deploy_id\" in globals():\n",
+        "        clients[\"model\"].delete_model(name=model_to_deploy_id)\n",
+        "except Exception as e:\n",
+        "    print(e)\n",
+        "\n",
+        "# Delete the endpoint using the Vertex fully qualified identifier for the endpoint\n",
+        "try:\n",
+        "    if delete_endpoint and \"endpoint_id\" in globals():\n",
+        "        clients[\"endpoint\"].delete_endpoint(name=endpoint_id)\n",
+        "except Exception as e:\n",
+        "    print(e)\n",
+        "\n",
+        "# Delete the batch job using the Vertex fully qualified identifier for the batch job\n",
+        "try:\n",
+        "    if delete_batchjob and \"batch_job_id\" in globals():\n",
+        "        clients[\"job\"].delete_batch_prediction_job(name=batch_job_id)\n",
+        "except Exception as e:\n",
+        "    print(e)\n",
+        "\n",
+        "# Delete the custom job using the Vertex fully qualified identifier for the custom job\n",
+        "try:\n",
+        "    if delete_customjob and \"job_id\" in globals():\n",
+        "        clients[\"job\"].delete_custom_job(name=job_id)\n",
+        "except Exception as e:\n",
+        "    print(e)\n",
+        "\n",
+        "# Delete the hyperparameter tuning job using the Vertex fully qualified identifier for the hyperparameter tuning job\n",
+        "try:\n",
+        "    if delete_hptjob and \"hpt_job_id\" in globals():\n",
+        "        clients[\"job\"].delete_hyperparameter_tuning_job(name=hpt_job_id)\n",
+        "except Exception as e:\n",
+        "    print(e)\n",
+        "\n",
+        "if delete_bucket and \"BUCKET_NAME\" in globals():\n",
+        "    ! gsutil rm -r $BUCKET_NAME"
       ]
     }
   ],

notebooks/community/ml_ops/stage2/get_started_vertex_distributed_training.ipynb

@@ -8,7 +8,7 @@
       },
       "outputs": [],
       "source": [
-        "# Copyright 2022 Google LLC\n",
+        "# Copyright 2021 Google LLC\n",
         "#\n",
         "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
         "# you may not use this file except in compliance with the License.\n",
@@ -29,14 +29,9 @@
         "id": "title:generic,gcp"
       },
       "source": [
-        "# E2E ML on GCP: MLOps stage 2 : experimentation: get started with Vertex AI Distributed Training\n",
+        "# E2E ML on GCP: MLOps stage 2 : experimentation: get started with Vertex Distributed Training\n",
         "\n",
         "<table align=\"left\">\n",
-        "      <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_distributed_training.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
-        "    </a>\n",
-        "  </td>\n",
         "  <td>\n",
         "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_distributed_training.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
@@ -44,9 +39,8 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage2/get_started_vertex_distributed_training.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "Open in Vertex AI Workbench\n",
+        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_distributed_training.ipynb\">\n",
+        "      Open in Google Cloud Notebooks\n",
         "    </a>\n",
         "  </td>\n",
         "</table>\n",
@@ -62,7 +56,18 @@
         "## Overview\n",
         "\n",
         "\n",
-        "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 2 : experimentation: get started with Vertex Distributed Training. Please note: There are incompatibilities between Colab and Docker and the Docker section may not work until resolved by the platform."
+        "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 2 : experimentation: get started with Vertex Distributed Training."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dataset:custom,boston,lrg"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the [Boston Housing Prices dataset](https://www.cs.toronto.edu/~delve/data/boston/bostonDetail.html). The version of the dataset you will use in this tutorial is built into TensorFlow. The trained model predicts the median price of a house in units of 1K USD."
       ]
     },
     {
@@ -121,86 +126,59 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "dataset:custom,boston,lrg"
+        "id": "install_mlops"
       },
       "source": [
-        "### Dataset\n",
+        "## Installations\n",
         "\n",
-        "The dataset used for this tutorial is the [Boston Housing Prices dataset](https://www.cs.toronto.edu/~delve/data/boston/bostonDetail.html). The version of the dataset you use in this tutorial is built into TensorFlow. The trained model predicts the median price of a house in units of 1K USD."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "d10166df7141"
-      },
-      "source": [
-        "### Costs\n",
-        " \n",
-        "This tutorial uses billable components of Google Cloud:\n",
-        "\n",
-        "Vertex AI\n",
-        "Cloud Storage\n",
-        "\n",
-        "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage pricing](https://cloud.google.com/storage/pricing), and use the [Pricing Calculator](https://cloud.google.com/products/calculator/),\n",
-        "        to generate a cost estimate based on your projected usage."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "XkYpRvOQyVYb"
-      },
-      "source": [
-        "## Installation\n",
-        "\n",
-        "Install the packages required for executing this notebook."
+        "Install *one time* the packages for executing the MLOps notebooks."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "xs_Kt8RcyXTC"
+        "id": "install_mlops"
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
-        "! pip3 install {USER_FLAG} --upgrade google-cloud-aiplatform -q"
+        "ONCE_ONLY = False\n",
+        "if ONCE_ONLY:\n",
+        "    ! pip3 install -U tensorflow==2.5 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-transform==1.2 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-io==0.18 $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-bigquery $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-logging $USER_FLAG\n",
+        "    ! pip3 install --upgrade apache-beam[gcp] $USER_FLAG\n",
+        "    ! pip3 install --upgrade pyarrow $USER_FLAG\n",
+        "    ! pip3 install --upgrade cloudml-hypertune $USER_FLAG\n",
+        "    ! pip3 install --upgrade kfp $USER_FLAG\n",
+        "    ! pip3 install --upgrade torchvision $USER_FLAG\n",
+        "    ! pip3 install --upgrade rpy2 $USER_FLAG"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "oQhwq1iozAxh"
+        "id": "restart"
       },
       "source": [
         "### Restart the kernel\n",
         "\n",
-        "After you install the additional packages, you need to restart the notebook kernel so it can find the packages."
+        "Once you've installed the additional packages, you need to restart the notebook kernel so it can find the packages."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "zo3YFZXLzCRJ"
+        "id": "restart"
       },
       "outputs": [],
       "source": [
-        "# Automatically restart kernel after installs\n",
         "import os\n",
         "\n",
         "if not os.getenv(\"IS_TESTING\"):\n",
@@ -211,32 +189,6 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "84cd83853240"
-      },
-      "source": [
-        "## Before you begin\n",
-        "\n",
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI, BigQuery, Compute Engine and Cloud Storage APIs](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,bigquery,compute_component,storage_component).\n",
-        "\n",
-        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -309,14 +261,11 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "qohAA9fJulvP"
+        "id": "region"
       },
       "outputs": [],
       "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
       ]
     },
     {
@@ -334,7 +283,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "8NKwwe7aulvQ"
+        "id": "timestamp"
       },
       "outputs": [],
       "source": [
@@ -343,82 +292,6 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "poKeKYG8ulvQ"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
-        "authenticated. Skip this step."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "MIpJGzF9ulvQ"
-      },
-      "source": [
-        "**If you are using Colab**, run the cell below and follow the instructions\n",
-        "when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "1. In the Cloud Console, go to the [**Create service account key**\n",
-        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
-        "\n",
-        "2. Click **Create service account**.\n",
-        "\n",
-        "3. In the **Service account name** field, enter a name, and\n",
-        "   click **Create**.\n",
-        "\n",
-        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
-        "into the filter box, and select\n",
-        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
-        "local environment.\n",
-        "\n",
-        "6. Enter the path to your service account key as the\n",
-        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "Vh6KDXB5ulvQ"
-      },
-      "outputs": [],
-      "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = False\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        IS_COLAB = True\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -442,8 +315,7 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -454,9 +326,8 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
-        "    BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
       ]
     },
     {
@@ -472,11 +343,11 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "Moosy2rOulvR"
+        "id": "create_bucket"
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_URI"
+        "! gsutil mb -l $REGION $BUCKET_NAME"
       ]
     },
     {
@@ -492,11 +363,11 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "56irx2CvulvS"
+        "id": "validate_bucket"
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_URI"
+        "! gsutil ls -al $BUCKET_NAME"
       ]
     },
     {
@@ -537,11 +408,11 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "wbvYPSTDulvS"
+        "id": "init_aip:mbsdk"
       },
       "outputs": [],
       "source": [
-        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
+        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_NAME)"
       ]
     },
     {
@@ -570,7 +441,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "PryARdnoulvT"
+        "id": "accelerators:training,prediction,ngpu,mbsdk"
       },
       "outputs": [],
       "source": [
@@ -612,14 +483,14 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "LhhUFw2nulvT"
+        "id": "container:training,prediction"
       },
       "outputs": [],
       "source": [
         "if os.getenv(\"IS_TESTING_TF\"):\n",
         "    TF = os.getenv(\"IS_TESTING_TF\")\n",
         "else:\n",
-        "    TF = \"2.5\".replace(\".\", \"-\")\n",
+        "    TF = \"2.1\".replace(\".\", \"-\")\n",
         "\n",
         "if TF[0] == \"2\":\n",
         "    if TRAIN_GPU:\n",
@@ -680,7 +551,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "vytMaukeulvT"
+        "id": "machine:training"
       },
       "outputs": [],
       "source": [
@@ -742,7 +613,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "mhw34XoOulvU"
+        "id": "create_custom_pp_training_job:mbsdk"
       },
       "outputs": [],
       "source": [
@@ -750,7 +621,7 @@
         "\n",
         "job = aip.CustomPythonPackageTrainingJob(\n",
         "    display_name=DISPLAY_NAME,\n",
-        "    python_package_gcs_uri=f\"{BUCKET_URI}/trainer_boston.tar.gz\",\n",
+        "    python_package_gcs_uri=f\"{BUCKET_NAME}/trainer_boston.tar.gz\",\n",
         "    python_module_name=\"trainer.task\",\n",
         "    container_uri=TRAIN_IMAGE,\n",
         "    model_serving_container_image_uri=DEPLOY_IMAGE,\n",
@@ -791,7 +662,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "IAaZpZyyulvU"
+        "id": "examine_training_package"
       },
       "outputs": [],
       "source": [
@@ -840,7 +711,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "zKzddzl6ulvV"
+        "id": "taskpy_contents:mirrored,boston"
       },
       "outputs": [],
       "source": [
@@ -896,13 +767,6 @@
         "    strategy = tf.distribute.MultiWorkerMirroredStrategy()\n",
         "    logging.info(\"Multi-worker Strategy distributed training\")\n",
         "    logging.info('TF_CONFIG = {}'.format(os.environ.get('TF_CONFIG', 'Not found')))\n",
-        "    # Single Machine, multiple TPU devices\n",
-        "elif args.distribute == 'tpu':\n",
-        "    cluster_resolver = tf.distribute.cluster_resolver.TPUClusterResolver(tpu=\"local\")\n",
-        "    tf.config.experimental_connect_to_cluster(cluster_resolver)\n",
-        "    tf.tpu.experimental.initialize_tpu_system(cluster_resolver)\n",
-        "    strategy = tf.distribute.TPUStrategy(cluster_resolver)\n",
-        "    print(\"All devices: \", tf.config.list_logical_devices('TPU'))\n",
         "\n",
         "logging.info('num_replicas_in_sync = {}'.format(strategy.num_replicas_in_sync))\n",
         "\n",
@@ -961,11 +825,8 @@
         "    else:\n",
         "        task_type, task_id = None, None\n",
         "\n",
-        "    if args.distribute==\"tpu\":\n",
-        "        save_locally = tf.saved_model.SaveOptions(experimental_io_device='/job:localhost')\n",
-        "        model.save(args.model_dir, options=save_locally)\n",
         "    # single, mirrored or primary for multiworker\n",
-        "    elif _is_chief(task_type, task_id):\n",
+        "    if _is_chief(task_type, task_id):\n",
         "        model.save(args.model_dir)\n",
         "    # non-primary workers for multi-workers\n",
         "    else:\n",
@@ -999,14 +860,14 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "LFUHioqTulvV"
+        "id": "tarball_training_script"
       },
       "outputs": [],
       "source": [
         "! rm -f custom.tar custom.tar.gz\n",
         "! tar cvf custom.tar custom\n",
         "! gzip custom.tar\n",
-        "! gsutil cp custom.tar.gz $BUCKET_URI/trainer_boston.tar.gz"
+        "! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_boston.tar.gz"
       ]
     },
     {
@@ -1024,11 +885,11 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "LnUX0UkvulvV"
+        "id": "run_custom_pp_training_job:mirrored"
       },
       "outputs": [],
       "source": [
-        "MODEL_DIR = BUCKET_URI\n",
+        "MODEL_DIR = BUCKET_NAME\n",
         "\n",
         "CMDARGS = [\"--epochs=5\", \"--batch_size=16\", \"--distribute=mirrored\"]\n",
         "\n",
@@ -1059,7 +920,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "iUWHFpPoulvW"
+        "id": "delete_job"
       },
       "outputs": [],
       "source": [
@@ -1081,7 +942,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "-0gqCUTEulvW"
+        "id": "model_delete:mbsdk"
       },
       "outputs": [],
       "source": [
@@ -1166,7 +1027,7 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "aXvPN8P6ulvX"
+        "id": "create_custom_pp_training_job:mbsdk"
       },
       "source": [
         "### Create and run custom training job\n",
@@ -1192,7 +1053,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "kYcFsVSEulvX"
+        "id": "create_custom_pp_training_job:mbsdk"
       },
       "outputs": [],
       "source": [
@@ -1200,7 +1061,7 @@
         "\n",
         "job = aip.CustomPythonPackageTrainingJob(\n",
         "    display_name=DISPLAY_NAME,\n",
-        "    python_package_gcs_uri=f\"{BUCKET_URI}/trainer_boston.tar.gz\",\n",
+        "    python_package_gcs_uri=f\"{BUCKET_NAME}/trainer_boston.tar.gz\",\n",
         "    python_module_name=\"trainer.task\",\n",
         "    container_uri=TRAIN_IMAGE,\n",
         "    model_serving_container_image_uri=DEPLOY_IMAGE,\n",
@@ -1223,11 +1084,11 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "GHRxPU32ulvX"
+        "id": "run_custom_pp_training_job:multiworker"
       },
       "outputs": [],
       "source": [
-        "MODEL_DIR = BUCKET_URI\n",
+        "MODEL_DIR = BUCKET_NAME\n",
         "\n",
         "CMDARGS = [\"--epochs=5\", \"--batch_size=16\", \"--distribute=multiworker\"]\n",
         "\n",
@@ -1250,7 +1111,7 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "92D_hbuVulvX"
+        "id": "delete_job"
       },
       "source": [
         "### Delete a custom training job\n",
@@ -1262,7 +1123,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "CqrfWkB3ulvX"
+        "id": "delete_job"
       },
       "outputs": [],
       "source": [
@@ -1314,13 +1175,14 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "pGI2viDAulvY"
+        "id": "write_docker_file:training,multiworker"
       },
       "outputs": [],
       "source": [
         "%%writefile custom/Dockerfile\n",
         "\n",
         "FROM gcr.io/deeplearning-platform-release/tf2-gpu.2-5\n",
+        "WORKDIR /root\n",
         "\n",
         "WORKDIR /\n",
         "\n",
@@ -1346,7 +1208,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "7P8cdlFtulvY"
+        "id": "name_container:training"
       },
       "outputs": [],
       "source": [
@@ -1366,15 +1228,11 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "jmw5cakNulvY"
+        "id": "build_container:training"
       },
       "outputs": [],
       "source": [
-        "if not IS_COLAB:\n",
-        "    ! docker build custom -t $TRAIN_IMAGE\n",
-        "else:\n",
-        "    # install docker daemon\n",
-        "    ! apt-get -qq install docker.io"
+        "! docker build custom -t $TRAIN_IMAGE"
       ]
     },
     {
@@ -1392,12 +1250,11 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "jJGLjU-TulvZ"
+        "id": "test_container:training"
       },
       "outputs": [],
       "source": [
-        "if not IS_COLAB:\n",
-        "    ! docker run $TRAIN_IMAGE --epochs=5 --model-dir=./"
+        "! docker run $TRAIN_IMAGE --epochs=5 --model-dir=./"
       ]
     },
     {
@@ -1415,42 +1272,11 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "GAXGjae7ulvZ"
+        "id": "register_container:training"
       },
       "outputs": [],
       "source": [
-        "if not IS_COLAB:\n",
-        "    ! docker push $TRAIN_IMAGE"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "f50e9c553fb7"
-      },
-      "source": [
-        "*Executes in Colab*"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "a7e8c98f1e56"
-      },
-      "outputs": [],
-      "source": [
-        "%%bash -s $IS_COLAB $TRAIN_IMAGE\n",
-        "if [ $1 == \"False\" ]; then\n",
-        "  exit 0\n",
-        "fi\n",
-        "set -x\n",
-        "dockerd -b none --iptables=0 -l warn &\n",
-        "for i in $(seq 5); do [ ! -S \"/var/run/docker.sock\" ] && sleep 2 || break; done\n",
-        "docker build custom -t $2\n",
-        "docker run $2 --epochs=5 --model-dir=./\n",
-        "docker push $2\n",
-        "kill $(jobs -p)"
+        "! docker push $TRAIN_IMAGE"
       ]
     },
     {
@@ -1470,13 +1296,13 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "CEAnXBzCulvZ"
+        "id": "worker_pool_primary"
       },
       "outputs": [],
       "source": [
         "PRIMARY_COMPUTE = \"n2-highcpu-64\"\n",
         "\n",
-        "MODEL_DIR = BUCKET_URI\n",
+        "MODEL_DIR = BUCKET_NAME\n",
         "\n",
         "CMDARGS = [\n",
         "    \"--model-dir=\" + MODEL_DIR,\n",
@@ -1513,7 +1339,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "6dchPSfNulvZ"
+        "id": "worker_pool_training"
       },
       "outputs": [],
       "source": [
@@ -1549,7 +1375,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "m2VgmqEOulva"
+        "id": "custom_job:worker_pool"
       },
       "outputs": [],
       "source": [
@@ -1573,7 +1399,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "hg8vnI_Wulva"
+        "id": "run_custom_job:multiworker"
       },
       "outputs": [],
       "source": [
@@ -1587,7 +1413,7 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "WT76Sc-culva"
+        "id": "delete_job"
       },
       "source": [
         "### Delete a custom training job\n",
@@ -1599,7 +1425,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "I_IxVfuDulva"
+        "id": "delete_job"
       },
       "outputs": [],
       "source": [
@@ -1648,7 +1474,7 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "L8Av8ATVulvb"
+        "id": "custom_job:worker_pool"
       },
       "source": [
         "### Create CustomJob with worker pool specifications\n",
@@ -1664,7 +1490,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "TUWEP1Lmulvb"
+        "id": "custom_job:worker_pool"
       },
       "outputs": [],
       "source": [
@@ -1676,7 +1502,7 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "_95FH8jeulvb"
+        "id": "run_custom_job:multiworker"
       },
       "source": [
         "### Run the CustomJob\n",
@@ -1688,7 +1514,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "IEbrY05Gulvb"
+        "id": "run_custom_job:multiworker"
       },
       "outputs": [],
       "source": [
@@ -1702,7 +1528,7 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "8R2Bnmwmulvb"
+        "id": "delete_job"
       },
       "source": [
         "### Delete a custom training job\n",
@@ -1714,7 +1540,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "s1geVE3Lulvb"
+        "id": "delete_job"
       },
       "outputs": [],
       "source": [
@@ -1757,14 +1583,14 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "nQVPtknpulvb"
+        "id": "docker_write:tpu"
       },
       "outputs": [],
       "source": [
         "%%writefile custom/Dockerfile\n",
         "FROM python:3.8\n",
         "\n",
-        "WORKDIR /\n",
+        "WORKDIR /root\n",
         "\n",
         "# Copies the trainer code to the docker image.\n",
         "COPY trainer /trainer\n",
@@ -1796,11 +1622,11 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "J_d_zEXUulvc"
+        "id": "docker_push:tpu"
       },
       "outputs": [],
       "source": [
-        "TRAIN_IMAGE = \"gcr.io/\" + PROJECT_ID + \"/tpu-train:latest\"\n",
+        "TRAIN_IMAGE = f\"gcr.io/\" + PROJECT_ID + \"/tpu-train:latest\"\n",
         "\n",
         "os.chdir(\"custom\")\n",
         "! docker build --quiet --tag={TRAIN_IMAGE} .\n",
@@ -1827,7 +1653,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "d514eU7lulvc"
+        "id": "worker_pool_tpu"
       },
       "outputs": [],
       "source": [
@@ -1875,7 +1701,7 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "RruSqNfrulvc"
+        "id": "custom_job:worker_pool"
       },
       "source": [
         "### Create CustomJob with worker pool specifications\n",
@@ -1891,7 +1717,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "2QvSqbbHulvc"
+        "id": "custom_job:worker_pool"
       },
       "outputs": [],
       "source": [
@@ -1903,7 +1729,7 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "Iw4L3UIfulvd"
+        "id": "run_custom_job:multiworker"
       },
       "source": [
         "### Run the CustomJob\n",
@@ -1915,7 +1741,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "zmqCNS78ulvd"
+        "id": "run_custom_job:multiworker"
       },
       "outputs": [],
       "source": [
@@ -1929,7 +1755,7 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "gWZoH9QKulvd"
+        "id": "delete_job"
       },
       "source": [
         "### Delete a custom training job\n",
@@ -1941,7 +1767,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "Lt8BJ4iBulvd"
+        "id": "delete_job"
       },
       "outputs": [],
       "source": [
@@ -1961,7 +1787,13 @@
         "\n",
         "Otherwise, you can delete the individual resources you created in this tutorial:\n",
         "\n",
-        "\n",
+        "- Dataset\n",
+        "- Pipeline\n",
+        "- Model\n",
+        "- Endpoint\n",
+        "- Batch Job\n",
+        "- Custom Job\n",
+        "- Hyperparameter Tuning Job\n",
         "- Cloud Storage Bucket"
       ]
     },
@@ -1969,15 +1801,70 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "U98Wzc01ulvd"
+        "id": "cleanup"
       },
       "outputs": [],
       "source": [
-        "# Set this to true only if you'd like to delete your bucket\n",
-        "delete_bucket = False\n",
+        "delete_dataset = True\n",
+        "delete_pipeline = True\n",
+        "delete_model = True\n",
+        "delete_endpoint = True\n",
+        "delete_batchjob = True\n",
+        "delete_customjob = True\n",
+        "delete_hptjob = True\n",
+        "delete_bucket = True\n",
         "\n",
-        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
-        "    ! gsutil rm -r $BUCKET_URI"
+        "# Delete the dataset using the Vertex fully qualified identifier for the dataset\n",
+        "try:\n",
+        "    if delete_dataset and \"dataset_id\" in globals():\n",
+        "        clients[\"dataset\"].delete_dataset(name=dataset_id)\n",
+        "except Exception as e:\n",
+        "    print(e)\n",
+        "\n",
+        "# Delete the training pipeline using the Vertex fully qualified identifier for the pipeline\n",
+        "try:\n",
+        "    if delete_pipeline and \"pipeline_id\" in globals():\n",
+        "        clients[\"pipeline\"].delete_training_pipeline(name=pipeline_id)\n",
+        "except Exception as e:\n",
+        "    print(e)\n",
+        "\n",
+        "# Delete the model using the Vertex fully qualified identifier for the model\n",
+        "try:\n",
+        "    if delete_model and \"model_to_deploy_id\" in globals():\n",
+        "        clients[\"model\"].delete_model(name=model_to_deploy_id)\n",
+        "except Exception as e:\n",
+        "    print(e)\n",
+        "\n",
+        "# Delete the endpoint using the Vertex fully qualified identifier for the endpoint\n",
+        "try:\n",
+        "    if delete_endpoint and \"endpoint_id\" in globals():\n",
+        "        clients[\"endpoint\"].delete_endpoint(name=endpoint_id)\n",
+        "except Exception as e:\n",
+        "    print(e)\n",
+        "\n",
+        "# Delete the batch job using the Vertex fully qualified identifier for the batch job\n",
+        "try:\n",
+        "    if delete_batchjob and \"batch_job_id\" in globals():\n",
+        "        clients[\"job\"].delete_batch_prediction_job(name=batch_job_id)\n",
+        "except Exception as e:\n",
+        "    print(e)\n",
+        "\n",
+        "# Delete the custom job using the Vertex fully qualified identifier for the custom job\n",
+        "try:\n",
+        "    if delete_customjob and \"job_id\" in globals():\n",
+        "        clients[\"job\"].delete_custom_job(name=job_id)\n",
+        "except Exception as e:\n",
+        "    print(e)\n",
+        "\n",
+        "# Delete the hyperparameter tuning job using the Vertex fully qualified identifier for the hyperparameter tuning job\n",
+        "try:\n",
+        "    if delete_hptjob and \"hpt_job_id\" in globals():\n",
+        "        clients[\"job\"].delete_hyperparameter_tuning_job(name=hpt_job_id)\n",
+        "except Exception as e:\n",
+        "    print(e)\n",
+        "\n",
+        "if delete_bucket and \"BUCKET_NAME\" in globals():\n",
+        "    ! gsutil rm -r $BUCKET_NAME"
       ]
     }
   ],

notebooks/community/ml_ops/stage2/get_started_vertex_feature_store.ipynb

@@ -29,7 +29,7 @@
         "id": "title:generic,gcp"
       },
       "source": [
-        "# E2E ML on GCP: MLOps stage 2 : experimentation: get started with Vertex AI Feature Store\n",
+        "# E2E ML on GCP: MLOps stage 2 : experimentation: get started with Feature Store\n",
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
@@ -38,20 +38,11 @@
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
-        "    \n",
         "  <td>\n",
-        "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_feature_store.ipynb\">\n",
-        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
-        "        </a>\n",
-        "  </td>\n",
-        "    \n",
-        "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage2/get_started_vertex_feature_store.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
+        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_feature_store.ipynb\">\n",
+        "      Open in Google Cloud Notebooks\n",
         "    </a>\n",
         "  </td>\n",
-        "    \n",
         "</table>\n",
         "<br/><br/><br/>"
       ]
@@ -68,6 +59,19 @@
         "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 2 : experimentation: get started with Feature Store."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dataset:movies,lbn,avro"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the Movie Recommendations. The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket, in Avro format.\n",
+        "\n",
+        "The dataset predicts whether a persons will watch a movie."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -76,7 +80,7 @@
       "source": [
         "### Objective\n",
         "\n",
-        "In this tutorial, you learn how to use `Vertex AI Feature Store` when training and predicting with `Vertex AI`.\n",
+        "In this tutorial, you learn how to use `Vertex AI Feature Store` for when training and prediction with `Vertex AI`.\n",
         "\n",
         "This tutorial uses the following Google Cloud ML services:\n",
         "\n",
@@ -94,35 +98,6 @@
         "- Perform batch serving from a `Featurestore` resource."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "dataset:movies,lbn,avro"
-      },
-      "source": [
-        "### Dataset\n",
-        "\n",
-        "The dataset used for this tutorial is the `Movie Recommendations` dataset. The version of the dataset you use in this tutorial is stored in a public Cloud Storage bucket, in Avro format.\n",
-        "\n",
-        "This dataset is used to predict whether a person watches a movie or not."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "81c777b8ad32"
-      },
-      "source": [
-        "### Costs\n",
-        "This tutorial uses billable components of Google Cloud:\n",
-        "\n",
-        "- Vertex AI\n",
-        "- Cloud Storage\n",
-        "- BigQuery\n",
-        "\n",
-        "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing), [Cloud Storage pricing](https://cloud.google.com/storage/pricing) and [BigQuery pricing](https://cloud.google.com/bigquery/pricing) and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -131,7 +106,7 @@
       "source": [
         "## Installations\n",
         "\n",
-        "Install the following packages for further running this notebook."
+        "Install *one time* the packages for executing the MLOps notebooks."
       ]
     },
     {
@@ -142,21 +117,24 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
-        "# Install the dependecies\n",
-        "! pip3 install --upgrade google-cloud-aiplatform google-cloud-bigquery pyarrow avro $USER_FLAG -q"
+        "ONCE_ONLY = False\n",
+        "if ONCE_ONLY:\n",
+        "    ! pip3 install -U tensorflow==2.5 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-transform==1.2 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-io==0.18 $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-bigquery $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-logging $USER_FLAG\n",
+        "    ! pip3 install --upgrade apache-beam[gcp] $USER_FLAG\n",
+        "    ! pip3 install --upgrade pyarrow $USER_FLAG\n",
+        "    ! pip3 install --upgrade cloudml-hypertune $USER_FLAG\n",
+        "    ! pip3 install --upgrade kfp $USER_FLAG\n",
+        "    ! pip3 install --upgrade torchvision $USER_FLAG\n",
+        "    ! pip3 install --upgrade rpy2 $USER_FLAG\n",
+        "    ! pip3 install --upgrade python-tabulate $USER_FLAG\n",
+        "    ! pip3 install -U opencv-python-headless==4.5.2.52 $USER_FLAG"
       ]
     },
     {
@@ -188,32 +166,6 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "project_id"
-      },
-      "source": [
-        "## Before you begin\n",
-        "\n",
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI, Compute Engine, Cloud Storage and Cloud Logging APIs](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component,storage_component,logging).\n",
-        "\n",
-        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -290,10 +242,7 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
       ]
     },
     {
@@ -320,72 +269,15 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "29b110b44457"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
-        "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
-        "\n",
-        "1. **Click Create service account**.\n",
-        "\n",
-        "2. In the **Service account name** field, enter a name, and click **Create**.\n",
-        "\n",
-        "3. In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex\" into the filter box, and select **Vertex Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "4. Click Create. A JSON file that contains your key downloads to your local environment.\n",
-        "\n",
-        "5. Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "89788a802687"
-      },
-      "outputs": [],
-      "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = \"google.colab\" in sys.modules\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "setup_vars"
       },
       "source": [
+        "### Set up variables\n",
+        "\n",
+        "Next, set up some variables used throughout the tutorial.\n",
         "### Import libraries and define constants"
       ]
     },
@@ -397,7 +289,28 @@
       },
       "outputs": [],
       "source": [
-        "import google.cloud.aiplatform as aiplatform\n",
+        "import google.cloud.aiplatform as aip"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "import_bq"
+      },
+      "source": [
+        "#### Import BigQuery\n",
+        "\n",
+        "Import the BigQuery package into your Python environment."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "import_bq"
+      },
+      "outputs": [],
+      "source": [
         "from google.cloud import bigquery"
       ]
     },
@@ -407,7 +320,9 @@
         "id": "init_bq"
       },
       "source": [
-        "Initialize Vertex AI and BigQuery clients."
+        "### Create BigQuery client\n",
+        "\n",
+        "Create the BigQuery client."
       ]
     },
     {
@@ -418,8 +333,7 @@
       },
       "outputs": [],
       "source": [
-        "aiplatform.init(project=PROJECT_ID)\n",
-        "bqclient = bigquery.Client(project=PROJECT_ID)"
+        "bqclient = bigquery.Client()"
       ]
     },
     {
@@ -436,11 +350,11 @@
         "\n",
         "Now it's time to do a live prediction. You get a transaction from the cash register, but all it has is the credit card number and this transaction. It does not have the enriched data the model needs. During serving, the credit card number is used as an index to Feature Store to get the enriched data needed for the model.\n",
         "\n",
-        "On the other hand, let's say the enriched data the model was trained on was timestamped on June 1st. The current transaction is from June 15th. Assume that the user has made other transactions between June 1st and 15th, and the enriched data has been continuously updated in Feature Store. But the model was trained on June 1st data. FeatureStore knows the version number and serves the June 1st version to the model (not the current June 15th). Otherwise, if you used June 15th data, you would have training-serving skew.\n",
+        "Next problem. Let's say the enriched data the model was trained on was timestamp June 1. This transaction is June 15. Assume that the user has made other transactions between June 1 and 15, and the enriched data has been continuously updated in Feature Store. But the model was trained on June 1st data. FeatureStore knows the version number and serves the June 1 version to the model (not the current June 15); otherwise, if you used June 15 data you have training-serving skew.\n",
         "\n",
-        "Another problem here is the data drift. Things change and suddenly one day, everybody is buying toilet paper! There is a significant change in the distribution of existing enriched data from the distribution that the deployed model was trained on. FeatureStore can detect changes/thresholds in distribution changes and trigger a notification for retraining the model.\n",
+        "Next problem, data drift. Things change, suddenly one day everybody is buying toilet paper! There is a significant change in the distribution of the current stored enriched data from the distribution that the deployed model was trained on. FeatureStore can detect changes/thresholds in distribution changes and trigger a notification for retraining the model.\n",
         "\n",
-        "Learn more about [Vertex AI Feature Store API](https://cloud.google.com/vertex-ai/docs/featurestore)."
+        "Learn more about [Vertex AI Feature Store API](https://cloud.google.com/vertex-ai/docs/featurestore)"
       ]
     },
     {
@@ -455,9 +369,9 @@
         "\n",
         "        Featurestore -> EntityType -> Feature\n",
         "\n",
-        "- `Featurestore`: the place to store your features.\n",
+        "- `Featurestore`: the place to store your features\n",
         "- `EntityType`: under a `Featurestore`, an `EntityType` describes an object to be modeled, real one or virtual one.\n",
-        "- `Feature`: under an `EntityType`, a `Feature` describes an attribute of the `EntityType`.\n",
+        "- `Feature`: under an `EntityType`, a `Feature` describes an attribute of the `EntityType`\n",
         "\n",
         "Learn more about [Vertex AI Feature Store data model](https://cloud.google.com/vertex-ai/docs/featurestore/concepts).\n",
         "\n",
@@ -491,9 +405,9 @@
       "outputs": [],
       "source": [
         "# Represents featurestore resource path.\n",
-        "FEATURESTORE_NAME = \"movies_\" + TIMESTAMP\n",
+        "FEATURESTORE_NAME = \"movies\"\n",
         "\n",
-        "featurestore = aiplatform.Featurestore.create(\n",
+        "featurestore = aip.Featurestore.create(\n",
         "    featurestore_id=FEATURESTORE_NAME,\n",
         "    online_store_fixed_node_count=1,\n",
         "    project=PROJECT_ID,\n",
@@ -522,7 +436,7 @@
       },
       "outputs": [],
       "source": [
-        "for featurestore in aiplatform.Featurestore.list():\n",
+        "for featurestore in aip.Featurestore.list():\n",
         "    print(featurestore)"
       ]
     },
@@ -549,7 +463,7 @@
       },
       "outputs": [],
       "source": [
-        "featurestore = featurestore = aiplatform.Featurestore(\n",
+        "featurestore = featurestore = aip.Featurestore(\n",
         "    featurestore_name=FEATURESTORE_NAME, project=PROJECT_ID, location=REGION\n",
         ")\n",
         "print(featurestore)"
@@ -608,7 +522,7 @@
       "outputs": [],
       "source": [
         "def create_features(featurestore_name, entity_name, features):\n",
-        "    entity_type = aiplatform.EntityType(\n",
+        "    entity_type = aip.EntityType(\n",
         "        entity_type_name=entity_name, featurestore_id=featurestore_name\n",
         "    )\n",
         "\n",
@@ -659,7 +573,7 @@
       },
       "outputs": [],
       "source": [
-        "for featurestore in aiplatform.Featurestore.list():\n",
+        "for featurestore in aip.Featurestore.list():\n",
         "    print(featurestore)"
       ]
     },
@@ -671,7 +585,7 @@
       "source": [
         "### Search `Feature` resources using a filter\n",
         "\n",
-        "You can narrow your search of `Feature` resources using the method `list_features()` and specifying a `filter` string."
+        "You can narrow your search of `Feature` resources using the method `list_features()` and specifying a `filter` filter."
       ]
     },
     {
@@ -723,26 +637,17 @@
       },
       "outputs": [],
       "source": [
-        "features = aiplatform.Feature.search(query=\"value_type=DOUBLE\")\n",
+        "features = aip.Feature.search(query=\"value_type=DOUBLE\")\n",
         "print(\"By data type\")\n",
         "for feature in features:\n",
         "    print(features)\n",
         "\n",
-        "aiplatform.Feature.search(query=\"feature_id=title\")\n",
+        "aip.Feature.search(query=\"feature_id=title\")\n",
         "print(\"By Name\")\n",
         "for feature in features:\n",
         "    print(features)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "277e9884cf37"
-      },
-      "source": [
-        "Define paths to the feature data."
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -772,7 +677,7 @@
         "\n",
         "### Data layout\n",
         "\n",
-        "Each imported `EntityType` resource data must have an ID. Also, each `EntityType` resource data item can optionally have a timestamp, sepecifying when the feature values were generated.\n",
+        "Each imported `EntityType` resource data must have an ID; also, each `EntityType` resource data item can optionally have a timestamp, sepecifying when the feature values were generated.\n",
         "\n",
         "When importing, specify the following in your request:\n",
         "\n",
@@ -780,7 +685,7 @@
         "- Data source URL\n",
         "- Destination: featurestore/entity types/features to be imported\n",
         "\n",
-        "The feature values for `Movie Recommendations` dataset are in Avro format. The Avro schemas are as follows:\n",
+        "The feature values for the movies dataset are in Avro format. The Avro schemas are as follows:\n",
         "\n",
         "**Users entity**:\n",
         "\n",
@@ -892,7 +797,7 @@
       "source": [
         "#### Delete the entity types and corresponding features and feature values\n",
         "\n",
-        "Now, in preparation to repeat the process of importing feature values but from a dataframe this time, you delete the existing entity types, and the corresponding content."
+        "Next, in preparation to repeat importing feature values from a dataframe, you first delete the existing entity types, and corresponding content."
       ]
     },
     {
@@ -917,7 +822,7 @@
       "source": [
         "## Create entity types for your `Featurestore` resource\n",
         "\n",
-        "Next, you create the `EntityType` resources again for your `Featurestore` resource using the `create_entity_type()` method, with the following parameters:\n",
+        "Next, you create the `EntityType` resources for your `Featurestore` resource using the `create_entity_type()` method, with the following parameters:\n",
         "\n",
         "- `entity_type_id`: The name of the `EntityType` resource.\n",
         "- `description`:  A description of the entity type."
@@ -946,7 +851,7 @@
       "source": [
         "### Add `Feature` resources for your `EntityType` resources\n",
         "\n",
-        "Further, you create the `Feature` resources again for each of the `EntityType` resources in your `Featurestore` resource using the `create_feature()` method, with the following parameters:\n",
+        "Next, you create the `Feature` resources for each of the `EntityType` resources in your `Featurestore` resource using the `create_feature()` method, with the following parameters:\n",
         "\n",
         "- `feature_id`: The name of the `Feature` resource.\n",
         "- `description`: A description of the feature.\n",
@@ -962,7 +867,7 @@
       "outputs": [],
       "source": [
         "def create_features(featurestore_name, entity_name, features):\n",
-        "    entity_type = aiplatform.EntityType(\n",
+        "    entity_type = aip.EntityType(\n",
         "        entity_type_name=entity_name, featurestore_id=featurestore_name\n",
         "    )\n",
         "\n",
@@ -994,15 +899,6 @@
         ")"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "8715a3f719c8"
-      },
-      "source": [
-        "Now, copy the `users` and `movies` data into avro files."
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -1145,7 +1041,7 @@
       "source": [
         "## Batch Serving\n",
         "\n",
-        "The Vertex AI Feature Store's batch serving service is optimized for serving large batches of features in real-time with high throughput, typically for training a model or batch prediction.\n",
+        "The Vertex AI Feature Store batch serving service is optimized for serving large batches of features in real-time with high-throughput, typically for training a model or batch prediction.\n",
         "\n",
         "One can batch serve to the following destinations:\n",
         "\n",
@@ -1197,7 +1093,7 @@
         "\n",
         "You batch serve entity data items to a BigQuery table using the `read_serve_to_bq()` method, with the following parameters:\n",
         "\n",
-        "- `bq_destination_output_uri`: The destination BigQuery table to receive the served features.\n",
+        "- `bq_destination_output_uri`: The destination BigQuery table to serve the features to.\n",
         "- `serving_feature_ids`: A dictionary of entity type and corresponding features to serve.\n",
         "- `read_instances_uri`: A Cloud Storage location to read the entity data items from.\n",
         "\n",
@@ -1230,7 +1126,6 @@
         "id": "delete_bq_dataset"
       },
       "source": [
-        "## Cleaning up\n",
         "### Delete a BigQuery dataset\n",
         "\n",
         "Use the method `delete_dataset()` to delete a BigQuery dataset along with all its tables, by setting the parameter `delete_contents` to `True`."

notebooks/community/ml_ops/stage2/get_started_vertex_tensorboard.ipynb

@@ -8,7 +8,7 @@
       },
       "outputs": [],
       "source": [
-        "# Copyright 2022 Google LLC\n",
+        "# Copyright 2021 Google LLC\n",
         "#\n",
         "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
         "# you may not use this file except in compliance with the License.\n",
@@ -29,14 +29,9 @@
         "id": "title:generic,gcp"
       },
       "source": [
-        "# E2E ML on GCP: MLOps stage 2 : experimentation: get started with Vertex AI TensorBoard\n",
+        "# E2E ML on GCP: MLOps stage 2 : experimentation: get started with Vertex Tensorboard\n",
         "\n",
         "<table align=\"left\">\n",
-        "   <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_tensorboard.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
-        "    </a>\n",
-        "  </td>\n",
         "  <td>\n",
         "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_tensorboard.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
@@ -44,9 +39,8 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/notebook_template.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
+        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_tensorboard.ipynb\">\n",
+        "      Open in Google Cloud Notebooks\n",
         "    </a>\n",
         "  </td>\n",
         "</table>\n",
@@ -62,7 +56,7 @@
         "## Overview\n",
         "\n",
         "\n",
-        "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 2 : experimentation: get started with Vertex AI TensorBoard."
+        "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 2 : experimentation: get started with Vertex Tensorboard."
       ]
     },
     {
@@ -83,7 +77,7 @@
         "The steps performed include:\n",
         "\n",
         "- Create a TensorBoard callback when training a model.\n",
-        "- Using TensorBoard with locally trained model.\n",
+        "- Using Tensorboard with locally trained model.\n",
         "- Using Vertex AI TensorBoard with Vertex AI Training."
       ]
     },
@@ -95,7 +89,7 @@
       "source": [
         "### Recommendations\n",
         "\n",
-        "When doing E2E MLOps on Google Cloud, the following are the best practices for visualizing your training with TensorBoard.\n",
+        "When doing E2E MLOps on Google Cloud, the following best practices for visualizing your training with TensorBoard.\n",
         "\n",
         "#### Local TensorBoard\n",
         "\n",
@@ -103,86 +97,13 @@
         "\n",
         "#### Cloud TensorBoard\n",
         "\n",
-        "Use the tensorboard.dev, when doing training on the cloud -- unless you have a privacy issue.\n",
+        "Use the Tensorboard.dev, when doing training on the cloud -- unless you have a privacy issue.\n",
         "\n",
         "#### Experiments\n",
         "\n",
         "Use Vertex AI TensorBoard when you have a privacy issue or doing experiments to compare results for different experiment configurations."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "03bfd1274241"
-      },
-      "source": [
-        "### Dataset\n",
-        "\n",
-        "In this tutorial you use the MNIST dataset. The version of the dataset is built into the TF.Keras framework. The dataset predicts which digit an image is, between 0 .. 9."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "b132d4ef86d6"
-      },
-      "source": [
-        "### Costs \n",
-        "\n",
-        "This tutorial uses billable components of Google Cloud:\n",
-        "\n",
-        "* Vertex AI\n",
-        "* Cloud Storage\n",
-        "\n",
-        "Learn about [Vertex AI\n",
-        "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
-        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
-        "Calculator](https://cloud.google.com/products/calculator/)\n",
-        "to generate a cost estimate based on your projected usage."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "56cb7f08a9e8"
-      },
-      "source": [
-        "### Set up your local development environment\n",
-        "\n",
-        "**If you are using Colab or Google Cloud Notebooks**, your environment already meets\n",
-        "all the requirements to run this notebook. You can skip this step.\n",
-        "\n",
-        "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
-        "You need the following:\n",
-        "\n",
-        "* The Google Cloud SDK\n",
-        "* Git\n",
-        "* Python 3\n",
-        "* virtualenv\n",
-        "* Jupyter notebook running in a virtual environment with Python 3\n",
-        "\n",
-        "The Google Cloud guide to [Setting up a Python development\n",
-        "environment](https://cloud.google.com/python/setup) and the [Jupyter\n",
-        "installation guide](https://jupyter.org/install) provide detailed instructions\n",
-        "for meeting these requirements. The following steps provide a condensed set of\n",
-        "instructions:\n",
-        "\n",
-        "1. [Install and initialize the Cloud SDK.](https://cloud.google.com/sdk/docs/)\n",
-        "\n",
-        "1. [Install Python 3.](https://cloud.google.com/python/setup#installing_python)\n",
-        "\n",
-        "1. [Install\n",
-        "   virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv)\n",
-        "   and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
-        "\n",
-        "1. To install Jupyter, run `pip3 install jupyter` on the\n",
-        "command-line in a terminal shell.\n",
-        "\n",
-        "1. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
-        "\n",
-        "1. Open this notebook in the Jupyter Notebook Dashboard.\n"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -191,32 +112,31 @@
       "source": [
         "## Installations\n",
         "\n",
-        "Install the following packages for executing this notebook."
+        "Install *one time* the packages for executing the MLOps notebooks."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "020040f91150"
+        "id": "install_mlops"
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
-        "! pip3 install -U tensorflow==2.8 $USER_FLAG -q\n",
-        "! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG -q"
+        "ONCE_ONLY = False\n",
+        "if ONCE_ONLY:\n",
+        "    ! pip3 install -U tensorflow==2.5 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-transform==1.2 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-io==0.18 $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-bigquery $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-logging $USER_FLAG\n",
+        "    ! pip3 install --upgrade apache-beam[gcp] $USER_FLAG\n",
+        "    ! pip3 install --upgrade pyarrow $USER_FLAG\n",
+        "    ! pip3 install --upgrade cloudml-hypertune $USER_FLAG\n",
+        "    ! pip3 install --upgrade kfp $USER_FLAG"
       ]
     },
     {
@@ -248,32 +168,6 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "2721ef0202d9"
-      },
-      "source": [
-        "## Before you begin\n",
-        "\n",
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com). \n",
-        "\n",
-        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -350,10 +244,7 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
       ]
     },
     {
@@ -380,82 +271,6 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "2700e693f1b3"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
-        "authenticated. Skip this step."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "885395904904"
-      },
-      "source": [
-        "**If you are using Colab**, run the cell below and follow the instructions\n",
-        "when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "1. In the Cloud Console, go to the [**Create service account key**\n",
-        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
-        "\n",
-        "2. Click **Create service account**.\n",
-        "\n",
-        "3. In the **Service account name** field, enter a name, and\n",
-        "   click **Create**.\n",
-        "\n",
-        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
-        "into the filter box, and select\n",
-        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
-        "local environment.\n",
-        "\n",
-        "6. Enter the path to your service account key as the\n",
-        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "eff327d0552b"
-      },
-      "outputs": [],
-      "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = False\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        IS_COLAB = True\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -479,7 +294,7 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_URI = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
+        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -490,8 +305,8 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
       ]
     },
     {
@@ -511,7 +326,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_URI"
+        "! gsutil mb -l $REGION $BUCKET_NAME"
       ]
     },
     {
@@ -531,7 +346,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_URI"
+        "! gsutil ls -al $BUCKET_NAME"
       ]
     },
     {
@@ -569,16 +384,9 @@
         "    or SERVICE_ACCOUNT is None\n",
         "    or SERVICE_ACCOUNT == \"[your-service-account]\"\n",
         "):\n",
-        "    # Get your service account from gcloud\n",
-        "    if not IS_COLAB:\n",
-        "        shell_output = !gcloud auth list 2>/dev/null\n",
-        "        SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
-        "\n",
-        "    if IS_COLAB:\n",
-        "        shell_output = ! gcloud projects describe  $PROJECT_ID\n",
-        "        project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n",
-        "        SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n",
-        "\n",
+        "    # Get your GCP project id from gcloud\n",
+        "    shell_output = !gcloud auth list 2>/dev/null\n",
+        "    SERVICE_ACCOUNT = shell_output[2].strip()\n",
         "    print(\"Service Account:\", SERVICE_ACCOUNT)"
       ]
     },
@@ -602,7 +410,7 @@
       },
       "outputs": [],
       "source": [
-        "import google.cloud.aiplatform as aiplatform"
+        "import google.cloud.aiplatform as aip"
       ]
     },
     {
@@ -646,7 +454,7 @@
       },
       "outputs": [],
       "source": [
-        "aiplatform.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
+        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_NAME)"
       ]
     },
     {
@@ -676,15 +484,13 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
         "if os.getenv(\"IS_TESTING_TRAIN_GPU\"):\n",
         "    TRAIN_GPU, TRAIN_NGPU = (\n",
-        "        aiplatform.gapic.AcceleratorType.NVIDIA_TESLA_K80,\n",
+        "        aip.gapic.AcceleratorType.NVIDIA_TESLA_K80,\n",
         "        int(os.getenv(\"IS_TESTING_TRAIN_GPU\")),\n",
         "    )\n",
         "else:\n",
-        "    TRAIN_GPU, TRAIN_NGPU = (aiplatform.gapic.AcceleratorType.NVIDIA_TESLA_K80, 1)"
+        "    TRAIN_GPU, TRAIN_NGPU = (aip.gapic.AcceleratorType.NVIDIA_TESLA_K80, 1)"
       ]
     },
     {
@@ -785,9 +591,9 @@
       "source": [
         "## Training with TensorBoard\n",
         "\n",
-        "TensorBoard provides the means to visualize your training in-real time and to visualize the results (metrics).\n",
+        "Tensorboard provides the means to visualize your training in-real time and to visualize the results (metrics).\n",
         "\n",
-        "You can use TensorBoard in conjunction with local training, cloud training and with `Vertex AI Training`, which is referred to as `Vertex AI TensorBoard`"
+        "You can use Tensorboard in conjunction with local training, cloud training and with `Vertex AI Training`, which is referred to as `Vertex AI TensorBoard`"
       ]
     },
     {
@@ -857,9 +663,9 @@
         "\n",
         "You can upload your TensorBoard logs and share with others using `tensorboard dev` command. Once uploaded, a URL is returned to open up the TensorBoard instance in a brower for visualizing.\n",
         "\n",
-        "*Note:* Your TensorBoard instance is publicly visible.\n",
+        "*Note:* Your TensorBoard instance is publicly visable.\n",
         "\n",
-        "*Note:* This cell is for demonstration purposes and must be ran in a terminal shell. In this example, while running within a notebook, the command will freeze since it is waiting for an interactive yes/no input. You can kill the command with a Ctrl C or kernel interupt.\n",
+        "*Note:* In this example, while running within a notebook, the command will freeze since it is waiting for an interactive yes/no input. You can kill the command with a Ctrl C or kernel interupt.\n",
         "\n",
         "Learn more about [What is TensorBoard.dev](https://tensorboard.dev/)."
       ]
@@ -872,7 +678,7 @@
       },
       "outputs": [],
       "source": [
-        "! tensorboard dev upload --logdir logs \\\n",
+        "! tensorboard dev upload --logdir {LOG_DIR} \\\n",
         "  --name \"Simple experiment with MNIST\" \\\n",
         "  --description \"Training results\" \\\n",
         "  --one_shot"
@@ -900,7 +706,7 @@
       "outputs": [],
       "source": [
         "TENSORBOARD_DISPLAY_NAME = \"example\"\n",
-        "tensorboard = aiplatform.Tensorboard.create(display_name=TENSORBOARD_DISPLAY_NAME)\n",
+        "tensorboard = aip.Tensorboard.create(display_name=TENSORBOARD_DISPLAY_NAME)\n",
         "tensorboard_resource_name = tensorboard.gca_resource.name\n",
         "print(\"TensorBoard resource name:\", tensorboard_resource_name)"
       ]
@@ -940,9 +746,9 @@
         "\n",
         "url = output[1].split(' ')[-1]\n",
         "\n",
-        "#print(url)\n",
+        "print(url)\n",
         "\n",
-        "from IPython.display import display, HTML\n",
+        "from IPython.core.display import display, HTML\n",
         "display(HTML(\"<a href='\" + url + \"'>click here for TensorBoard instance</a>\"))"
       ]
     },
@@ -1147,7 +953,7 @@
         "! rm -f custom.tar custom.tar.gz\n",
         "! tar cvf custom.tar custom\n",
         "! gzip custom.tar\n",
-        "! gsutil cp custom.tar.gz $BUCKET_URI/trainer_example.tar.gz"
+        "! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_example.tar.gz"
       ]
     },
     {
@@ -1179,7 +985,7 @@
       },
       "outputs": [],
       "source": [
-        "job = aiplatform.CustomTrainingJob(\n",
+        "job = aip.CustomTrainingJob(\n",
         "    display_name=\"example_\" + TIMESTAMP,\n",
         "    script_path=\"custom/trainer/task.py\",\n",
         "    container_uri=TRAIN_IMAGE,\n",
@@ -1215,7 +1021,7 @@
       },
       "outputs": [],
       "source": [
-        "MODEL_DIR = \"{}/{}\".format(BUCKET_URI, TIMESTAMP)\n",
+        "MODEL_DIR = \"{}/{}\".format(BUCKET_NAME, TIMESTAMP)\n",
         "\n",
         "EPOCHS = 20\n",
         "STEPS = 100\n",
@@ -1337,8 +1143,14 @@
         "\n",
         "Otherwise, you can delete the individual resources you created in this tutorial:\n",
         "\n",
+        "- Dataset\n",
+        "- Pipeline\n",
         "- Model\n",
+        "- Endpoint\n",
+        "- AutoML Training Job\n",
+        "- Batch Job\n",
         "- Custom Job\n",
+        "- Hyperparameter Tuning Job\n",
         "- Cloud Storage Bucket"
       ]
     },
@@ -1350,14 +1162,61 @@
       },
       "outputs": [],
       "source": [
-        "# Delete the custom training job\n",
-        "job.delete()\n",
+        "delete_all = True\n",
         "\n",
-        "# Set this to true only if you'd like to delete your bucket\n",
-        "delete_bucket = False\n",
+        "if delete_all:\n",
+        "    # Delete the dataset using the Vertex dataset object\n",
+        "    try:\n",
+        "        if \"dataset\" in globals():\n",
+        "            dataset.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
         "\n",
-        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
-        "    ! gsutil rm -r $BUCKET_URI"
+        "    # Delete the model using the Vertex model object\n",
+        "    try:\n",
+        "        if \"model\" in globals():\n",
+        "            model.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the endpoint using the Vertex endpoint object\n",
+        "    try:\n",
+        "        if \"endpoint\" in globals():\n",
+        "            endpoint.undeploy_all()\n",
+        "            endpoint.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the AutoML or Pipeline training job\n",
+        "    try:\n",
+        "        if \"dag\" in globals():\n",
+        "            dag.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the custom training job\n",
+        "    try:\n",
+        "        if \"job\" in globals():\n",
+        "            job.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the batch prediction job using the Vertex batch prediction object\n",
+        "    try:\n",
+        "        if \"batch_predict_job\" in globals():\n",
+        "            batch_predict_job.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the hyperparameter tuning job using the Vertex hyperparameter tuning object\n",
+        "    try:\n",
+        "        if \"hpt_job\" in globals():\n",
+        "            hpt_job.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    if \"BUCKET_NAME\" in globals():\n",
+        "        ! gsutil rm -r $BUCKET_NAME"
       ]
     }
   ],

notebooks/community/ml_ops/stage2/get_started_vertex_training.ipynb

@@ -29,7 +29,7 @@
         "id": "title:generic,gcp"
       },
       "source": [
-        "# E2E ML on GCP: MLOps stage 2 : experimentation: get started with Vertex AI Training\n",
+        "# E2E ML on GCP: MLOps stage 2 : experimentation: get started with Vertex Training\n",
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
@@ -39,14 +39,8 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training.ipynb\">\n",
-        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
-        "        </a>\n",
-        "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage2/get_started_vertex_training.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
+        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training.ipynb\">\n",
+        "      Open in Google Cloud Notebooks\n",
         "    </a>\n",
         "  </td>\n",
         "</table>\n",
@@ -62,7 +56,18 @@
         "## Overview\n",
         "\n",
         "\n",
-        "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 2 : experimentation: get started with Vertex AI Training."
+        "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 2 : experimentation: get started with Vertex Training."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dataset:custom,boston,lrg"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the [Boston Housing Prices dataset](https://www.cs.toronto.edu/~delve/data/boston/bostonDetail.html). The version of the dataset you will use in this tutorial is built into TensorFlow. The trained model predicts the median price of a house in units of 1K USD."
       ]
     },
     {
@@ -114,38 +119,6 @@
         "CustomJob"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "dataset:custom,boston,lrg"
-      },
-      "source": [
-        "### Dataset\n",
-        "\n",
-        "The dataset used for this tutorial is the [Boston Housing Prices dataset](https://www.cs.toronto.edu/~delve/data/boston/bostonDetail.html). The version of the dataset this tutorial is built into TensorFlow. The trained model predicts the median price of a house in units of 1K USD."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "c480fc50ec3c"
-      },
-      "source": [
-        "### Costs \n",
-        "\n",
-        "This tutorial uses billable components of Google Cloud:\n",
-        "\n",
-        "* Vertex AI\n",
-        "* Cloud Storage\n",
-        "\n",
-        "\n",
-        "Learn about [Vertex AI\n",
-        "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
-        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
-        "Calculator](https://cloud.google.com/products/calculator/)\n",
-        "to generate a cost estimate based on your projected usage."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -154,7 +127,7 @@
       "source": [
         "## Installations\n",
         "\n",
-        "Install the packages required for executing this notebook"
+        "Install *one time* the packages for executing the MLOps notebooks."
       ]
     },
     {
@@ -165,20 +138,20 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
-        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG -q"
+        "ONCE_ONLY = False\n",
+        "if ONCE_ONLY:\n",
+        "    ! pip3 install -U tensorflow==2.5 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-transform==1.2 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-io==0.18 $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-bigquery $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-logging $USER_FLAG\n",
+        "    ! pip3 install --upgrade apache-beam[gcp] $USER_FLAG\n",
+        "    ! pip3 install --upgrade pyarrow $USER_FLAG\n",
+        "    ! pip3 install --upgrade cloudml-hypertune $USER_FLAG\n",
+        "    ! pip3 install --upgrade kfp $USER_FLAG"
       ]
     },
     {
@@ -200,6 +173,8 @@
       },
       "outputs": [],
       "source": [
+        "import os\n",
+        "\n",
         "if not os.getenv(\"IS_TESTING\"):\n",
         "    # Automatically restart kernel after installs\n",
         "    import IPython\n",
@@ -208,36 +183,6 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "before_you_begin"
-      },
-      "source": [
-        "## Before you begin\n",
-        "\n",
-        "### GPU runtime\n",
-        "\n",
-        "*Make sure you're running this notebook in a GPU runtime if you have that option. In Colab, select* **Runtime > Change Runtime Type > GPU**\n",
-        "\n",
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "2. [Make sure that billing is enabled for your project.](https://cloud.google.com/billing/docs/how-to/modify-project)\n",
-        "\n",
-        "3. [Enable the following APIs: Vertex AI APIs, Compute Engine APIs, and Cloud Storage.](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component,storage-component.googleapis.com)\n",
-        "\n",
-        "4. If you are running this notebook locally, you need to install the [Cloud SDK]((https://cloud.google.com/sdk)).\n",
-        "\n",
-        "5. Enter your project ID in the cell below. Then run the  cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$`."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -314,10 +259,7 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
       ]
     },
     {
@@ -344,67 +286,6 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "gcp_authenticate"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
-        "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
-        "\n",
-        "**Click Create service account**.\n",
-        "\n",
-        "In the **Service account name** field, enter a name, and click **Create**.\n",
-        "\n",
-        "In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex\" into the filter box, and select **Vertex Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "Click Create. A JSON file that contains your key downloads to your local environment.\n",
-        "\n",
-        "Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "gcp_authenticate"
-      },
-      "outputs": [],
-      "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = False\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        IS_COLAB = True\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -428,8 +309,7 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -440,9 +320,8 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
-        "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
       ]
     },
     {
@@ -462,7 +341,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_URI"
+        "! gsutil mb -l $REGION $BUCKET_NAME"
       ]
     },
     {
@@ -482,7 +361,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_URI"
+        "! gsutil ls -al $BUCKET_NAME"
       ]
     },
     {
@@ -505,7 +384,7 @@
       },
       "outputs": [],
       "source": [
-        "import google.cloud.aiplatform as aiplatform"
+        "import google.cloud.aiplatform as aip"
       ]
     },
     {
@@ -527,7 +406,7 @@
       },
       "outputs": [],
       "source": [
-        "aiplatform.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
+        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_NAME)"
       ]
     },
     {
@@ -562,7 +441,7 @@
       "source": [
         "if os.getenv(\"IS_TESTING_TRAIN_GPU\"):\n",
         "    TRAIN_GPU, TRAIN_NGPU = (\n",
-        "        aiplatform.gapic.AcceleratorType.NVIDIA_TESLA_K80,\n",
+        "        aip.gapic.AcceleratorType.NVIDIA_TESLA_K80,\n",
         "        int(os.getenv(\"IS_TESTING_TRAIN_GPU\")),\n",
         "    )\n",
         "else:\n",
@@ -570,7 +449,7 @@
         "\n",
         "if os.getenv(\"IS_TESTING_DEPLOY_GPU\"):\n",
         "    DEPLOY_GPU, DEPLOY_NGPU = (\n",
-        "        aiplatform.gapic.AcceleratorType.NVIDIA_TESLA_K80,\n",
+        "        aip.gapic.AcceleratorType.NVIDIA_TESLA_K80,\n",
         "        int(os.getenv(\"IS_TESTING_DEPLOY_GPU\")),\n",
         "    )\n",
         "else:\n",
@@ -605,7 +484,7 @@
         "if os.getenv(\"IS_TESTING_TF\"):\n",
         "    TF = os.getenv(\"IS_TESTING_TF\")\n",
         "else:\n",
-        "    TF = \"2.5\".replace(\".\", \"-\")\n",
+        "    TF = \"2.1\".replace(\".\", \"-\")\n",
         "\n",
         "if TF[0] == \"2\":\n",
         "    if TRAIN_GPU:\n",
@@ -723,7 +602,7 @@
         "DISPLAY_NAME = \"boston_\" + TIMESTAMP\n",
         "REQUIREMENTS = [\"tensorflow==2.3\"]\n",
         "\n",
-        "job = aiplatform.CustomTrainingJob(\n",
+        "job = aip.CustomTrainingJob(\n",
         "    display_name=DISPLAY_NAME,\n",
         "    script_path=\"task.py\",\n",
         "    requirements=REQUIREMENTS,\n",
@@ -814,12 +693,12 @@
       "outputs": [],
       "source": [
         "CMDARGS = [\n",
-        "    \"--model-dir=\" + BUCKET_URI,\n",
+        "    \"--model-dir=\" + BUCKET_NAME,\n",
         "]\n",
         "\n",
         "job.run(args=CMDARGS, replica_count=1, machine_type=TRAIN_COMPUTE, sync=True)\n",
         "\n",
-        "! gsutil cat {BUCKET_URI}/test.txt"
+        "! gsutil cat {BUCKET_NAME}/test.txt"
       ]
     },
     {
@@ -889,9 +768,9 @@
       "source": [
         "DISPLAY_NAME = \"boston_\" + TIMESTAMP\n",
         "\n",
-        "job = aiplatform.CustomPythonPackageTrainingJob(\n",
+        "job = aip.CustomPythonPackageTrainingJob(\n",
         "    display_name=DISPLAY_NAME,\n",
-        "    python_package_gcs_uri=f\"{BUCKET_URI}/trainer_boston.tar.gz\",\n",
+        "    python_package_gcs_uri=f\"{BUCKET_NAME}/trainer_boston.tar.gz\",\n",
         "    python_module_name=\"trainer.task\",\n",
         "    container_uri=TRAIN_IMAGE,\n",
         ")"
@@ -1021,7 +900,7 @@
         "! rm -f custom.tar custom.tar.gz\n",
         "! tar cvf custom.tar custom\n",
         "! gzip custom.tar\n",
-        "! gsutil cp custom.tar.gz $BUCKET_URI/trainer_boston.tar.gz"
+        "! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_boston.tar.gz"
       ]
     },
     {
@@ -1043,11 +922,11 @@
       },
       "outputs": [],
       "source": [
-        "CMDARGS = [\"--model-dir=\" + BUCKET_URI, \"--epochs=5\"]\n",
+        "CMDARGS = [\"--model-dir=\" + BUCKET_NAME, \"--epochs=5\"]\n",
         "\n",
         "job.run(args=CMDARGS, replica_count=1, machine_type=TRAIN_COMPUTE, sync=True)\n",
         "\n",
-        "! gsutil cat {BUCKET_URI}/test.txt"
+        "! gsutil cat {BUCKET_NAME}/test.txt"
       ]
     },
     {
@@ -1272,11 +1151,7 @@
       },
       "outputs": [],
       "source": [
-        "if not IS_COLAB:\n",
-        "    ! docker build custom -t $TRAIN_IMAGE\n",
-        "else:\n",
-        "    # install docker daemon\n",
-        "    ! apt-get -qq install docker.io"
+        "! docker build custom -t $TRAIN_IMAGE"
       ]
     },
     {
@@ -1298,8 +1173,7 @@
       },
       "outputs": [],
       "source": [
-        "if not IS_COLAB:\n",
-        "    ! docker run $TRAIN_IMAGE --epochs=5 --model-dir=./"
+        "! docker run $TRAIN_IMAGE --epochs=5 --model-dir=./"
       ]
     },
     {
@@ -1321,38 +1195,7 @@
       },
       "outputs": [],
       "source": [
-        "if not IS_COLAB:\n",
-        "    ! docker push $TRAIN_IMAGE"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "f50e9c553fb7"
-      },
-      "source": [
-        "*Executes in Colab*"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "a7e8c98f1e56"
-      },
-      "outputs": [],
-      "source": [
-        "%%bash -s $IS_COLAB $TRAIN_IMAGE\n",
-        "if [ $1 == \"False\" ]; then\n",
-        "  exit 0\n",
-        "fi\n",
-        "set -x\n",
-        "dockerd -b none --iptables=0 -l warn &\n",
-        "for i in $(seq 5); do [ ! -S \"/var/run/docker.sock\" ] && sleep 2 || break; done\n",
-        "docker build custom -t $2\n",
-        "docker run $2 --epochs=5 --model-dir=./\n",
-        "docker push $2\n",
-        "kill $(jobs -p)"
+        "! docker push $TRAIN_IMAGE"
       ]
     },
     {
@@ -1386,7 +1229,7 @@
       },
       "outputs": [],
       "source": [
-        "job = aiplatform.CustomContainerTrainingJob(\n",
+        "job = aip.CustomContainerTrainingJob(\n",
         "    display_name=\"boston_\" + TIMESTAMP,\n",
         "    container_uri=TRAIN_IMAGE,\n",
         "    command=[\"python3\", \"trainer/task.py\"],\n",
@@ -1414,11 +1257,11 @@
       },
       "outputs": [],
       "source": [
-        "CMDARGS = [\"--model-dir=\" + BUCKET_URI, \"--epochs=5\"]\n",
+        "CMDARGS = [\"--model-dir=\" + BUCKET_NAME, \"--epochs=5\"]\n",
         "\n",
         "job.run(args=CMDARGS, replica_count=1, machine_type=TRAIN_COMPUTE, sync=True)\n",
         "\n",
-        "! gsutil cat {BUCKET_URI}/test.txt"
+        "! gsutil cat {BUCKET_NAME}/test.txt"
       ]
     },
     {
@@ -1623,7 +1466,7 @@
         "! rm -f custom.tar custom.tar.gz\n",
         "! tar cvf custom.tar custom\n",
         "! gzip custom.tar\n",
-        "! gsutil cp custom.tar.gz $BUCKET_URI/trainer_boston.tar.gz"
+        "! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_boston.tar.gz"
       ]
     },
     {
@@ -1653,7 +1496,7 @@
         "if os.getenv(\"IS_TESTING_TF\"):\n",
         "    TF = os.getenv(\"IS_TESTING_TF\")\n",
         "else:\n",
-        "    TF = \"2.5\".replace(\".\", \"-\")\n",
+        "    TF = \"2.1\".replace(\".\", \"-\")\n",
         "\n",
         "if TF[0] == \"2\":\n",
         "    if TRAIN_GPU:\n",
@@ -1708,9 +1551,9 @@
       "source": [
         "DISPLAY_NAME = \"boston_\" + TIMESTAMP\n",
         "\n",
-        "job = aiplatform.CustomPythonPackageTrainingJob(\n",
+        "job = aip.CustomPythonPackageTrainingJob(\n",
         "    display_name=DISPLAY_NAME,\n",
-        "    python_package_gcs_uri=f\"{BUCKET_URI}/trainer_boston.tar.gz\",\n",
+        "    python_package_gcs_uri=f\"{BUCKET_NAME}/trainer_boston.tar.gz\",\n",
         "    python_module_name=\"trainer.task\",\n",
         "    container_uri=TRAIN_IMAGE,\n",
         "    model_serving_container_image_uri=DEPLOY_IMAGE,\n",
@@ -1744,12 +1587,12 @@
       },
       "outputs": [],
       "source": [
-        "MODEL_DIR = \"{}/{}\".format(BUCKET_URI, TIMESTAMP)\n",
+        "MODEL_DIR = \"{}/{}\".format(BUCKET_NAME, TIMESTAMP)\n",
         "\n",
         "EPOCHS = 20\n",
         "STEPS = 100\n",
         "\n",
-        "DIRECT = False\n",
+        "DIRECT = True\n",
         "if DIRECT:\n",
         "    CMDARGS = [\n",
         "        \"--model-dir=\" + MODEL_DIR,\n",
@@ -1915,7 +1758,17 @@
         "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
         "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
         "\n",
-        "Otherwise, you can delete the individual resources you created in this tutorial."
+        "Otherwise, you can delete the individual resources you created in this tutorial:\n",
+        "\n",
+        "- Dataset\n",
+        "- Pipeline\n",
+        "- Model\n",
+        "- Endpoint\n",
+        "- AutoML Training Job\n",
+        "- Batch Job\n",
+        "- Custom Job\n",
+        "- Hyperparameter Tuning Job\n",
+        "- Cloud Storage Bucket"
       ]
     },
     {
@@ -1926,24 +1779,61 @@
       },
       "outputs": [],
       "source": [
-        "delete_bucket = False\n",
-        "delete_model = True\n",
-        "delete_job = True\n",
+        "delete_all = True\n",
         "\n",
-        "if delete_model:\n",
+        "if delete_all:\n",
+        "    # Delete the dataset using the Vertex dataset object\n",
         "    try:\n",
-        "        model.delete()\n",
+        "        if \"dataset\" in globals():\n",
+        "            dataset.delete()\n",
         "    except Exception as e:\n",
         "        print(e)\n",
         "\n",
-        "if delete_job:\n",
+        "    # Delete the model using the Vertex model object\n",
         "    try:\n",
-        "        job.delete()\n",
+        "        if \"model\" in globals():\n",
+        "            model.delete()\n",
         "    except Exception as e:\n",
         "        print(e)\n",
         "\n",
-        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
-        "    ! gsutil rm -rf {BUCKET_URI}"
+        "    # Delete the endpoint using the Vertex endpoint object\n",
+        "    try:\n",
+        "        if \"endpoint\" in globals():\n",
+        "            endpoint.undeploy_all()\n",
+        "            endpoint.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the AutoML or Pipeline training job\n",
+        "    try:\n",
+        "        if \"dag\" in globals():\n",
+        "            dag.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the custom training job\n",
+        "    try:\n",
+        "        if \"job\" in globals():\n",
+        "            job.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the batch prediction job using the Vertex batch prediction object\n",
+        "    try:\n",
+        "        if \"batch_predict_job\" in globals():\n",
+        "            batch_predict_job.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the hyperparameter tuning job using the Vertex hyperparameter tuning object\n",
+        "    try:\n",
+        "        if \"hpt_job\" in globals():\n",
+        "            hpt_job.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    if \"BUCKET_NAME\" in globals():\n",
+        "        ! gsutil rm -r $BUCKET_NAME"
       ]
     }
   ],

notebooks/community/ml_ops/stage2/get_started_vertex_training_pytorch.ipynb

@@ -29,24 +29,18 @@
         "id": "title:generic,gcp"
       },
       "source": [
-        "# E2E ML on GCP: MLOps stage 2 : experimentation: get started with Vertex AI Training for Pytorch\n",
+        "# E2E ML on GCP: MLOps stage 2 : experimentation: get started with Vertex Training for Pytorch\n",
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_pytorch.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
-        "    </a>\n",
-        "  </td>\n",
-        "  <td>\n",
         "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_pytorch.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/notebook_template.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
+        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_pytorch.ipynb\">\n",
+        "      Open in Google Cloud Notebooks\n",
         "    </a>\n",
         "  </td>\n",
         "</table>\n",
@@ -62,7 +56,18 @@
         "## Overview\n",
         "\n",
         "\n",
-        "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 2 : experimentation: get started with Vertex AI Training for Pytorch."
+        "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 2 : experimentation: get started with Vertex Training for Pytorch."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dataset:pytorch,cifar10,icn"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the [CIFAR10 dataset](https://pytorch.org/vision/stable/datasets.html#cifar) from [Pytorch Datasets](https://pytorch.org/vision/stable/datasets.html). The version of the dataset you will use is built into TensorFlow. The trained model predicts which type of class an image is from ten classes: airplane, automobile, bird, cat, deer, dog, frog, horse, ship, or truck."
       ]
     },
     {
@@ -77,9 +82,8 @@
         "\n",
         "This tutorial uses the following Google Cloud ML services:\n",
         "\n",
-        "* `Vertex AI Training`\n",
-        "* `Vertex AI Model` resource\n",
-        "\n",
+        "- `Vertex AI Training`\n",
+        "- `Vertex AI Model` resource\n",
         "\n",
         "The steps performed include:\n",
         "\n",
@@ -89,86 +93,6 @@
         "- Create a `Vertex AI Model` resource."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "dataset:pytorch,cifar10,icn"
-      },
-      "source": [
-        "### Dataset\n",
-        "\n",
-        "The dataset used for this tutorial is the [CIFAR10 dataset](https://pytorch.org/vision/stable/datasets.html#cifar) from [Pytorch Datasets](https://pytorch.org/vision/stable/datasets.html). The version of the dataset is built into TensorFlow. The trained model predicts which type of class an image is from ten classes: airplane, automobile, bird, cat, deer, dog, frog, horse, ship, or truck."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "85ee859437ed"
-      },
-      "source": [
-        "### Costs \n",
-        "\n",
-        "This tutorial uses billable components of Google Cloud:\n",
-        "\n",
-        "* Vertex AI\n",
-        "* Cloud Storage\n",
-        "\n",
-        "Learn about [Vertex AI\n",
-        "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
-        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
-        "Calculator](https://cloud.google.com/products/calculator/)\n",
-        "to generate a cost estimate based on your projected usage."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "5cd61a5dd9db"
-      },
-      "source": [
-        "### Set up your local development environment\n",
-        "\n",
-        "**If you are using Colab or Google Cloud Notebooks**, your environment already meets\n",
-        "all the requirements to run this notebook. You can skip this step."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "7e689ee0bc3c"
-      },
-      "source": [
-        "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
-        "You need the following:\n",
-        "\n",
-        "* The Google Cloud SDK\n",
-        "* Git\n",
-        "* Python 3\n",
-        "* virtualenv\n",
-        "* Jupyter notebook running in a virtual environment with Python 3\n",
-        "\n",
-        "The Google Cloud guide to [Setting up a Python development\n",
-        "environment](https://cloud.google.com/python/setup) and the [Jupyter\n",
-        "installation guide](https://jupyter.org/install) provide detailed instructions\n",
-        "for meeting these requirements. The following steps provide a condensed set of\n",
-        "instructions:\n",
-        "\n",
-        "1. [Install and initialize the Cloud SDK.](https://cloud.google.com/sdk/docs/)\n",
-        "\n",
-        "1. [Install Python 3.](https://cloud.google.com/python/setup#installing_python)\n",
-        "\n",
-        "1. [Install\n",
-        "   virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv)\n",
-        "   and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
-        "\n",
-        "1. To install Jupyter, run `pip3 install jupyter` on the\n",
-        "command-line in a terminal shell.\n",
-        "\n",
-        "1. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
-        "\n",
-        "1. Open this notebook in the Jupyter Notebook Dashboard."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -177,7 +101,7 @@
       "source": [
         "## Installations\n",
         "\n",
-        "Install the following packages to execute this notebook."
+        "Install *one time* the packages for executing the MLOps notebooks."
       ]
     },
     {
@@ -188,22 +112,22 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
-        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG -q\n",
-        "! pip3 install --upgrade cloudml-hypertune $USER_FLAG -q\n",
-        "! pip3 install --upgrade torchvision $USER_FLAG -q"
+        "ONCE_ONLY = False\n",
+        "if ONCE_ONLY:\n",
+        "    ! pip3 install -U tensorflow==2.5 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-transform==1.2 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-io==0.18 $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-bigquery $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-logging $USER_FLAG\n",
+        "    ! pip3 install --upgrade apache-beam[gcp] $USER_FLAG\n",
+        "    ! pip3 install --upgrade pyarrow $USER_FLAG\n",
+        "    ! pip3 install --upgrade cloudml-hypertune $USER_FLAG\n",
+        "    ! pip3 install --upgrade kfp $USER_FLAG\n",
+        "    ! pip3 install --upgrade torchvision $USER_FLAG\n",
+        "    ! pip3 install --upgrade rpy2 $USER_FLAG"
       ]
     },
     {
@@ -235,32 +159,6 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "84cd83853240"
-      },
-      "source": [
-        "## Before you begin\n",
-        "\n",
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI, BigQuery, Compute Engine and Cloud Storage APIs](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,bigquery,compute_component,storage_component).\n",
-        "\n",
-        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -337,10 +235,7 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
       ]
     },
     {
@@ -367,67 +262,6 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "77c385f0db59"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
-        "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
-        "\n",
-        "1. **Click Create service account**.\n",
-        "\n",
-        "2. In the **Service account name** field, enter a name, and click **Create**.\n",
-        "\n",
-        "3. In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex AI\" into the filter box, and select **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "4. Click Create. A JSON file that contains your key downloads to your local environment.\n",
-        "\n",
-        "5. Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "535223fa4b84"
-      },
-      "outputs": [],
-      "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = False\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        IS_COLAB = True\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -451,7 +285,7 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_URI = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
+        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -462,8 +296,8 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
       ]
     },
     {
@@ -483,7 +317,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_URI"
+        "! gsutil mb -l $REGION $BUCKET_NAME"
       ]
     },
     {
@@ -503,7 +337,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_URI"
+        "! gsutil ls -al $BUCKET_NAME"
       ]
     },
     {
@@ -526,7 +360,7 @@
       },
       "outputs": [],
       "source": [
-        "import google.cloud.aiplatform as aiplatform"
+        "import google.cloud.aiplatform as aip"
       ]
     },
     {
@@ -548,7 +382,7 @@
       },
       "outputs": [],
       "source": [
-        "aiplatform.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
+        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_NAME)"
       ]
     },
     {
@@ -578,15 +412,13 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
         "if os.getenv(\"IS_TESTING_TRAIN_GPU\"):\n",
         "    TRAIN_GPU, TRAIN_NGPU = (\n",
-        "        aiplatform.gapic.AcceleratorType.NVIDIA_TESLA_K80,\n",
+        "        aip.gapic.AcceleratorType.NVIDIA_TESLA_K80,\n",
         "        int(os.getenv(\"IS_TESTING_TRAIN_GPU\")),\n",
         "    )\n",
         "else:\n",
-        "    TRAIN_GPU, TRAIN_NGPU = (aiplatform.gapic.AcceleratorType.NVIDIA_TESLA_K80, 1)"
+        "    TRAIN_GPU, TRAIN_NGPU = (aip.gapic.AcceleratorType.NVIDIA_TESLA_K80, 1)"
       ]
     },
     {
@@ -729,7 +561,6 @@
         "# Add package information\n",
         "! touch custom/README.md\n",
         "\n",
-        "# Instructions for installing package into environment of the docker image\n",
         "setup_cfg = \"[egg_info]\\n\\ntag_build =\\n\\ntag_date = 0\"\n",
         "! echo \"$setup_cfg\" > custom/setup.cfg\n",
         "\n",
@@ -1060,7 +891,7 @@
         "! rm -f custom.tar custom.tar.gz\n",
         "! tar cvf custom.tar custom\n",
         "! gzip custom.tar\n",
-        "! gsutil cp custom.tar.gz $BUCKET_URI/trainer_cifar10.tar.gz"
+        "! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_cifar10.tar.gz"
       ]
     },
     {
@@ -1071,7 +902,7 @@
       "source": [
         "### Make Pytorch container for prediction\n",
         "\n",
-        "Currently, Vertex AI does not have a predefined container for making predictions with a deployed Pytorch model. No problem, you can assemble your own custom container. Typically, one would base the container on the `Torch Server`. For demonstration purpose, you build a placeholder container (not complete) that includes the latest `Torch Server` image, and push it to the `Container Registry`."
+        "Currently, Vertex AI does not have a prefined container for making predictions with a deployed Pytorch model. No problem, you can assemble your own custom container. Typically, one would base the container on the `Torch Server`. For demonstration purpose, you build a placeholder container (not complete) that includes the latest `Torch Server` image, and push it to the `Container Registry`."
       ]
     },
     {
@@ -1101,52 +932,11 @@
       "source": [
         "APP_NAME = \"cifar10\"\n",
         "DEPLOY_IMAGE = f\"gcr.io/{PROJECT_ID}/pytorch_predict_{APP_NAME}\"\n",
-        "print(DEPLOY_IMAGE)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "85739262f629"
-      },
-      "outputs": [],
-      "source": [
-        "if not IS_COLAB:\n",
-        "    ! docker build --tag=$DEPLOY_IMAGE ./\n",
-        "    ! docker push $DEPLOY_IMAGE\n",
-        "else:\n",
-        "    # install docker daemon\n",
-        "    ! apt-get -qq install docker.io"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "f50e9c553fb7"
-      },
-      "source": [
-        "*Executes in Colab*"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "a7e8c98f1e56"
-      },
-      "outputs": [],
-      "source": [
-        "%%bash -s $IS_COLAB $DEPLOY_IMAGE\n",
-        "if [ $1 == \"False\" ]; then\n",
-        "  exit 0\n",
-        "fi\n",
-        "set -x\n",
-        "dockerd -b none --iptables=0 -l warn &\n",
-        "for i in $(seq 5); do [ ! -S \"/var/run/docker.sock\" ] && sleep 2 || break; done\n",
-        "docker build --tag=$2 ./\n",
-        "docker push $2\n",
-        "kill $(jobs -p)"
+        "print(DEPLOY_IMAGE)\n",
+        "\n",
+        "! docker build --tag=$DEPLOY_IMAGE ./\n",
+        "\n",
+        "! docker push $DEPLOY_IMAGE"
       ]
     },
     {
@@ -1184,9 +974,9 @@
       "source": [
         "DISPLAY_NAME = \"cifar10_\" + TIMESTAMP\n",
         "\n",
-        "job = aiplatform.CustomPythonPackageTrainingJob(\n",
+        "job = aip.CustomPythonPackageTrainingJob(\n",
         "    display_name=DISPLAY_NAME,\n",
-        "    python_package_gcs_uri=f\"{BUCKET_URI}/trainer_cifar10.tar.gz\",\n",
+        "    python_package_gcs_uri=f\"{BUCKET_NAME}/trainer_cifar10.tar.gz\",\n",
         "    python_module_name=\"trainer.task\",\n",
         "    container_uri=TRAIN_IMAGE,\n",
         "    model_serving_container_image_uri=DEPLOY_IMAGE,\n",
@@ -1219,7 +1009,7 @@
       },
       "outputs": [],
       "source": [
-        "MODEL_DIR = \"{}/{}\".format(BUCKET_URI, TIMESTAMP)\n",
+        "MODEL_DIR = \"{}/{}\".format(BUCKET_NAME, TIMESTAMP)\n",
         "\n",
         "DIRECT = False\n",
         "if DIRECT:\n",
@@ -1331,7 +1121,7 @@
       "source": [
         "### Delete a custom training job\n",
         "\n",
-        "After a training job is completed, you can delete the training job with the method `delete()`.  Prior to completion, a training job can be cancelled with the method `cancel()`."
+        "After a training job is completed, you can delete the training job with the method `delete()`.  Prior to completion, a training job can be canceled with the method `cancel()`."
       ]
     },
     {
@@ -1358,7 +1148,14 @@
         "\n",
         "Otherwise, you can delete the individual resources you created in this tutorial:\n",
         "\n",
+        "- Dataset\n",
+        "- Pipeline\n",
         "- Model\n",
+        "- Endpoint\n",
+        "- AutoML Training Job\n",
+        "- Batch Job\n",
+        "- Custom Job\n",
+        "- Hyperparameter Tuning Job\n",
         "- Cloud Storage Bucket"
       ]
     },
@@ -1370,12 +1167,61 @@
       },
       "outputs": [],
       "source": [
-        "# Delete the model using the Vertex model object\n",
-        "model.delete()\n",
+        "delete_all = True\n",
         "\n",
-        "delete_bucket = False\n",
-        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
-        "    ! gsutil rm -r $BUCKET_URI"
+        "if delete_all:\n",
+        "    # Delete the dataset using the Vertex dataset object\n",
+        "    try:\n",
+        "        if \"dataset\" in globals():\n",
+        "            dataset.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the model using the Vertex model object\n",
+        "    try:\n",
+        "        if \"model\" in globals():\n",
+        "            model.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the endpoint using the Vertex endpoint object\n",
+        "    try:\n",
+        "        if \"endpoint\" in globals():\n",
+        "            endpoint.undeploy_all()\n",
+        "            endpoint.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the AutoML or Pipeline training job\n",
+        "    try:\n",
+        "        if \"dag\" in globals():\n",
+        "            dag.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the custom training job\n",
+        "    try:\n",
+        "        if \"job\" in globals():\n",
+        "            job.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the batch prediction job using the Vertex batch prediction object\n",
+        "    try:\n",
+        "        if \"batch_predict_job\" in globals():\n",
+        "            batch_predict_job.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the hyperparameter tuning job using the Vertex hyperparameter tuning object\n",
+        "    try:\n",
+        "        if \"hpt_job\" in globals():\n",
+        "            hpt_job.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    if \"BUCKET_NAME\" in globals():\n",
+        "        ! gsutil rm -r $BUCKET_NAME"
       ]
     }
   ],

notebooks/community/ml_ops/stage2/get_started_vertex_training_r.ipynb

@@ -29,7 +29,7 @@
         "id": "title:generic,gcp"
       },
       "source": [
-        "# E2E ML on GCP: MLOps stage 2 : experimentation: get started with Vertex AI Training for R\n",
+        "# E2E ML on GCP: MLOps stage 2 : experimentation: get started with Vertex Training for R\n",
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
@@ -39,14 +39,8 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_r.ipynb\">\n",
-        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
-        "        </a>\n",
-        "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_r.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
+        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_r.ipynb\">\n",
+        "      Open in Google Cloud Notebooks\n",
         "    </a>\n",
         "  </td>\n",
         "</table>\n",
@@ -62,7 +56,18 @@
         "## Overview\n",
         "\n",
         "\n",
-        "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 2 : experimentation: get started with Vertex AI Training for R. Please note that this notebook should be ran only in R notebook image (e.g., R4.1)."
+        "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 2 : experimentation: get started with Vertex Training for R."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dataset:r,iris,lcn"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the Iris dataset built into the R package. This dataset does not require any feature engineering. The trained model predicts the type of Iris flower species from a class of three species: setosa, virginica, or versicolor."
       ]
     },
     {
@@ -93,39 +98,6 @@
         "- Train a R model using `Vertex AI Trainingh` service with the R-to-Python training package."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "dataset:r,iris,lcn"
-      },
-      "source": [
-        "### Dataset\n",
-        "\n",
-        "The dataset used for this tutorial is the Iris dataset built into the R package. This dataset does not require any feature engineering. The trained model predicts the type of Iris flower species from a class of three species: setosa, virginica, or versicolor."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "0c997d8d92ce"
-      },
-      "source": [
-        "### Costs \n",
-        "\n",
-        "\n",
-        "This tutorial uses billable components of Google Cloud:\n",
-        "\n",
-        "* Vertex AI\n",
-        "* Cloud Storage\n",
-        "\n",
-        "\n",
-        "Learn about [Vertex AI\n",
-        "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
-        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
-        "Calculator](https://cloud.google.com/products/calculator/)\n",
-        "to generate a cost estimate based on your projected usage."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -134,32 +106,33 @@
       "source": [
         "## Installations\n",
         "\n",
-        "Install the packages required for executing this notebook."
+        "Install *one time* the packages for executing the MLOps notebooks."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "1fd00fa70a2a"
+        "id": "install_mlops"
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
-        "! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG -q\n",
-        "! pip3 install --upgrade rpy2 $USER_FLAG -q"
+        "ONCE_ONLY = False\n",
+        "if ONCE_ONLY:\n",
+        "    ! pip3 install -U tensorflow==2.5 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-transform==1.2 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-io==0.18 $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-bigquery $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-logging $USER_FLAG\n",
+        "    ! pip3 install --upgrade apache-beam[gcp] $USER_FLAG\n",
+        "    ! pip3 install --upgrade pyarrow $USER_FLAG\n",
+        "    ! pip3 install --upgrade cloudml-hypertune $USER_FLAG\n",
+        "    ! pip3 install --upgrade kfp $USER_FLAG\n",
+        "    ! pip3 install --upgrade torchvision $USER_FLAG\n",
+        "    ! pip3 install --upgrade rpy2 $USER_FLAG"
       ]
     },
     {
@@ -191,39 +164,6 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "0e3cab0cc491"
-      },
-      "source": [
-        "## Before you begin"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "be929e7b4d76"
-      },
-      "source": [
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com). \n",
-        "\n",
-        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -300,9 +240,7 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"[your-region]\"  # @param {type:\"string\"}\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
       ]
     },
     {
@@ -329,67 +267,6 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "3ffa6b6c7cdb"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
-        "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
-        "\n",
-        "1. **Click Create service account**.\n",
-        "\n",
-        "2. In the **Service account name** field, enter a name, and click **Create**.\n",
-        "\n",
-        "3. In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex AI\" into the filter box, and select **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "4. Click Create. A JSON file that contains your key downloads to your local environment.\n",
-        "\n",
-        "5. Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "2b72272258fc"
-      },
-      "outputs": [],
-      "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = False\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        IS_COLAB = True\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -400,7 +277,7 @@
         "\n",
         "**The following steps are required, regardless of your notebook environment.**\n",
         "\n",
-        "When you initialize the Vertex AI SDK for Python, you specify a Cloud Storage staging bucket. The staging bucket is where all the data associated with your dataset and model resources are retained across sessions.\n",
+        "When you initialize the Vertex SDK for Python, you specify a Cloud Storage staging bucket. The staging bucket is where all the data associated with your dataset and model resources are retained across sessions.\n",
         "\n",
         "Set the name of your Cloud Storage bucket below. Bucket names must be globally unique across all Google Cloud projects, including those outside of your organization."
       ]
@@ -413,7 +290,7 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_URI = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
+        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -424,8 +301,8 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
       ]
     },
     {
@@ -445,7 +322,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_URI"
+        "! gsutil mb -l $REGION $BUCKET_NAME"
       ]
     },
     {
@@ -465,7 +342,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_URI"
+        "! gsutil ls -al $BUCKET_NAME"
       ]
     },
     {
@@ -488,8 +365,6 @@
       },
       "outputs": [],
       "source": [
-        "import traceback\n",
-        "\n",
         "import google.cloud.aiplatform as aip"
       ]
     },
@@ -512,7 +387,7 @@
       },
       "outputs": [],
       "source": [
-        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
+        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_NAME)"
       ]
     },
     {
@@ -1089,17 +964,14 @@
       },
       "outputs": [],
       "source": [
-        "try:\n",
-        "    INSTANCES = [\n",
-        "        {\"sepal_width\": 1, \"sepal_length\": 2, \"petal_width\": 3, \"petal_length\": 1},\n",
-        "        {\"sepal_width\": 4, \"sepal_length\": 2, \"petal_width\": 1, \"petal_length\": 1},\n",
-        "    ]\n",
+        "INSTANCES = [\n",
+        "    {\"sepal_width\": 1, \"sepal_length\": 2, \"petal_width\": 3, \"petal_length\": 1},\n",
+        "    {\"sepal_width\": 4, \"sepal_length\": 2, \"petal_width\": 1, \"petal_length\": 1},\n",
+        "]\n",
         "\n",
-        "    prediction = endpoint.predict(instances=INSTANCES)\n",
+        "prediction = endpoint.predict(instances=INSTANCES)\n",
         "\n",
-        "    print(prediction)\n",
-        "except:\n",
-        "    traceback.print_exc()"
+        "print(prediction)"
       ]
     },
     {
@@ -1370,7 +1242,7 @@
       },
       "outputs": [],
       "source": [
-        "CMDARGS = [\"--model-dir=\" + BUCKET_URI]\n",
+        "CMDARGS = [\"--model-dir=\" + BUCKET_NAME]\n",
         "\n",
         "job.run(args=CMDARGS, replica_count=1, machine_type=TRAIN_COMPUTE, sync=True)"
       ]
@@ -1410,9 +1282,14 @@
         "\n",
         "Otherwise, you can delete the individual resources you created in this tutorial:\n",
         "\n",
-        "- Model (Already deleted in previous cells)\n",
-        "- Endpoint (Already deleted in previous cells)\n",
-        "- Custom Job (Already deleted in previous cells)\n",
+        "- Dataset\n",
+        "- Pipeline\n",
+        "- Model\n",
+        "- Endpoint\n",
+        "- AutoML Training Job\n",
+        "- Batch Job\n",
+        "- Custom Job\n",
+        "- Hyperparameter Tuning Job\n",
         "- Cloud Storage Bucket"
       ]
     },
@@ -1424,10 +1301,61 @@
       },
       "outputs": [],
       "source": [
-        "delete_bucket = False\n",
+        "delete_all = True\n",
         "\n",
-        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
-        "    ! gsutil rm -r $BUCKET_URI"
+        "if delete_all:\n",
+        "    # Delete the dataset using the Vertex dataset object\n",
+        "    try:\n",
+        "        if \"dataset\" in globals():\n",
+        "            dataset.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the model using the Vertex model object\n",
+        "    try:\n",
+        "        if \"model\" in globals():\n",
+        "            model.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the endpoint using the Vertex endpoint object\n",
+        "    try:\n",
+        "        if \"endpoint\" in globals():\n",
+        "            endpoint.undeploy_all()\n",
+        "            endpoint.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the AutoML or Pipeline training job\n",
+        "    try:\n",
+        "        if \"dag\" in globals():\n",
+        "            dag.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the custom training job\n",
+        "    try:\n",
+        "        if \"job\" in globals():\n",
+        "            job.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the batch prediction job using the Vertex batch prediction object\n",
+        "    try:\n",
+        "        if \"batch_predict_job\" in globals():\n",
+        "            batch_predict_job.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the hyperparameter tuning job using the Vertex hyperparameter tuning object\n",
+        "    try:\n",
+        "        if \"hpt_job\" in globals():\n",
+        "            hpt_job.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    if \"BUCKET_NAME\" in globals():\n",
+        "        ! gsutil rm -r $BUCKET_NAME"
       ]
     }
   ],

notebooks/community/ml_ops/stage2/get_started_vertex_training_sklearn.ipynb

@@ -29,24 +29,18 @@
         "id": "title:generic,gcp"
       },
       "source": [
-        "# E2E ML on GCP: MLOps stage 2 : experimentation: get started with Vertex AI Training for Scikit-Learn\n",
+        "# E2E ML on GCP: MLOps stage 2 : experimentation: get started with Vertex Training for Scikit-Learn\n",
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_sklearn.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
-        "    </a>\n",
-        "  </td>\n",
-        "  <td>\n",
         "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_sklearn.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_sklearn.ipynb\">\n",
-        "     <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "Open in Vertex AI Workbench\n",
+        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_sklearn.ipynb\">\n",
+        "      Open in Google Cloud Notebooks\n",
         "    </a>\n",
         "  </td>\n",
         "</table>\n",
@@ -62,7 +56,18 @@
         "## Overview\n",
         "\n",
         "\n",
-        "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 2 : experimentation: get started with Vertex AI Training for scikit-Learn."
+        "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 2 : experimentation: get started with Vertex Training for Scikit-Learn."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dataset:custom,newsaggr,tcn"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the [News Aggregation](https://archive.ics.uci.edu/ml/datasets/News+Aggregator) from [ICS Machine Learning Datasets](https://archive.ics.uci.edu/ml/datasets.php). The trained model predicts the news category of the news article."
       ]
     },
     {
@@ -88,112 +93,41 @@
         "- Create a `Vertex AI Model` resource."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "dataset:custom,newsaggr,tcn"
-      },
-      "source": [
-        "### Dataset\n",
-        "\n",
-        "The dataset used for this tutorial is the [News Aggregation](https://archive.ics.uci.edu/ml/datasets/News+Aggregator) from [ICS Machine Learning Datasets](https://archive.ics.uci.edu/ml/datasets.php). The trained model predicts the news category of the news article."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "b132d4ef86d6"
-      },
-      "source": [
-        "### Costs \n",
-        "\n",
-        "This tutorial uses billable components of Google Cloud:\n",
-        "\n",
-        "* Vertex AI\n",
-        "* Cloud Storage\n",
-        "\n",
-        "Learn about [Vertex AI\n",
-        "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
-        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
-        "Calculator](https://cloud.google.com/products/calculator/)\n",
-        "to generate a cost estimate based on your projected usage."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "94a148f11da5"
-      },
-      "source": [
-        "### Set up your local development environment\n",
-        "\n",
-        "**If you are using Colab or Google Cloud Notebooks**, your environment already meets\n",
-        "all the requirements to run this notebook. You can skip this step.\n",
-        "\n",
-        "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
-        "You need the following:\n",
-        "\n",
-        "* The Google Cloud SDK\n",
-        "* Git\n",
-        "* Python 3\n",
-        "* virtualenv\n",
-        "* Jupyter notebook running in a virtual environment with Python 3\n",
-        "\n",
-        "The Google Cloud guide to [Setting up a Python development\n",
-        "environment](https://cloud.google.com/python/setup) and the [Jupyter\n",
-        "installation guide](https://jupyter.org/install) provide detailed instructions\n",
-        "for meeting these requirements. The following steps provide a condensed set of\n",
-        "instructions:\n",
-        "\n",
-        "1. [Install and initialize the Cloud SDK.](https://cloud.google.com/sdk/docs/)\n",
-        "\n",
-        "1. [Install Python 3.](https://cloud.google.com/python/setup#installing_python)\n",
-        "\n",
-        "1. [Install\n",
-        "   virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv)\n",
-        "   and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
-        "\n",
-        "1. To install Jupyter, run `pip3 install jupyter` on the\n",
-        "command-line in a terminal shell.\n",
-        "\n",
-        "1. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
-        "\n",
-        "1. Open this notebook in the Jupyter Notebook Dashboard.\n"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "install_mlops"
       },
       "source": [
-        "## Installation\n",
+        "## Installations\n",
         "\n",
-        "Install the following packages for executing this notebook."
+        "Install *one time* the packages for executing the MLOps notebooks."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "78168417490e"
+        "id": "install_mlops"
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
-        "! pip3 install {USER_FLAG} --upgrade google-cloud-aiplatform -q"
+        "ONCE_ONLY = False\n",
+        "if ONCE_ONLY:\n",
+        "    ! pip3 install -U tensorflow==2.5 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-transform==1.2 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-io==0.18 $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-bigquery $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-logging $USER_FLAG\n",
+        "    ! pip3 install --upgrade apache-beam[gcp] $USER_FLAG\n",
+        "    ! pip3 install --upgrade pyarrow $USER_FLAG\n",
+        "    ! pip3 install --upgrade cloudml-hypertune $USER_FLAG\n",
+        "    ! pip3 install --upgrade kfp $USER_FLAG\n",
+        "    ! pip3 install --upgrade torchvision $USER_FLAG\n",
+        "    ! pip3 install --upgrade rpy2 $USER_FLAG"
       ]
     },
     {
@@ -225,32 +159,6 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "2721ef0202d9"
-      },
-      "source": [
-        "## Before you begin\n",
-        "\n",
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com). \n",
-        "\n",
-        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -327,10 +235,7 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
       ]
     },
     {
@@ -357,67 +262,6 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "77c385f0db59"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
-        "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
-        "\n",
-        "1. **Click Create service account**.\n",
-        "\n",
-        "2. In the **Service account name** field, enter a name, and click **Create**.\n",
-        "\n",
-        "3. In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex AI\" into the filter box, and select **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "4. Click Create. A JSON file that contains your key downloads to your local environment.\n",
-        "\n",
-        "5. Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "535223fa4b84"
-      },
-      "outputs": [],
-      "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = False\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        IS_COLAB = True\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -441,7 +285,7 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_URI = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
+        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -452,8 +296,8 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
       ]
     },
     {
@@ -473,7 +317,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_URI"
+        "! gsutil mb -l $REGION $BUCKET_NAME"
       ]
     },
     {
@@ -493,7 +337,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_URI"
+        "! gsutil ls -al $BUCKET_NAME"
       ]
     },
     {
@@ -538,7 +382,7 @@
       },
       "outputs": [],
       "source": [
-        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
+        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_NAME)"
       ]
     },
     {
@@ -571,8 +415,6 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
         "if os.getenv(\"IS_TESTING_TRAIN_GPU\"):\n",
         "    TRAIN_GPU, TRAIN_NGPU = (\n",
         "        aip.gapic.AcceleratorType.NVIDIA_TESLA_K80,\n",
@@ -675,9 +517,9 @@
         "id": "sklearn_intro"
       },
       "source": [
-        "## Introduction to scikit-learn training\n",
+        "## Introduction to Scikit-learn training\n",
         "\n",
-        "Once you have trained a scikit-learn model, you will want to save it at a Cloud Storage location, so it can subsequently be uploaded to a `Vertex AI Model` resource. The Scikit-learn package does not have support to save the model to a Cloud Storage location. Instead, you will do the following steps to save to a Cloud Storage location.\n",
+        "Once you have trained a Scikit-learn model, you will want to save it at a Cloud Storage location, so it can subsequently be uploaded to a `Vertex AI Model` resource. The Scikit-learn package does not have support to save the model to a Cloud Storage location. Instead, you will do the following steps to save to a Cloud Storage location.\n",
         "\n",
         "1. Save the in-memory model to the local filesystem in pickle format (e.g., model.pkl).\n",
         "2. Create a Cloud Storage storage client.\n",
@@ -941,7 +783,7 @@
         "! rm -f custom.tar custom.tar.gz\n",
         "! tar cvf custom.tar custom\n",
         "! gzip custom.tar\n",
-        "! gsutil cp custom.tar.gz $BUCKET_URI/trainer_newsaggr.tar.gz"
+        "! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_newsaggr.tar.gz"
       ]
     },
     {
@@ -981,7 +823,7 @@
         "\n",
         "job = aip.CustomPythonPackageTrainingJob(\n",
         "    display_name=DISPLAY_NAME,\n",
-        "    python_package_gcs_uri=f\"{BUCKET_URI}/trainer_newsaggr.tar.gz\",\n",
+        "    python_package_gcs_uri=f\"{BUCKET_NAME}/trainer_newsaggr.tar.gz\",\n",
         "    python_module_name=\"trainer.task\",\n",
         "    container_uri=TRAIN_IMAGE,\n",
         "    model_serving_container_image_uri=DEPLOY_IMAGE,\n",
@@ -1015,7 +857,7 @@
       },
       "outputs": [],
       "source": [
-        "MODEL_DIR = \"{}/{}\".format(BUCKET_URI, TIMESTAMP)\n",
+        "MODEL_DIR = \"{}/{}\".format(BUCKET_NAME, TIMESTAMP)\n",
         "DATASET_URL = \"https://archive.ics.uci.edu/ml/machine-learning-databases/00359/NewsAggregatorDataset.zip\"\n",
         "\n",
         "DIRECT = False\n",
@@ -1160,8 +1002,14 @@
         "\n",
         "Otherwise, you can delete the individual resources you created in this tutorial:\n",
         "\n",
-        "\n",
+        "- Dataset\n",
+        "- Pipeline\n",
         "- Model\n",
+        "- Endpoint\n",
+        "- AutoML Training Job\n",
+        "- Batch Job\n",
+        "- Custom Job\n",
+        "- Hyperparameter Tuning Job\n",
         "- Cloud Storage Bucket"
       ]
     },
@@ -1169,16 +1017,65 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "b413063dfdcf"
+        "id": "cleanup:mbsdk"
       },
       "outputs": [],
       "source": [
-        "# Delete the model using the Vertex model object\n",
-        "model.delete()\n",
+        "delete_all = True\n",
         "\n",
-        "delete_bucket = False\n",
-        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
-        "    ! gsutil rm -r $BUCKET_URI"
+        "if delete_all:\n",
+        "    # Delete the dataset using the Vertex dataset object\n",
+        "    try:\n",
+        "        if \"dataset\" in globals():\n",
+        "            dataset.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the model using the Vertex model object\n",
+        "    try:\n",
+        "        if \"model\" in globals():\n",
+        "            model.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the endpoint using the Vertex endpoint object\n",
+        "    try:\n",
+        "        if \"endpoint\" in globals():\n",
+        "            endpoint.undeploy_all()\n",
+        "            endpoint.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the AutoML or Pipeline training job\n",
+        "    try:\n",
+        "        if \"dag\" in globals():\n",
+        "            dag.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the custom training job\n",
+        "    try:\n",
+        "        if \"job\" in globals():\n",
+        "            job.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the batch prediction job using the Vertex batch prediction object\n",
+        "    try:\n",
+        "        if \"batch_predict_job\" in globals():\n",
+        "            batch_predict_job.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the hyperparameter tuning job using the Vertex hyperparameter tuning object\n",
+        "    try:\n",
+        "        if \"hpt_job\" in globals():\n",
+        "            hpt_job.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    if \"BUCKET_NAME\" in globals():\n",
+        "        ! gsutil rm -r $BUCKET_NAME"
       ]
     }
   ],

notebooks/community/ml_ops/stage2/get_started_vertex_training_xgboost.ipynb

@@ -29,15 +29,9 @@
         "id": "title:generic,gcp"
       },
       "source": [
-        "# E2E ML on GCP: MLOps stage 2 : experimentation: get started with Vertex AI Training for XGBoost\n",
+        "# E2E ML on GCP: MLOps stage 2 : experimentation: get started with Vertex Training for XGBoost\n",
         "\n",
         "<table align=\"left\">\n",
-        "    <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_xgboost.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
-        "    </a>\n",
-        "  </td>\n",
-        "    \n",
         "  <td>\n",
         "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_xgboost.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
@@ -45,9 +39,8 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_xgboost.ipynb\">\n",
-        "    <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
+        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_xgboost.ipynb\">\n",
+        "      Open in Google Cloud Notebooks\n",
         "    </a>\n",
         "  </td>\n",
         "</table>\n",
@@ -63,7 +56,18 @@
         "## Overview\n",
         "\n",
         "\n",
-        "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 2 : experimentation: get started with Vertex AI Training for XGBoost."
+        "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 2 : experimentation: get started with Vertex Training for XGBoost."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dataset:iris,lcn"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the [Iris dataset](https://www.tensorflow.org/datasets/catalog/iris) from [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/overview). This dataset does not require any feature engineering. The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket. The trained model predicts the type of Iris flower species from a class of three species: setosa, virginica, or versicolor."
       ]
     },
     {
@@ -89,38 +93,6 @@
         "- Create a `Vertex AI Model` resource."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "dataset:iris,lcn"
-      },
-      "source": [
-        "### Dataset\n",
-        "\n",
-        "The dataset used for this tutorial is the [Iris dataset](https://www.tensorflow.org/datasets/catalog/iris) from [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/overview). This dataset does not require any feature engineering. The version of the dataset in this tutorial is stored in a public Cloud Storage bucket. The trained model predicts the type of Iris flower species from a class of three species: setosa, virginica, or versicolor."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "4fc0ad661ebb"
-      },
-      "source": [
-        "### Costs \n",
-        "\n",
-        "This tutorial uses billable components of Google Cloud:\n",
-        "\n",
-        "* Vertex AI\n",
-        "* Cloud Storage\n",
-        "\n",
-        "\n",
-        "Learn about [Vertex AI\n",
-        "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
-        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
-        "Calculator](https://cloud.google.com/products/calculator/)\n",
-        "to generate a cost estimate based on your projected usage."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -129,53 +101,54 @@
       "source": [
         "## Installations\n",
         "\n",
-        "Install the following packages to execute this notebook."
+        "Install *one time* the packages for executing the MLOps notebooks."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "ncRJ_Dfdox9L"
+        "id": "install_mlops"
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
-        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG -q"
+        "ONCE_ONLY = False\n",
+        "if ONCE_ONLY:\n",
+        "    ! pip3 install -U tensorflow==2.5 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-transform==1.2 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-io==0.18 $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-bigquery $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-logging $USER_FLAG\n",
+        "    ! pip3 install --upgrade apache-beam[gcp] $USER_FLAG\n",
+        "    ! pip3 install --upgrade pyarrow $USER_FLAG\n",
+        "    ! pip3 install --upgrade cloudml-hypertune $USER_FLAG\n",
+        "    ! pip3 install --upgrade kfp $USER_FLAG\n",
+        "    ! pip3 install --upgrade torchvision $USER_FLAG\n",
+        "    ! pip3 install --upgrade rpy2 $USER_FLAG"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "oQhwq1iozAxh"
+        "id": "restart"
       },
       "source": [
         "### Restart the kernel\n",
         "\n",
-        "After you install the additional packages, you need to restart the notebook kernel so it can find the packages."
+        "Once you've installed the additional packages, you need to restart the notebook kernel so it can find the packages."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "zo3YFZXLzCRJ"
+        "id": "restart"
       },
       "outputs": [],
       "source": [
-        "# Automatically restart kernel after installs\n",
         "import os\n",
         "\n",
         "if not os.getenv(\"IS_TESTING\"):\n",
@@ -186,32 +159,6 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "2721ef0202d9"
-      },
-      "source": [
-        "## Before you begin\n",
-        "\n",
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com). \n",
-        "\n",
-        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -284,14 +231,11 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "sKBTnvJpox9P"
+        "id": "region"
       },
       "outputs": [],
       "source": [
-        "REGION = \"[your-region]\"  # @param {type:\"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\"  # @param {type: \"string\"}"
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
       ]
     },
     {
@@ -309,7 +253,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "JYtXOocrox9Q"
+        "id": "timestamp"
       },
       "outputs": [],
       "source": [
@@ -318,67 +262,6 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "77c385f0db59"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
-        "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
-        "\n",
-        "1. **Click Create service account**.\n",
-        "\n",
-        "2. In the **Service account name** field, enter a name, and click **Create**.\n",
-        "\n",
-        "3. In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex AI\" into the filter box, and select **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "4. Click Create. A JSON file that contains your key downloads to your local environment.\n",
-        "\n",
-        "5. Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "NNc5Bf_NpPTq"
-      },
-      "outputs": [],
-      "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = False\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        IS_COLAB = True\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -389,7 +272,7 @@
         "\n",
         "**The following steps are required, regardless of your notebook environment.**\n",
         "\n",
-        "When you initialize the Vertex AI SDK for Python, you specify a Cloud Storage staging bucket. The staging bucket is where all the data associated with your dataset and model resources are retained across sessions.\n",
+        "When you initialize the Vertex SDK for Python, you specify a Cloud Storage staging bucket. The staging bucket is where all the data associated with your dataset and model resources are retained across sessions.\n",
         "\n",
         "Set the name of your Cloud Storage bucket below. Bucket names must be globally unique across all Google Cloud projects, including those outside of your organization."
       ]
@@ -402,7 +285,7 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_URI = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
+        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -413,8 +296,8 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
       ]
     },
     {
@@ -430,11 +313,11 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "aO4sKJfFox9R"
+        "id": "create_bucket"
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_URI"
+        "! gsutil mb -l $REGION $BUCKET_NAME"
       ]
     },
     {
@@ -450,11 +333,11 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "yWnghzKFox9S"
+        "id": "validate_bucket"
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_URI"
+        "! gsutil ls -al $BUCKET_NAME"
       ]
     },
     {
@@ -495,11 +378,11 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "JZg2sszQox9T"
+        "id": "init_aip:mbsdk"
       },
       "outputs": [],
       "source": [
-        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
+        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_NAME)"
       ]
     },
     {
@@ -528,12 +411,10 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "cQUrG4Mbox9T"
+        "id": "accelerators:training,cpu,prediction,cpu,mbsdk"
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
         "if os.getenv(\"IS_TESTING_TRAIN_GPU\"):\n",
         "    TRAIN_GPU, TRAIN_NGPU = (\n",
         "        aip.gapic.AcceleratorType.NVIDIA_TESLA_K80,\n",
@@ -572,7 +453,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "XujRA5ueox9U"
+        "id": "container:training,prediction,xgboost"
       },
       "outputs": [],
       "source": [
@@ -616,7 +497,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "UMPFgENkox9U"
+        "id": "machine:training"
       },
       "outputs": [],
       "source": [
@@ -680,7 +561,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "f4wS4eISox9V"
+        "id": "examine_training_package:xgboost"
       },
       "outputs": [],
       "source": [
@@ -735,7 +616,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "WiSnFuDoox9W"
+        "id": "taskpy_contents:iris,xgboost"
       },
       "outputs": [],
       "source": [
@@ -850,14 +731,14 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "dnmdycf6ox9X"
+        "id": "tarball_training_script"
       },
       "outputs": [],
       "source": [
         "! rm -f custom.tar custom.tar.gz\n",
         "! tar cvf custom.tar custom\n",
         "! gzip custom.tar\n",
-        "! gsutil cp custom.tar.gz $BUCKET_URI/trainer_iris.tar.gz"
+        "! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_iris.tar.gz"
       ]
     },
     {
@@ -889,7 +770,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "rVEMz1xqox9X"
+        "id": "create_custom_pp_training_job:mbsdk"
       },
       "outputs": [],
       "source": [
@@ -897,7 +778,7 @@
         "\n",
         "job = aip.CustomPythonPackageTrainingJob(\n",
         "    display_name=DISPLAY_NAME,\n",
-        "    python_package_gcs_uri=f\"{BUCKET_URI}/trainer_iris.tar.gz\",\n",
+        "    python_package_gcs_uri=f\"{BUCKET_NAME}/trainer_iris.tar.gz\",\n",
         "    python_module_name=\"trainer.task\",\n",
         "    container_uri=TRAIN_IMAGE,\n",
         "    model_serving_container_image_uri=DEPLOY_IMAGE,\n",
@@ -928,11 +809,11 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "AoUfpBqVox9Y"
+        "id": "prepare_custom_cmdargs:iris,xgboost"
       },
       "outputs": [],
       "source": [
-        "MODEL_DIR = \"{}/{}\".format(BUCKET_URI, TIMESTAMP)\n",
+        "MODEL_DIR = \"{}/{}\".format(BUCKET_NAME, TIMESTAMP)\n",
         "DATASET_DIR = \"gs://cloud-samples-data/ai-platform/iris\"\n",
         "\n",
         "ROUNDS = 20\n",
@@ -977,7 +858,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "JCruQq1aox9Y"
+        "id": "run_custom_job:mbsdk"
       },
       "outputs": [],
       "source": [
@@ -1018,7 +899,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "KBM_KLMSox9Y"
+        "id": "list_job"
       },
       "outputs": [],
       "source": [
@@ -1041,7 +922,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "lHPMHbSyox9Z"
+        "id": "custom_job_wait:mbsdk"
       },
       "outputs": [],
       "source": [
@@ -1063,7 +944,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "tlYg7Sp-ox9Z"
+        "id": "delete_job"
       },
       "outputs": [],
       "source": [
@@ -1083,7 +964,14 @@
         "\n",
         "Otherwise, you can delete the individual resources you created in this tutorial:\n",
         "\n",
-        "- Custom Job (Custome Training job is remove in previous step)\n",
+        "- Dataset\n",
+        "- Pipeline\n",
+        "- Model\n",
+        "- Endpoint\n",
+        "- AutoML Training Job\n",
+        "- Batch Job\n",
+        "- Custom Job\n",
+        "- Hyperparameter Tuning Job\n",
         "- Cloud Storage Bucket"
       ]
     },
@@ -1091,14 +979,65 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "JyWy23gDox9a"
+        "id": "cleanup:mbsdk"
       },
       "outputs": [],
       "source": [
-        "delete_bucket = False\n",
+        "delete_all = True\n",
         "\n",
-        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
-        "    ! gsutil rm -r $BUCKET_URI"
+        "if delete_all:\n",
+        "    # Delete the dataset using the Vertex dataset object\n",
+        "    try:\n",
+        "        if \"dataset\" in globals():\n",
+        "            dataset.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the model using the Vertex model object\n",
+        "    try:\n",
+        "        if \"model\" in globals():\n",
+        "            model.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the endpoint using the Vertex endpoint object\n",
+        "    try:\n",
+        "        if \"endpoint\" in globals():\n",
+        "            endpoint.undeploy_all()\n",
+        "            endpoint.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the AutoML or Pipeline training job\n",
+        "    try:\n",
+        "        if \"dag\" in globals():\n",
+        "            dag.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the custom training job\n",
+        "    try:\n",
+        "        if \"job\" in globals():\n",
+        "            job.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the batch prediction job using the Vertex batch prediction object\n",
+        "    try:\n",
+        "        if \"batch_predict_job\" in globals():\n",
+        "            batch_predict_job.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the hyperparameter tuning job using the Vertex hyperparameter tuning object\n",
+        "    try:\n",
+        "        if \"hpt_job\" in globals():\n",
+        "            hpt_job.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    if \"BUCKET_NAME\" in globals():\n",
+        "        ! gsutil rm -r $BUCKET_NAME"
       ]
     }
   ],

notebooks/community/ml_ops/stage2/get_started_vertex_vizier.ipynb

@@ -29,7 +29,7 @@
         "id": "title:generic,gcp"
       },
       "source": [
-        "# E2E ML on GCP: MLOps stage 2 : experimentation: get started with Vertex AI Vizier\n",
+        "# E2E ML on GCP: MLOps stage 2 : experimentation: get started with Vertex Vizier\n",
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
@@ -39,12 +39,7 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_vizier.ipynb\">\n",
-        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
-        "        </a>\n",
-        "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage2/get_started_vertex_vizier.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_vizier.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
@@ -65,6 +60,17 @@
         "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 2 : experimentation: get started with Vertex Vizier."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dataset:custom,boston,lrg"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the [Boston Housing Prices dataset](https://www.cs.toronto.edu/~delve/data/boston/bostonDetail.html). The version of the dataset you will use in this tutorial is built into TensorFlow. The trained model predicts the median price of a house in units of 1K USD."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -123,38 +129,6 @@
         "- multiple of objectives"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "dataset:custom,boston,lrg"
-      },
-      "source": [
-        "### Dataset\n",
-        "\n",
-        "The dataset used for this tutorial is the [Boston Housing Prices dataset](https://www.cs.toronto.edu/~delve/data/boston/bostonDetail.html). The version of the dataset in this tutorial is built into TensorFlow. The trained model predicts the median price of a house in units of 1K USD."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "c480fc50ec3c"
-      },
-      "source": [
-        "### Costs \n",
-        "\n",
-        "This tutorial uses billable components of Google Cloud:\n",
-        "\n",
-        "* Vertex AI\n",
-        "* Cloud Storage\n",
-        "\n",
-        "\n",
-        "Learn about [Vertex AI\n",
-        "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
-        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
-        "Calculator](https://cloud.google.com/products/calculator/)\n",
-        "to generate a cost estimate based on your projected usage."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -163,7 +137,7 @@
       "source": [
         "## Installations\n",
         "\n",
-        "Install the packages required for executing this notebook."
+        "Install *one time* the packages for executing the MLOps notebooks."
       ]
     },
     {
@@ -176,18 +150,25 @@
       "source": [
         "import os\n",
         "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
+        "# The Google Cloud Notebook product has specific requirements\n",
+        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
         "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "# Google Cloud Notebook requires dependencies to be installed with '--user'\n",
         "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
-        "! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG -q"
+        "if IS_GOOGLE_CLOUD_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "install_mlops"
+      },
+      "outputs": [],
+      "source": [
+        "! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG\n",
+        "    "
       ]
     },
     {
@@ -219,32 +200,6 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "2721ef0202d9"
-      },
-      "source": [
-        "## Before you begin\n",
-        "\n",
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com). \n",
-        "\n",
-        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -350,67 +305,6 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "gcp_authenticate"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
-        "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
-        "\n",
-        "**Click Create service account**.\n",
-        "\n",
-        "In the **Service account name** field, enter a name, and click **Create**.\n",
-        "\n",
-        "In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex\" into the filter box, and select **Vertex Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "Click Create. A JSON file that contains your key downloads to your local environment.\n",
-        "\n",
-        "Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "gcp_authenticate"
-      },
-      "outputs": [],
-      "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = False\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        IS_COLAB = True\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -1296,8 +1190,7 @@
         "Use the class `CustomJob` to create a custom job, such as for hyperparameter tuning, with the following parameters:\n",
         "\n",
         "- `display_name`: A human readable name for the custom job.\n",
-        "- `worker_pool_specs`: The specification for the corresponding VM instances.\n",
-        "- `base_output_dir`: The Cloud Storage location for storing the model artifacts."
+        "- `worker_pool_specs`: The specification for the corresponding VM instances."
       ]
     },
     {
@@ -1309,9 +1202,7 @@
       "outputs": [],
       "source": [
         "job = aip.CustomJob(\n",
-        "    display_name=\"boston_\" + TIMESTAMP,\n",
-        "    worker_pool_specs=worker_pool_spec,\n",
-        "    base_output_dir=MODEL_DIR,\n",
+        "    display_name=\"boston_\" + TIMESTAMP, worker_pool_specs=worker_pool_spec\n",
         ")"
       ]
     },
@@ -1444,32 +1335,6 @@
         "print(best)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "get_best_model"
-      },
-      "source": [
-        "### Get the Best Model\n",
-        "\n",
-        "If you used the method of having the service tell the tuning script where to save the model artifacts (`DIRECT = False`), then the model artifacts for the best model are saved at:\n",
-        "\n",
-        "    MODEL_DIR/<best_trial_id>/model"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "get_best_model"
-      },
-      "outputs": [],
-      "source": [
-        "BEST_MODEL_DIR = MODEL_DIR + \"/\" + best[0] + \"/model\"\n",
-        "\n",
-        "! gsutil ls {BEST_MODEL_DIR}"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -1733,8 +1598,6 @@
       },
       "outputs": [],
       "source": [
-        "delete_bucket = False\n",
-        "\n",
         "if os.getenv(\"IS_TESTING\"):\n",
         "    ! gsutil rm -r $BUCKET_URI"
       ]

notebooks/community/ml_ops/stage2/get_started_with_cmek_training.ipynb

@@ -38,14 +38,8 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_with_cmek_training.ipynb\">\n",
-        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
-        "        </a>\n",
-        "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage2/get_started_with_cmek_training.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
+        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_with_cmek_training.ipynb\">\n",
+        "      Open in Google Cloud Notebooks\n",
         "    </a>\n",
         "  </td>\n",
         "</table>\n",
@@ -64,6 +58,17 @@
         "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 2 : experimentation: get started with AutoML training with a customer managed encyrption key CMEK."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dataset:flowers,icn"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the [Flowers dataset](https://www.tensorflow.org/datasets/catalog/tf_flowers) from [TensorFlow](https://www.tensorflow.org/datasets/catalog/overview). The version of the dataset you will use in this tutorial is stored in a public #(GCS) bucket. The trained model predicts the type of flower an image is from a class of five flowers: daisy, dandelion, rose, sunflower, or tulip.\n"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -86,37 +91,6 @@
         "- Train an AutoML model with CMEK encryption."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "dataset:flowers,icn"
-      },
-      "source": [
-        "### Dataset\n",
-        "\n",
-        "The dataset used for this tutorial is the [Flowers dataset](https://www.tensorflow.org/datasets/catalog/tf_flowers) from [TensorFlow](https://www.tensorflow.org/datasets/catalog/overview). The version of the dataset in this tutorial is stored in a public #(GCS) bucket. The trained model predicts the type of flower an image is from a class of five flowers: daisy, dandelion, rose, sunflower, or tulip.\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "5e2eba58ad71"
-      },
-      "source": [
-        "### Costs \n",
-        "\n",
-        "This tutorial uses billable components of Google Cloud:\n",
-        "\n",
-        "* Vertex AI\n",
-        "* Cloud Storage\n",
-        "\n",
-        "Learn about [Vertex AI\n",
-        "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
-        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
-        "Calculator](https://cloud.google.com/products/calculator/)\n",
-        "to generate a cost estimate based on your projected usage."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -136,21 +110,10 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
+        "USER_FLAG = \"--user\"\n",
         "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
-        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG -q\n",
-        "! pip3 install --upgrade google-cloud-kms $USER_FLAG -q"
+        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG\n",
+        "! pip3 install --upgrade google-cloud-kms $USER_FLAG"
       ]
     },
     {
@@ -182,39 +145,6 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "013daf3de88e"
-      },
-      "source": [
-        "## Before you begin"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "d1afc945645f"
-      },
-      "source": [
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com). \n",
-        "\n",
-        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -318,82 +248,6 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "d35af059208d"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
-        "authenticated. Skip this step."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "a00567d0660a"
-      },
-      "source": [
-        "**If you are using Colab**, run the cell below and follow the instructions\n",
-        "when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "1. In the Cloud Console, go to the [**Create service account key**\n",
-        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
-        "\n",
-        "2. Click **Create service account**.\n",
-        "\n",
-        "3. In the **Service account name** field, enter a name, and\n",
-        "   click **Create**.\n",
-        "\n",
-        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
-        "into the filter box, and select\n",
-        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
-        "local environment.\n",
-        "\n",
-        "6. Enter the path to your service account key as the\n",
-        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "40160162ea4c"
-      },
-      "outputs": [],
-      "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = False\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        IS_COLAB = True\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -417,7 +271,7 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_URI = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
+        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -428,8 +282,8 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
       ]
     },
     {
@@ -449,7 +303,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_URI"
+        "! gsutil mb -l $REGION $BUCKET_NAME"
       ]
     },
     {
@@ -469,7 +323,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_URI"
+        "! gsutil ls -al $BUCKET_NAME"
       ]
     },
     {
@@ -492,7 +346,7 @@
       },
       "outputs": [],
       "source": [
-        "import google.cloud.aiplatform as aiplatform\n",
+        "import google.cloud.aiplatform as aip\n",
         "from google.cloud import kms"
       ]
     },
@@ -515,7 +369,7 @@
       },
       "outputs": [],
       "source": [
-        "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=BUCKET_URI)"
+        "aip.init(project=PROJECT_ID, location=REGION, staging_bucket=BUCKET_NAME)"
       ]
     },
     {
@@ -530,7 +384,7 @@
         "\n",
         "### Enable KMS API\n",
         "\n",
-        "First, you enable the [Cloud Key Management Service (KMS)](https://console.cloud.google.com/flows/enableapi?apiid=cloudkms.googleapis.com)\n",
+        "First, you enble the [Cloud Key Management Service (KMS)](https://console.cloud.google.com/flows/enableapi?apiid=cloudkms.googleapis.com)\n",
         "\n",
         "Learn more about [Customer managed encryption keys (CMEK)](https://cloud.google.com/vertex-ai/docs/general/cmek)"
       ]
@@ -695,8 +549,6 @@
         "\n",
         "Next, you set permissions for your Vertex AI service account to encrypt and decrypt resources using your key.\n",
         "\n",
-        "Note: Compute Engine default service account which is used by this notebook instance for authentication purposes during Google API calls, should be granted the role of Cloud KMS Admin.\n",
-        "\n",
         "Learn more about [Grant Vertex AI permissions](https://cloud.google.com/vertex-ai/docs/general/cmek#grant_permissions)"
       ]
     },
@@ -786,9 +638,9 @@
       },
       "outputs": [],
       "source": [
-        "aiplatform.init(\n",
+        "aip.init(\n",
         "    project=PROJECT_ID,\n",
-        "    staging_bucket=BUCKET_URI,\n",
+        "    staging_bucket=BUCKET_NAME,\n",
         "    location=REGION,\n",
         "    encryption_spec_key_name=ENCRYPTION_SPEC_KEY_NAME,\n",
         ")"
@@ -837,10 +689,10 @@
       },
       "outputs": [],
       "source": [
-        "dataset = aiplatform.ImageDataset.create(\n",
+        "dataset = aip.ImageDataset.create(\n",
         "    display_name=\"flowers_\" + TIMESTAMP,\n",
         "    gcs_source=[IMPORT_FILE],\n",
-        "    import_schema_uri=aiplatform.schema.dataset.ioformat.image.single_label_classification,\n",
+        "    import_schema_uri=aip.schema.dataset.ioformat.image.single_label_classification,\n",
         ")\n",
         "\n",
         "print(dataset.resource_name)"
@@ -875,7 +727,7 @@
         "\n",
         "# This will take around half an hour to run\n",
         "model = job.run(\n",
-        "    dataset=dataset,\n",
+        "    dataset=ds,\n",
         "    model_display_name=\"flowers_\" + TIMESTAMP,\n",
         "    training_fraction_split=0.6,\n",
         "    validation_fraction_split=0.2,\n",
@@ -975,25 +827,6 @@
         "endpoint.undeploy_all()"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "ba77c4e02355"
-      },
-      "source": [
-        "## Cleaning up\n",
-        "\n",
-        "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
-        "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
-        "\n",
-        "Otherwise, you can delete the individual resources you created in this tutorial:\n",
-        "\n",
-        "- Model\n",
-        "- Dataset\n",
-        "- Cloud Storage Bucket\n",
-        "- Endpoint"
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -1002,54 +835,17 @@
       },
       "outputs": [],
       "source": [
-        "# Delete endpoint resource\n",
+        "# missing\n",
         "endpoint.delete()\n",
-        "\n",
-        "# Delete model resource\n",
         "model.delete()\n",
-        "\n",
-        "# Delete dataset resource\n",
         "dataset.delete()\n",
         "\n",
-        "# Set this to true only if you'd like to delete your bucket\n",
-        "delete_bucket = False\n",
-        "\n",
-        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
-        "    ! gsutil rm -r $BUCKET_URI"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "d88c8a053a90"
-      },
-      "source": [
-        "## Destroying CMEK by providing key-version value and other parameters."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "5d8d2fc34346"
-      },
-      "outputs": [],
-      "source": [
-        "! gcloud kms keys versions destroy 1 \\\n",
-        "  --key {KEY_ID} \\\n",
+        "! gcloud kms keys versions destroy key-version \\\n",
+        "  --key key {KEY_ID} \\\n",
         "  --keyring={KEY_RING_ID} \\\n",
         "  --location={REGION} "
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "f7e42e642ad3"
-      },
-      "source": [
-        "## List of keys "
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,

notebooks/community/ml_ops/stage2/get_started_with_logging.ipynb

@@ -1,710 +0,0 @@
-{
-  "cells": [
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "copyright"
-      },
-      "outputs": [],
-      "source": [
-        "# Copyright 2022 Google LLC\n",
-        "#\n",
-        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
-        "# you may not use this file except in compliance with the License.\n",
-        "# You may obtain a copy of the License at\n",
-        "#\n",
-        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
-        "#\n",
-        "# Unless required by applicable law or agreed to in writing, software\n",
-        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
-        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
-        "# See the License for the specific language governing permissions and\n",
-        "# limitations under the License."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "title:generic,gcp"
-      },
-      "source": [
-        "# E2E ML on GCP: MLOps stage 2 : experimentation: get started with Logging\n",
-        "\n",
-        "<table align=\"left\">\n",
-        "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_with_logging.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
-        "      View on GitHub\n",
-        "    </a>\n",
-        "  </td>\n",
-        "    <td>\n",
-        "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_with_logging.ipynb\">\n",
-        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
-        "        </a>\n",
-        "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage2/get_started_with_logging.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
-        "    </a>\n",
-        "  </td>\n",
-        "</table>\n",
-        "<br/><br/><br/>"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "overview:mlops"
-      },
-      "source": [
-        "## Overview\n",
-        "\n",
-        "\n",
-        "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 2 : experimentation: get started with Logging."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "objective:mlops,stage2,get_started_vertex_experiments"
-      },
-      "source": [
-        "### Objective\n",
-        "\n",
-        "In this tutorial, you learn how to use Python and Cloud logging awhen training with `Vertex AI`.\n",
-        "\n",
-        "This tutorial uses the following Google Cloud ML services:\n",
-        "\n",
-        "- `Cloud Logging`\n",
-        "\n",
-        "The steps performed include:\n",
-        "\n",
-        "- Use Python logging to log training configuration/results locally.\n",
-        "- Use Google Cloud Logging to log training configuration/results in cloud storage."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "recommendation:mlops,stage2,logging"
-      },
-      "source": [
-        "### Recommendations\n",
-        "\n",
-        "When doing E2E MLOps on Google Cloud, the following are some of the best practices for logging data when experimenting or formally training a model.\n",
-        "\n",
-        "#### Python Logging\n",
-        "\n",
-        "Use Python's logging package when doing ad-hoc training locally.\n",
-        "\n",
-        "#### Cloud Logging\n",
-        "\n",
-        "Use `Google Cloud Logging` when doing training on the cloud.\n",
-        "\n",
-        "#### Experiments\n",
-        "\n",
-        "Use Vertex AI Experiments in conjunction with logging when performing experiments to compare results for different experiment configurations."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "5341f31587c8"
-      },
-      "source": [
-        "### Dataset\n",
-        "\n",
-        "This tutorial does not use a dataset. References to example datasets is for demonstration purposes."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "41512a89f379"
-      },
-      "source": [
-        "### Costs\n",
-        "This tutorial uses billable components of Google Cloud:\n",
-        "\n",
-        "- Vertex AI\n",
-        "\n",
-        "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing) and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "install_mlops"
-      },
-      "source": [
-        "## Installations\n",
-        "\n",
-        "Install the following packages for executing this notebook."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "install_mlops"
-      },
-      "outputs": [],
-      "source": [
-        "import os\n",
-        "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
-        "! pip3 install --upgrade google-cloud-logging $USER_FLAG -q"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "restart"
-      },
-      "source": [
-        "### Restart the kernel\n",
-        "\n",
-        "Once you've installed the additional packages, you need to restart the notebook kernel so it can find the packages."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "restart"
-      },
-      "outputs": [],
-      "source": [
-        "import os\n",
-        "\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    # Automatically restart kernel after installs\n",
-        "    import IPython\n",
-        "\n",
-        "    app = IPython.Application.instance()\n",
-        "    app.kernel.do_shutdown(True)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "project_id"
-      },
-      "source": [
-        "## Before you begin\n",
-        "\n",
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI, Compute Engine, Cloud Storage and Cloud Logging APIs](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component,storage_component,logging).\n",
-        "\n",
-        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "5aee4379e8e5"
-      },
-      "source": [
-        "#### Set your project ID\n",
-        "\n",
-        "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "set_project_id"
-      },
-      "outputs": [],
-      "source": [
-        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "autoset_project_id"
-      },
-      "outputs": [],
-      "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
-        "    # Get your GCP project id from gcloud\n",
-        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
-        "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID:\", PROJECT_ID)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "set_gcloud_project_id"
-      },
-      "outputs": [],
-      "source": [
-        "! gcloud config set project $PROJECT_ID"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "region"
-      },
-      "source": [
-        "#### Region\n",
-        "\n",
-        "You can also change the `REGION` variable, which is used for operations\n",
-        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
-        "\n",
-        "- Americas: `us-central1`\n",
-        "- Europe: `europe-west4`\n",
-        "- Asia Pacific: `asia-east1`\n",
-        "\n",
-        "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
-        "\n",
-        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "region"
-      },
-      "outputs": [],
-      "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "timestamp"
-      },
-      "source": [
-        "#### Timestamp\n",
-        "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "timestamp"
-      },
-      "outputs": [],
-      "source": [
-        "from datetime import datetime\n",
-        "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "f3bd8c0d0469"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
-        "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
-        "\n",
-        "1. **Click Create service account**.\n",
-        "\n",
-        "2. In the **Service account name** field, enter a name, and click **Create**.\n",
-        "\n",
-        "3. In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex AI\" into the filter box, and select **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "4. Click Create. A JSON file that contains your key downloads to your local environment.\n",
-        "\n",
-        "5. Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "e0953a00668e"
-      },
-      "outputs": [],
-      "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = \"google.colab\" in sys.modules\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "setup_vars"
-      },
-      "source": [
-        "### Set up variables\n",
-        "\n",
-        "Next, set up some variables used throughout the tutorial.\n",
-        "### Import libraries"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "import_aip:mbsdk"
-      },
-      "outputs": [],
-      "source": [
-        "import logging\n",
-        "\n",
-        "import google.cloud.aiplatform as aiplatform"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "init_aip:mbsdk,region"
-      },
-      "source": [
-        "### Initialize Vertex AI SDK for Python\n",
-        "\n",
-        "Initialize the Vertex AI SDK for Python for your project and corresponding bucket."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "init_aip:mbsdk,region"
-      },
-      "outputs": [],
-      "source": [
-        "aiplatform.init(project=PROJECT_ID, location=REGION)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "python_logging"
-      },
-      "source": [
-        "## Python Logging\n",
-        "\n",
-        "The Python logging package is widely used for logging within Python scripts. Commonly used features:\n",
-        "\n",
-        "- Set logging levels.\n",
-        "- Send log output to console.\n",
-        "- Send log output to a file.\n",
-        "\n",
-        "### Logging Levels in Python Logging\n",
-        "\n",
-        "The logging levels in order (from least to highest) and each level inclusive of the previous level are :\n",
-        "\n",
-        "1. Informational\n",
-        "2. Warnings\n",
-        "3. Errors\n",
-        "4. Debugging\n",
-        "\n",
-        "By default, the logging level is set to error level.\n",
-        "\n",
-        "### Logging output to console\n",
-        "\n",
-        "By default, the Python logging package outputs to the console. Note, in the example the debug log message is not outputted since the default logging level is set to error."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "python_logging"
-      },
-      "outputs": [],
-      "source": [
-        "def logging_examples():\n",
-        "    logging.info(\"Model training started...\")\n",
-        "    logging.warning(\"Using older version of package ...\")\n",
-        "    logging.error(\"Training was terminated ...\")\n",
-        "    logging.debug(\"Hyperparameters were ...\")\n",
-        "\n",
-        "\n",
-        "logging_examples()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "python_logging_level"
-      },
-      "source": [
-        "### Setting logging level\n",
-        "\n",
-        "To set the logging level, you get the logging handler using `getLogger()`. You can have multiple logging handles. When `getLogger()` is called without any arguments, it gets the default handler named ROOT. With the handler, you set the logging level with the method `setLevel()`."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "python_logging_level"
-      },
-      "outputs": [],
-      "source": [
-        "logging.getLogger().setLevel(logging.DEBUG)\n",
-        "\n",
-        "logging_examples()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "python_logging_remove"
-      },
-      "source": [
-        "### Clearing handlers\n",
-        "\n",
-        "At times, you may desire to reconfigure your logging. A common practice in this case is to first remove all existing logging handles for a fresh start."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "python_logging_remove"
-      },
-      "outputs": [],
-      "source": [
-        "for handler in logging.root.handlers[:]:\n",
-        "    logging.root.removeHandler(handler)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "python_logging_file"
-      },
-      "source": [
-        "### Output to a local file\n",
-        "\n",
-        "You can preserve your logging output to a file that is local to where the Python script is running with the method `BasicConfig()`, that takes the following parameters:\n",
-        "\n",
-        "- `filename`: The file path to the local file to write the log output to.\n",
-        "- `level`: Sets the level of logging that is written to the logging file.\n",
-        "\n",
-        "*Note:* You cannot use a Cloud Storage bucket as the output file."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "python_logging_file"
-      },
-      "outputs": [],
-      "source": [
-        "logging.basicConfig(filename=\"mylog.log\", level=logging.DEBUG)\n",
-        "\n",
-        "logging_examples()\n",
-        "\n",
-        "! cat mylog.log"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "cloud_logging"
-      },
-      "source": [
-        "## Logging with Google Cloud Logging\n",
-        "\n",
-        "You can preserve and retrieve your logging output to `Google Cloud Logging` service. Commonly used features:\n",
-        "\n",
-        "- Set logging levels.\n",
-        "- Send log output to storage.\n",
-        "- Retrieve log output from storage.\n",
-        "\n",
-        "### Logging Levels in Cloud Logging\n",
-        "\n",
-        "The logging levels in order (from least to highest) are, with each level inclusive of the previous level:\n",
-        "\n",
-        "1. Informational\n",
-        "2. Warnings\n",
-        "3. Errors\n",
-        "4. Debugging\n",
-        "\n",
-        "By default, the logging level is set to warning level.\n",
-        "\n",
-        "### Configurable and storing log data.\n",
-        "\n",
-        "To use the `Google Cloud Logging` service, you do the following steps:\n",
-        "\n",
-        "1. Create a client to the service.\n",
-        "2. Obtain a handler for the service.\n",
-        "3. Create a logger instance and set logging level.\n",
-        "4. Attach logger instance to the service.\n",
-        "\n",
-        "Learn more about [Logging client libraries](https://cloud.google.com/logging/docs/reference/libraries)."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "cloud_logging"
-      },
-      "outputs": [],
-      "source": [
-        "import google.cloud.logging\n",
-        "from google.cloud.logging.handlers import CloudLoggingHandler\n",
-        "\n",
-        "# Connect to the Cloud Logging service\n",
-        "cl_client = google.cloud.logging.Client(project=PROJECT_ID)\n",
-        "handler = CloudLoggingHandler(cl_client, name=\"mylog\")\n",
-        "\n",
-        "# Create a logger instance and logging level\n",
-        "cloud_logger = logging.getLogger(\"cloudLogger\")\n",
-        "cloud_logger.setLevel(logging.INFO)\n",
-        "\n",
-        "# Attach the logger instance to the service.\n",
-        "cloud_logger.addHandler(handler)\n",
-        "\n",
-        "# Log something\n",
-        "cloud_logger.error(\"bad news\")"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "cloud_logging_write"
-      },
-      "source": [
-        "### Logging output\n",
-        "\n",
-        "Logging output at specific levels is identical to Python logging with respect to method and method names. The only difference is that you use your instance of the cloud logger in place of logging."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "cloud_logging_write"
-      },
-      "outputs": [],
-      "source": [
-        "cloud_logger.info(\"Model training started...\")\n",
-        "cloud_logger.warning(\"Using older version of package ...\")\n",
-        "cloud_logger.error(\"Training was terminated ...\")\n",
-        "cloud_logger.debug(\"Hyperparameters were ...\")"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "cloud_logging_list"
-      },
-      "source": [
-        "### Get logging entries\n",
-        "\n",
-        "To get the logged output, you:\n",
-        "\n",
-        "1. Retrieve the log handle to the service.\n",
-        "2. Using the handle, call the method `list_entries()`.\n",
-        "3. Iterate through the entries."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "cloud_logging_list"
-      },
-      "outputs": [],
-      "source": [
-        "logger = cl_client.logger(\"mylog\")\n",
-        "\n",
-        "for entry in logger.list_entries():\n",
-        "    timestamp = entry.timestamp.isoformat()\n",
-        "    print(\"* {}: {}: {}\".format(timestamp, entry.severity, entry.payload))"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "cleanup:mbsdk"
-      },
-      "source": [
-        "# Cleaning up\n",
-        "\n",
-        "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
-        "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
-        "\n",
-        "Otherwise, you can delete the individual resources you created in this tutorial."
-      ]
-    }
-  ],
-  "metadata": {
-    "colab": {
-      "name": "get_started_with_logging.ipynb",
-      "toc_visible": true
-    },
-    "kernelspec": {
-      "display_name": "Python 3",
-      "name": "python3"
-    }
-  },
-  "nbformat": 4,
-  "nbformat_minor": 0
-}