Merge branch 'main' into update_model_eval

.cloud-build/cleanup/cleanup.py

@@ -1,49 +1,45 @@
 from typing import List
-from ratemate import RateLimit
 from resource_cleanup_manager import (
-    DatasetResourceCleanupManager,
-    ModelResourceCleanupManager,
-    EndpointResourceCleanupManager,
-    ResourceCleanupManager,
+	ResourceCleanupManager,
+	DatasetResourceCleanupManager,
+	EndpointResourceCleanupManager,
+	ModelResourceCleanupManager,
 )
 
-rate_limit = RateLimit(max_count=25, per=60, greedy=False)
-
 
 def run_cleanup_managers(managers: List[ResourceCleanupManager], is_dry_run: bool):
-    for manager in managers:
-        type_name = manager.type_name
+	for manager in managers:
+		type_name = manager.type_name
 
-        print(f"Fetching {type_name}'s...")
-        resources = manager.list()
-        print(f"Found {len(resources)} {type_name}'s")
-        for resource in resources:
-            try:
-                if not manager.is_deletable(resource):
-                    continue
+		print(f"Fetching {type_name}'s...")
+		resources = manager.list()
+		print(f"Found {len(resources)} {type_name}'s")
+		for resource in resources:
+			if not manager.is_deletable(resource):
+				continue
 
-                if is_dry_run:
-                    resource_name = manager.resource_name(resource)
-                    print(f"Will delete '{type_name}': {resource_name}")
-                else:
-                    rate_limit.wait()  # wait before deleting
-                    manager.delete(resource)
-            except Exception as exception:
-                print(exception)
+			if is_dry_run:
+				resource_name = manager.resource_name(resource)
+				print(f"Will delete '{type_name}': {resource_name}")
+			else:
+				try:
+					manager.delete(resource)
+				except Exception as exception:
+					print(exception)
 
-        print("")
+		print("")
 
 
 is_dry_run = False
 
 if is_dry_run:
-    print("Starting cleanup in dry run mode...")
+	print("Starting cleanup in dry run mode...")
 
 # List of all cleanup managers
 managers = [
-    DatasetResourceCleanupManager(),
-    EndpointResourceCleanupManager(),
-    ModelResourceCleanupManager(),  # ModelResourceCleanupManager must follow EndpointResourceCleanupManager due to deployed models blocking model deletion.
+	DatasetResourceCleanupManager(),
+	EndpointResourceCleanupManager(),
+	ModelResourceCleanupManager(),
 ]
 
 run_cleanup_managers(managers=managers, is_dry_run=is_dry_run)

.cloud-build/cleanup/resource_cleanup_manager.py

@@ -1,9 +1,8 @@
 import abc
-from typing import Any, Type
-
 from google.cloud import aiplatform
-from google.cloud.aiplatform import base
+from typing import Any
 from proto.datetime_helpers import DatetimeWithNanoseconds
+from google.cloud.aiplatform import base
 
 # If a resource was updated within this number of seconds, do not delete.
 RESOURCE_UPDATE_BUFFER_IN_SECONDS = 60 * 60 * 8
@@ -41,7 +40,7 @@ class ResourceCleanupManager(abc.ABC):
         # Check that it wasn't created too recently, to prevent race conditions
         if time_difference <= RESOURCE_UPDATE_BUFFER_IN_SECONDS:
             print(
-                f"Skipping '{resource}' due to update_time being '{time_difference}', which is less than '{RESOURCE_UPDATE_BUFFER_IN_SECONDS}'."
+                f"Skipping '{resource}' due update_time being '{time_difference}', which is less than '{RESOURCE_UPDATE_BUFFER_IN_SECONDS}'."
             )
             return False
 
@@ -51,7 +50,7 @@ class ResourceCleanupManager(abc.ABC):
 class VertexAIResourceCleanupManager(ResourceCleanupManager):
     @property
     @abc.abstractmethod
-    def vertex_ai_resource(self) -> Type[base.VertexAiResourceNounWithFutureManager]:
+    def vertex_ai_resource(self) -> base.VertexAiResourceNounWithFutureManager:
         pass
 
     @property
@@ -61,9 +60,7 @@ class VertexAIResourceCleanupManager(ResourceCleanupManager):
     def list(self) -> Any:
         return self.vertex_ai_resource.list()
 
-    def resource_name(
-        self, resource: Type[base.VertexAiResourceNounWithFutureManager]
-    ) -> str:
+    def resource_name(self, resource: Any) -> str:
         return resource.display_name
 
     def delete(self, resource):
@@ -77,33 +74,12 @@ class VertexAIResourceCleanupManager(ResourceCleanupManager):
 
 class DatasetResourceCleanupManager(VertexAIResourceCleanupManager):
     vertex_ai_resource = aiplatform.datasets._Dataset
-    dataset_types = [
-        aiplatform.ImageDataset,
-        aiplatform.TabularDataset,
-        aiplatform.TextDataset,
-        aiplatform.TimeSeriesDataset,
-        aiplatform.VideoDataset,
-    ]
-
-    def list(self) -> Any:
-        return [
-            dataset
-            for dataset_type in self.dataset_types
-            for dataset in dataset_type.list()
-        ]
 
 
 class EndpointResourceCleanupManager(VertexAIResourceCleanupManager):
     vertex_ai_resource = aiplatform.Endpoint
 
     def delete(self, resource):
-        # TODO: Remove this once https://github.com/googleapis/python-aiplatform/issues/1441 is fixed
-        resource._sync_gca_resource()
-        for deployed_model_id in [
-            models.id for models in resource._gca_resource.deployed_models
-        ]:
-            resource._undeploy(deployed_model_id=deployed_model_id)
-
         resource.delete(force=True)
 
 

.cloud-build/execute_changed_notebooks_cli.py

@@ -17,7 +17,6 @@
 
 import argparse
 import pathlib
-
 import execute_changed_notebooks_helper
 
 
@@ -74,13 +73,6 @@ parser.add_argument(
     help="The GCP directory for storing executed notebooks.",
     required=True,
 )
-parser.add_argument(
-    "--timeout",
-    type=int,
-    help="Timeout in seconds",
-    default=86400,
-    required=False,
-)
 parser.add_argument(
     "--private_pool_id",
     type=str,
@@ -110,7 +102,6 @@ execute_changed_notebooks_helper.process_and_execute_notebooks(
     artifacts_bucket=args.artifacts_bucket,
     variable_project_id=args.variable_project_id,
     variable_region=args.variable_region,
-    private_pool_id=args.private_pool_id,
+    private_pool_id=args.private_pool_id if not "default" else None,
     should_parallelize=args.should_parallelize,
-    timeout=args.timeout,
 )

.cloud-build/execute_changed_notebooks_helper.py

@@ -17,24 +17,18 @@ import concurrent
 import dataclasses
 import datetime
 import functools
-import git
-import operator
 import os
 import pathlib
+import nbformat
 import re
 import subprocess
 from typing import List, Optional
-
-import execute_notebook_helper
-import execute_notebook_remote
-import nbformat
-from google.cloud.devtools.cloudbuild_v1.types import BuildOperationMetadata
-from ratemate import RateLimit
 from tabulate import tabulate
-from utils import NotebookProcessors, util
+import operator
 
-# A buffer so that workers finish before the orchestrating job
-WORKER_TIMEOUT_BUFFER_IN_SECONDS: int = 60 * 60
+import execute_notebook_remote
+from utils import util, NotebookProcessors
+from google.cloud.devtools.cloudbuild_v1.types import BuildOperationMetadata
 
 
 def format_timedelta(delta: datetime.timedelta) -> str:
@@ -109,9 +103,6 @@ def _create_tag(filepath: str) -> str:
     return tag
 
 
-rate_limit = RateLimit(max_count=50, per=60, greedy=True)
-
-
 def process_and_execute_notebook(
     container_uri: str,
     staging_bucket: str,
@@ -119,12 +110,9 @@ def process_and_execute_notebook(
     variable_project_id: str,
     variable_region: str,
     private_pool_id: Optional[str],
-    deadline: datetime,
     notebook: str,
     should_get_tail_logs: bool = False,
 ) -> NotebookExecutionResult:
-    rate_limit.wait()  # wait before creating the task
-
     print(f"Running notebook: {notebook}")
 
     # Create paths
@@ -157,20 +145,14 @@ def process_and_execute_notebook(
         # Upload the pre-processed code to a GCS bucket
         code_archive_uri = util.archive_code_and_upload(staging_bucket=staging_bucket)
 
-        # Calculate timeout in seconds
-        timeout_in_seconds = max(
-            int((deadline - datetime.datetime.now()).total_seconds()), 1
-        )
-
         operation = execute_notebook_remote.execute_notebook_remote(
             code_archive_uri=code_archive_uri,
             notebook_uri=notebook,
             notebook_output_uri=notebook_output_uri,
             container_uri=container_uri,
             tag=tag,
+            region=variable_region,
             private_pool_id=private_pool_id,
-            private_pool_region=variable_region,
-            timeout_in_seconds=timeout_in_seconds,
         )
 
         operation_metadata = BuildOperationMetadata(mapping=operation.metadata)
@@ -233,40 +215,20 @@ def get_changed_notebooks(
 
     # Find notebooks
     notebooks = []
-
-    # Instantiate GitPython objects
-    repo = git.Repo(os.getcwd())
-    index = repo.index
-
     if base_branch:
-        # Get the point at which this branch branches off from main
-        branching_commits = repo.merge_base("HEAD", f"origin/{base_branch}")
-
-        if len(branching_commits) > 0:
-            branching_commit = branching_commits[0]
-            print(f"Looking for notebooks that changed from branch: {branching_commit}")
-
-            notebooks = [
-                diff.b_path
-                for diff in index.diff(branching_commit, paths=test_paths)
-                if diff.b_path is not None
-            ]
-        else:
-            notebooks = []
+        print(f"Looking for notebooks that changed from branch: {base_branch}")
+        notebooks = subprocess.check_output(
+            ["git", "diff", "--name-only", f"origin/{base_branch}..."] + test_paths
+        )
     else:
         print(f"Looking for all notebooks.")
         notebooks = subprocess.check_output(["git", "ls-files"] + test_paths)
-        notebooks = notebooks.decode("utf-8").split("\n")
 
+    notebooks = notebooks.decode("utf-8").split("\n")
     notebooks = [notebook for notebook in notebooks if notebook.endswith(".ipynb")]
     notebooks = [notebook for notebook in notebooks if len(notebook) > 0]
     notebooks = [notebook for notebook in notebooks if pathlib.Path(notebook).exists()]
 
-    if len(notebooks) > 0:
-        print(f"Found {len(notebooks)} notebooks:")
-        for notebook in notebooks:
-            print(f"\t{notebook}")
-
     return notebooks
 
 
@@ -279,7 +241,6 @@ def process_and_execute_notebooks(
     variable_region: str,
     private_pool_id: Optional[str],
     should_parallelize: bool,
-    timeout: int,
 ):
     """
     Run the notebooks that exist under the folders defined in the test_paths_file.
@@ -306,27 +267,17 @@ def process_and_execute_notebooks(
             Required. The value for REGION to inject into notebooks.
         should_parallelize (bool):
             Required. Should run notebooks in parallel using a thread pool as opposed to in sequence.
-        timeout (str):
-            Required. Timeout string according to https://cloud.google.com/build/docs/build-config-file-schema#timeout.
     """
+    notebook_execution_results: List[NotebookExecutionResult] = []
 
-    # Calculate deadline
-    deadline = datetime.datetime.now() + datetime.timedelta(
-        seconds=max(timeout - WORKER_TIMEOUT_BUFFER_IN_SECONDS, 0)
-    )
-
-    if len(notebooks) > 1:
-        notebook_execution_results: List[NotebookExecutionResult] = []
-
+    if len(notebooks) > 0:
         print(f"Found {len(notebooks)} modified notebooks: {notebooks}")
 
         if should_parallelize and len(notebooks) > 1:
             print(
                 "Running notebooks in parallel, so no logs will be displayed. Please wait..."
             )
-            with concurrent.futures.ThreadPoolExecutor(max_workers=100) as executor:
-                print(f"Max workers: {executor._max_workers}")
-
+            with concurrent.futures.ThreadPoolExecutor(max_workers=None) as executor:
                 notebook_execution_results = list(
                     executor.map(
                         functools.partial(
@@ -337,7 +288,6 @@ def process_and_execute_notebooks(
                             variable_project_id,
                             variable_region,
                             private_pool_id,
-                            deadline,
                         ),
                         notebooks,
                     )
@@ -351,69 +301,47 @@ def process_and_execute_notebooks(
                     variable_project_id=variable_project_id,
                     variable_region=variable_region,
                     private_pool_id=private_pool_id,
-                    deadline=deadline,
                     notebook=notebook,
                 )
                 for notebook in notebooks
             ]
-
-        print("\n=== RESULTS ===\n")
-
-        results_sorted = sorted(
-            notebook_execution_results,
-            key=lambda result: result.is_pass,
-            reverse=True,
-        )
-
-        # Print results
-        print(
-            tabulate(
-                [
-                    [
-                        result.name,
-                        "PASSED" if result.is_pass else "FAILED",
-                        format_timedelta(result.duration),
-                        result.log_url,
-                        result.output_uri,
-                    ]
-                    for result in results_sorted
-                ],
-                headers=["build_tag", "status", "duration", "log_url", "output_url"],
-            )
-        )
-
-        print("\n=== END RESULTS===\n")
-
-        total_notebook_duration = functools.reduce(
-            operator.add,
-            [datetime.timedelta(seconds=0)]
-            + [result.duration for result in results_sorted],
-        )
-
-        print(
-            f"Cumulative notebook duration: {format_timedelta(total_notebook_duration)}"
-        )
-
-        # Raise error if any notebooks failed
-        if not all([result.is_pass for result in results_sorted]):
-            raise RuntimeError("Notebook failures detected. See logs for details")
-
-    elif len(notebooks) == 1:
-        notebook = notebooks[0]
-
-        # Pre-process notebook by substituting variable names
-        _process_notebook(
-            notebook_path=notebook,
-            variable_project_id=variable_project_id,
-            variable_region=variable_region,
-        )
-
-        execute_notebook_helper.execute_notebook(
-            notebook_source=notebook,
-            output_file_or_uri="/".join(
-                [artifacts_bucket, pathlib.Path(notebook).name]
-            ),
-            should_log_output=True,
-        )
     else:
         print("No notebooks modified in this pull request.")
+
+    print("\n=== RESULTS ===\n")
+
+    results_sorted = sorted(
+        notebook_execution_results,
+        key=lambda result: result.is_pass,
+        reverse=True,
+    )
+
+    # Print results
+    print(
+        tabulate(
+            [
+                [
+                    result.name,
+                    "PASSED" if result.is_pass else "FAILED",
+                    format_timedelta(result.duration),
+                    result.log_url,
+                ]
+                for result in results_sorted
+            ],
+            headers=["build_tag", "status", "duration", "log_url"],
+        )
+    )
+
+    print("\n=== END RESULTS===\n")
+
+    total_notebook_duration = functools.reduce(
+        operator.add,
+        [datetime.timedelta(seconds=0)]
+        + [result.duration for result in results_sorted],
+    )
+
+    print(f"Cumulative notebook duration: {format_timedelta(total_notebook_duration)}")
+
+    # Raise error if any notebooks failed
+    if not all([result.is_pass for result in results_sorted]):
+        raise RuntimeError("Notebook failures detected. See logs for details")

.cloud-build/execute_notebook_cli.py

@@ -16,7 +16,6 @@
 """A CLI to download (optional) and run a single notebook locally"""
 
 import argparse
-
 import execute_notebook_helper
 
 parser = argparse.ArgumentParser(description="Run a single notebook locally.")

.cloud-build/execute_notebook_helper.py

@@ -15,14 +15,14 @@
 
 """Methods to run a notebook locally"""
 
-import errno
-import os
-import shutil
 import sys
-
+import os
+import errno
 import papermill as pm
-from google.cloud.aiplatform import utils
+import shutil
+
 from utils import util
+from google.cloud.aiplatform import utils
 
 # This script is used to execute a notebook and write out the output notebook.
 

.cloud-build/execute_notebook_remote.py

@@ -16,18 +16,22 @@
 """Methods to run a notebook on Google Cloud Build"""
 
 from re import sub
-from typing import Optional
-
-import google.auth
-import yaml
-from google.api_core import client_options, operation
-from google.cloud.aiplatform import utils
-from google.cloud.devtools import cloudbuild_v1
-from google.cloud.devtools.cloudbuild_v1.types import Source, StorageSource
 from google.protobuf import duration_pb2
 from yaml.loader import FullLoader
 
+import google.auth
+from google.cloud.devtools import cloudbuild_v1
+from google.cloud.devtools.cloudbuild_v1.types import Source, StorageSource
+
+from typing import Optional
+import yaml
+
+from google.cloud.aiplatform import utils
+from google.api_core import operation, client_options
+
+
 CLOUD_BUILD_FILEPATH = ".cloud-build/notebook-execution-test-cloudbuild-single.yaml"
+TIMEOUT_IN_SECONDS = 86400
 SERVICE_BASE_PATH = "cloudbuild.googleapis.com"
 
 
@@ -36,14 +40,12 @@ def execute_notebook_remote(
     notebook_uri: str,
     notebook_output_uri: str,
     container_uri: str,
+    region: str,
     private_pool_id: Optional[str],
-    private_pool_region: Optional[str],
     tag: Optional[str],
-    timeout_in_seconds: Optional[int] = None,
 ) -> operation.Operation:
     """Create and execute a single notebook on Google Cloud Build"""
     # Load build steps from YAML
-
     cloudbuild_config = yaml.load(open(CLOUD_BUILD_FILEPATH), Loader=FullLoader)
 
     substitutions = {
@@ -55,14 +57,13 @@ def execute_notebook_remote(
     build = cloudbuild_v1.Build()
 
     options: Optional[client_options.ClientOptions] = None
-    if private_pool_id and private_pool_region:
-        # substitutions["_PRIVATE_POOL_NAME"] = private_pool_id
-        build.options = cloudbuild_config.get("options")
-        build.options.pool = {"name": private_pool_id}
+    if private_pool_id:
+        substitutions["_PRIVATE_POOL_NAME"] = private_pool_id
+        build.options = cloudbuild_config["options"]
 
         # Switch to the regional endpoint of the pool
         options = client_options.ClientOptions(
-            api_endpoint=f"{private_pool_region}-{SERVICE_BASE_PATH}"
+            api_endpoint=f"{region}-{SERVICE_BASE_PATH}"
         )
 
     # Authorize the client with Google defaults
@@ -84,8 +85,8 @@ def execute_notebook_remote(
 
     build.steps = cloudbuild_config["steps"]
     build.substitutions = substitutions
-    build.timeout = duration_pb2.Duration(seconds=timeout_in_seconds)
-    build.queue_ttl = duration_pb2.Duration(seconds=timeout_in_seconds)
+    build.timeout = duration_pb2.Duration(seconds=TIMEOUT_IN_SECONDS)
+    build.queue_ttl = duration_pb2.Duration(seconds=TIMEOUT_IN_SECONDS)
 
     if tag:
         build.tags = [tag]

.cloud-build/notebook-execution-test-cloudbuild-single.yaml

@@ -10,37 +10,22 @@ steps:
     entrypoint: /bin/sh
     args:
     - -c
-    - python3 .cloud-build/CheckPythonVersion.py
-  # Create a virtual environment
-  - name: ${_PYTHON_IMAGE}
-    entrypoint: /bin/sh
-    args:
-    - -c
-    - python3 -m venv workspace/env
+    - 'python3 .cloud-build/CheckPythonVersion.py'
   # Install Python dependencies
-  - name: ${_PYTHON_IMAGE}
-    entrypoint: /bin/sh
-    args:
-    - -c 
-    - . workspace/env/bin/activate &&
-      python3 -m pip install -U pip &&
-      python3 -m pip install -U -r .cloud-build/requirements.txt
-  # pip freeze
   - name: ${_PYTHON_IMAGE}
     entrypoint: /bin/sh
     args:
     - -c
-    - |
-      . workspace/env/bin/activate &&
-      python3 -m pip freeze    
+    - 'python3 -m pip install -U pip && python3 -m pip install -U --user -r .cloud-build/requirements.txt'
   # Install Python dependencies and run testing script
   - name: ${_PYTHON_IMAGE}
     entrypoint: /bin/sh
     args:
     - -c
-    - | 
-      . workspace/env/bin/activate &&
-      python3 .cloud-build/execute_notebook_cli.py --notebook_source "${_NOTEBOOK_GCS_URI}" --output_file_or_uri "${_NOTEBOOK_OUTPUT_GCS_URI}"
+    - 'python3 -m pip install -U pip && python3 -m pip freeze && python3 .cloud-build/execute_notebook_cli.py --notebook_source "${_NOTEBOOK_GCS_URI}" --output_file_or_uri "${_NOTEBOOK_OUTPUT_GCS_URI}"'
     env:
     - 'IS_TESTING=1'
-timeout: 86400s
+timeout: 86400s
+options:
+  pool:
+    name: ${_PRIVATE_POOL_NAME}

.cloud-build/notebook-execution-test-cloudbuild.yaml

@@ -4,47 +4,32 @@ steps:
     entrypoint: /bin/sh
     args:
     - -c
-    - gcloud config list
+    - 'gcloud config list'
   # Check the Python version
   - name: ${_PYTHON_IMAGE}
     entrypoint: /bin/sh
     args:
     - -c
-    - python3 .cloud-build/CheckPythonVersion.py
-  # Fetch full repo for diff purposes
-  - name: gcr.io/cloud-builders/git
-    args: [fetch, --unshallow]
-  # Create a virtual environment
+    - 'python3 .cloud-build/CheckPythonVersion.py'
+  # Fetch base branch if required
   - name: ${_PYTHON_IMAGE}
     entrypoint: /bin/sh
     args:
     - -c
-    - python3 -m venv workspace/env
+    - 'if [ -n "${_BASE_BRANCH}" ]; then git fetch origin "${_BASE_BRANCH}":refs/remotes/origin/"${_BASE_BRANCH}"; else echo "Skipping fetch."; fi'
   # Install Python dependencies
-  - name: ${_PYTHON_IMAGE}
-    entrypoint: /bin/sh
-    args:
-    - -c 
-    - . workspace/env/bin/activate &&
-      python3 -m pip install -U pip &&
-      python3 -m pip install -U -r .cloud-build/requirements.txt
-  # pip freeze
   - name: ${_PYTHON_IMAGE}
     entrypoint: /bin/sh
     args:
     - -c
-    - |
-      . workspace/env/bin/activate &&
-      python3 -m pip freeze
+    - 'python3 -m pip install -U pip && python3 -m pip install -U --user -r .cloud-build/requirements.txt'
   # Install Python dependencies and run testing script
   # TODO: Only pass in private_pool_id if it is set
   - name: ${_PYTHON_IMAGE}
     entrypoint: /bin/sh
     args:
     - -c
-    - | 
-      . workspace/env/bin/activate &&
-      python3 .cloud-build/execute_changed_notebooks_cli.py --test_paths_file "${_TEST_PATHS_FILE}" --base_branch "${_FORCED_BASE_BRANCH}" --container_uri ${_PYTHON_IMAGE} --staging_bucket ${_GCS_STAGING_BUCKET}  --artifacts_bucket ${_GCS_STAGING_BUCKET}/executed_notebooks/PR_${_PR_NUMBER}/BUILD_${BUILD_ID} --variable_project_id ${PROJECT_ID} --variable_region ${_GCP_REGION} `if [ ! -z "${_PRIVATE_POOL_NAME}" ]; then echo "--private_pool_id ${_PRIVATE_POOL_NAME}"; fi`
+    - 'python3 -m pip install -U pip && python3 -m pip freeze && python3 .cloud-build/execute_changed_notebooks_cli.py --test_paths_file "${_TEST_PATHS_FILE}" --base_branch "${_FORCED_BASE_BRANCH}" --container_uri ${_PYTHON_IMAGE} --staging_bucket ${_GCS_STAGING_BUCKET}  --artifacts_bucket ${_GCS_STAGING_BUCKET}/executed_notebooks/PR_${_PR_NUMBER}/BUILD_${BUILD_ID} --variable_project_id ${PROJECT_ID} --variable_region ${_GCP_REGION} `if [ ! -z "${_PRIVATE_POOL_NAME}" ]; then echo "--private_pool_id ${_PRIVATE_POOL_NAME}"; fi`'
     env:
     - 'IS_TESTING=1'
 timeout: 86400s

.cloud-build/requirements.txt

@@ -9,5 +9,4 @@ tabulate
 google-cloud-aiplatform
 google-cloud-storage
 google-cloud-build
-ratemate
-GitPython
+gcloud

.cloud-build/test_folders.txt

@@ -1,2 +1,3 @@
 notebooks/official
 notebooks/notebook_template.ipynb
+notebooks/community/ml_ops

.cloud-build/utils/NotebookProcessors.py

@@ -13,10 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Dict
-
 from nbconvert.preprocessors import Preprocessor
-
+from typing import Dict
 from . import UpdateNotebookVariables as update_notebook_variables
 
 
@@ -62,4 +60,4 @@ class UpdateVariablesPreprocessor(Preprocessor):
 
             executable_cells.append(cell)
         notebook.cells = executable_cells
-        return notebook, resources
+        return notebook, resources

.cloud-build/utils/UpdateNotebookVariables.py

@@ -78,4 +78,4 @@ def test_region():
         variable_name="REGION",
         variable_value="us-central1",
     )
-    assert new_content == 'REGION = "us-central1"  # @param {type:"string"}'
+    assert new_content == 'REGION = "us-central1"  # @param {type:"string"}'

.cloud-build/utils/util.py

@@ -1,13 +1,13 @@
+from datetime import datetime
+from typing import Optional
+from google.cloud import storage
+from google.cloud.aiplatform import utils
+from google.auth import credentials as auth_credentials
 import os
+
 import subprocess
 import tarfile
 import uuid
-from datetime import datetime
-from typing import Optional
-
-from google.auth import credentials as auth_credentials
-from google.cloud import storage
-from google.cloud.aiplatform import utils
 
 
 def download_file(bucket_name: str, blob_name: str, destination_file: str) -> str:
@@ -57,4 +57,4 @@ def archive_code_and_upload(staging_bucket: str):
 
     print(f"Uploaded source code archive to {source_archived_file_gcs}")
 
-    return source_archived_file_gcs
+    return source_archived_file_gcs

.github/PULL_REQUEST_TEMPLATE.md

@@ -2,17 +2,17 @@ If you are opening a PR for `Official Notebooks` under the [notebooks/official](
 - [ ] Use the [notebook template](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/notebook_template.ipynb) as a starting point.
 - [ ] Follow the style and grammar rules outlined in the above notebook template.
 - [ ] Verify the notebook runs successfully in Colab since the automated tests cannot guarantee this even when it passes.
-- [ ] Passes all the required automated checks. You can locally test for formatting and linting with these [instructions](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/CONTRIBUTING.md#code-quality-checks).
+- [ ] Passes all the required automated checks. You can locally test for formatting and linting with these [instructions](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/docs/contributing.md#code-quality-checks).
 - [ ] You have consulted with a tech writer to see if tech writer review is necessary. If so, the notebook has been reviewed by a tech writer, and they have approved it.
-- [ ] This notebook has been added to the [CODEOWNERS](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/CODEOWNERS) file under the `Official Notebooks` section, pointing to the author or the author's team.
+- [ ] This notebook has been added to the [CODEOWNERS](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/docs/CODEOWNERS) file under `# Official Notebooks` section, pointing to the author or the author's team.
 - [ ] The Jupyter notebook cleans up any artifacts it has created (datasets, ML models, endpoints, etc) so as not to eat up unnecessary resources.
 
 
 If you are opening a PR for `Community Notebooks` under the [notebooks/community](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/notebooks/community) folder:
-- [ ] This notebook has been added to the [CODEOWNERS](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/CODEOWNERS) file under the `Community Notebooks` section, pointing to the author or the author's team.
-- [ ] Passes all the required formatting and linting checks. You can locally test with these [instructions](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/CONTRIBUTING.md#code-quality-checks).
+- [ ] This notebook has been added to the [CODEOWNERS](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/docs/CODEOWNERS) file under the `# Community Notebooks` section, pointing to the author or the author's team.
+- [ ] Passes all the required formatting and linting checks. You can locally test with these [instructions](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/docs/contributing.md#code-quality-checks).
 
 If you are opening a PR for `Community Content` under the [community-content](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/community-content) folder:
 - [ ] Make sure your main `Content Directory Name` is descriptive, informative, and includes some of the key products and attributes of your content, so that it is differentiable from other content
-- [ ] The main content directory has been added to the [CODEOWNERS](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/community-content/CODEOWNERS) file under the `Community Content` section, pointing to the author or the author's team.
-- [ ] Passes all the required formatting and linting checks. You can locally test with these [instructions](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/CONTRIBUTING.md#code-quality-checks).
+- [ ] The main content directory has been added to the [CODEOWNERS](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/docs/CODEOWNERS) file under the `# Community Content` section, pointing to the author or the author's team.
+- [ ] Passes all the required formatting and linting checks. You can locally test with these [instructions](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/docs/contributing.md#code-quality-checks).

.github/workflows/ci.yaml

@@ -7,9 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Set up Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: '3.x'      
+      uses: actions/setup-python@v3
     - name: Fetch pull request branch
       uses: actions/checkout@v3
       with:

.github/workflows/linter/requirements.txt

@@ -3,8 +3,7 @@ ipython
 jupyter
 nbconvert
 black==22.3.0
-pyupgrade==2.34.0
+pyupgrade==2.31.1
 isort==5.10.1
 flake8==4.0.1
 nbqa==1.3.1
-

CONTRIBUTING.md

@@ -48,8 +48,8 @@ then you will need to manually address them before submitting your PR.
 nbqa black "$notebook"
 nbqa pyupgrade "$notebook"
 nbqa isort "$notebook"
-nbqa flake8 "$notebook" --extend-ignore=W391,E501,F821,E402,F404,W503,E203,E722,W293,W291
 python3 -m tensorflow_docs.tools.nbfmt --remove_outputs "$notebook"
+nbqa flake8 "$notebook" --extend-ignore=W391,E501,F821,E402,F404,W503,E203,E722,W293,W291
 ```
 
 ## Code Reviews

community-content/tf_agents_bandits_movie_recommendation_with_kfp_and_vertex_sdk/mlops_pipeline_tf_agents_bandits_movie_recommendation/src/logger/requirements.txt

@@ -1,4 +1,4 @@
 google-cloud-bigquery==2.20.0
-tensorflow==2.7.2
+tensorflow==2.5.3
 pillow==9.0.1
 tf-agents==0.8.0

community-content/tf_agents_bandits_movie_recommendation_with_kfp_and_vertex_sdk/mlops_pipeline_tf_agents_bandits_movie_recommendation/src/prediction_container/requirements.txt

@@ -1,4 +1,4 @@
 google-cloud-pubsub==2.5.0
 pillow==9.0.1
 tf-agents==0.8.0
-tensorflow==2.7.2
+tensorflow==2.5.3

community-content/tf_agents_bandits_movie_recommendation_with_kfp_and_vertex_sdk/mlops_pipeline_tf_agents_bandits_movie_recommendation/src/simulator/requirements.txt

@@ -1,5 +1,5 @@
 dataclasses==0.6
 google-cloud-aiplatform==1.8.1
-tensorflow==2.7.2
+tensorflow==2.5.3
 pillow==9.0.1
 tf-agents==0.8.0

community-content/tf_keras_image_classification_distributed_multi_worker_with_vertex_sdk/trainer/requirements.txt

@@ -1 +1 @@
-tensorflow==2.7.2
+tensorflow==2.5.3

community-content/tf_keras_text_classification_distributed_single_worker_gpus_with_gcloud_local_run_and_vertex_sdk/trainer/requirements.txt

@@ -1 +1 @@
-tensorflow==2.7.2
+tensorflow==2.5.3

notebooks/README.md

@@ -1,5 +1,5 @@
-The [official](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/notebooks/official) folder contains notebooks organized by Google Cloud product. These are tested weekly and maintained by Google.
+The [official](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/notebooks/official) folder contains notebooks organized by Google Cloud product.
 
-The [community](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/notebooks/community) folder contains notebooks that may be created by Google or external contributors. They are not necessary maintained.
+The [community](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/notebooks/community) folder contains notebooks that aren't officially supported by Google.
 
 Contributions to the repo should use the [notebook template](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/notebook_template.ipynb) as a starting point.

notebooks/community/CODEOWNERS

@@ -12,17 +12,11 @@
 /managed_notebooks/
 /sdk/SDK_FBProphet_Forecasting_Online.ipynb @brianchunkang
 /pipelines/google_cloud_pipeline_components_TPU_model_train_upload_deploy.ipynb @brianchunkang
-/explainable_ai/SDK_Custom_Container_XAI.ipynb @brianchunkang
 /matching_engine/sdk_matching_engine_for_indexing.ipynb @ivanmkc
 /matching_engine/matching_engine_for_indexing.ipynb @yinghsienwu
 /sdk/pytorch_lightning_custom_container_training.ipynb @brianchunkang
 /tensorboard @yfang1
 /feature_store @nayaknishant @morgandu
-/prediction @googleapis/vertex-prediction-team
 /vertex_endpoints/tf_hub_obj_detection/deploy_tfhub_object_detection_on_vertex_endpoints.ipynb @entrpn
 /vertex_endpoints/nvidia-triton/nvidia-triton-custom-container-prediction.ipynb @RajeshThallam
 /vertex_endpoints/optimized_tensorflow_runtime @vlasenkoalexey
-/notebooks/community/ml_ops/stage2/get_started_with_visionapi_and_automl.ipynb @mansari
-/notebooks/community/neo4j/graph_paysim.ipynb @benofben @laeg
-/notebooks/community/ml_ops/stage1/get_started_with_visionapi_and_vertex_datasets.ipynb @mansari
-/notebooks/community/pipelines/google_cloud_pipeline_components_bqml_pipeline_demand_forecasting.ipynb @inardini

notebooks/community/feature_store/mobile_gaming/mobile_gaming_feature_store.ipynb

@@ -32,18 +32,18 @@
         "<table align=\"left\">\n",
         "\n",
         "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/vertex-ai-samples/blob/main/notebooks/community/feature_store/mobile_gaming/mobile_gaming_feature_store.ipynb\">\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/notebook_template.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/feature_store/mobile_gaming/mobile_gaming_feature_store.ipynb\">\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/notebook_template.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/feature_store/mobile_gaming/mobile_gaming_feature_store.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/notebook_template.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
@@ -54,52 +54,52 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "tvgnzT1CKxrO"
+        "id": "7FZeBEwdXS4d"
       },
       "source": [
         "## Overview\n",
-        " \n",
-        "Imagine you are a member of the Data Science team working on the same Mobile Gaming application reported in the [Churn prediction for game developers using Google Analytics 4 (GA4) and BigQuery ML](https://cloud.google.com/blog/topics/developers-practitioners/churn-prediction-game-developers-using-google-analytics-4-ga4-and-bigquery-ml) blog post.\n",
-        " \n",
-        "Business wants to use that information in real-time to take immediate intervention actions in-game to prevent churn. In particular, for each player, they want to provide gaming incentives like new items or bonus packs depending on the customer demographic,  behavioral information and the resulting propensity of return.\n",
-        " \n",
-        "Last year, Google Cloud announced Vertex AI, a managed machine learning (ML) platform that allows data science teams to accelerate the deployment and maintenance of ML models. One of the platform building blocks is Vertex AI Feature store which provides a managed service for low latency scalable feature serving. Also it is a centralized feature repository with easy APIs to search & discover features and feature monitoring capabilities to track drift and other quality issues.\n",
-        " \n",
-        "In this notebook, we will show how the role of Vertex AI Feature Store in a ready to production scenario when the user's activities within the first 24 hours of last engagement and the gaming platform would consume in order to improve UX. Below you can find the high level picture of the system\n",
-        " \n",
+        "\n",
+        "Imagine you are a member of the Data Science team working on the same Mobile Gaming application reported in the [Churn prediction for game developers using Google Analytics 4 (GA4) and BigQuery ML](https://cloud.google.com/blog/topics/developers-practitioners/churn-prediction-game-developers-using-google-analytics-4-ga4-and-bigquery-ml) blog post. \n",
+        "\n",
+        "Business wants to use that information in real-time to take immediate intervention actions in-game to prevent churn. In particular, for each player, they want to provide gaming incentives like new items or bonus packs depending on the customer demographic,  behavioral information and the resulting propensity of return. \n",
+        "\n",
+        "Last year, Google Cloud announced Vertex AI, a managed machine learning (ML) platform that allows data science teams to accelerate the deployment and maintenance of ML models. One of the platform building blocks is Vertex AI Feature store which provides a managed service for low latency scalable feature serving. Also it is a centralized feature repository with easy APIs to search & discover features and feature monitoring capabilities to track drift and other quality issues. \n",
+        "\n",
+        "In this notebook, we will show how the role of Vertex AI Feature Store in a ready to production scenario when the user's activities within the first 24 hours of last engagment and the gaming platform would consume in order to improver UX. Below you can find the high level picture of the system\n",
+        "\n",
         "<img src=\"./assets/mobile_gaming_architecture_1.png\">\n",
-        " \n",
-        " \n",
+        "\n",
+        "\n",
         "### Dataset\n",
-        " \n",
+        "\n",
         "The dataset is the public sample export data from an actual mobile game app called \"Flood It!\" (Android, iOS)\n",
-        " \n",
+        "\n",
         "### Objective\n",
-        " \n",
+        "\n",
         "In the following notebook, you will learn how Vertex AI Feature store\n",
-        " \n",
-        "1.  Provide a centralized feature repository with easy APIs to search & discover features and fetch them for training/serving.\n",
-        " \n",
-        "2.  Simplify deployments of models for Online Prediction, via low latency scalable feature serving.\n",
-        " \n",
-        "3.  Mitigate training serving skew and data leakage by performing point in time lookups to fetch historical data for training.\n",
-        " \n",
+        "\n",
+        "1.   Provide a centralized feature repository with easy APIs to search & discover features and fetch them for training/serving. \n",
+        "\n",
+        "2.   Simplify deployments of models for Online Prediction, via low latency scalable feature serving.\n",
+        "\n",
+        "3.   Mitigate training serving skew and data leakage by performing point in time lookups to fetch historical data for training.\n",
+        "\n",
         "**Notice that we assume that already know how to set up a Vertex AI Feature store. In case you are not, please check out [this detailed notebook](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/feature_store/gapic-feature-store.ipynb).**\n",
-        " \n",
-        " \n",
-        "### Costs\n",
-        " \n",
+        "\n",
+        "\n",
+        "### Costs \n",
+        "\n",
         "This tutorial uses billable components of Google Cloud:\n",
-        " \n",
+        "\n",
         "* Vertex AI\n",
         "* BigQuery\n",
         "* Cloud Storage\n",
-        " \n",
+        "\n",
         "Learn about [Vertex AI\n",
         "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
         "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
         "Calculator](https://cloud.google.com/products/calculator/)\n",
-        "to generate a cost estimate based on your projected usage.\n"
+        "to generate a cost estimate based on your projected usage."
       ]
     },
     {
@@ -110,7 +110,7 @@
       "source": [
         "### Set up your local development environment\n",
         "\n",
-        "**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets\n",
+        "**If you are using Colab or Google Cloud Notebooks**, your environment already meets\n",
         "all the requirements to run this notebook. You can skip this step."
       ]
     },
@@ -159,7 +159,7 @@
       "source": [
         "### Install additional packages\n",
         "\n",
-        "Install additional package dependencies not installed in your notebook environment, such as XGBoost. Use the latest major GA version of each package."
+        "Install additional package dependencies not installed in your notebook environment, such as {XGBoost, AdaNet, or TensorFlow Hub TODO: Replace with relevant packages for the tutorial}. Use the latest major GA version of each package."
       ]
     },
     {
@@ -172,15 +172,12 @@
       "source": [
         "import os\n",
         "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
+        "# The Google Cloud Notebook product has specific requirements\n",
+        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
         "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "# Google Cloud Notebook requires dependencies to be installed with '--user'\n",
         "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "if IS_GOOGLE_CLOUD_NOTEBOOK:\n",
         "    USER_FLAG = \"--user\""
       ]
     },
@@ -188,11 +185,11 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "_vr6BYED_5my"
+        "id": "SzEo6DeE2GOP"
       },
       "outputs": [],
       "source": [
-        "! pip3 install {USER_FLAG} --upgrade pip -q\n",
+        "! pip3 install {USER_FLAG} --upgrade pip\n",
         "! pip3 install {USER_FLAG} --upgrade google-cloud-aiplatform==1.11.0 -q --no-warn-conflicts\n",
         "! pip3 install {USER_FLAG} git+https://github.com/googleapis/python-aiplatform.git@main # For features monitoring\n",
         "! pip3 install {USER_FLAG} --upgrade google-cloud-bigquery==2.24.0 -q --no-warn-conflicts\n",
@@ -252,7 +249,7 @@
         "\n",
         "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
         "\n",
-        "1. [Enable the APIs](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,notebooks.googleapis.com, ). \n",
+        "1. [Enable the Vertex AI API and Compute Engine API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component). \n",
         "\n",
         "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
         "\n",
@@ -310,76 +307,18 @@
       "outputs": [],
       "source": [
         "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
-        "    PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
+        "    PROJECT_ID = \"\"  # @param {type:\"string\"}"
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "set_gcloud_project_id"
+        "id": "dEjRdjxBuDsi"
       },
       "outputs": [],
       "source": [
-        "! gcloud config set project $PROJECT_ID"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "23988890fef6"
-      },
-      "source": [
-        "#### Get your project number (Optional)\n",
-        "\n",
-        "Now that the project ID is set, you get your corresponding project number."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "2d6950574e1d"
-      },
-      "outputs": [],
-      "source": [
-        "shell_output = ! gcloud projects list --filter=\"PROJECT_ID:'{PROJECT_ID}'\" --format='value(PROJECT_NUMBER)'\n",
-        "PROJECT_NUMBER = shell_output[0]\n",
-        "print(\"Project Number:\", PROJECT_NUMBER)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "region"
-      },
-      "source": [
-        "#### Region\n",
-        "\n",
-        "You can also change the `REGION` variable, which is used for operations\n",
-        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
-        "\n",
-        "- Americas: `us-central1`\n",
-        "- Europe: `europe-west4`\n",
-        "- Asia Pacific: `asia-east1`\n",
-        "\n",
-        "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
-        "\n",
-        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "jIcZV7-C2RrX"
-      },
-      "outputs": [],
-      "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "!gcloud config set project $PROJECT_ID #change it"
       ]
     },
     {
@@ -414,7 +353,7 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
+        "**If you are using Google Cloud Notebooks**, your environment is already\n",
         "authenticated. Skip this step."
       ]
     },
@@ -437,13 +376,9 @@
         "3. In the **Service account name** field, enter a name, and\n",
         "   click **Create**.\n",
         "\n",
-        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list and add the following roles:\n",
-        "   - BigQuery Admin\n",
-        "   - Storage Admin\n",
-        "   - Storage Object Admin\n",
-        "   - Vertex AI Administrator\n",
-        "   - Vertex AI Feature Store Admin\n",
-        "\n",
+        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
+        "into the filter box, and select\n",
+        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
         "\n",
         "5. Click *Create*. A JSON file that contains your key downloads to your\n",
         "local environment.\n",
@@ -460,19 +395,19 @@
       },
       "outputs": [],
       "source": [
+        "import os\n",
+        "import sys\n",
+        "\n",
         "# If you are running this notebook in Colab, run this cell and follow the\n",
         "# instructions to authenticate your GCP account. This provides access to your\n",
         "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
         "# requests.\n",
         "\n",
-        "import os\n",
-        "import sys\n",
+        "# The Google Cloud Notebook product has specific requirements\n",
+        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
         "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = \"google.colab\" in sys.modules\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
+        "# If on Google Cloud Notebooks, then don't execute this code\n",
+        "if not IS_GOOGLE_CLOUD_NOTEBOOK:\n",
         "    if \"google.colab\" in sys.modules:\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
@@ -512,8 +447,8 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+        "BUCKET_URI = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "REGION = \"[your-region]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -524,9 +459,11 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"-aip-\" + TIMESTAMP\n",
-        "    BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"-aip-\" + TIMESTAMP\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -549,6 +486,26 @@
         "! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "994afa65eaa2"
+      },
+      "source": [
+        "Run the following cell to grant access to your Cloud Storage resources from Vertex AI Feature store"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "psP1rPU9TRnX"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil uniformbucketlevelaccess set on $BUCKET_URI"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -569,78 +526,6 @@
         "! gsutil ls -al $BUCKET_URI"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "set_service_account"
-      },
-      "source": [
-        "#### Service Account (Optional)\n",
-        "\n",
-        "If you do not want to use your project's Compute Engine service account, set `SERVICE_ACCOUNT` to another service account ID."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "MQVV9haf2Rra"
-      },
-      "outputs": [],
-      "source": [
-        "SERVICE_ACCOUNT = \"[your-service-account]\"  # @param {type:\"string\"}"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "autoset_service_account"
-      },
-      "outputs": [],
-      "source": [
-        "if (\n",
-        "    SERVICE_ACCOUNT == \"\"\n",
-        "    or SERVICE_ACCOUNT is None\n",
-        "    or SERVICE_ACCOUNT == \"[your-service-account]\"\n",
-        "):\n",
-        "    # Get your service account from gcloud\n",
-        "    if not IS_COLAB:\n",
-        "        shell_output = !gcloud auth list 2>/dev/null\n",
-        "        SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
-        "\n",
-        "    else:  # IS_COLAB:\n",
-        "        shell_output = ! gcloud projects describe  $PROJECT_ID\n",
-        "        project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n",
-        "        SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n",
-        "\n",
-        "    print(\"Service Account:\", SERVICE_ACCOUNT)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "set_service_account:pipelines"
-      },
-      "source": [
-        "#### Set service account access\n",
-        "\n",
-        "Run the following commands to grant your service account access. You only need to run this step once per service account."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "U4UpQThc2Rrb"
-      },
-      "outputs": [],
-      "source": [
-        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
-        "\n",
-        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -652,18 +537,6 @@
         "You create the BigQuery dataset to store the data along the demo."
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "8615339fa4ca"
-      },
-      "outputs": [],
-      "source": [
-        "BQ_DATASET = \"Mobile_Gaming\"  # @param {type:\"string\"}\n",
-        "LOCATION = \"US\""
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -672,6 +545,9 @@
       },
       "outputs": [],
       "source": [
+        "BQ_DATASET = \"Mobile_Gaming\"  # @param {type:\"string\"}\n",
+        "LOCATION = \"US\"\n",
+        "\n",
         "!bq mk --location=$LOCATION --dataset $PROJECT_ID:$BQ_DATASET"
       ]
     },
@@ -724,9 +600,12 @@
       "outputs": [],
       "source": [
         "# Data Engineering and Feature Engineering\n",
-        "TODAY = \"2022-06-16\"\n",
+        "TODAY = \"2018-10-03\"\n",
+        "TOMORROW = \"2018-10-04\"\n",
         "LABEL_TABLE = f\"label_table_{TODAY}\".replace(\"-\", \"\")\n",
-        "FEATURES_TABLE = f\"wide_features_table_{TODAY}\"  # @param {type:\"string\"}\n",
+        "FEATURES_TABLE = \"wide_features_table\"  # @param {type:\"string\"}\n",
+        "FEATURES_TABLE_TODAY = f\"wide_features_table_{TODAY}\".replace(\"-\", \"\")\n",
+        "FEATURES_TABLE_TOMORROW = f\"wide_features_table_{TOMORROW}\".replace(\"-\", \"\")\n",
         "FEATURESTORE_ID = \"mobile_gaming\"  # @param {type:\"string\"}\n",
         "ENTITY_TYPE_ID = \"user\"\n",
         "\n",
@@ -1068,37 +947,13 @@
         "You will cover those steps in details below."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "init_aip:mbsdk,all"
-      },
-      "source": [
-        "### Initialize Vertex AI SDK for Python\n",
-        "\n",
-        "Initialize the Vertex AI SDK for Python for your project and corresponding bucket."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "poLJ0fV52Rrc"
-      },
-      "outputs": [],
-      "source": [
-        "vertex_ai.init(project=PROJECT_ID, location=REGION, staging_bucket=BUCKET_URI)"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "4ffd54e97270"
       },
       "source": [
-        "### Initialize BigQuery SDK for Python\n",
-        "\n",
-        "Initialize the BigQuery AI SDK for Python for your project and corresponding bucket."
+        "## Initiate clients"
       ]
     },
     {
@@ -1109,53 +964,55 @@
       },
       "outputs": [],
       "source": [
-        "bq_client = bigquery.Client(project=PROJECT_ID, location=LOCATION)"
+        "bq_client = bigquery.Client(project=PROJECT_ID, location=LOCATION)\n",
+        "vertex_ai.init(project=PROJECT_ID, location=REGION, staging_bucket=BUCKET_URI)"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "WnUQO2IHC9pZ"
+        "id": "zmMWIpCwsET9"
       },
       "source": [
         "## Identify users and build your features\n",
-        " \n",
-        "This section we will have static features we want to fetch from Vertex AI Feature Store. In particular, we will cover the following steps:\n",
-        " \n",
+        "\n",
+        "This section we will static features we want to fetch from Vertex AI Feature Store. In particular, we will cover the following steps:\n",
+        "\n",
         "1. Identify users, process demographic features and process behavioral features within the last 24 hours using **BigQuery**\n",
-        " \n",
+        "\n",
         "2. Set up the feature store\n",
-        " \n",
+        "\n",
         "3. Register features using **Vertex AI Feature Store** and the SDK.\n",
-        " \n",
-        "Below you have a picture that shows the process.\n",
-        " \n",
+        "\n",
+        "Below you have a picture that shows the process. \n",
+        "\n",
         "<img src=\"./assets/feature_store_ingestion_2.png\">\n",
-        " \n",
-        " \n",
-        "The original dataset contains raw event data we cannot ingest in the feature store as they are. We need to pre-process the raw data in order to get user features.\n",
-        " \n",
-        "**Notice we simulate those transformations in different points of time (today and tomorrow).**\n"
+        "\n",
+        "\n",
+        "\n",
+        "The original dataset contains raw event data we cannot ingest in the feature store as they are. We need to pre-process the raw data in order to get user features. \n",
+        "\n",
+        "**Notice we simulate those transformations in different point of time (today and tomorrow).**\n"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "e9zIrwhpDF2q"
+        "id": "8avYy5QOv02s"
       },
       "source": [
         "### Label, Demographic and Behavioral Transformations\n",
-        " \n",
-        "This section is based on the [Churn prediction for game developers using Google Analytics 4 (GA4) and BigQuery ML](https://cloud.google.com/blog/topics/developers-practitioners/churn-prediction-game-developers-using-google-analytics-4-ga4-and-bigquery-ml?utm_source=linkedin&utm_medium=unpaidsoc&utm_campaign=FY21-Q2-Google-Cloud-Tech-Blog&utm_content=google-analytics-4&utm_term=-) blog article by Minhaz Kazi and Polong Lin.\n",
-        " \n",
-        "You will adapt it to turn a batch churn prediction (using features within the first 24h user of first engagement) into a real-time churn prediction (using features within the first 6h user of last engagement).\n"
+        "\n",
+        "This section is based on the [Churn prediction for game developers using Google Analytics 4 (GA4) and BigQuery ML](https://cloud.google.com/blog/topics/developers-practitioners/churn-prediction-game-developers-using-google-analytics-4-ga4-and-bigquery-ml?utm_source=linkedin&utm_medium=unpaidsoc&utm_campaign=FY21-Q2-Google-Cloud-Tech-Blog&utm_content=google-analytics-4&utm_term=-) blog article by Minhaz Kazi and Polong Lin. \n",
+        "\n",
+        "You will adapt it in order to turn a batch churn prediction (using features within the first 24h user of first engagment) in a real-time churn prediction (using features within the first 24h user of last engagment)."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "RQX5m8UiC_px"
+        "id": "YO28RAITh6L-"
       },
       "outputs": [],
       "source": [
@@ -1182,27 +1039,27 @@
         "  SELECT\n",
         "    event_timestamp,\n",
         "    user_pseudo_id,\n",
-        "    SUM(IF(event_name = 'user_engagement', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 21600000000 PRECEDING\n",
+        "    SUM(IF(event_name = 'user_engagement', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 86400000000 PRECEDING\n",
         "      AND CURRENT ROW ) AS cnt_user_engagement,\n",
-        "    SUM(IF(event_name = 'level_start_quickplay', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 21600000000 PRECEDING\n",
+        "    SUM(IF(event_name = 'level_start_quickplay', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 86400000000 PRECEDING\n",
         "      AND CURRENT ROW ) AS cnt_level_start_quickplay,\n",
-        "    SUM(IF(event_name = 'level_end_quickplay', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 21600000000 PRECEDING\n",
+        "    SUM(IF(event_name = 'level_end_quickplay', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 86400000000 PRECEDING\n",
         "      AND CURRENT ROW ) AS cnt_level_end_quickplay,\n",
-        "    SUM(IF(event_name = 'level_complete_quickplay', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 21600000000 PRECEDING\n",
+        "    SUM(IF(event_name = 'level_complete_quickplay', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 86400000000 PRECEDING\n",
         "      AND CURRENT ROW ) AS cnt_level_complete_quickplay,\n",
-        "    SUM(IF(event_name = 'level_reset_quickplay', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 21600000000 PRECEDING\n",
+        "    SUM(IF(event_name = 'level_reset_quickplay', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 86400000000 PRECEDING\n",
         "      AND CURRENT ROW ) AS cnt_level_reset_quickplay,\n",
-        "    SUM(IF(event_name = 'post_score', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 21600000000 PRECEDING\n",
+        "    SUM(IF(event_name = 'post_score', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 86400000000 PRECEDING\n",
         "      AND CURRENT ROW ) AS cnt_post_score,\n",
-        "    SUM(IF(event_name = 'spend_virtual_currency', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 21600000000 PRECEDING\n",
+        "    SUM(IF(event_name = 'spend_virtual_currency', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 86400000000 PRECEDING\n",
         "      AND CURRENT ROW ) AS cnt_spend_virtual_currency,\n",
-        "    SUM(IF(event_name = 'ad_reward', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 21600000000 PRECEDING\n",
+        "    SUM(IF(event_name = 'ad_reward', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 86400000000 PRECEDING\n",
         "      AND CURRENT ROW ) AS cnt_ad_reward,\n",
-        "    SUM(IF(event_name = 'challenge_a_friend', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 21600000000 PRECEDING\n",
+        "    SUM(IF(event_name = 'challenge_a_friend', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 86400000000 PRECEDING\n",
         "      AND CURRENT ROW ) AS cnt_challenge_a_friend,\n",
-        "    SUM(IF(event_name = 'completed_5_levels', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 21600000000 PRECEDING\n",
+        "    SUM(IF(event_name = 'completed_5_levels', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 86400000000 PRECEDING\n",
         "      AND CURRENT ROW ) AS cnt_completed_5_levels,\n",
-        "    SUM(IF(event_name = 'use_extra_steps', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 21600000000 PRECEDING\n",
+        "    SUM(IF(event_name = 'use_extra_steps', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 86400000000 PRECEDING\n",
         "      AND CURRENT ROW ) AS cnt_use_extra_steps,\n",
         "  FROM (\n",
         "    SELECT\n",
@@ -1214,7 +1071,7 @@
         "\n",
         "SELECT\n",
         "    -- PARSE_TIMESTAMP('%Y-%m-%d %H:%M:%S', CONCAT('{TODAY}', ' ', STRING(TIME_TRUNC(CURRENT_TIME(), SECOND))), 'UTC') as timestamp,\n",
-        "    TIMESTAMP_ADD(PARSE_TIMESTAMP('%Y-%m-%d %H:%M:%S', FORMAT_TIMESTAMP('%Y-%m-%d %H:%M:%S', TIMESTAMP_MICROS(beh.event_timestamp))), INTERVAL 1351 DAY) AS timestamp,\n",
+        "    PARSE_TIMESTAMP('%Y-%m-%d %H:%M:%S', FORMAT_TIMESTAMP('%Y-%m-%d %H:%M:%S', TIMESTAMP_MICROS(beh.event_timestamp))) AS timestamp,\n",
         "    dem.*,\n",
         "    CAST(IFNULL(beh.cnt_user_engagement, 0) AS FLOAT64)  AS cnt_user_engagement,\n",
         "    CAST(IFNULL(beh.cnt_level_start_quickplay, 0) AS FLOAT64) AS cnt_level_start_quickplay,\n",
@@ -1240,7 +1097,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "oGYxLCSnD068"
+        "id": "Z6CjIOmDsET-"
       },
       "outputs": [],
       "source": [
@@ -1250,31 +1107,31 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "xQLIlsTCD_nk"
+        "id": "Lx__2-assET-"
       },
       "source": [
         "## Create a Vertex AI Feature store and ingest your features\n",
-        " \n",
-        "Now you have a wide table of features. It is time to ingest them into the feature store.\n",
-        " \n",
+        "\n",
+        "Now you have the wide table of features. It is time to ingest them into the feature store. \n",
+        "\n",
         "Before to moving on, you may have a question: **Why do I need a feature store**\n",
         "in this scenario at that point?\n",
-        " \n",
-        "One of the reasons would be to make those features accessible across teams by calculating once and reuse them many times. And in order to make it possible you need also be able to monitor those features over time to guarantee freshness and in case have a new feature engineering run to refresh them.\n",
-        " \n",
-        "If it is not your case, I will give even more reasons about why you should consider a feature store in the following sections. Just keep following me for now.\n",
-        " \n",
-        "One of the most important things is related to its data model. As you can see in the picture below, Vertex AI Feature Store organizes resources hierarchically in the following order: `Featurestore -> EntityType -> Feature`. You must create these resources before you can ingest data into Vertex AI Feature Store.\n",
-        " \n",
+        "\n",
+        "One of the reason would be to make those features accessable across team by calculating once and reuse them many times. And in order to make it possible you need also be able to monitor those features over time to guarantee freshness and in case have a new feature engineerign run to refresh them. \n",
+        "\n",
+        "If it is not your case, I will give even more reasons about why you should consider feature store in the following sections. Just keep following me for now.\n",
+        "\n",
+        "One of the most important thing is related to its data model. As you can see in the picture below, Vertex AI Feature Store organizes resources hierarchically in the following order: `Featurestore -> EntityType -> Feature`. You must create these resources before you can ingest data into Vertex AI Feature Store.\n",
+        "\n",
         "<img src=\"./assets/feature_store_data_model_3.png\">\n",
-        " \n",
-        "In our case we are going to create **mobile_gaming** featurestore resource containing **user** entity type and all its associated **features** such as country or the number of times a user challenged a friend (cnt_challenge_a_friend).\n"
+        "\n",
+        "In our case we are going to create **mobile_gaming** featurestore resource containing **user** entity type and all its associated **features** such as country or the number of times a user challenged a friend (cnt_challenge_a_friend)."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "VR7BJEozED_Q"
+        "id": "8dNlxda2sET_"
       },
       "source": [
         "### Create featurestore, ```mobile_gaming```\n",
@@ -1286,7 +1143,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "vUFqtYU-EDTR"
+        "id": "t2en8I7TSe4b"
       },
       "outputs": [],
       "source": [
@@ -1307,7 +1164,7 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "mUlCwfdpEHJG"
+        "id": "rN-vlvPUsET_"
       },
       "source": [
         "### Create the ```User``` entity type and its features\n",
@@ -1319,7 +1176,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "PnCU1wBND3W7"
+        "id": "CbZ2RQ5XbuRq"
       },
       "outputs": [],
       "source": [
@@ -1337,7 +1194,7 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "bT9LXzu1EOvW"
+        "id": "B2PIAprPmnhB"
       },
       "source": [
         "### Set Feature Monitoring\n",
@@ -1351,7 +1208,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "8WBlYUkOERaI"
+        "id": "N6im2c3ymiwC"
       },
       "outputs": [],
       "source": [
@@ -1374,7 +1231,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "92X4-7PFETj5"
+        "id": "gp9xaLQXn0CS"
       },
       "outputs": [],
       "source": [
@@ -1397,18 +1254,18 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "hxAuZjt3EWFo"
+        "id": "ustwKOMle8Qp"
       },
       "source": [
         "### Create features\n",
         "\n",
-        "In order to ingest features, you need to provide feature configuration and create them as featurestore resources."
+        "In order to ingest features, you need to provide feature configuration and create them as featurestore resources.\n"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "hRXO2I5VEYwt"
+        "id": "ijeZCTKIfCRL"
       },
       "source": [
         "#### Create Feature configuration\n",
@@ -1421,7 +1278,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "K26NEYZIEbvE"
+        "id": "vX_uYmjUgd9x"
       },
       "outputs": [],
       "source": [
@@ -1502,7 +1359,7 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "FjzMd1XbEfdo"
+        "id": "ErkruXPJkPuy"
       },
       "source": [
         "#### Create features using `batch_create_features` method\n",
@@ -1514,7 +1371,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "nqlgCDI9pbCD"
+        "id": "ZsCAO_IfsEUC"
       },
       "outputs": [],
       "source": [
@@ -1531,19 +1388,19 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "7zpFV7wAppkC"
+        "id": "9WisJk18qqgs"
       },
       "source": [
         "### Search features\n",
         "\n",
-        "Vertex AI Feature store supports searching capabilities. Below you have a simple example that shows how to filter a feature based on its name. "
+        "Vertex AI Feature store supports serching capabilities. Below you have a simple example that show how to filter a feature based on its name. "
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "BJXYLLOfppCL"
+        "id": "JzqyarMZqvZS"
       },
       "outputs": [],
       "source": [
@@ -1555,7 +1412,7 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "is9C_6-QpxG3"
+        "id": "ugtBfW5gsEUD"
       },
       "source": [
         "## Ingest features \n",
@@ -1590,7 +1447,7 @@
         "        entity_id_field=ENTITY_ID_FIELD,\n",
         "        disable_online_serving=False,\n",
         "        worker_count=10,\n",
-        "        sync=False,\n",
+        "        sync=True,\n",
         "    )\n",
         "except RuntimeError as error:\n",
         "    print(error)"
@@ -1599,7 +1456,7 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "8lCMpDPGp-oQ"
+        "id": "3Yv8MenWXrRX"
       },
       "source": [
         "# Train and deploy a real-time churn ML model using Vertex AI Training and Endpoints\n",
@@ -1610,34 +1467,34 @@
         "\n",
         "<img src=\"./assets/train_model_4.png\">\n",
         "\n",
-        "Let's dive into each step of this process."
+        "Let's dive into each step of this process.\n"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "VMrvnuyjqGfY"
+        "id": "saZZ3zWKX1YK"
       },
       "source": [
-        "## Fetch training data with point-in-time query using BigQuery and Vertex AI Feature store \n",
-        " \n",
-        "As we mentioned above, in real time churn prediction, it is so important defining the label you want to predict with your model.\n",
-        " \n",
-        "Let's assume that you decide to predict the churn probability over the next hour. So now you have your label. Next step is to define your training sample. But let's think about that for a second.\n",
-        " \n",
-        "In that churn real time system, you have a high volume of transactions you could use to calculate those features which keep floating and are collected constantly over time. It implies that you always get fresh data to reconstruct features. And depending on when you decide to calculate one feature or another you can end up with a set of features that are not aligned in time.\n",
-        " \n",
+        "## Fetch training data with point-in-time query using BigQuery and Vertex AI Feature store  \n",
+        "\n",
+        "As we mentioned above, in real time churn prediction, it is so important defining the label you want to predict with your model. \n",
+        "\n",
+        "Let's assume that you decide to predict the churn probability over the last 24 hr. So now you have your label. Next step is to define your training sample. But let's think about that for a second. \n",
+        "\n",
+        "In that churn real time system, you have a high volume of transactions you could use to calculate those features which keep floating and are collected constantly over time. It implies that you always get fresh data to reconstruct features. And depending on when you decide to calculate one feature or another you can end up with a set of features that are not aligned in time. \n",
+        "\n",
         "When you have labels available, it would be incredibly difficult to say which set of features contains the most up to date historical information associated with the label you want to predict. And, when you are not able to guarantee that, the performance of your model would be badly affected because you serve no representative features of the data and the label from the field when it goes live. So you need a way to get the most updated features you calculated over time before the label becomes available in order to avoid this informational skew.\n",
-        " \n",
-        "**With the Vertex AI Feature store, you can fetch feature values corresponding to a particular timestamp thanks to point-in-time lookup capability.** In our case, it would be the timestamp associated with the label you want to predict with your model. In this way, you will avoid data leakage and you will get the most updated features to train your model.\n",
-        " \n",
-        "Let's see how to do that.\n"
+        "\n",
+        "**With the Vertex AI Feature store, you can fetch feature values corresponding to a particular timestamp thanks to point-in-time lookup capability.** In our case, it would be the timestamp associated to the label you want to predict with your model. In this way, you will avoid data leakage and you will get the most updated features to train your model. \n",
+        "\n",
+        "Let's see how to do that. \n"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "RE_Pvmu-qdDt"
+        "id": "YHNbIHqFcQiM"
       },
       "source": [
         "### Define query for reading instances at a specific point in time\n",
@@ -1649,7 +1506,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "bUDVw7l-qF2x"
+        "id": "DGUm0bYqhVV4"
       },
       "outputs": [],
       "source": [
@@ -1661,13 +1518,13 @@
         "  # get training threshold ----------------------------------------------------------------------------------\n",
         "  get_training_threshold AS (\n",
         "  SELECT\n",
-        "    (MAX(event_timestamp) - 10800000000) AS training_thrs\n",
+        "    (MAX(event_timestamp) - 86400000000) AS training_thrs\n",
         "  FROM\n",
         "    `firebase-public-project.analytics_153293282.events_*`\n",
         "  WHERE\n",
         "    event_name=\"user_engagement\"\n",
         "    AND\n",
-        "    TIMESTAMP_ADD(PARSE_TIMESTAMP('%Y-%m-%d %H:%M:%S', FORMAT_TIMESTAMP('%Y-%m-%d %H:%M:%S', TIMESTAMP_MICROS(event_timestamp))), INTERVAL 1351 DAY) < '{TODAY}'),\n",
+        "    PARSE_TIMESTAMP('%Y-%m-%d %H:%M:%S', FORMAT_TIMESTAMP('%Y-%m-%d %H:%M:%S', TIMESTAMP_MICROS(event_timestamp))) < '{TODAY}'),\n",
         "\n",
         "  # query to create label -----------------------------------------------------------------------------------\n",
         "  get_label AS (\n",
@@ -1692,7 +1549,7 @@
         "    WHERE\n",
         "      event_name=\"user_engagement\"\n",
         "    AND\n",
-        "    TIMESTAMP_ADD(PARSE_TIMESTAMP('%Y-%m-%d %H:%M:%S', FORMAT_TIMESTAMP('%Y-%m-%d %H:%M:%S', TIMESTAMP_MICROS(event_timestamp))), INTERVAL 1351 DAY) < '{TODAY}'\n",
+        "    PARSE_TIMESTAMP('%Y-%m-%d %H:%M:%S', FORMAT_TIMESTAMP('%Y-%m-%d %H:%M:%S', TIMESTAMP_MICROS(event_timestamp))) < '{TODAY}'\n",
         "    GROUP BY\n",
         "      user_pseudo_id )\n",
         "  GROUP BY\n",
@@ -1812,7 +1669,7 @@
       "source": [
         "!mkdir -m 777 -p trainer data/ingest data/raw model config\n",
         "!gsutil -m cp -r $GCS_DESTINATION_OUTPUT_URI/*.csv data/ingest\n",
-        "!head -n 2000 data/ingest/*.csv > data/raw/sample.csv"
+        "!head -n 1000 data/ingest/*.csv > data/raw/sample.csv"
       ]
     },
     {
@@ -2214,7 +2071,7 @@
       },
       "outputs": [],
       "source": [
-        "TRAIN_JOB_RESOURCE_NAME = \"[your-train-job-resource-name]\"  # @param {type:\"string\"}"
+        "TRAIN_JOB_RESOURCE_NAME = \"\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -2309,32 +2166,32 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "1TNzL_EGrVUm"
+        "id": "7c9330928aa1"
       },
       "source": [
         "# Serve ML features at scale with low latency\n",
-        " \n",
-        "At that time, you are ready **to deploy our simple model which would requires fetching preprocessed attributes as input features in real time**.\n",
-        " \n",
+        "\n",
+        "At that time, you are ready **to deploy our simple model which would requires fetching preprocessed attributes as input features in real time**. \n",
+        "\n",
         "Below you can see how it works\n",
-        " \n",
-        "<center><img src=\"./assets/online_serving_5.png\" width=\"800\"/></center>\n",
-        " \n",
-        "But think about those features for a second.\n",
-        " \n",
-        "Your behavioral features used to train your model, they cannot be computed when you are going to serve the model online.\n",
-        " \n",
-        "How could you compute the number of times a user challenged a friend within the last 24 hours on the fly?\n",
-        " \n",
-        "You need to be computed this feature on the server side and serve it with low latency. And because Bigquery is not optimized for those read operations, we need a different service that allows singleton lookup where the result is a single row with many columns.\n",
-        " \n",
-        "Also, even if it was not the case, when you deploy a model that requires preprocessing your data, you need to be sure to reproduce the same preprocessing steps you had when you trained it. If you are not able to do that a skew between training and serving data would happen and it will badly affect your model performance (and in the worst scenario break your serving system).\n",
-        " \n",
-        "You need a way to mitigate that in a way you don't need to implement those preprocessing steps online but just serve the same aggregated features you already have for training to generate online prediction.\n",
-        " \n",
-        "These are other valuable reasons to introduce Vertex AI Feature Store. With it, you have a service which helps you to serve features at scale with low latency as they were available at training time mitigating in that way possible training-serving skew.\n",
-        " \n",
-        "Now that you know **why you need a feature store**, let's conclude this journey by deploying your model using a feature store to retrieve features online, pass them to the endpoint and generate predictions.\n"
+        "\n",
+        "<img src=\"./assets/online_serving_5.png\" width=\"600\">\n",
+        "\n",
+        "But think about those features for a second. \n",
+        "\n",
+        "Your behavioral features used to trained your model, they cannot be computed when you are going to serve the model online. \n",
+        "\n",
+        "How could you compute the number of time a user challenged a friend withing the last 24 hours on the fly?\n",
+        "\n",
+        "You simply can't do that. You need to be computed this feature on the server side and serve it with low latency. And becuase Bigquery is not optimized for those read operations, we need a different service that allows singleton lookup where the result is a single row with many columns.\n",
+        "\n",
+        "Also, even if it was not the case, when you deploy a model that requires preprocessing your data, you need to be sure to reproduce the same preprocessing steps you had when you trained it. If you are not able to do that a skew between training and serving data would happen and it will affect badly your model performance (and in the worst scenario break your serving system). \n",
+        "\n",
+        "You need a way to mitigate that in a way you don't need to implement those preprocessing steps online but just serve the same aggregated features you already have for training to generate online prediction. \n",
+        "\n",
+        "These are other valuable reasons to introduce Vertex AI Feature Store. With it, you have a service which helps you to serve feature at scale with low latency as they were available at training time mitigating in that way possible training-serving skew.\n",
+        "\n",
+        "Now that you know **why you need a feature store**, let's closing this journey by deploying your model and use feature store to retrieve features online, pass them to endpoint and generate predictions.\n"
       ]
     },
     {
@@ -2364,13 +2221,13 @@
       },
       "outputs": [],
       "source": [
-        "simulate_prediction(endpoint=endpoint, n_requests=10, latency=1)"
+        "simulate_prediction(endpoint=endpoint, n_requests=1000, latency=1)"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "8d3S1d1urZOy"
+        "id": "TpV-iwP9qw9c"
       },
       "source": [
         "## Cleaning up\n",
@@ -2410,12 +2267,11 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "FMXT2akXrZOy"
+        "id": "sx_vKniMq9ZX"
       },
       "outputs": [],
       "source": [
         "# Delete bucket\n",
-        "delete_bucket = False\n",
         "if (delete_bucket or os.getenv(\"IS_TESTING\")) and \"BUCKET_URI\" in globals():\n",
         "    ! gsutil -m rm -r $BUCKET_URI"
       ]

notebooks/community/managed_notebooks/fraud_detection/fraud-detection-model.ipynb

@@ -9,57 +9,6 @@
         "# Build a fraud detection model on Vertex AI"
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "5fcd3e4da897"
-      },
-      "outputs": [],
-      "source": [
-        "# Copyright 2022 Google LLC\n",
-        "#\n",
-        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
-        "# you may not use this file except in compliance with the License.\n",
-        "# You may obtain a copy of the License at\n",
-        "#\n",
-        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
-        "#\n",
-        "# Unless required by applicable law or agreed to in writing, software\n",
-        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
-        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
-        "# See the License for the specific language governing permissions and\n",
-        "# limitations under the License."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "05c670d35496"
-      },
-      "source": [
-        "<table align=\"left\">\n",
-        "\n",
-        "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/fraud_detection/fraud-detection-model.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
-        "    </a>\n",
-        "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/fraud_detection/fraud-detection-model.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
-        "      View on GitHub\n",
-        "    </a>\n",
-        "  </td>\n",
-        "   <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/fraud_detection/fraud-detection-model.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
-        "    </a>\n",
-        "  </td>\n",
-        "</table>"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -91,7 +40,9 @@
         "## Overview\n",
         "<a name=\"section-1\"></a>\n",
         "\n",
-        "This tutorial shows you how to build, deploy, and analyze predictions from a simple [random forest](https://en.wikipedia.org/wiki/Random_forest) model using tools like scikit-learn, Vertex AI, and the [What-IF Tool (WIT)](https://cloud.google.com/ai-platform/prediction/docs/using-what-if-tool) on a synthetic fraud transaction dataset to solve a financial fraud detection problem.\n"
+        "This tutorial shows you how to build, deploy, and analyze predictions from a simple [random forest](https://en.wikipedia.org/wiki/Random_forest) model using tools like scikit-learn, Vertex AI, and the [What-IF Tool (WIT)](https://cloud.google.com/ai-platform/prediction/docs/using-what-if-tool) on a synthetic fraud transaction dataset to solve a financial fraud detection problem.\n",
+        "\n",
+        "*Note: This notebook file was designed to run in a [Vertex AI Workbench managed notebooks](https://cloud.google.com/vertex-ai/docs/workbench/managed/create-instance) instance using the `TensorFlow 2 (Local)` kernel. Some components of this notebook may not work in other notebook environments.*"
       ]
     },
     {
@@ -151,62 +102,13 @@
         "to generate a cost estimate based on your projected usage. "
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "1ba37fa1511f"
-      },
-      "source": [
-        "### Set up your local development environment\n",
-        "\n",
-        "**If you are using Colab or Google Cloud Notebooks**, your environment already meets\n",
-        "all the requirements to run this notebook. You can skip this step."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "cd1bc75a1cb2"
-      },
-      "source": [
-        "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
-        "You need the following:\n",
-        "\n",
-        "* The Google Cloud SDK\n",
-        "* Git\n",
-        "* Python 3\n",
-        "* virtualenv\n",
-        "* Jupyter notebook running in a virtual environment with Python 3\n",
-        "\n",
-        "The Google Cloud guide to [Setting up a Python development\n",
-        "environment](https://cloud.google.com/python/setup) and the [Jupyter\n",
-        "installation guide](https://jupyter.org/install) provide detailed instructions\n",
-        "for meeting these requirements. The following steps provide a condensed set of\n",
-        "instructions:\n",
-        "\n",
-        "1. [Install and initialize the Cloud SDK.](https://cloud.google.com/sdk/docs/)\n",
-        "\n",
-        "1. [Install Python 3.](https://cloud.google.com/python/setup#installing_python)\n",
-        "\n",
-        "1. [Install\n",
-        "   virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv)\n",
-        "   and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
-        "\n",
-        "1. To install Jupyter, run `pip3 install jupyter` on the\n",
-        "command-line in a terminal shell.\n",
-        "\n",
-        "1. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
-        "\n",
-        "1. Open this notebook in the Jupyter Notebook Dashboard."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "611991f03b38"
       },
       "source": [
-        "## Install additional packages"
+        "## Installation"
       ]
     },
     {
@@ -235,7 +137,7 @@
       "source": [
         "Install the latest version of the Vertex AI client library.\n",
         "\n",
-        "Run the following command in your notebook environment to install the Vertex SDK for Python:"
+        "Run the following command in your virtual environment to install the Vertex SDK for Python:"
       ]
     },
     {
@@ -249,168 +151,6 @@
         "! pip install {USER_FLAG} --upgrade google-cloud-aiplatform"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "1969a1cc46cf"
-      },
-      "source": [
-        "Run the following command in your notebook environment to install witwidget:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "8b10e59b0911"
-      },
-      "outputs": [],
-      "source": [
-        "! pip install {USER_FLAG} witwidget"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "4099ce79705a"
-      },
-      "source": [
-        "Run the following command in your notebook environment to install joblib:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "1e56d524753a"
-      },
-      "outputs": [],
-      "source": [
-        "! pip install {USER_FLAG} joblib"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "b87ee3041f7d"
-      },
-      "source": [
-        "Run the following command in your notebook environment to install scikit-learn:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "c3ebecd9bd72"
-      },
-      "outputs": [],
-      "source": [
-        "! pip install {USER_FLAG} scikit-learn"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "b624b5163531"
-      },
-      "source": [
-        "Run the following command in your notebook environment to install fsspec:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "79c7a64b04de"
-      },
-      "outputs": [],
-      "source": [
-        "! pip install {USER_FLAG} fsspec"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "5593090dcf0a"
-      },
-      "source": [
-        "Run the following command in your notebook environment to install gcsfs:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "7bf981bc5bf6"
-      },
-      "outputs": [],
-      "source": [
-        "! pip install {USER_FLAG} gcsfs"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "1c7b2a25df27"
-      },
-      "source": [
-        "### Restart the kernel\n",
-        "\n",
-        "After you install the additional packages, you need to restart the notebook kernel so it can find the packages."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "2117d92e6766"
-      },
-      "outputs": [],
-      "source": [
-        "# Automatically restart kernel after installs\n",
-        "\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    # Automatically restart kernel after installs\n",
-        "    import IPython\n",
-        "\n",
-        "    app = IPython.Application.instance()\n",
-        "    app.kernel.do_shutdown(True)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "2d9b3731b3e0"
-      },
-      "source": [
-        "## Before you begin"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "7a5cb1df1ef7"
-      },
-      "source": [
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com). {TODO: Update the APIs needed for your tutorial. Edit the API names, and update the link to append the API IDs, separating each one with a comma. For example, container.googleapis.com,cloudbuild.googleapis.com}\n",
-        "\n",
-        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -430,8 +170,6 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
         "PROJECT_ID = \"\"\n",
         "\n",
         "# Get your Google Cloud project ID from gcloud\n",
@@ -462,17 +200,6 @@
         "    PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "b11114d77c5f"
-      },
-      "outputs": [],
-      "source": [
-        "! gcloud config set project $PROJECT_ID"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -497,81 +224,6 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "0c7f603fcdcf"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Google Cloud Notebooks**, your environment is already\n",
-        "authenticated. Skip this step."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "72bf8f7c9ab3"
-      },
-      "source": [
-        "**If you are using Colab**, run the cell below and follow the instructions\n",
-        "when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "1. In the Cloud Console, go to the [**Create service account key**\n",
-        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
-        "\n",
-        "2. Click **Create service account**.\n",
-        "\n",
-        "3. In the **Service account name** field, enter a name, and\n",
-        "   click **Create**.\n",
-        "\n",
-        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
-        "into the filter box, and select\n",
-        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
-        "local environment.\n",
-        "\n",
-        "6. Enter the path to your service account key as the\n",
-        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "da63f3587ef9"
-      },
-      "outputs": [],
-      "source": [
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "# The Google Cloud Notebook product has specific requirements\n",
-        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
-        "\n",
-        "# If on Google Cloud Notebooks, then don't execute this code\n",
-        "if not IS_GOOGLE_CLOUD_NOTEBOOK:\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -606,7 +258,7 @@
       "outputs": [],
       "source": [
         "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "REGION = \"[your-region]\"  # @param {type:\"string\"}"
+        "REGION = \"us-central1\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -619,9 +271,7 @@
       "source": [
         "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
         "    BUCKET_NAME = PROJECT_ID + \"-vertex-ai-\" + TIMESTAMP\n",
-        "BUCKET_URI = f\"gs://{BUCKET_NAME}\"\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
     {
@@ -696,7 +346,7 @@
         "import matplotlib.pyplot as plt\n",
         "import numpy as np\n",
         "import pandas as pd\n",
-        "from google.cloud import aiplatform, storage\n",
+        "from google.cloud import storage\n",
         "from sklearn.ensemble import RandomForestClassifier\n",
         "from sklearn.metrics import (average_precision_score, classification_report,\n",
         "                             confusion_matrix, f1_score)\n",
@@ -937,11 +587,9 @@
       },
       "outputs": [],
       "source": [
-        "print(\"before initiating\")\n",
-        "forest = RandomForestClassifier(verbose=1)\n",
-        "print(\"after initiating\")\n",
-        "forest.fit(X_train, y_train)\n",
-        "print(\"after fitting\")"
+        "%%time\n",
+        "forest = RandomForestClassifier()\n",
+        "forest.fit(X_train, y_train)"
       ]
     },
     {
@@ -964,9 +612,7 @@
       },
       "outputs": [],
       "source": [
-        "print(\"before predicting\")\n",
         "y_prob = forest.predict_proba(X_test)\n",
-        "print(\"after predicting y_prob\")\n",
         "y_pred = forest.predict(X_test)\n",
         "\n",
         "print(\"AUPRC :\", (average_precision_score(y_test, y_prob[:, 1])))\n",
@@ -976,8 +622,7 @@
         "print(confusion_matrix(y_test, y_pred))\n",
         "\n",
         "print(\"classification_report\")\n",
-        "print(classification_report(y_test, y_pred))\n",
-        "print(\"after printing classification_report\")"
+        "print(classification_report(y_test, y_pred))"
       ]
     },
     {
@@ -1033,7 +678,7 @@
         "BLOB_PATH = \"[your-blob-path]\"\n",
         "BLOB_NAME = os.path.join(BLOB_PATH, FILE_NAME)\n",
         "\n",
-        "bucket = storage.Client(PROJECT_ID).bucket(BUCKET_NAME)\n",
+        "bucket = storage.Client().bucket(BUCKET_NAME)\n",
         "blob = bucket.blob(BLOB_NAME)\n",
         "blob.upload_from_filename(FILE_NAME)"
       ]
@@ -1057,10 +702,7 @@
       "outputs": [],
       "source": [
         "MODEL_DISPLAY_NAME = \"[your-model-display-name]\"\n",
-        "ARTIFACT_GCS_PATH = f\"{BUCKET_URI}/{BLOB_PATH}\"\n",
-        "SERVING_CONTAINER_IMAGE_URI = (\n",
-        "    \"us-docker.pkg.dev/vertex-ai/prediction/sklearn-cpu.1-0:latest\"\n",
-        ")"
+        "ARTIFACT_GCS_PATH = f\"{BUCKET_URI}/{BLOB_PATH}\""
       ]
     },
     {
@@ -1072,13 +714,14 @@
       "outputs": [],
       "source": [
         "# Create a Vertex AI model resource\n",
+        "from google.cloud import aiplatform\n",
         "\n",
         "aiplatform.init(project=PROJECT_ID, location=REGION)\n",
         "\n",
         "model = aiplatform.Model.upload(\n",
         "    display_name=MODEL_DISPLAY_NAME,\n",
         "    artifact_uri=ARTIFACT_GCS_PATH,\n",
-        "    serving_container_image_uri=SERVING_CONTAINER_IMAGE_URI,\n",
+        "    serving_container_image_uri=\"us-docker.pkg.dev/vertex-ai/prediction/sklearn-cpu.0-24:latest\",\n",
         ")\n",
         "\n",
         "model.wait()\n",
@@ -1154,6 +797,10 @@
       },
       "outputs": [],
       "source": [
+        "# Uncomment if starting over without model and endpoint references\n",
+        "# model = aiplatform.Model('[your-model-resource-name]')\n",
+        "# endpoint = aiplatform.Endpoint('[your-endpoint-resource-name]')\n",
+        "\n",
         "# deploy the model to the endpoint\n",
         "model.deploy(\n",
         "    endpoint=endpoint,\n",
@@ -1167,6 +814,26 @@
         "print(model.resource_name)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "602a1a615bb0"
+      },
+      "source": [
+        "Save the ID of the deployed model. The ID of the deployed model can also be checked by using the `endpoint.list_models()` method."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "84a1da5b5e93"
+      },
+      "outputs": [],
+      "source": [
+        "DEPLOYED_MODEL_ID = \"[your-deployed-model-id]\""
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -1176,7 +843,7 @@
         "## What-If Tool \n",
         "<a name=\"section-11\"></a>\n",
         "\n",
-        "The What-If Tool can be used to analyze the model predictions on a test data. See a [brief introduction to the What-If Tool](https://pair-code.github.io/what-if-tool/). In this tutorial, the What-If Tool will be configured and run on the model trained locally, and on the model deployed on Vertex AI Endpoint in the previous steps.\n",
+        "The What-If Tool can be used to analyze the model predictions on a test data. See a [brief introduction to the What-If Tool](https://pair-code.github.io/what-if-tool/). In this tutorial, the What-If Tool will be configured and run on the model trained locally, and on the model deployed on Vertex AI Endpoints in the previous steps.\n",
         "\n",
         "[WitConfigBuilder](https://github.com/PAIR-code/what-if-tool/blob/master/witwidget/notebook/visualization.py#L30) provides the  `set_ai_platform_model()` method to configure the What-If Tool with a model deployed as a version on Ai Platform models. This feature currently supports Ai Platform only but not Vertex AI models. Fortunately, there is also an option to pass a custom function for generating predictions through the `set_custom_predict_fn()` method where either the locally trained model or a function that returns predictions from a Vertex AI model can be passed."
       ]
@@ -1303,27 +970,6 @@
         "WitWidget(config_builder, height=400)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "c446b1263b34"
-      },
-      "source": [
-        "## Undeploy the model\n",
-        "When you are done doing predictions, you undeploy the model from the Endpoint resouce. This deprovisions all compute resources and ends billing for the deployed model."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "72eb599403d4"
-      },
-      "outputs": [],
-      "source": [
-        "endpoint.undeploy_all()"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -1340,6 +986,18 @@
         "Otherwise, you can delete the individual resources you created in this tutorial:"
       ]
     },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "46061cbb656d"
+      },
+      "outputs": [],
+      "source": [
+        "# undeploy the model\n",
+        "endpoint.undeploy(deployed_model_id=DEPLOYED_MODEL_ID)"
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -1372,9 +1030,8 @@
       },
       "outputs": [],
       "source": [
-        "delete_bucket = True\n",
-        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
-        "    ! gsutil rm -r $BUCKET_URI"
+        "# uncomment to remove the contents of the Cloud Storage bucket\n",
+        "# ! gsutil -m rm -r $BUCKET_NAME"
       ]
     }
   ],

notebooks/community/managed_notebooks/subscriber_churn_prediction/telecom-subscriber-churn-prediction.ipynb

@@ -1,56 +1,12 @@
 {
   "cells": [
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "18ebbd838e32"
-      },
-      "outputs": [],
-      "source": [
-        "# Copyright 2022 Google LLC\n",
-        "#\n",
-        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
-        "# you may not use this file except in compliance with the License.\n",
-        "# You may obtain a copy of the License at\n",
-        "#\n",
-        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
-        "#\n",
-        "# Unless required by applicable law or agreed to in writing, software\n",
-        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
-        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
-        "# See the License for the specific language governing permissions and\n",
-        "# limitations under the License."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "64f7165bd1ac"
       },
       "source": [
-        "# Telecom subscriber churn prediction on Vertex AI\n",
-        "\n",
-        "<table align=\"left\">\n",
-        "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/subscriber_churn_prediction/telecom-subscriber-churn-prediction.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
-        "      View on GitHub\n",
-        "    </a>\n",
-        "  </td>\n",
-        "    <td>\n",
-        "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/subscriber_churn_prediction/telecom-subscriber-churn-prediction.ipynb\">\n",
-        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\\\" alt=\"Colab logo\"> Run in Colab\n",
-        "        </a>\n",
-        "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/subscriber_churn_prediction/telecom-subscriber-churn-prediction.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
-        "    </a>\n",
-        "  </td>\n",
-        "</table>\n",
-        "<br/><br/><br/>"
+        "# Telecom subscriber churn prediction on Vertex AI"
       ]
     },
     {
@@ -83,7 +39,9 @@
         "## Overview\n",
         "<a name=\"section-1\"></a>\n",
         "\n",
-        "This example demonstrates building a subscriber churn prediction model on a [telecom customer churn dataset](https://www.kaggle.com/c/customer-churn-prediction-2020/overview). The generated churn model is further deployed to Vertex AI Endpoints and explanations are generated using the Explainable AI feature of Vertex AI. "
+        "This example demonstrates building a subscriber churn prediction model on a [telecom customer churn dataset](https://www.kaggle.com/c/customer-churn-prediction-2020/overview). The generated churn model is further deployed to Vertex AI Endpoints and explanations are generated using the Explainable AI feature of Vertex AI. \n",
+        "\n",
+        "*Note: This notebook file was designed to run in a [Vertex AI Workbench managed notebooks](https://cloud.google.com/vertex-ai/docs/workbench/managed/create-instance) instance using the `Python (Local)` kernel. Some components of this notebook may not work in other notebook environments.*"
       ]
     },
     {
@@ -95,7 +53,7 @@
         "## Dataset\n",
         "<a name=\"section-2\"></a>\n",
         "\n",
-        "The dataset used in this tutorial is Telecom-Customer Churn dataset publicly available on Kaggle. See [Customer Churn Prediction 2020](https://www.kaggle.com/c/customer-churn-prediction-2020/data). This dataset is used to build and deploy a churn prediction model using Vertex AI in this notebook."
+        "The dataset used in this tutorial is publicly available at Kaggle. See [Customer Churn Prediction 2020](https://www.kaggle.com/c/customer-churn-prediction-2020/data). "
       ]
     },
     {
@@ -107,7 +65,7 @@
         "## Objective\n",
         "<a name=\"section-3\"></a>\n",
         "\n",
-        "This tutorial shows you how to do exploratory data analysis, preprocess data, train, deploy and get predictions from a churn prediction model on a tabular churn dataset. The objectives of this tutorial are as follows:\n",
+        "This tutorial shows you how to do exploratory data analysis, preprocess data, and train a churn prediction model on a tabular churn dataset. The steps include the following:\n",
         "\n",
         "- Load data from a Cloud Storage path\n",
         "- Perform exploratory data analysis (EDA)\n",
@@ -149,9 +107,7 @@
         "id": "44b8ae8e2d19"
       },
       "source": [
-        "## Installation\n",
-        "\n",
-        "Install the following packages to run this notebook."
+        "## Installation"
       ]
     },
     {
@@ -173,6 +129,17 @@
         "    USER_FLAG = \"--user\""
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "606337930991"
+      },
+      "source": [
+        "Install the latest version of the Vertex AI client library.\n",
+        "\n",
+        "Run the following command in your virtual environment to install the Vertex SDK for Python:"
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -181,43 +148,67 @@
       },
       "outputs": [],
       "source": [
-        "! pip install {USER_FLAG} --upgrade google-cloud-aiplatform \\\n",
-        "                                    google-cloud-storage \\\n",
-        "                                    category_encoders \\\n",
-        "                                    seaborn \\\n",
-        "                                    sklearn \\\n",
-        "                                    pandas \\\n",
-        "                                    fsspec \\\n",
-        "                                    gcsfs -q"
+        "! pip install {USER_FLAG} --upgrade google-cloud-aiplatform"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "b24902cde81b"
+        "id": "e67139e68463"
       },
       "source": [
-        "### Restart the kernel\n",
-        "\n",
-        "Once you've installed the additional packages, you need to restart the notebook kernel so it can find the packages."
+        "Install the Cloud Storage library:"
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "c61d171395d7"
+        "id": "2ad918f94f5d"
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    # Automatically restart kernel after installs\n",
-        "    import IPython\n",
-        "\n",
-        "    app = IPython.Application.instance()\n",
-        "    app.kernel.do_shutdown(True)"
+        "! pip install {USER_FLAG} --upgrade google-cloud-storage"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "eb0c1e24a8f0"
+      },
+      "source": [
+        "Install the `category_encoders` library:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "deb95a7f2104"
+      },
+      "outputs": [],
+      "source": [
+        "! pip install --upgrade category_encoders"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "184560c1b742"
+      },
+      "source": [
+        "Install the `seaborn` library for the EDA step. If a Vertex AI Workbench managed notebooks instance is being used, this step is optional as the library is already available in the `Python (Local)` kernel."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "0d99cdcdc470"
+      },
+      "outputs": [],
+      "source": [
+        "! pip install --upgrade seaborn"
       ]
     },
     {
@@ -252,7 +243,7 @@
         "id": "96ff17f75e21"
       },
       "source": [
-        "### Set your project ID\n",
+        "#### Set your project ID\n",
         "\n",
         "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
       ]
@@ -265,13 +256,11 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
         "PROJECT_ID = \"\"\n",
         "\n",
         "# Get your Google Cloud project ID from gcloud\n",
         "if not os.getenv(\"IS_TESTING\"):\n",
-        "    shell_output = !gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "    shell_output=!gcloud config list --format 'value(core.project)' 2>/dev/null\n",
         "    PROJECT_ID = shell_output[0]\n",
         "    print(\"Project ID: \", PROJECT_ID)"
       ]
@@ -297,58 +286,13 @@
         "    PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "f2e3c0f2cbfb"
-      },
-      "outputs": [],
-      "source": [
-        "! gcloud config set project $PROJECT_ID"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "60d535f443ac"
-      },
-      "source": [
-        "### Region\n",
-        "\n",
-        "You can also change the `REGION` variable, which is used for operations\n",
-        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
-        "\n",
-        "- Americas: `us-central1`\n",
-        "- Europe: `europe-west4`\n",
-        "- Asia Pacific: `asia-east1`\n",
-        "\n",
-        "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
-        "\n",
-        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "3aaadaaf9b30"
-      },
-      "outputs": [],
-      "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "e663bd062c6f"
       },
       "source": [
-        "### Timestamp\n",
+        "#### Timestamp\n",
         "\n",
         "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append it onto the name of resources you create in this tutorial."
       ]
@@ -366,63 +310,6 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "3ffa6b6c7cdb"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Google Cloud Notebooks**, your environment is already authenticated. Skip this step.\n",
-        "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
-        "\n",
-        "1. **Click Create service account**.\n",
-        "\n",
-        "2. In the **Service account name** field, enter a name, and click **Create**.\n",
-        "\n",
-        "3. In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex AI\" into the filter box, and select **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "4. Click Create. A JSON file that contains your key downloads to your local environment.\n",
-        "\n",
-        "5. Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "2b72272258fc"
-      },
-      "outputs": [],
-      "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Google Cloud Notebook, then don't execute this code\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -440,7 +327,12 @@
         "online predictions.\n",
         "\n",
         "Set the name of your Cloud Storage bucket below. It must be unique across all\n",
-        "Cloud Storage buckets."
+        "Cloud Storage buckets.\n",
+        "\n",
+        "You may also change the `REGION` variable, which is used for operations\n",
+        "throughout the rest of this notebook. Make sure to [choose a region where Vertex AI services are\n",
+        "available](https://cloud.google.com/vertex-ai/docs/general/locations#available_regions). You may\n",
+        "not use a Multi-Regional Storage bucket for training with Vertex AI."
       ]
     },
     {
@@ -451,8 +343,8 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "REGION = \"[your-region]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -463,9 +355,8 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
-        "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
       ]
     },
     {
@@ -485,7 +376,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_URI"
+        "! gsutil mb -l $REGION $BUCKET_NAME"
       ]
     },
     {
@@ -505,7 +396,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_URI"
+        "! gsutil ls -al $BUCKET_NAME"
       ]
     },
     {
@@ -559,13 +450,7 @@
         "id": "e37354341588"
       },
       "source": [
-        "### Load data from Cloud Storage using Pandas\n",
-        "\n",
-        "The Telecom-Customer Churn dataset from [Kaggle](https://www.kaggle.com/c/customer-churn-prediction-2020/overview) is made available on a public Cloud Storage bucket at: \n",
-        "\n",
-        "```gs://cloud-samples-data/vertex-ai/managed_notebooks/telecom_churn_prediction/train.csv```\n",
-        "\n",
-        "Use Pandas to read data directly from the URI."
+        "### Load data from Cloud Storage path using Pandas"
       ]
     },
     {
@@ -1231,8 +1116,6 @@
         "    \"[your-blob-path]\"  # leave blank if no folders inside the bucket are needed.\n",
         ")\n",
         "\n",
-        "if BLOB_PATH == (\"[your-blob-path]\"):\n",
-        "    BLOB_PATH = \"\"\n",
         "\n",
         "BLOB_NAME = BLOB_PATH + FILE_NAME\n",
         "\n",
@@ -1250,9 +1133,7 @@
         "## Create a model with Explainable AI support in Vertex AI\n",
         "<a name=\"section-9\"></a>\n",
         "\n",
-        "Before creating a model, configure the explanations for the model. For further details, see [Configuring explanations in Vertex AI](https://cloud.google.com/vertex-ai/docs/explainable-ai/configuring-explanations#scikit-learn-and-xgboost-pre-built-containers).\n",
-        "\n",
-        "Set a display name for the model resource."
+        "Before creating a model, configure the explanations for the model. For further details, see [Configuring explanations in Vertex AI](https://cloud.google.com/vertex-ai/docs/explainable-ai/configuring-explanations#scikit-learn-and-xgboost-pre-built-containers)."
       ]
     },
     {
@@ -1263,13 +1144,10 @@
       },
       "outputs": [],
       "source": [
-        "# Set the model display name\n",
-        "MODEL_DISPLAY_NAME = \"[your-model-display-name]\"  # @param {type:\"string\"}\n",
-        "\n",
-        "if MODEL_DISPLAY_NAME == \"[your-model-display-name]\":\n",
-        "    MODEL_DISPLAY_NAME = \"subscriber_churn_model\"\n",
-        "\n",
+        "MODEL_DISPLAY_NAME = \"[your-model-display-name]\"\n",
         "ARTIFACT_GCS_PATH = f\"gs://{BUCKET_NAME}/{BLOB_PATH}\"\n",
+        "PROJECT = \"[your-project-id]\"\n",
+        "LOCATION = REGION\n",
         "\n",
         "# Feature-name(Inp_feature) and Output-name(Model_output) can be arbitrary\n",
         "exp_metadata = {\"inputs\": {\"Inp_feature\": {}}, \"outputs\": {\"Model_output\": {}}}"
@@ -1283,20 +1161,17 @@
       },
       "outputs": [],
       "source": [
-        "from google.cloud.aiplatform_v1.types import SampledShapleyAttribution\n",
         "# Create a Vertex AI model resource with support for explanations\n",
-        "from google.cloud.aiplatform_v1.types.explanation import ExplanationParameters\n",
         "\n",
-        "aiplatform.init(project=PROJECT_ID, location=REGION)\n",
+        "aiplatform.init(project=PROJECT, location=LOCATION)\n",
+        "explanation_parameters = {\"sampledShapleyAttribution\": {\"pathCount\": 25}}\n",
         "\n",
         "model = aiplatform.Model.upload(\n",
         "    display_name=MODEL_DISPLAY_NAME,\n",
         "    artifact_uri=ARTIFACT_GCS_PATH,\n",
-        "    serving_container_image_uri=\"us-docker.pkg.dev/vertex-ai/prediction/sklearn-cpu.1-0:latest\",\n",
+        "    serving_container_image_uri=\"us-docker.pkg.dev/vertex-ai/prediction/sklearn-cpu.0-24:latest\",\n",
         "    explanation_metadata=exp_metadata,\n",
-        "    explanation_parameters=ExplanationParameters(\n",
-        "        sampled_shapley_attribution=SampledShapleyAttribution(path_count=25)\n",
-        "    ),\n",
+        "    explanation_parameters=explanation_parameters,\n",
         ")\n",
         "\n",
         "model.wait()\n",
@@ -1317,7 +1192,7 @@
         "gcloud beta ai models upload \\\n",
         "  --region=$REGION \\\n",
         "  --display-name=$MODEL_DISPLAY_NAME \\\n",
-        "  --container-image-uri=\"us-docker.pkg.dev/vertex-ai/prediction/sklearn-cpu.1-0:latest\" \\\n",
+        "  --container-image-uri=\"us-docker.pkg.dev/vertex-ai/prediction/sklearn-cpu.0-24:latest\" \\\n",
         "  --artifact-uri=$ARTIFACT_GCS_PATH \\\n",
         "  --explanation-method=sampled-shapley \\\n",
         "  --explanation-path-count=25 \\\n",
@@ -1342,9 +1217,7 @@
       },
       "outputs": [],
       "source": [
-        "ENDPOINT_DISPLAY_NAME = \"[your-endpoint-display-name]\"  # @param {type:\"string\"}\n",
-        "if ENDPOINT_DISPLAY_NAME == \"[your-endpoint-display-name]\":\n",
-        "    ENDPOINT_DISPLAY_NAME = \"subsc_churn_endpoint\""
+        "ENDPOINT_DISPLAY_NAME = \"[your-endpoint-display-name]\""
       ]
     },
     {
@@ -1356,13 +1229,33 @@
       "outputs": [],
       "source": [
         "endpoint = aiplatform.Endpoint.create(\n",
-        "    display_name=ENDPOINT_DISPLAY_NAME, project=PROJECT_ID, location=REGION\n",
+        "    display_name=ENDPOINT_DISPLAY_NAME, project=PROJECT, location=LOCATION\n",
         ")\n",
         "\n",
         "print(endpoint.display_name)\n",
         "print(endpoint.resource_name)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ae4c69ef8a8c"
+      },
+      "source": [
+        "Save the endpoint ID after the endpoint is created."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6aa73d9a88d3"
+      },
+      "outputs": [],
+      "source": [
+        "ENDPOINT_ID = \"[your-endpoint-id]\""
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -1382,11 +1275,8 @@
       },
       "outputs": [],
       "source": [
-        "DEPLOYED_MODEL_NAME = \"[deployment-model-name]\"  # @param {type:\"string\"}\n",
-        "MACHINE_TYPE = \"n1-standard-4\"\n",
-        "\n",
-        "if DEPLOYED_MODEL_NAME == \"[deployment-model-name]\":\n",
-        "    DEPLOYED_MODEL_NAME = \"subsc_churn_deployment\""
+        "DEPLOYED_MODEL_NAME = \"[deployment-model-name]\"\n",
+        "MACHINE_TYPE = \"n1-standard-4\""
       ]
     },
     {
@@ -1416,7 +1306,7 @@
         "id": "359c43e630cb"
       },
       "source": [
-        "To ensure the model is deployed, the ID of the deployed model can be checked using the `endpoint.list_models()` method."
+        "Save the ID of the deployed model. The ID of the deployed model can also checked using the `endpoint.list_models()` method."
       ]
     },
     {
@@ -1427,7 +1317,7 @@
       },
       "outputs": [],
       "source": [
-        "endpoint.list_models()"
+        "DEPLOYED_MODEL_ID = \"[your-deployed-model-id]\""
       ]
     },
     {
@@ -1446,7 +1336,7 @@
         "id": "7b50c31e0552"
       },
       "source": [
-        "Get explanations for a test instance from the hosted model."
+        "Get explanations for some test instances from the hosted model."
       ]
     },
     {
@@ -1457,8 +1347,8 @@
       },
       "outputs": [],
       "source": [
-        "# format a test instance as the request's payload\n",
-        "test_json = [X_test.iloc[0].tolist()]"
+        "# format the top 2 test instances as the request's payload\n",
+        "test_json = {\"instances\": [X_test.iloc[0].tolist(), X_test.iloc[1].tolist()]}"
       ]
     },
     {
@@ -1495,13 +1385,15 @@
         "    return\n",
         "\n",
         "\n",
-        "def explain_tabular_sample(project: str, location: str, endpoint, instances: list):\n",
+        "def explain_tabular_sample(\n",
+        "    project: str, location: str, endpoint_id: str, instances: list\n",
+        "):\n",
         "    \"\"\"\n",
         "    Function to make an explanation request for the specified payload and generate feature attribution plots\n",
         "    \"\"\"\n",
         "    aiplatform.init(project=project, location=location)\n",
         "\n",
-        "    # endpoint = aiplatform.Endpoint(endpoint_id)\n",
+        "    endpoint = aiplatform.Endpoint(endpoint_id)\n",
         "\n",
         "    response = endpoint.explain(instances=instances)\n",
         "    print(\"#\" * 10 + \"Explanations\" + \"#\" * 10)\n",
@@ -1530,8 +1422,8 @@
         "    return response\n",
         "\n",
         "\n",
-        "# Get explanations for the test instance\n",
-        "prediction = explain_tabular_sample(PROJECT_ID, REGION, endpoint, test_json)"
+        "test_json = [X_test.iloc[0].tolist(), X_test.iloc[1].tolist()]\n",
+        "prediction = explain_tabular_sample(PROJECT, LOCATION, ENDPOINT_ID, test_json)"
       ]
     },
     {
@@ -1546,12 +1438,7 @@
         "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
         "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
         "\n",
-        "Otherwise, you can delete the individual resources you created in this tutorial:\n",
-        "* Vertex AI Model\n",
-        "* Vertex AI Endpoint\n",
-        "* Cloud Storage bucket\n",
-        "\n",
-        "Set `delete_bucket` to *True* to delete the Cloud Storage bucket."
+        "Otherwise, you can delete the individual resources you created in this tutorial:"
       ]
     },
     {
@@ -1562,8 +1449,8 @@
       },
       "outputs": [],
       "source": [
-        "# Undeploy model\n",
-        "endpoint.undeploy_all()"
+        "# undeploy the model\n",
+        "endpoint.undeploy(deployed_model_id=DEPLOYED_MODEL_ID)"
       ]
     },
     {
@@ -1574,7 +1461,7 @@
       },
       "outputs": [],
       "source": [
-        "# Delete the endpoint\n",
+        "# delete the endpoint\n",
         "endpoint.delete()"
       ]
     },
@@ -1586,7 +1473,7 @@
       },
       "outputs": [],
       "source": [
-        "# Delete the model\n",
+        "# delete the model\n",
         "model.delete()"
       ]
     },
@@ -1598,10 +1485,8 @@
       },
       "outputs": [],
       "source": [
-        "# Delete the Cloud Storage bucket\n",
-        "delete_bucket = True\n",
-        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
-        "    ! gsutil -m rm -r $BUCKET_URI"
+        "# remove the contents of the Cloud Storage bucket\n",
+        "! gsutil -m rm -r $BUCKET_NAME"
       ]
     }
   ],

notebooks/community/matching_engine/sdk_matching_engine_for_indexing.ipynb

@@ -97,9 +97,11 @@
       "source": [
         "## Before you begin\n",
         "\n",
-        "* **Prepare a VPC network**.  To reduce any network overhead that might lead to unnecessary increase in overhead latency, it is best to call the ANN endpoints from your VPC via a direct [VPC Peering](https://cloud.google.com/vertex-ai/docs/general/vpc-peering) connection. \n",
-        "  * The following section describes how to setup a VPC Peering connection if you don't have one. \n",
-        "  * This is a one-time initial setup task. You can also reuse existing VPC network and skip this section."
+        "* **Prepare a VPC network**.  To reduce any network overhead that might lead to unnecessary increase in overhead latency, it is best to call the ANN endpoints from your VPC via a direct [VPC Peering](https://cloud.google.com/vertex-ai/docs/general/vpc-peering) connection. The following section describes how to setup a VPC Peering connection if you don't have one. This is a one-time initial setup task. You can also reuse existing VPC network and skip this section.\n",
+        "* **WARNING:** The match service gRPC API (to create online queries against your deployed index) has to be executed in a Google Cloud Notebook instance that is created with the following requirements:\n",
+        "  * **In the same region as where your ANN service is deployed** (for example, if you set `REGION = \"us-central1\"` as same as the tutorial, the notebook instance has to be in `us-central1`).\n",
+        "  * **Make sure you select the VPC network you created for ANN service** (instead of using the \"default\" one). That is, you will have to create the VPC network below and then create a new notebook instance that uses that VPC.  \n",
+        "  * If you run it in the colab or a Google Cloud Notebook instance in a different VPC network or region, the gRPC API will fail to peer the network (InactiveRPCError)."
       ]
     },
     {
@@ -110,11 +112,11 @@
       },
       "outputs": [],
       "source": [
-        "PROJECT_ID = \"python-docs-samples-tests\"  # @param {type:\"string\"}\n",
+        "PROJECT_ID = \"<your_project_id>\"  # @param {type:\"string\"}\n",
         "\n",
-        "NETWORK_NAME = \"ann-vpc-network\"  # @param {type:\"string\"}\n",
+        "NETWORK_NAME = \"ucaip-haystack-vpc-network\"  # @param {type:\"string\"}\n",
         "\n",
-        "PEERING_RANGE_NAME = \"ann-haystack-range\""
+        "PEERING_RANGE_NAME = \"ucaip-haystack-range\""
       ]
     },
     {
@@ -138,10 +140,9 @@
         "! gcloud compute firewall-rules create {NETWORK_NAME}-allow-ssh --network {NETWORK_NAME} --priority 65534 --project {PROJECT_ID} --allow tcp:22\n",
         "\n",
         "# Reserve IP range\n",
-        "! gcloud compute addresses create {PEERING_RANGE_NAME} --global --prefix-length=16 --network={NETWORK_NAME} --purpose=VPC_PEERING --project={PROJECT_ID} --description=\"peering range\"\n",
+        "! gcloud compute addresses create {PEERING_RANGE_NAME} --global --prefix-length=16 --network={NETWORK_NAME} --purpose=VPC_PEERING --project={PROJECT_ID} --description=\"peering range for uCAIP Haystack.\"\n",
         "\n",
         "# Set up peering with service networking\n",
-        "# Your account must have the \"Compute Network Admin\" role to run the following.\n",
         "! gcloud services vpc-peerings connect --service=servicenetworking.googleapis.com --network={NETWORK_NAME} --ranges={PEERING_RANGE_NAME} --project={PROJECT_ID}"
       ]
     },
@@ -151,21 +152,7 @@
         "id": "d3uj8x73nDX_"
       },
       "source": [
-        "* Authentication: Rerun the `gcloud auth login` command in the Vertex AI Workbench notebook terminal when you are logged out and need the credential again."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "d5de53b31bf1"
-      },
-      "source": [
-        "## Make sure the following cells are run from inside the VPC network that you created in the previous step.\n",
-        "\n",
-        "* **WARNING:** The MatchingIndexEndpoint.match method (to create online queries against your deployed index) has to be executed in a Vertex AI Workbench notebook instance that is created with the following requirements:\n",
-        "  * **In the same region as where your ANN service is deployed** (for example, if you set `REGION = \"us-central1\"` as same as the tutorial, the notebook instance has to be in `us-central1`).\n",
-        "  * **Make sure you select the VPC network you created for ANN service** (instead of using the \"default\" one). That is, you will have to create the VPC network below and then create a new notebook instance that uses that VPC.  \n",
-        "  * If you run it in the colab or a Vertex AI Workbench notebook instance in a different VPC network or region, \"Create Online Queries\" section will fail."
+        "* Authentication: `$ gcloud auth login` rerun this in Google Cloud Notebook terminal when you are logged out and need the credential again."
       ]
     },
     {
@@ -176,7 +163,7 @@
       "source": [
         "### Installation\n",
         "\n",
-        "Download and install the latest version of the Vertex SDK for Python."
+        "Download and install the latest (preview) version of the Vertex SDK for Python."
       ]
     },
     {
@@ -187,7 +174,7 @@
       },
       "outputs": [],
       "source": [
-        "! pip install -U google-cloud-aiplatform"
+        "! pip install -U git+https://github.com/ivanmkc/python-aiplatform.git@imkc--matching-engine"
       ]
     },
     {
@@ -284,7 +271,7 @@
       "source": [
         "import os\n",
         "\n",
-        "PROJECT_ID = \"python-docs-samples-tests\"\n",
+        "PROJECT_ID = \"\"\n",
         "\n",
         "# Get your Google Cloud project ID from gcloud\n",
         "if not os.getenv(\"IS_TESTING\"):\n",
@@ -346,7 +333,7 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using a Vertex AI Workbench notebook**, your environment is already\n",
+        "**If you are using Google Cloud Notebooks**, your environment is already\n",
         "authenticated. Skip this step."
       ]
     },
@@ -396,13 +383,11 @@
         "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
         "# requests.\n",
         "\n",
-        "# The Vertex AI Workbench notebook product has specific requirements\n",
-        "IS_VERTEX_AI_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
+        "# The Google Cloud Notebook product has specific requirements\n",
+        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
         "\n",
-        "# If on a Vertex AI Workbench notebook, then don't execute this code\n",
-        "if not IS_VERTEX_AI_WORKBENCH_NOTEBOOK:\n",
+        "# If on Google Cloud Notebooks, then don't execute this code\n",
+        "if not IS_GOOGLE_CLOUD_NOTEBOOK:\n",
         "    if \"google.colab\" in sys.modules:\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
@@ -554,12 +539,12 @@
         "id": "lR6Wwv-hCCN-"
       },
       "source": [
-        "## Prepare the data\n",
+        "## Prepare the Data\n",
         "\n",
         "The GloVe dataset consists of a set of pre-trained embeddings. The embeddings are split into a \"train\" split, and a \"test\" split.\n",
         "We will create a vector search index from the \"train\" split, and use the embedding vectors in the \"test\" split as query vectors to test the vector search index.\n",
         "\n",
-        "**Note:** While the data split uses the term \"train\", these are pre-trained embeddings and therefore are ready to be indexed for search. The terms \"train\" and \"test\" split are used just to be consistent with machine learning terminology.\n",
+        "NOTE: While the data split uses the term \"train\", these are pre-trained embeddings and thus are ready to be indexed for search. The terms \"train\" and \"test\" split are used just to be consistent with usual machine learning terminology.\n",
         "\n",
         "Download the GloVe dataset.\n"
       ]
@@ -747,28 +732,6 @@
         "INDEX_RESOURCE_NAME"
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "0f1a9fbecabb"
-      },
-      "outputs": [],
-      "source": [
-        "Using the resource name, you can retrieve an existing MatchingEngineIndex."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "1ddb70647d98"
-      },
-      "outputs": [],
-      "source": [
-        "tree_ah_index = aiplatform.MatchingEngineIndex(INDEX_RESOURCE_NAME)"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -777,7 +740,7 @@
       "source": [
         "### Create Brute Force Index (for Ground Truth)\n",
         "\n",
-        "The brute force index uses a naive brute force method to find the nearest neighbors. This method is not fast or efficient. Hence brute force indices are not recommended for production usage. They are to be used to find the \"ground truth\" set of neighbors, so that the \"ground truth\" set can be used to measure recall of the indices being tuned for production usage. To ensure an apples to apples comparison, the `distanceMeasureType` and `dimensions` of the brute force index should match those of the production indices being tuned.\n",
+        "The brute force index uses a naive brute force method to find the nearest neighbors. This method is not fast or efficient. Hence brute force indices are not recommended for production usage. They are to be used to find the \"ground truth\" set of neighbors, so that the \"ground truth\" set can be used to measure recall of the indices being tuned for production usage. To ensure an apples to apples comparison, the `distanceMeasureType` and `featureNormType`, `dimensions` of the brute force index should match those of the production indices being tuned.\n",
         "\n",
         "Create the brute force index configuration:"
       ]
@@ -812,19 +775,6 @@
         "INDEX_BRUTE_FORCE_RESOURCE_NAME"
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "865fcad494d7"
-      },
-      "outputs": [],
-      "source": [
-        "brute_force_index = aiplatform.MatchingEngineIndex(\n",
-        "    \"projects/1012616486416/locations/us-central1/indexes/6738176690918260736\"\n",
-        ")"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -989,7 +939,7 @@
       },
       "outputs": [],
       "source": [
-        "DEPLOYED_INDEX_ID = f\"tree_ah_glove_deployed_{TIMESTAMP}\""
+        "DEPLOYED_INDEX_ID = \"tree_ah_glove_deployed\""
       ]
     },
     {
@@ -1024,7 +974,7 @@
       },
       "outputs": [],
       "source": [
-        "DEPLOYED_BRUTE_FORCE_INDEX_ID = f\"glove_brute_force_deployed_{TIMESTAMP}\""
+        "DEPLOYED_BRUTE_FORCE_INDEX_ID = \"glove_brute_force_deployed\""
       ]
     },
     {
@@ -1071,13 +1021,344 @@
       "outputs": [],
       "source": [
         "# Test query\n",
+        "query = [\n",
+        "    -0.11333,\n",
+        "    0.48402,\n",
+        "    0.090771,\n",
+        "    -0.22439,\n",
+        "    0.034206,\n",
+        "    -0.55831,\n",
+        "    0.041849,\n",
+        "    -0.53573,\n",
+        "    0.18809,\n",
+        "    -0.58722,\n",
+        "    0.015313,\n",
+        "    -0.014555,\n",
+        "    0.80842,\n",
+        "    -0.038519,\n",
+        "    0.75348,\n",
+        "    0.70502,\n",
+        "    -0.17863,\n",
+        "    0.3222,\n",
+        "    0.67575,\n",
+        "    0.67198,\n",
+        "    0.26044,\n",
+        "    0.4187,\n",
+        "    -0.34122,\n",
+        "    0.2286,\n",
+        "    -0.53529,\n",
+        "    1.2582,\n",
+        "    -0.091543,\n",
+        "    0.19716,\n",
+        "    -0.037454,\n",
+        "    -0.3336,\n",
+        "    0.31399,\n",
+        "    0.36488,\n",
+        "    0.71263,\n",
+        "    0.1307,\n",
+        "    -0.24654,\n",
+        "    -0.52445,\n",
+        "    -0.036091,\n",
+        "    0.55068,\n",
+        "    0.10017,\n",
+        "    0.48095,\n",
+        "    0.71104,\n",
+        "    -0.053462,\n",
+        "    0.22325,\n",
+        "    0.30917,\n",
+        "    -0.39926,\n",
+        "    0.036634,\n",
+        "    -0.35431,\n",
+        "    -0.42795,\n",
+        "    0.46444,\n",
+        "    0.25586,\n",
+        "    0.68257,\n",
+        "    -0.20821,\n",
+        "    0.38433,\n",
+        "    0.055773,\n",
+        "    -0.2539,\n",
+        "    -0.20804,\n",
+        "    0.52522,\n",
+        "    -0.11399,\n",
+        "    -0.3253,\n",
+        "    -0.44104,\n",
+        "    0.17528,\n",
+        "    0.62255,\n",
+        "    0.50237,\n",
+        "    -0.7607,\n",
+        "    -0.071786,\n",
+        "    0.0080131,\n",
+        "    -0.13286,\n",
+        "    0.50097,\n",
+        "    0.18824,\n",
+        "    -0.54722,\n",
+        "    -0.42664,\n",
+        "    0.4292,\n",
+        "    0.14877,\n",
+        "    -0.0072514,\n",
+        "    -0.16484,\n",
+        "    -0.059798,\n",
+        "    0.9895,\n",
+        "    -0.61738,\n",
+        "    0.054169,\n",
+        "    0.48424,\n",
+        "    -0.35084,\n",
+        "    -0.27053,\n",
+        "    0.37829,\n",
+        "    0.11503,\n",
+        "    -0.39613,\n",
+        "    0.24266,\n",
+        "    0.39147,\n",
+        "    -0.075256,\n",
+        "    0.65093,\n",
+        "    -0.20822,\n",
+        "    -0.17456,\n",
+        "    0.53571,\n",
+        "    -0.16537,\n",
+        "    0.13582,\n",
+        "    -0.56016,\n",
+        "    0.016964,\n",
+        "    0.1277,\n",
+        "    0.94071,\n",
+        "    -0.22608,\n",
+        "    -0.021106,\n",
+        "]\n",
+        "\n",
         "response = my_index_endpoint.match(\n",
-        "    deployed_index_id=DEPLOYED_INDEX_ID, queries=test[:1], num_neighbors=NUM_NEIGHBOURS\n",
+        "    deployed_index_id=DEPLOYED_INDEX_ID, queries=[query], num_neighbors=NUM_NEIGHBOURS\n",
         ")\n",
         "\n",
         "response"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "_mNwdU9_B_Ez"
+      },
+      "source": [
+        "### Batch Query\n",
+        "\n",
+        "You can run multiple queries in a single match call:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "A0XL0PJ1GoM9"
+      },
+      "outputs": [],
+      "source": [
+        "# Test query\n",
+        "queries = [\n",
+        "    [\n",
+        "        -0.11333,\n",
+        "        0.48402,\n",
+        "        0.090771,\n",
+        "        -0.22439,\n",
+        "        0.034206,\n",
+        "        -0.55831,\n",
+        "        0.041849,\n",
+        "        -0.53573,\n",
+        "        0.18809,\n",
+        "        -0.58722,\n",
+        "        0.015313,\n",
+        "        -0.014555,\n",
+        "        0.80842,\n",
+        "        -0.038519,\n",
+        "        0.75348,\n",
+        "        0.70502,\n",
+        "        -0.17863,\n",
+        "        0.3222,\n",
+        "        0.67575,\n",
+        "        0.67198,\n",
+        "        0.26044,\n",
+        "        0.4187,\n",
+        "        -0.34122,\n",
+        "        0.2286,\n",
+        "        -0.53529,\n",
+        "        1.2582,\n",
+        "        -0.091543,\n",
+        "        0.19716,\n",
+        "        -0.037454,\n",
+        "        -0.3336,\n",
+        "        0.31399,\n",
+        "        0.36488,\n",
+        "        0.71263,\n",
+        "        0.1307,\n",
+        "        -0.24654,\n",
+        "        -0.52445,\n",
+        "        -0.036091,\n",
+        "        0.55068,\n",
+        "        0.10017,\n",
+        "        0.48095,\n",
+        "        0.71104,\n",
+        "        -0.053462,\n",
+        "        0.22325,\n",
+        "        0.30917,\n",
+        "        -0.39926,\n",
+        "        0.036634,\n",
+        "        -0.35431,\n",
+        "        -0.42795,\n",
+        "        0.46444,\n",
+        "        0.25586,\n",
+        "        0.68257,\n",
+        "        -0.20821,\n",
+        "        0.38433,\n",
+        "        0.055773,\n",
+        "        -0.2539,\n",
+        "        -0.20804,\n",
+        "        0.52522,\n",
+        "        -0.11399,\n",
+        "        -0.3253,\n",
+        "        -0.44104,\n",
+        "        0.17528,\n",
+        "        0.62255,\n",
+        "        0.50237,\n",
+        "        -0.7607,\n",
+        "        -0.071786,\n",
+        "        0.0080131,\n",
+        "        -0.13286,\n",
+        "        0.50097,\n",
+        "        0.18824,\n",
+        "        -0.54722,\n",
+        "        -0.42664,\n",
+        "        0.4292,\n",
+        "        0.14877,\n",
+        "        -0.0072514,\n",
+        "        -0.16484,\n",
+        "        -0.059798,\n",
+        "        0.9895,\n",
+        "        -0.61738,\n",
+        "        0.054169,\n",
+        "        0.48424,\n",
+        "        -0.35084,\n",
+        "        -0.27053,\n",
+        "        0.37829,\n",
+        "        0.11503,\n",
+        "        -0.39613,\n",
+        "        0.24266,\n",
+        "        0.39147,\n",
+        "        -0.075256,\n",
+        "        0.65093,\n",
+        "        -0.20822,\n",
+        "        -0.17456,\n",
+        "        0.53571,\n",
+        "        -0.16537,\n",
+        "        0.13582,\n",
+        "        -0.56016,\n",
+        "        0.016964,\n",
+        "        0.1277,\n",
+        "        0.94071,\n",
+        "        -0.22608,\n",
+        "        -0.021106,\n",
+        "    ],\n",
+        "    [\n",
+        "        -0.99544,\n",
+        "        -2.3651,\n",
+        "        -0.24332,\n",
+        "        -1.0321,\n",
+        "        0.42052,\n",
+        "        -1.1817,\n",
+        "        -0.16451,\n",
+        "        -1.683,\n",
+        "        0.49673,\n",
+        "        -0.27258,\n",
+        "        -0.025397,\n",
+        "        0.34188,\n",
+        "        1.5523,\n",
+        "        1.3532,\n",
+        "        0.33297,\n",
+        "        -0.0056677,\n",
+        "        -0.76525,\n",
+        "        0.49587,\n",
+        "        1.2211,\n",
+        "        0.83394,\n",
+        "        -0.20031,\n",
+        "        -0.59657,\n",
+        "        0.38485,\n",
+        "        -0.23487,\n",
+        "        -1.0725,\n",
+        "        0.95856,\n",
+        "        0.16161,\n",
+        "        -1.2496,\n",
+        "        1.6751,\n",
+        "        0.73899,\n",
+        "        0.051347,\n",
+        "        -0.42702,\n",
+        "        0.16257,\n",
+        "        -0.16772,\n",
+        "        0.40146,\n",
+        "        0.29837,\n",
+        "        0.96204,\n",
+        "        -0.36232,\n",
+        "        -0.47848,\n",
+        "        0.78278,\n",
+        "        0.14834,\n",
+        "        1.3407,\n",
+        "        0.47834,\n",
+        "        -0.39083,\n",
+        "        -1.037,\n",
+        "        -0.24643,\n",
+        "        -0.75841,\n",
+        "        0.7669,\n",
+        "        -0.37363,\n",
+        "        0.52741,\n",
+        "        0.018563,\n",
+        "        -0.51301,\n",
+        "        0.97674,\n",
+        "        0.55232,\n",
+        "        1.1584,\n",
+        "        0.73715,\n",
+        "        1.3055,\n",
+        "        -0.44743,\n",
+        "        -0.15961,\n",
+        "        0.85006,\n",
+        "        -0.34092,\n",
+        "        -0.67667,\n",
+        "        0.2317,\n",
+        "        1.5582,\n",
+        "        1.2308,\n",
+        "        -0.62213,\n",
+        "        -0.032801,\n",
+        "        0.1206,\n",
+        "        -0.25899,\n",
+        "        -0.02756,\n",
+        "        -0.52814,\n",
+        "        -0.93523,\n",
+        "        0.58434,\n",
+        "        -0.24799,\n",
+        "        0.37692,\n",
+        "        0.86527,\n",
+        "        0.069626,\n",
+        "        1.3096,\n",
+        "        0.29975,\n",
+        "        -1.3651,\n",
+        "        -0.32048,\n",
+        "        -0.13741,\n",
+        "        0.33329,\n",
+        "        -1.9113,\n",
+        "        -0.60222,\n",
+        "        -0.23921,\n",
+        "        0.12664,\n",
+        "        -0.47961,\n",
+        "        -0.89531,\n",
+        "        0.62054,\n",
+        "        0.40869,\n",
+        "        -0.08503,\n",
+        "        0.6413,\n",
+        "        -0.84044,\n",
+        "        -0.74325,\n",
+        "        -0.19426,\n",
+        "        0.098722,\n",
+        "        0.32648,\n",
+        "        -0.67621,\n",
+        "        -0.62692,\n",
+        "    ],\n",
+        "]"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -1086,7 +1367,7 @@
       "source": [
         "### Compute Recall\n",
         "\n",
-        "Use the deployed brute force Index as the ground truth to calculate the recall of ANN Index. Note that you can run multiple queries in a single match call."
+        "Use deployed brute force Index as the ground truth to calculate the recall of ANN Index:"
       ]
     },
     {
@@ -1119,20 +1400,18 @@
       "outputs": [],
       "source": [
         "# Calculate recall by determining how many neighbors were correctly retrieved as compared to the brute-force option.\n",
-        "recalled_neighbors = 0\n",
+        "correct_neighbors = 0\n",
         "for tree_ah_neighbors, brute_force_neighbors in zip(\n",
         "    tree_ah_response_test, brute_force_response_test\n",
         "):\n",
         "    tree_ah_neighbor_ids = [neighbor.id for neighbor in tree_ah_neighbors]\n",
         "    brute_force_neighbor_ids = [neighbor.id for neighbor in brute_force_neighbors]\n",
         "\n",
-        "    recalled_neighbors += len(\n",
+        "    correct_neighbors += len(\n",
         "        set(tree_ah_neighbor_ids).intersection(brute_force_neighbor_ids)\n",
         "    )\n",
         "\n",
-        "recall = recalled_neighbors / len(\n",
-        "    [neighbor for neighbors in brute_force_response_test for neighbor in neighbors]\n",
-        ")\n",
+        "recall = correct_neighbors / (len(test) * NUM_NEIGHBOURS)\n",
         "\n",
         "print(\"Recall: {}\".format(recall))"
       ]

notebooks/community/ml_metadata/sdk-metric-parameter-tracking-for-locally-trained-models.ipynb

@@ -0,0 +1,877 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ur8xi4C7S06n"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2021 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "JAPoU8Sm5E6e"
+      },
+      "source": [
+        "<table align=\"left\">\n",
+        "\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/ml_metadata/sdk-metric-parameter-tracking-for-locally-trained-models.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/ml_metadata/sdk-metric-parameter-tracking-for-locally-trained-models.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WBFL9LagqmwT"
+      },
+      "source": [
+        "#Vertex AI: Track parameters and metrics for locally trained models"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tvgnzT1CKxrO"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "This notebook demonstrates how to track metrics and parameters for ML training jobs and analyze this metadata using Vertex SDK for Python.\n",
+        "\n",
+        "### Dataset\n",
+        "\n",
+        "In this notebook, we will train a simple distributed neural network (DNN) model to predict automobile's miles per gallon (MPG) based on automobile information in the [auto-mpg dataset](https://www.kaggle.com/devanshbesain/exploration-and-analysis-auto-mpg).\n",
+        "\n",
+        "### Objective\n",
+        "\n",
+        "In this notebook, you will learn how to use Vertex SDK for Python to:\n",
+        "\n",
+        "    * Track parameters and metrics for a locally trainined model.\n",
+        "    * Extract and perform analysis for all parameters and metrics within an Experiment.\n",
+        "\n",
+        "### Costs \n",
+        "\n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Cloud Storage\n",
+        "\n",
+        "\n",
+        "Learn about [Vertex AI\n",
+        "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
+        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
+        "Calculator](https://cloud.google.com/products/calculator/)\n",
+        "to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ze4-nDLfK4pw"
+      },
+      "source": [
+        "### Set up your local development environment\n",
+        "\n",
+        "**If you are using Colab or Google Cloud Notebooks**, your environment already meets\n",
+        "all the requirements to run this notebook. You can skip this step."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gCuSR8GkAgzl"
+      },
+      "source": [
+        "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
+        "You need the following:\n",
+        "\n",
+        "* The Google Cloud SDK\n",
+        "* Git\n",
+        "* Python 3\n",
+        "* virtualenv\n",
+        "* Jupyter notebook running in a virtual environment with Python 3\n",
+        "\n",
+        "The Google Cloud guide to [Setting up a Python development\n",
+        "environment](https://cloud.google.com/python/setup) and the [Jupyter\n",
+        "installation guide](https://jupyter.org/install) provide detailed instructions\n",
+        "for meeting these requirements. The following steps provide a condensed set of\n",
+        "instructions:\n",
+        "\n",
+        "1. [Install and initialize the Cloud SDK.](https://cloud.google.com/sdk/docs/)\n",
+        "\n",
+        "1. [Install Python 3.](https://cloud.google.com/python/setup#installing_python)\n",
+        "\n",
+        "1. [Install\n",
+        "   virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv)\n",
+        "   and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
+        "\n",
+        "1. To install Jupyter, run `pip install jupyter` on the\n",
+        "command-line in a terminal shell.\n",
+        "\n",
+        "1. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
+        "\n",
+        "1. Open this notebook in the Jupyter Notebook Dashboard."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "i7EUnXsZhAGF"
+      },
+      "source": [
+        "### Install additional packages\n",
+        "\n",
+        "Run the following commands to install the Vertex SDK for Python."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "IaYsrh0Tc17L"
+      },
+      "outputs": [],
+      "source": [
+        "import sys\n",
+        "\n",
+        "if \"google.colab\" in sys.modules:\n",
+        "    USER_FLAG = \"\"\n",
+        "else:\n",
+        "    USER_FLAG = \"--user\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "wyy5Lbnzg5fi"
+      },
+      "outputs": [],
+      "source": [
+        "!python3 -m pip install {USER_FLAG} google-cloud-aiplatform --upgrade"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "hhq5zEbGg0XX"
+      },
+      "source": [
+        "### Restart the kernel\n",
+        "\n",
+        "After you install the additional packages, you need to restart the notebook kernel so it can find the packages."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "EzrelQZ22IZj"
+      },
+      "outputs": [],
+      "source": [
+        "# Automatically restart kernel after installs\n",
+        "import os\n",
+        "\n",
+        "if not os.getenv(\"IS_TESTING\"):\n",
+        "    # Automatically restart kernel after installs\n",
+        "    import IPython\n",
+        "\n",
+        "    app = IPython.Application.instance()\n",
+        "    app.kernel.do_shutdown(True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "lWEdiXsJg0XY"
+      },
+      "source": [
+        "## Before you begin\n",
+        "\n",
+        "### Select a GPU runtime\n",
+        "\n",
+        "**Make sure you're running this notebook in a GPU runtime if you have that option. In Colab, select \"Runtime --> Change runtime type > GPU\"**"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "BF1j6f9HApxa"
+      },
+      "source": [
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com).\n",
+        "\n",
+        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WReHDGG5g0XY"
+      },
+      "source": [
+        "#### Set your project ID\n",
+        "\n",
+        "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "oM1iC_MfAts1"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "PROJECT_ID = \"\"\n",
+        "\n",
+        "# Get your Google Cloud project ID from gcloud\n",
+        "if not os.getenv(\"IS_TESTING\"):\n",
+        "    shell_output=!gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "    PROJECT_ID = shell_output[0]\n",
+        "    print(\"Project ID: \", PROJECT_ID)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "qJYoRfYng0XZ"
+      },
+      "source": [
+        "Otherwise, set your project ID here."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "riG_qUokg0XZ"
+      },
+      "outputs": [],
+      "source": [
+        "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
+        "    PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "06571eb4063b"
+      },
+      "source": [
+        "#### Timestamp\n",
+        "\n",
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append it onto the name of resources you create in this tutorial."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "697568e92bd6"
+      },
+      "outputs": [],
+      "source": [
+        "from datetime import datetime\n",
+        "\n",
+        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dr--iN2kAylZ"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using Google Cloud Notebooks**, your environment is already\n",
+        "authenticated. Skip this step."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "sBCra4QMA2wR"
+      },
+      "source": [
+        "**If you are using Colab**, run the cell below and follow the instructions\n",
+        "when prompted to authenticate your account via oAuth.\n",
+        "\n",
+        "**Otherwise**, follow these steps:\n",
+        "\n",
+        "1. In the Cloud Console, go to the [**Create service account key**\n",
+        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
+        "\n",
+        "2. Click **Create service account**.\n",
+        "\n",
+        "3. In the **Service account name** field, enter a name, and\n",
+        "   click **Create**.\n",
+        "\n",
+        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
+        "into the filter box, and select\n",
+        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "\n",
+        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
+        "local environment.\n",
+        "\n",
+        "6. Enter the path to your service account key as the\n",
+        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "PyQmSRbKA8r-"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "import sys\n",
+        "\n",
+        "# If you are running this notebook in Colab, run this cell and follow the\n",
+        "# instructions to authenticate your GCP account. This provides access to your\n",
+        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
+        "# requests.\n",
+        "\n",
+        "# If on Google Cloud Notebooks, then don't execute this code\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
+        "    if \"google.colab\" in sys.modules:\n",
+        "        from google.colab import auth as google_auth\n",
+        "\n",
+        "        google_auth.authenticate_user()\n",
+        "\n",
+        "    # If you are running this notebook locally, replace the string below with the\n",
+        "    # path to your service account key and run this cell to authenticate your GCP\n",
+        "    # account.\n",
+        "    elif not os.getenv(\"IS_TESTING\"):\n",
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "XoEqT2Y4DJmf"
+      },
+      "source": [
+        "### Import libraries and define constants"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Y9Uo3tifg1kx"
+      },
+      "source": [
+        "Import required libraries."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "pRUOFELefqf1"
+      },
+      "outputs": [],
+      "source": [
+        "import matplotlib.pyplot as plt\n",
+        "import pandas as pd\n",
+        "from google.cloud import aiplatform\n",
+        "from tensorflow.python.keras import Sequential, layers\n",
+        "from tensorflow.python.keras.utils import data_utils"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "xtXZWmYqJ1bh"
+      },
+      "source": [
+        "Define some constants"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "JIOrI-hoJ46P"
+      },
+      "outputs": [],
+      "source": [
+        "EXPERIMENT_NAME = \"\"  # @param {type:\"string\"}\n",
+        "REGION = \"[your-region]\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "jWQLXXNVN4Lv"
+      },
+      "source": [
+        "If EXEPERIMENT_NAME is not set, set a default one below:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Q1QInYWOKsmo"
+      },
+      "outputs": [],
+      "source": [
+        "if EXPERIMENT_NAME == \"\" or EXPERIMENT_NAME is None:\n",
+        "    EXPERIMENT_NAME = \"my-experiment-\" + TIMESTAMP"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Xuny18aMcWDb"
+      },
+      "source": [
+        "## Concepts\n",
+        "\n",
+        "To better understanding how parameters and metrics are stored and organized, we'd like to introduce the following concepts:\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "NThDci5bp0Uw"
+      },
+      "source": [
+        "### Experiment\n",
+        "Experiments describe a context that groups your runs and the artifacts you create into a logical session. For example, in this notebook you create an Experiment and log data to that experiment."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "SAyRR3Ydp4X5"
+      },
+      "source": [
+        "### Run\n",
+        "A run represents a single path/avenue that you executed while performing an experiment. A run includes artifacts that you used as inputs or outputs, and parameters that you used in this execution. An Experiment can contain multiple runs. "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "l1YW2pgyegFP"
+      },
+      "source": [
+        "## Getting started tracking parameters and metrics\n",
+        "\n",
+        "You can use the Vertex SDK for Python to track metrics and parameters for models trained locally. \n",
+        "\n",
+        "In the following example, you train a simple distributed neural network (DNN) model to predict automobile's miles per gallon (MPG) based on automobile information in the [auto-mpg dataset](https://www.kaggle.com/devanshbesain/exploration-and-analysis-auto-mpg)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "KPY41M9_AhZU"
+      },
+      "source": [
+        "### Load and process the training dataset"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "bfMQSmRuUuX-"
+      },
+      "source": [
+        "Download and process the dataset."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "RiQuMv4bmpuV"
+      },
+      "outputs": [],
+      "source": [
+        "def read_data(uri):\n",
+        "    dataset_path = data_utils.get_file(\"auto-mpg.data\", uri)\n",
+        "    column_names = [\n",
+        "        \"MPG\",\n",
+        "        \"Cylinders\",\n",
+        "        \"Displacement\",\n",
+        "        \"Horsepower\",\n",
+        "        \"Weight\",\n",
+        "        \"Acceleration\",\n",
+        "        \"Model Year\",\n",
+        "        \"Origin\",\n",
+        "    ]\n",
+        "    raw_dataset = pd.read_csv(\n",
+        "        dataset_path,\n",
+        "        names=column_names,\n",
+        "        na_values=\"?\",\n",
+        "        comment=\"\\t\",\n",
+        "        sep=\" \",\n",
+        "        skipinitialspace=True,\n",
+        "    )\n",
+        "    dataset = raw_dataset.dropna()\n",
+        "    dataset[\"Origin\"] = dataset[\"Origin\"].map(\n",
+        "        lambda x: {1: \"USA\", 2: \"Europe\", 3: \"Japan\"}.get(x)\n",
+        "    )\n",
+        "    dataset = pd.get_dummies(dataset, prefix=\"\", prefix_sep=\"\")\n",
+        "    return dataset\n",
+        "\n",
+        "\n",
+        "dataset = read_data(\n",
+        "    \"http://archive.ics.uci.edu/ml/machine-learning-databases/auto-mpg/auto-mpg.data\"\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Y06J7A7yU21t"
+      },
+      "source": [
+        "Split dataset for training and testing."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "p5JBCBKyH-NC"
+      },
+      "outputs": [],
+      "source": [
+        "def train_test_split(dataset, split_frac=0.8, random_state=0):\n",
+        "    train_dataset = dataset.sample(frac=split_frac, random_state=random_state)\n",
+        "    test_dataset = dataset.drop(train_dataset.index)\n",
+        "    train_labels = train_dataset.pop(\"MPG\")\n",
+        "    test_labels = test_dataset.pop(\"MPG\")\n",
+        "\n",
+        "    return train_dataset, test_dataset, train_labels, test_labels\n",
+        "\n",
+        "\n",
+        "train_dataset, test_dataset, train_labels, test_labels = train_test_split(dataset)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gaNNTFPaU7KT"
+      },
+      "source": [
+        "Normalize the features in the dataset for better model performance."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "VGq5QCoyIEWJ"
+      },
+      "outputs": [],
+      "source": [
+        "def normalize_dataset(train_dataset, test_dataset):\n",
+        "    train_stats = train_dataset.describe()\n",
+        "    train_stats = train_stats.transpose()\n",
+        "\n",
+        "    def norm(x):\n",
+        "        return (x - train_stats[\"mean\"]) / train_stats[\"std\"]\n",
+        "\n",
+        "    normed_train_data = norm(train_dataset)\n",
+        "    normed_test_data = norm(test_dataset)\n",
+        "\n",
+        "    return normed_train_data, normed_test_data\n",
+        "\n",
+        "\n",
+        "normed_train_data, normed_test_data = normalize_dataset(train_dataset, test_dataset)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "UBXUgxgqA_GB"
+      },
+      "source": [
+        "### Define ML model and training function"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "66odBYKrIN4q"
+      },
+      "outputs": [],
+      "source": [
+        "def train(\n",
+        "    train_data,\n",
+        "    train_labels,\n",
+        "    num_units=64,\n",
+        "    activation=\"relu\",\n",
+        "    dropout_rate=0.0,\n",
+        "    validation_split=0.2,\n",
+        "    epochs=1000,\n",
+        "):\n",
+        "\n",
+        "    model = Sequential(\n",
+        "        [\n",
+        "            layers.Dense(\n",
+        "                num_units,\n",
+        "                activation=activation,\n",
+        "                input_shape=[len(train_dataset.keys())],\n",
+        "            ),\n",
+        "            layers.Dropout(rate=dropout_rate),\n",
+        "            layers.Dense(num_units, activation=activation),\n",
+        "            layers.Dense(1),\n",
+        "        ]\n",
+        "    )\n",
+        "\n",
+        "    model.compile(loss=\"mse\", optimizer=\"adam\", metrics=[\"mae\", \"mse\"])\n",
+        "    print(model.summary())\n",
+        "\n",
+        "    history = model.fit(\n",
+        "        train_data, train_labels, epochs=epochs, validation_split=validation_split\n",
+        "    )\n",
+        "\n",
+        "    return model, history"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "O8XJZB3gR8eL"
+      },
+      "source": [
+        "### Initialize the Vertex AI SDK for Python and create an Experiment\n",
+        "\n",
+        "Initialize the *client* for Vertex AI and create an experiment."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "o_wnT10RJ7-W"
+      },
+      "outputs": [],
+      "source": [
+        "aiplatform.init(project=PROJECT_ID, location=REGION, experiment=EXPERIMENT_NAME)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "u-iTnzt3B6Z_"
+      },
+      "source": [
+        "### Start several model training runs\n",
+        "\n",
+        "Training parameters and metrics are logged for each run."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "i2wnpu8_7JfV"
+      },
+      "outputs": [],
+      "source": [
+        "parameters = [\n",
+        "    {\"num_units\": 16, \"epochs\": 3, \"dropout_rate\": 0.1},\n",
+        "    {\"num_units\": 16, \"epochs\": 10, \"dropout_rate\": 0.1},\n",
+        "    {\"num_units\": 16, \"epochs\": 10, \"dropout_rate\": 0.2},\n",
+        "    {\"num_units\": 32, \"epochs\": 10, \"dropout_rate\": 0.1},\n",
+        "    {\"num_units\": 32, \"epochs\": 10, \"dropout_rate\": 0.2},\n",
+        "]\n",
+        "\n",
+        "for i, params in enumerate(parameters):\n",
+        "    aiplatform.start_run(run=f\"auto-mpg-local-run-{i}\")\n",
+        "    aiplatform.log_params(params)\n",
+        "    model, history = train(\n",
+        "        normed_train_data,\n",
+        "        train_labels,\n",
+        "        num_units=params[\"num_units\"],\n",
+        "        activation=\"relu\",\n",
+        "        epochs=params[\"epochs\"],\n",
+        "        dropout_rate=params[\"dropout_rate\"],\n",
+        "    )\n",
+        "    aiplatform.log_metrics(\n",
+        "        {metric: values[-1] for metric, values in history.history.items()}\n",
+        "    )\n",
+        "\n",
+        "    loss, mae, mse = model.evaluate(normed_test_data, test_labels, verbose=2)\n",
+        "    aiplatform.log_metrics({\"eval_loss\": loss, \"eval_mae\": mae, \"eval_mse\": mse})"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "jZLrJZTfL7tE"
+      },
+      "source": [
+        "### Extract parameters and metrics into a dataframe for analysis"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "A1PqKxlpOZa2"
+      },
+      "source": [
+        "We can also extract all parameters and metrics associated with any Experiment into a dataframe for further analysis."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "jbRf1WoH_vbY"
+      },
+      "outputs": [],
+      "source": [
+        "experiment_df = aiplatform.get_experiment_df()\n",
+        "experiment_df"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "EYuYgqVCMKU1"
+      },
+      "source": [
+        "### Visualizing an experiment's parameters and metrics"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "r8orCj8iJuO1"
+      },
+      "outputs": [],
+      "source": [
+        "plt.rcParams[\"figure.figsize\"] = [15, 5]\n",
+        "\n",
+        "ax = pd.plotting.parallel_coordinates(\n",
+        "    experiment_df.reset_index(level=0),\n",
+        "    \"run_name\",\n",
+        "    cols=[\n",
+        "        \"param.num_units\",\n",
+        "        \"param.dropout_rate\",\n",
+        "        \"param.epochs\",\n",
+        "        \"metric.loss\",\n",
+        "        \"metric.val_loss\",\n",
+        "        \"metric.eval_loss\",\n",
+        "    ],\n",
+        "    color=[\"blue\", \"green\", \"pink\", \"red\"],\n",
+        ")\n",
+        "ax.set_yscale(\"symlog\")\n",
+        "ax.legend(bbox_to_anchor=(1.0, 0.5))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WTHvPMweMlP1"
+      },
+      "source": [
+        "## Visualizing experiments in Cloud Console"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "F19_5lw0MqXv"
+      },
+      "source": [
+        "Run the following to get the URL of Vertex AI Experiments for your project.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "GmN9vE9pqqzt"
+      },
+      "outputs": [],
+      "source": [
+        "print(\"Vertex AI Experiments:\")\n",
+        "print(\n",
+        "    f\"https://console.cloud.google.com/ai/platform/experiments/experiments?folder=&organizationId=&project={PROJECT_ID}\"\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "TpV-iwP9qw9c"
+      },
+      "source": [
+        "## Cleaning up\n",
+        "\n",
+        "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
+        "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial."
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "sdk-metric-parameter-tracking-for-locally-trained-models.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/community/ml_ops/stage1/README.md

@@ -22,7 +22,7 @@ The first stage in MLOps is the collection and preparation for the purpose of de
 - Data is preprocessed for training and evaluation using Dataflow.
 - Data augmentation is performed on-the-fly and is coupled with model feeding.
 
-<img src='stage1v2.png'>
+<img src='stage1.jpg'>
 
 ## Notebooks
 
@@ -76,7 +76,7 @@ The steps performed include:
     - image data
 ```
 
-[Get Started with Data Labeling](get_started_with_data_labeling.ipynb)
+[Get Started with Data Labeling](get_started_data_labeling.ipynb)
 
 ```
 The steps performed include:
@@ -88,17 +88,6 @@ The steps performed include:
 - Cancel a data labeling job.
 ```
 
-[Get Started with Vision API and Vertex AI Datasets](get_started_with_visionapi_and_vertex_datasets.ipynb)
-
-```
-The steps performed include:
-
-- Using Vision API to perform Optical Character Recognition (OCR) to extract text from PDF files.
-- Processing the results and saving them to text files.
-- Generating a Vertex AI Dataset import file.
-- Creating a new unlabelled text entity extraction Vertex AI Dataset resource in Vertex AI.
-```
-
 ### E2E Stage Example
 
 [Stage 1: Data Management](mlops_data_management.ipynb)

notebooks/community/ml_ops/stage1/get_started_bq_datasets.ipynb

@@ -34,16 +34,17 @@
         "<table align=\"left\">\n",
         "  <td>\n",
         "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_bq_datasets.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
-        "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_bq_datasets.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
-        "    </a>\n",
+        "  </td>\n",
+        "    <td>\n",
+        "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_bq_datasets.ipynb\">\n",
+        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\\\" alt=\"Colab logo\"> Run in Colab\n",
+        "        </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage1/get_started_bq_datasets.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_bq_datasets.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
@@ -173,24 +174,20 @@
       "source": [
         "import os\n",
         "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
+        "# The Google Cloud Notebook product has specific requirements\n",
+        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
         "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "# Google Cloud Notebook requires dependencies to be installed with '--user'\n",
         "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "if IS_GOOGLE_CLOUD_NOTEBOOK:\n",
         "    USER_FLAG = \"--user\"\n",
         "\n",
         "# Install the packages\n",
-        "! pip3 install --upgrade pyarrow $USER_FLAG -q\n",
-        "! pip3 install --upgrade google-cloud-bigquery $USER_FLAG -q\n",
-        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG -q\n",
-        "! pip3 install -U xgboost $USER_FLAG -q\n",
-        "! pip3 install -U tensorflow $USER_FLAG -q\n",
-        "! pip3 install -U tensorflow-io==0.18 $USER_FLAG -q"
+        "! pip3 install --upgrade pyarrow $USER_FLAG\n",
+        "! pip3 install --upgrade google-cloud-bigquery $USER_FLAG\n",
+        "! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG\n",
+        "! pip3 install -U xgboost $USER_FLAG\n",
+        "! pip3 install -U tensorflow-io==0.18 $USER_FLAG"
       ]
     },
     {
@@ -222,14 +219,21 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "a47846030fef"
+      },
+      "source": [
+        "## Before you begin"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "84cd83853240"
       },
       "source": [
-        "## Before you begin\n",
-        "\n",
         "### Set up your Google Cloud project\n",
         "\n",
         "**The following steps are required, regardless of your notebook environment.**\n",
@@ -362,7 +366,7 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
+        "**If you are using Google Cloud Notebooks**, your environment is already authenticated. Skip this step.\n",
         "\n",
         "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
         "\n",
@@ -389,21 +393,20 @@
       },
       "outputs": [],
       "source": [
+        "import os\n",
+        "import sys\n",
+        "\n",
         "# If you are running this notebook in Colab, run this cell and follow the\n",
         "# instructions to authenticate your GCP account. This provides access to your\n",
         "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
         "# requests.\n",
         "\n",
-        "import os\n",
-        "import sys\n",
+        "# The Google Cloud Notebook product has specific requirements\n",
+        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
         "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = False\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
+        "# If on Google Cloud Notebooks, then don't execute this code\n",
+        "if not IS_GOOGLE_CLOUD_NOTEBOOK:\n",
         "    if \"google.colab\" in sys.modules:\n",
-        "        IS_COLAB = True\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
         "        google_auth.authenticate_user()\n",
@@ -438,8 +441,7 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+        "BUCKET_URI = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -451,8 +453,7 @@
       "outputs": [],
       "source": [
         "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
-        "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
+        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
       ]
     },
     {
@@ -1132,7 +1133,7 @@
       "source": [
         "dataframe[\"station_number\"] = pd.to_numeric(dataframe[\"station_number\"])\n",
         "labels = dataframe[\"mean_temp\"]\n",
-        "data = dataframe.drop([\"mean_temp\"], axis=1)\n",
+        "data = dataframe.drop(4)\n",
         "\n",
         "dtrain = xgb.DMatrix(data, label=labels)"
       ]
@@ -1193,7 +1194,6 @@
         "\n",
         "# Delete the dataset using the Vertex dataset object\n",
         "dataset.delete()\n",
-        "\n",
         "# Delete the temporary BigQuery dataset\n",
         "! bq rm -r -f $PROJECT_ID:$DATASET_ID\n",
         "\n",

notebooks/community/ml_ops/stage1/get_started_dataflow.ipynb

@@ -39,14 +39,8 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_dataflow.ipynb\">\n",
-        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\\\" alt=\"Colab logo\"> Run in Colab\n",
-        "        </a>\n",
-        "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_dataflow.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
+        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_dataflow.ipynb\">\n",
+        "      Open in Google Cloud Notebooks\n",
         "    </a>\n",
         "  </td>\n",
         "</table>\n",
@@ -145,7 +139,7 @@
       "source": [
         "## Installations\n",
         "\n",
-        "Install the following packages to execute this notebook."
+        "Install *one time* the packages for executing the MLOps notebooks."
       ]
     },
     {
@@ -156,26 +150,20 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
-        "! pip3 install -U tensorflow==2.5 $USER_FLAG -q\n",
-        "! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG -q\n",
-        "! pip3 install -U tensorflow-transform==1.2 $USER_FLAG -q\n",
-        "! pip3 install -U tensorflow-io==0.18 $USER_FLAG -q\n",
-        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG -q\n",
-        "! pip3 install --upgrade google-cloud-bigquery $USER_FLAG -q\n",
-        "! pip3 install --upgrade apache-beam[gcp] $USER_FLAG -q"
+        "ONCE_ONLY = False\n",
+        "if ONCE_ONLY:\n",
+        "    ! pip3 install -U tensorflow==2.5 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-transform==1.2 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-io==0.18 $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-bigquery $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-logging $USER_FLAG\n",
+        "    ! pip3 install --upgrade apache-beam[gcp] $USER_FLAG\n",
+        "    ! pip3 install --upgrade pyarrow $USER_FLAG\n",
+        "    ! pip3 install --upgrade cloudml-hypertune $USER_FLAG\n",
+        "    ! pip3 install --upgrade kfp $USER_FLAG"
       ]
     },
     {
@@ -207,32 +195,6 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "84cd83853240"
-      },
-      "source": [
-        "## Before you begin\n",
-        "\n",
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI, BigQuery, Compute Engine and Cloud Storage APIs](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,bigquery,compute_component,storage_component).\n",
-        "\n",
-        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -309,10 +271,7 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
       ]
     },
     {
@@ -339,67 +298,6 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "77c385f0db59"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
-        "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
-        "\n",
-        "1. **Click Create service account**.\n",
-        "\n",
-        "2. In the **Service account name** field, enter a name, and click **Create**.\n",
-        "\n",
-        "3. In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex AI\" into the filter box, and select **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "4. Click Create. A JSON file that contains your key downloads to your local environment.\n",
-        "\n",
-        "5. Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "535223fa4b84"
-      },
-      "outputs": [],
-      "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = False\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        IS_COLAB = True\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -428,8 +326,7 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -440,9 +337,8 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
-        "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
       ]
     },
     {
@@ -462,7 +358,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_URI"
+        "! gsutil mb -l $REGION $BUCKET_NAME"
       ]
     },
     {
@@ -482,7 +378,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_URI"
+        "! gsutil ls -al $BUCKET_NAME"
       ]
     },
     {
@@ -505,7 +401,7 @@
       },
       "outputs": [],
       "source": [
-        "import google.cloud.aiplatform as aiplatform"
+        "import google.cloud.aiplatform as aip"
       ]
     },
     {
@@ -659,7 +555,7 @@
       },
       "outputs": [],
       "source": [
-        "aiplatform.init(project=PROJECT_ID, location=REGION)"
+        "aip.init(project=PROJECT_ID, location=REGION)"
       ]
     },
     {
@@ -1108,7 +1004,7 @@
       },
       "outputs": [],
       "source": [
-        "SCHEMA_LOCATION = BUCKET_URI + \"/schema.txt\"\n",
+        "SCHEMA_LOCATION = BUCKET_NAME + \"/schema.txt\"\n",
         "\n",
         "# When running Apache Beam directly (file is directly accessed)\n",
         "tfdv.write_schema_text(output_path=SCHEMA_LOCATION, schema=schema)\n",
@@ -1253,7 +1149,7 @@
         "            )\n",
         "\n",
         "\n",
-        "EXPORTED_DATA_PREFIX = os.path.join(BUCKET_URI, \"exported_data\")\n",
+        "EXPORTED_DATA_PREFIX = os.path.join(BUCKET_NAME, \"exported_data\")\n",
         "\n",
         "QUERY_STRING = \"SELECT {},{} FROM {} LIMIT 500\".format(\n",
         "    \"CAST(station_number as STRING) AS station_number,year,month,day\",\n",
@@ -1266,7 +1162,7 @@
         "    \"runner\": RUNNER,\n",
         "    \"raw_data_query\": QUERY_STRING,\n",
         "    \"exported_data_prefix\": EXPORTED_DATA_PREFIX,\n",
-        "    \"temp_location\": os.path.join(BUCKET_URI, \"temp\"),\n",
+        "    \"temp_location\": os.path.join(BUCKET_NAME, \"temp\"),\n",
         "    \"project\": PROJECT_ID,\n",
         "    \"region\": REGION,\n",
         "    \"setup_file\": \"./setup.py\",\n",
@@ -1291,7 +1187,17 @@
         "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
         "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
         "\n",
-        "Otherwise, you can delete the individual resources you created in this tutorial."
+        "Otherwise, you can delete the individual resources you created in this tutorial:\n",
+        "\n",
+        "- Dataset\n",
+        "- Pipeline\n",
+        "- Model\n",
+        "- Endpoint\n",
+        "- AutoML Training Job\n",
+        "- Batch Job\n",
+        "- Custom Job\n",
+        "- Hyperparameter Tuning Job\n",
+        "- Cloud Storage Bucket"
       ]
     },
     {
@@ -1302,11 +1208,61 @@
       },
       "outputs": [],
       "source": [
-        "delete_storage = True\n",
+        "delete_all = True\n",
         "\n",
-        "if delete_storage or os.getenv(\"IS_TESTING\"):\n",
-        "    if \"BUCKET_URI\" in globals():\n",
-        "        ! gsutil rm -r $BUCKET_URI"
+        "if delete_all:\n",
+        "    # Delete the dataset using the Vertex dataset object\n",
+        "    try:\n",
+        "        if \"dataset\" in globals():\n",
+        "            dataset.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the model using the Vertex model object\n",
+        "    try:\n",
+        "        if \"model\" in globals():\n",
+        "            model.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the endpoint using the Vertex endpoint object\n",
+        "    try:\n",
+        "        if \"endpoint\" in globals():\n",
+        "            endpoint.undeploy_all()\n",
+        "            endpoint.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the AutoML or Pipeline training job\n",
+        "    try:\n",
+        "        if \"dag\" in globals():\n",
+        "            dag.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the custom training job\n",
+        "    try:\n",
+        "        if \"job\" in globals():\n",
+        "            job.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the batch prediction job using the Vertex batch prediction object\n",
+        "    try:\n",
+        "        if \"batch_predict_job\" in globals():\n",
+        "            batch_predict_job.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the hyperparameter tuning job using the Vertex hyperparameter tuning object\n",
+        "    try:\n",
+        "        if \"hpt_job\" in globals():\n",
+        "            hpt_job.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    if \"BUCKET_NAME\" in globals():\n",
+        "        ! gsutil rm -r $BUCKET_NAME"
       ]
     }
   ],

notebooks/community/ml_ops/stage1/get_started_vertex_datasets.ipynb

@@ -8,7 +8,7 @@
       },
       "outputs": [],
       "source": [
-        "# Copyright 2022 Google LLC\n",
+        "# Copyright 2021 Google LLC\n",
         "#\n",
         "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
         "# you may not use this file except in compliance with the License.\n",
@@ -34,21 +34,15 @@
         "<table align=\"left\">\n",
         "  <td>\n",
         "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_vertex_datasets.ipynb\">\n",
-        "     <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
         "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_vertex_datasets.ipynb\">\n",
-        "<img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
-        "      View on GitHub\n",
+        "      Open in Google Cloud Notebooks\n",
         "    </a>\n",
         "  </td>\n",
-        "   <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage1/get_started_vertex_datasets.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
-        "    </a>\n",
-        "  </td>   \n",
         "</table>\n",
         "<br/><br/><br/>"
       ]
@@ -136,17 +130,7 @@
         "            - Create a tf.data.Dataset generator from the CSV index file.\n",
         "        - If text strings are in text files:\n",
         "            - Using the JSON index file, convert the text files and labels to TFRecords.\n",
-        "            - Create a tf.data.Dataset from the TFRecords.\n",
-        "\n",
-        "    \n",
-        "### Costs\n",
-        "This tutorial uses billable components of Google Cloud:\n",
-        "\n",
-        "- Vertex AI\n",
-        "- Cloud Storage\n",
-        "- BigQuery\n",
-        "\n",
-        "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing), [Cloud Storage pricing](https://cloud.google.com/storage/pricing) and [BigQuery pricing](https://cloud.google.com/bigquery/pricing) and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
+        "            - Create a tf.data.Dataset from the TFRecords."
       ]
     },
     {
@@ -157,7 +141,7 @@
       "source": [
         "## Installations\n",
         "\n",
-        "Install the packages required for executing this notebook."
+        "Install *one time* the packages for executing the MLOps notebooks."
       ]
     },
     {
@@ -168,27 +152,20 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
-        "! pip3 install -U tensorflow $USER_FLAG -q\n",
-        "! pip3 install -U tensorflow-data-validation $USER_FLAG -q\n",
-        "! pip3 install -U tensorflow-transform $USER_FLAG -q\n",
-        "! pip3 install -U tensorflow-io $USER_FLAG -q\n",
-        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG -q\n",
-        "! pip3 install --upgrade google-cloud-bigquery $USER_FLAG -q\n",
-        "! pip3 install -U tensorflow-io==0.18 $USER_FLAG -q\n",
-        "! pip3 install --upgrade db-dtypes $USER_FLAG -q! pip3 install --upgrade future $USER_FLAG -q"
+        "ONCE_ONLY = False\n",
+        "if ONCE_ONLY:\n",
+        "    ! pip3 install -U tensorflow==2.5 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-transform==1.2 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-io==0.18 $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-bigquery $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-logging $USER_FLAG\n",
+        "    ! pip3 install --upgrade apache-beam[gcp] $USER_FLAG\n",
+        "    ! pip3 install --upgrade pyarrow $USER_FLAG\n",
+        "    ! pip3 install --upgrade cloudml-hypertune $USER_FLAG\n",
+        "    ! pip3 install --upgrade kfp $USER_FLAG"
       ]
     },
     {
@@ -220,30 +197,6 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "cb082379ed5b"
-      },
-      "source": [
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com). \n",
-        "\n",
-        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -263,24 +216,7 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
-        "PROJECT_ID = \"\"\n",
-        "\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    # Get your Google Cloud project ID from gcloud\n",
-        "    shell_output = !gcloud config list --format 'value(core.project)' 2>/dev/null\n",
-        "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID: \", PROJECT_ID)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "37c0a68ff20d"
-      },
-      "source": [
-        "Otherwise, set your project ID here."
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -291,15 +227,18 @@
       },
       "outputs": [],
       "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
-        "    PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
+        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
+        "    # Get your GCP project id from gcloud\n",
+        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "    PROJECT_ID = shell_output[0]\n",
+        "    print(\"Project ID:\", PROJECT_ID)"
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "c021ca495967"
+        "id": "set_gcloud_project_id"
       },
       "outputs": [],
       "source": [
@@ -334,10 +273,7 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
       ]
     },
     {
@@ -364,66 +300,6 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "927085b84a07"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
-        "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
-        "\n",
-        "**Click Create service account**.\n",
-        "\n",
-        "In the **Service account name** field, enter a name, and click **Create**.\n",
-        "\n",
-        "In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex\" into the filter box, and select **Vertex Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "Click Create. A JSON file that contains your key downloads to your local environment.\n",
-        "\n",
-        "Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "89788a802687"
-      },
-      "outputs": [],
-      "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = \"google.colab\" in sys.modules\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -452,7 +328,7 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_URI = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
+        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -463,8 +339,8 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
       ]
     },
     {
@@ -484,7 +360,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_URI"
+        "! gsutil mb -l $REGION $BUCKET_NAME"
       ]
     },
     {
@@ -504,7 +380,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_URI"
+        "! gsutil ls -al $BUCKET_NAME"
       ]
     },
     {
@@ -516,11 +392,7 @@
         "### Set up variables\n",
         "\n",
         "Next, set up some variables used throughout the tutorial.\n",
-        "### Import libraries and define constants\n",
-        "\n",
-        "Import the BigQuery package, TensorFlow Data Validation (TFDV) package and TensorFlow Data Validation package into your Python environment. \n",
-        "\n",
-        "Import TensorFlow Transform (TFT) package and pandas into your Python environment."
+        "### Import libraries and define constants"
       ]
     },
     {
@@ -531,13 +403,97 @@
       },
       "outputs": [],
       "source": [
-        "import google.cloud.aiplatform as aip\n",
-        "import pandas as pd\n",
-        "import tensorflow_data_validation as tfdv\n",
-        "import tensorflow_transform as tft\n",
+        "import google.cloud.aiplatform as aip"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "import_bq"
+      },
+      "source": [
+        "#### Import BigQuery\n",
+        "\n",
+        "Import the BigQuery package into your Python environment."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "import_bq"
+      },
+      "outputs": [],
+      "source": [
         "from google.cloud import bigquery"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "import_tfdv"
+      },
+      "source": [
+        "#### Import TensorFlow Data Validation\n",
+        "\n",
+        "Import the TensorFlow Data Validation (TFDV) package into your Python environment."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "import_tfdv"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow_data_validation as tfdv"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "import_tft"
+      },
+      "source": [
+        "#### Import TensorFlow Transform\n",
+        "\n",
+        "Import the TensorFlow Transform (TFT) package into your Python environment."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "import_tft"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow_transform as tft"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "import_pandas"
+      },
+      "source": [
+        "#### Import pandas\n",
+        "\n",
+        "Import the pandas package into your Python environment."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "import_pandas"
+      },
+      "outputs": [],
+      "source": [
+        "import pandas as pd"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -557,7 +513,7 @@
       },
       "outputs": [],
       "source": [
-        "aip.init(project=PROJECT_ID, location=REGION, staging_bucket=BUCKET_URI)"
+        "aip.init(project=PROJECT_ID, location=REGION)"
       ]
     },
     {
@@ -611,13 +567,26 @@
         "Learn more about [All dataset documentation](https://cloud.google.com/vertex-ai/docs/datasets/datasets)"
       ]
     },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "import_file:flowers,csv,icn"
+      },
+      "outputs": [],
+      "source": [
+        "IMPORT_FILE = (\n",
+        "    \"gs://cloud-samples-data/vision/automl_classification/flowers/all_data_v2.csv\"\n",
+        ")"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "create_dataset:image,icn"
       },
       "source": [
-        "### Create an Image Dataset\n",
+        "### Create the Dataset\n",
         "\n",
         "Next, create the `Dataset` resource using the `create` method for the `ImageDataset` class, which takes the following parameters:\n",
         "\n",
@@ -632,19 +601,6 @@
         "Learn more about [ImageDataset](https://cloud.google.com/vertex-ai/docs/datasets/prepare-image)."
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "import_file:flowers,csv,icn"
-      },
-      "outputs": [],
-      "source": [
-        "IMPORT_FILE = (\n",
-        "    \"gs://cloud-samples-data/vision/automl_classification/flowers/all_data_v2.csv\"\n",
-        ")"
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -662,13 +618,24 @@
         "print(dataset.resource_name)"
       ]
     },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "import_file:hmdb,csv,vcn"
+      },
+      "outputs": [],
+      "source": [
+        "IMPORT_FILE = \"gs://automl-video-demo-data/hmdb_split1_5classes_train_inf.csv\""
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "create_dataset:video,vcn"
       },
       "source": [
-        "### Create a Video Dataset\n",
+        "### Create the Dataset\n",
         "\n",
         "Next, create the `Dataset` resource using the `create` method for the `VideoDataset` class, which takes the following parameters:\n",
         "\n",
@@ -682,17 +649,6 @@
         "Learn more about [VideoDataset](https://cloud.google.com/vertex-ai/docs/datasets/prepare-video)."
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "import_file:hmdb,csv,vcn"
-      },
-      "outputs": [],
-      "source": [
-        "IMPORT_FILE = \"gs://automl-video-demo-data/hmdb_split1_5classes_train_inf.csv\""
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -710,13 +666,24 @@
         "print(dataset.resource_name)"
       ]
     },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "import_file:happydb,csv,tcn"
+      },
+      "outputs": [],
+      "source": [
+        "IMPORT_FILE = \"gs://cloud-ml-data/NL-classification/happiness.csv\""
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "create_dataset:text,tcn"
       },
       "source": [
-        "### Create a Text Dataset\n",
+        "### Create the Dataset\n",
         "\n",
         "Next, create the `Dataset` resource using the `create` method for the `TextDataset` class, which takes the following parameters:\n",
         "\n",
@@ -731,17 +698,6 @@
         "Learn more about [TextDataset](https://cloud.google.com/vertex-ai/docs/datasets/prepare-text)."
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "import_file:happydb,csv,tcn"
-      },
-      "outputs": [],
-      "source": [
-        "IMPORT_FILE = \"gs://cloud-ml-data/NL-classification/happiness.csv\""
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -759,24 +715,6 @@
         "print(dataset.resource_name)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "create_dataset:tabular,bq,lrg,v2"
-      },
-      "source": [
-        "### Create a Tabular Dataset\n",
-        "\n",
-        "#### CSV input data\n",
-        "\n",
-        "Next, create the `Dataset` resource using the `create` method for the `TabularDataset` class for CSV input data, which takes the following parameters:\n",
-        "\n",
-        "- `display_name`: The human readable name for the `Dataset` resource.\n",
-        "- `gcs_source`: A list of one or more dataset index files to import the data items into the `Dataset` resource.\n",
-        "\n",
-        "Learn more about [TabularDataset from CSV files](https://cloud.google.com/vertex-ai/docs/datasets/create-dataset-api#aiplatform_create_dataset_tabular_gcs_sample-python)"
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -785,50 +723,27 @@
       },
       "outputs": [],
       "source": [
-        "IMPORT_FILE = \"gs://cloud-samples-data/tables/iris_1000.csv\""
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "create_dataset:tabular,bq,lrg,v2"
-      },
-      "outputs": [],
-      "source": [
-        "dataset = aip.TabularDataset.create(\n",
-        "    display_name=\"example\" + \"_\" + TIMESTAMP, gcs_source=[IMPORT_FILE]\n",
-        ")\n",
-        "\n",
-        "print(dataset.resource_name)"
+        "IMPORT_FILE = \"bq://bigquery-public-data.samples.gsod\"\n",
+        "BQ_TABLE = \"bigquery-public-data.samples.gsod\""
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "854dd1e0195c"
+        "id": "create_dataset:tabular,bq,lrg,v2"
       },
       "source": [
-        "#### BigQuery input data\n",
+        "### Create the Dataset\n",
         "\n",
-        "Next, create the `Dataset` resource using the `create` method for the `TabularDataset` class for BigQuery table input, which takes the following parameters:\n",
+        "#### CSV input data\n",
+        "\n",
+        "Next, create the `Dataset` resource using the `create` method for the `TabularDataset` class, which takes the following parameters:\n",
         "\n",
         "- `display_name`: The human readable name for the `Dataset` resource.\n",
-        "- `bq_source`: A list of one or more BigQuery tables to import the data items into the `Dataset` resource.\n",
+        "- `gcs_source`: A list of one or more dataset index files to import the data items into the `Dataset` resource.\n",
+        "- `labels`: User defined metadata. In this example, you store the location of the Cloud Storage bucket containing the user defined data.\n",
         "\n",
-        "Learn more about [TabularDataset from BigQuery table](https://cloud.google.com/vertex-ai/docs/datasets/create-dataset-api#aiplatform_create_dataset_tabular_bigquery_sample-pythonn)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "86343c146300"
-      },
-      "outputs": [],
-      "source": [
-        "IMPORT_FILE = \"bq://bigquery-public-data.samples.gsod\"\n",
-        "BQ_TABLE = \"bigquery-public-data.samples.gsod\""
+        "Learn more about [TabularDataset from CSV files](https://cloud.google.com/vertex-ai/docs/datasets/create-dataset-api#aiplatform_create_dataset_tabular_gcs_sample-python)"
       ]
     },
     {
@@ -846,82 +761,6 @@
         "print(dataset.resource_name)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "82e9fe20ce71"
-      },
-      "source": [
-        "#### Dataframe input data\n",
-        "\n",
-        "Next, create the `Dataset` resource using the `create_from_dataframe` method for the `TabularDataset` class for pandas dataframe input, which takes the following parameters:\n",
-        "\n",
-        "- `display_name`: The human readable name for the `Dataset` resource.\n",
-        "- `df_source`: The pandas dataframe to import the data items into the `Dataset` resource.\n",
-        "- `staging_path`: The BigQuery table to store the imported data."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "3805f945ffdd"
-      },
-      "outputs": [],
-      "source": [
-        "# Download the table.\n",
-        "table = bigquery.TableReference.from_string(BQ_TABLE)\n",
-        "\n",
-        "rows = bqclient.list_rows(\n",
-        "    table,\n",
-        "    max_results=10000,\n",
-        "    selected_fields=[\n",
-        "        bigquery.SchemaField(\"station_number\", \"STRING\"),\n",
-        "        bigquery.SchemaField(\"year\", \"INTEGER\"),\n",
-        "        bigquery.SchemaField(\"month\", \"INTEGER\"),\n",
-        "        bigquery.SchemaField(\"day\", \"INTEGER\"),\n",
-        "        bigquery.SchemaField(\"mean_temp\", \"FLOAT\"),\n",
-        "    ],\n",
-        ")\n",
-        "\n",
-        "dataframe = rows.to_dataframe()\n",
-        "print(dataframe.head())"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "create_dataset:tabular,bq,lrg,v2"
-      },
-      "outputs": [],
-      "source": [
-        "dataset = aip.TabularDataset.create_from_dataframe(\n",
-        "    display_name=\"example\" + \"_\" + TIMESTAMP,\n",
-        "    df_source=dataframe,\n",
-        "    staging_path=f\"bq://{PROJECT_ID}.samples.gsod\",\n",
-        ")\n",
-        "\n",
-        "print(dataset.resource_name)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "create_dataset:tabular,forecast,v2"
-      },
-      "source": [
-        "### Create a Time Series Dataset\n",
-        "\n",
-        "Next, create the `Dataset` resource using the `create` method for the `TimeSeriesDataset` class, which takes the following parameters:\n",
-        "\n",
-        "- `display_name`: The human readable name for the `Dataset` resource.\n",
-        "- `gcs_source`: A list of one or more dataset index files to import the data items into the `Dataset` resource.\n",
-        "- `bq_source`: Alternatively, import data items from a BigQuery table into the `Dataset` resource.\n",
-        "\n",
-        "Learn more about [TimeSeriesDataset](https://cloud.google.com/vertex-ai/docs/datasets/prepare-tabular)."
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -933,6 +772,23 @@
         "IMPORT_FILE = \"gs://cloud-samples-data/ai-platform/covid/bigquery-public-covid-nyt-us-counties-train.csv\""
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "create_dataset:tabular,forecast,v2"
+      },
+      "source": [
+        "### Create the Dataset\n",
+        "\n",
+        "Next, create the `Dataset` resource using the `create` method for the `TimeSeriesDataset` class, which takes the following parameters:\n",
+        "\n",
+        "- `display_name`: The human readable name for the `Dataset` resource.\n",
+        "- `gcs_source`: A list of one or more dataset index files to import the data items into the `Dataset` resource.\n",
+        "- `bq_source`: Alternatively, import data items from a BigQuery table into the `Dataset` resource.\n",
+        "\n",
+        "Learn more about [TimeSeriesDataset](https://cloud.google.com/vertex-ai/docs/datasets/prepare-tabular)."
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -1293,9 +1149,9 @@
         "comps = BQ_TABLE.split(\".\")\n",
         "BQ_PROJECT_DATASET_TABLE = comps[0] + \":\" + comps[1] + \".\" + comps[2]\n",
         "\n",
-        "! bq --location=us extract --destination_format CSV $BQ_PROJECT_DATASET_TABLE $BUCKET_URI/mydata*.csv\n",
+        "! bq --location=us extract --destination_format CSV $BQ_PROJECT_DATASET_TABLE $BUCKET_NAME/mydata*.csv\n",
         "\n",
-        "IMPORT_FILES = ! gsutil ls $BUCKET_URI/mydata*.csv\n",
+        "IMPORT_FILES = ! gsutil ls $BUCKET_NAME/mydata*.csv\n",
         "\n",
         "print(IMPORT_FILES)\n",
         "\n",
@@ -1353,38 +1209,6 @@
         "To create a dataframe from multiple CSV sources, you read each CSV file and concatenate the dataframes together."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "bcd2e4e0703b"
-      },
-      "source": [
-        "If you are running this notebook on Colab, run the following cell to install packages fsspec and gcsfs."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "927bd3f92268"
-      },
-      "outputs": [],
-      "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Workbench AI Notebook, then don't execute this code\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        ! pip3 install fsspec\n",
-        "        ! pip3 install gcsfs"
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -1445,7 +1269,7 @@
       },
       "outputs": [],
       "source": [
-        "EXPORTED_DIR = f\"{BUCKET_URI}/exported\"\n",
+        "EXPORTED_DIR = f\"{BUCKET_NAME}/exported\"\n",
         "exported_files = dataset.export_data(output_dir=EXPORTED_DIR)\n",
         "\n",
         "! gsutil ls $EXPORTED_DIR"
@@ -1674,7 +1498,7 @@
         "    data = f.readlines()\n",
         "\n",
         "# The path to the TFRecord cached file.\n",
-        "GCS_TFRECORD_URI = BUCKET_URI + \"/flowers.tfrecord\"\n",
+        "GCS_TFRECORD_URI = BUCKET_NAME + \"/flowers.tfrecord\"\n",
         "\n",
         "# Create the TFRecord cached file\n",
         "with tf.io.TFRecordWriter(GCS_TFRECORD_URI) as writer:\n",
@@ -1708,7 +1532,14 @@
         "Otherwise, you can delete the individual resources you created in this tutorial:\n",
         "\n",
         "- Dataset\n",
-        "- Bucket"
+        "- Pipeline\n",
+        "- Model\n",
+        "- Endpoint\n",
+        "- AutoML Training Job\n",
+        "- Batch Job\n",
+        "- Custom Job\n",
+        "- Hyperparameter Tuning Job\n",
+        "- Cloud Storage Bucket"
       ]
     },
     {
@@ -1719,16 +1550,61 @@
       },
       "outputs": [],
       "source": [
-        "delete_bucket = False\n",
+        "delete_all = True\n",
         "\n",
-        "# Delete the dataset using the Vertex dataset object\n",
-        "datasets = aip.TabularDataset.list(filter=f'display_name=\"example_{TIMESTAMP}\"')\n",
-        "for dataset in datasets:\n",
-        "    dataset.delete()\n",
+        "if delete_all:\n",
+        "    # Delete the dataset using the Vertex dataset object\n",
+        "    try:\n",
+        "        if \"dataset\" in globals():\n",
+        "            dataset.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
         "\n",
-        "# Delete the bucket\n",
-        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
-        "    ! gsutil rm -r $BUCKET_URI"
+        "    # Delete the model using the Vertex model object\n",
+        "    try:\n",
+        "        if \"model\" in globals():\n",
+        "            model.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the endpoint using the Vertex endpoint object\n",
+        "    try:\n",
+        "        if \"endpoint\" in globals():\n",
+        "            endpoint.undeploy_all()\n",
+        "            endpoint.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the AutoML or Pipeline training job\n",
+        "    try:\n",
+        "        if \"dag\" in globals():\n",
+        "            dag.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the custom training job\n",
+        "    try:\n",
+        "        if \"job\" in globals():\n",
+        "            job.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the batch prediction job using the Vertex batch prediction object\n",
+        "    try:\n",
+        "        if \"batch_predict_job\" in globals():\n",
+        "            batch_predict_job.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the hyperparameter tuning job using the Vertex hyperparameter tuning object\n",
+        "    try:\n",
+        "        if \"hpt_job\" in globals():\n",
+        "            hpt_job.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    if \"BUCKET_NAME\" in globals():\n",
+        "        ! gsutil rm -r $BUCKET_NAME"
       ]
     }
   ],

notebooks/community/ml_ops/stage1/get_started_with_data_labeling.ipynb

@@ -44,7 +44,7 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage1/get_started_with_data_labeling.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_with_data_labeling.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
@@ -145,20 +145,17 @@
       "source": [
         "import os\n",
         "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
+        "# The Google Cloud Notebook product has specific requirements\n",
+        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
         "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "# Google Cloud Notebook requires dependencies to be installed with '--user'\n",
         "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "if IS_GOOGLE_CLOUD_NOTEBOOK:\n",
         "    USER_FLAG = \"--user\"\n",
         "\n",
         "# Install the packages\n",
-        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG -q\n",
-        "! pip3 install --upgrade google-cloud-storage $USER_FLAG -q"
+        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG\n",
+        "! pip3 install --upgrade google-cloud-storage $USER_FLAG"
       ]
     },
     {
@@ -210,7 +207,7 @@
         "\n",
         "2. [Make sure that billing is enabled for your project.](https://cloud.google.com/billing/docs/how-to/modify-project)\n",
         "\n",
-        "3. [Enable the Vertex AI APIs and Compute Engine APIs.](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component)\n",
+        "3. [Enable the Vertex AI APIs and Compute Engine APIs.](https://console.cloud.google.com/flows/enableapi?apiid=ml.googleapis.com,compute_component)\n",
         "\n",
         "4. [Google Cloud SDK](https://cloud.google.com/sdk) is already installed in Google Cloud Notebooks.\n",
         "\n",
@@ -362,7 +359,7 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
+        "**If you are using Google Cloud Notebooks**, your environment is already\n",
         "authenticated. Skip this step."
       ]
     },
@@ -404,21 +401,20 @@
       },
       "outputs": [],
       "source": [
+        "import os\n",
+        "import sys\n",
+        "\n",
         "# If you are running this notebook in Colab, run this cell and follow the\n",
         "# instructions to authenticate your GCP account. This provides access to your\n",
         "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
         "# requests.\n",
         "\n",
-        "import os\n",
-        "import sys\n",
+        "# The Google Cloud Notebook product has specific requirements\n",
+        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
         "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = False\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
+        "# If on Google Cloud Notebooks, then don't execute this code\n",
+        "if not IS_GOOGLE_CLOUD_NOTEBOOK:\n",
         "    if \"google.colab\" in sys.modules:\n",
-        "        IS_COLAB = True\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
         "        google_auth.authenticate_user()\n",
@@ -466,8 +462,7 @@
       "outputs": [],
       "source": [
         "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
-        "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
+        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
       ]
     },
     {

notebooks/community/ml_ops/stage1/get_started_with_visionapi_and_vertex_datasets.ipynb

@@ -1,994 +0,0 @@
-{
-  "cells": [
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "copyright"
-      },
-      "outputs": [],
-      "source": [
-        "# Copyright 2022 Google LLC\n",
-        "#\n",
-        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
-        "# you may not use this file except in compliance with the License.\n",
-        "# You may obtain a copy of the License at\n",
-        "#\n",
-        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
-        "#\n",
-        "# Unless required by applicable law or agreed to in writing, software\n",
-        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
-        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
-        "# See the License for the specific language governing permissions and\n",
-        "# limitations under the License."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "4JIDiHvGasba"
-      },
-      "source": [
-        "This notebook was contributed by [Mohammad Al-Ansari](https://github.com/Mansari)."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "2xDiUNIZINWp"
-      },
-      "source": [
-        "# E2E ML on GCP: MLOps stage 1 : data management: create an unlabelled Vertex AI AutoML text entity extraction dataset from PDFs using Vision API\n",
-        "\n",
-        "<table align=\"left\">\n",
-        "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_with_visionapi_and_vertex_datasets.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
-        "    </a>\n",
-        "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_with_visionapi_and_vertex_datasets.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
-        "      View on GitHub\n",
-        "    </a>\n",
-        "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_with_visionapi_and_vertex_datasets.ipynb\">\n",
-        "     <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
-        "    </a>\n",
-        "  </td>\n",
-        "</table>\n",
-        "<br/><br/><br/>"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "H0alLPo_A-LK"
-      },
-      "source": [
-        "## Overview\n",
-        "\n",
-        "This notebook will create an unlabelled `Vertex AI AutoML` text entity extraction dataset based on a collection of PDF files stored in a Cloud Storage bucket. \n",
-        "\n",
-        "The notebook can be modified to create different types of text datasets including sentiment analysis and classification."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "W4IBLTKOA5nl"
-      },
-      "source": [
-        "### Dataset\n",
-        "\n",
-        "The dataset used for this tutorial is the [Patent PDF Samples with Extracted Structured Data](https://console.cloud.google.com/marketplace/product/global-patents/labeled-patents) from Google Public Data Sets. \n",
-        "\n",
-        "This dataset includes data extracted from over 300 patent documents issued in the US and EU. The dataset includes links to Cloud Storage blobs for the first page of each patent, in addition to a number of extracted entities. \n",
-        "\n",
-        "The data is published as a [public dataset](https://cloud.google.com/bigquery/public-data) on `BigQuery`."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "3f8c2f702ccd"
-      },
-      "source": [
-        "### Objective\n",
-        "\n",
-        "In this tutorial, you learn to use `Vision API` to extract text from PDF files stored on a Cloud Storage bucket. You will then process the results and create an unlabelled `Vertex AI Dataset`, compatible with `AutoML`, for text entity extraction.\n",
-        "\n",
-        "You can then either use Google Cloud console to annotate / label the dataset, or create a labelling job as demonstrated in [this notebook](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_with_data_labeling.ipynb).\n",
-        "\n",
-        "This tutorial uses the following Google Cloud services:\n",
-        "\n",
-        "- `Vision AI`\n",
-        "- `Vertex AI AutoML`\n",
-        "\n",
-        "The steps performed include:\n",
-        "\n",
-        "1. Using `Vision API` to perform Optical Character Recognition (OCR) to extract text from PDF files.\n",
-        "2. Processing the results and saving them to text files.\n",
-        "3. Generating a `Vertex AI Dataset` import file.\n",
-        "4. Creating a new unlabelled text entity extraction `Vertex AI Dataset` resource in `Vertex AI`."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "CgLDJ419LPJs"
-      },
-      "source": [
-        "### Costs\n",
-        "\n",
-        "This tutorial uses billable components of Google Cloud:\n",
-        "\n",
-        "* Vision API\n",
-        "* Vertex AI\n",
-        "* Cloud Storage\n",
-        "\n",
-        "Learn about [Vertex AI\n",
-        "pricing](https://cloud.google.com/vertex-ai/pricing), [Vision API pricing](https://cloud.google.com/vision/pricing), [Cloud Storage\n",
-        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
-        "Calculator](https://cloud.google.com/products/calculator/)\n",
-        "to generate a cost estimate based on your projected usage."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "va2g7m9wLTjA"
-      },
-      "source": [
-        "### Set up your local development environment\n",
-        "\n",
-        "If you are using Colab or Vertex AI Workbench Notebooks, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
-        "\n",
-        "Otherwise, make sure your environment meets this notebook's requirements. You need the following:\n",
-        "\n",
-        "- The Vision API SDK\n",
-        "- The Vertex AI SDK\n",
-        "- The Cloud Storage SDK\n",
-        "- Git\n",
-        "- Python 3\n",
-        "- virtualenv\n",
-        "- Jupyter notebook running in a virtual environment with Python 3\n",
-        "\n",
-        "The Cloud Storage guide to [Setting up a Python development environment](https://cloud.google.com/python/setup) and the [Jupyter installation guide](https://jupyter.org/install) provide detailed instructions for meeting these requirements. The following steps provide a condensed set of instructions:\n",
-        "\n",
-        "1. [Install and initialize the SDKs](https://cloud.google.com/sdk/docs/).\n",
-        "\n",
-        "2. [Install Python 3](https://cloud.google.com/python/setup#installing_python).\n",
-        "\n",
-        "3. [Install virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv) and create a virtual environment that uses Python 3.  Activate the virtual environment.\n",
-        "\n",
-        "4. To install Jupyter, run `pip3 install jupyter` on the command-line in a terminal shell.\n",
-        "\n",
-        "5. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
-        "\n",
-        "6. Open this notebook in the Jupyter Notebook Dashboard.\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "X2tZAmugAe6h"
-      },
-      "source": [
-        "## Installation\n",
-        "\n",
-        "Install the packages required for executing this notebook. You can ignore errors for the `pip` dependecy resolver as they do not impact this notebook."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "BQOsJ1hZAZu0"
-      },
-      "outputs": [],
-      "source": [
-        "import os\n",
-        "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
-        "! pip3 install --upgrade google-cloud-storage google-cloud-vision google-cloud-aiplatform $USER_FLAG -q"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "yzvvcmCuAon3"
-      },
-      "source": [
-        "### Restart the kernel\n",
-        "\n",
-        "Once you've installed the additional packages, you need to restart the notebook kernel so it can find the packages."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "6qEonzbuAoI_"
-      },
-      "outputs": [],
-      "source": [
-        "import os\n",
-        "\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    # Automatically restart kernel after installs\n",
-        "    import IPython\n",
-        "\n",
-        "    app = IPython.Application.instance()\n",
-        "    app.kernel.do_shutdown(True)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "pGbbyN7rAuRM"
-      },
-      "source": [
-        "## Before you begin\n",
-        "\n",
-        "### GPU runtime\n",
-        "\n",
-        "*Make sure you're running this notebook in a GPU runtime if you have that option. In Colab, select* **Runtime > Change Runtime Type > GPU**\n",
-        "\n",
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "2. [Make sure that billing is enabled for your project.](https://cloud.google.com/billing/docs/how-to/modify-project)\n",
-        "\n",
-        "3. [Enable the following APIs: Vision API, Vertex AI APIs, Compute Engine APIs, and Cloud Storage.](https://console.cloud.google.com/flows/enableapi?apiid=vision.googleapis.com,aiplatform.googleapis.com,compute_component,storage-component.googleapis.com)\n",
-        "\n",
-        "4. If you are running this notebook locally, you will need to install the [Cloud SDK]((https://cloud.google.com/sdk)).\n",
-        "\n",
-        "5. Enter your project ID in the cell below. Then run the  cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$`."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "AE97adtnAzrr"
-      },
-      "outputs": [],
-      "source": [
-        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "nWlzLu5ELxWd"
-      },
-      "outputs": [],
-      "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
-        "    # Get your GCP project id from gcloud\n",
-        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
-        "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID:\", PROJECT_ID)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "GB5b27r0LxqE"
-      },
-      "outputs": [],
-      "source": [
-        "! gcloud config set project $PROJECT_ID"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "pMJdU1K5xG7D"
-      },
-      "source": [
-        "### Regions\n",
-        "\n",
-        "#### Vision AI\n",
-        "\n",
-        "You can now specify continent-level data storage and Optical Character Regonition (OCR) processing by setting the `VISION_AI_REGION` variable. You can select one of the following options:\n",
-        "\n",
-        "* USA country only: `us`\n",
-        "* The European Union: `eu`\n",
-        "\n",
-        "Learn more about [Vision AI regions for OCR](https://cloud.google.com/vision/docs/pdf#regionalization)\n",
-        "\n",
-        "#### Vertex AI\n",
-        "\n",
-        "You can also change the `VERTEX_AI_REGION` variable, which is used for operations throughout the rest of this notebook.  Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
-        "\n",
-        "- Americas: `us-central1`\n",
-        "- Europe: `europe-west4`\n",
-        "- Asia Pacific: `asia-east1`\n",
-        "\n",
-        "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
-        "\n",
-        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "5EhEAOK5xIKc"
-      },
-      "outputs": [],
-      "source": [
-        "VISION_AI_REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if VISION_AI_REGION == \"[your-region]\":\n",
-        "    VISION_AI_REGION = \"us\"\n",
-        "\n",
-        "VERTEX_AI_REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if VERTEX_AI_REGION == \"[your-region]\":\n",
-        "    VERTEX_AI_REGION = \"us-central1\""
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "xkgvWoXkxM1r"
-      },
-      "source": [
-        "### Timestamp\n",
-        "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append onto the name of resources which will be created in this tutorial."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "gr0HTpQZxNy4"
-      },
-      "outputs": [],
-      "source": [
-        "from datetime import datetime\n",
-        "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "AA-ns5CcBA9U"
-      },
-      "source": [
-        "### Vertex AI dataset import schema\n",
-        "\n",
-        "This constant tells Vertex AI the schema for importing the dataset. In this tutorial you are going to use the value for text extraction, but you can also change it to any of the values below for other use cases:\n",
-        "\n",
-        "- \n",
-        "`aiplatform.schema.dataset.ioformat.text.single_label_classification`\n",
-        "\n",
-        "- \n",
-        "`aiplatform.schema.dataset.ioformat.text.multi_label_classification`\n",
-        "\n",
-        "- \n",
-        "`aiplatform.schema.dataset.ioformat.text.extraction`\n",
-        "\n",
-        "- \n",
-        "`aiplatform.schema.dataset.ioformat.text.sentiment`\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "jnOb6Pp-4w5P"
-      },
-      "outputs": [],
-      "source": [
-        "from google.cloud import aiplatform\n",
-        "\n",
-        "DATASET_IMPORT_SCHEMA = aiplatform.schema.dataset.ioformat.text.extraction"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "ekbg-G7UA-bK"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Vertex AI Workbench**, your environment is already authenticated. Skip this step. If you receive errors still, you may have to grant the service account that is your Workbench notebook is running under access to the services listed below.\n",
-        "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
-        "\n",
-        "**Click Create service account**.\n",
-        "\n",
-        "In the **Service account name** field, enter a name, and click **Create**.\n",
-        "\n",
-        "In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex\" into the filter box, and select **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "Click Create. A JSON file that contains your key downloads to your local environment.\n",
-        "\n",
-        "Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "lCRrULxKBAfa"
-      },
-      "outputs": [],
-      "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = \"google.colab\" in sys.modules\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "rHB6fbonMMbI"
-      },
-      "source": [
-        "### Create a Cloud Storage bucket\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "When you initialize the Vertex AI SDK for Python, you specify a Cloud Storage staging bucket. The staging bucket is where all the data associated with your dataset and model resources are retained across sessions. This bucket will be also used to store the output of the Vision API SDK PDF-to-text conversion process.\n",
-        "\n",
-        "Set the name of your Cloud Storage bucket below. Bucket names must be globally unique across all Google Cloud projects, including those outside of your organization."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "ZSM5j0nfMOVK"
-      },
-      "outputs": [],
-      "source": [
-        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "i6H2iQX2MP-s"
-      },
-      "outputs": [],
-      "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
-        "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "AOsnYE5cMQX4"
-      },
-      "source": [
-        "**Only if your bucket doesn't already exist**: Run the following cell to create your Cloud Storage bucket."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "33RgSjhyMR6C"
-      },
-      "outputs": [],
-      "source": [
-        "! gsutil mb -l $VERTEX_AI_REGION -p $PROJECT_ID $BUCKET_URI"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "UpKfi0VfMTwe"
-      },
-      "source": [
-        "Finally, validate access to your Cloud Storage bucket by examining its contents:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "G9dMjMnkMVNt"
-      },
-      "outputs": [],
-      "source": [
-        "! gsutil ls -al $BUCKET_URI"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "k2qH7YCI0vnG"
-      },
-      "source": [
-        "### Set up variables\n",
-        "\n",
-        "Next, set up some variables used throughout the tutorial.\n",
-        "\n",
-        "### Import libraries and define constants"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "TB5-_2Xh01NH"
-      },
-      "outputs": [],
-      "source": [
-        "import os\n",
-        "\n",
-        "from google.cloud import aiplatform, storage, vision"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "-v7gY_KABIn8"
-      },
-      "source": [
-        "### Initialize Vision API SDK for Python\n",
-        "\n",
-        "Initialize the `Vision AI` SDK for Python for your project and region."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "DRbf--kWBLpx"
-      },
-      "outputs": [],
-      "source": [
-        "vision_client_options = {\n",
-        "    \"quota_project_id\": PROJECT_ID,\n",
-        "    \"api_endpoint\": f\"{VISION_AI_REGION}-vision.googleapis.com\",\n",
-        "}\n",
-        "vision_client = vision.ImageAnnotatorClient(client_options=vision_client_options)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "CA4nNVbBZ25d"
-      },
-      "source": [
-        "### Initialize Vertex AI SDK for Python\n",
-        "\n",
-        "Initialize the `Vertex AI` SDK for Python for your project, region and corresponding bucket."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "awWpNW1vZ6uV"
-      },
-      "outputs": [],
-      "source": [
-        "aiplatform.init(\n",
-        "    project=PROJECT_ID, location=VERTEX_AI_REGION, staging_bucket=BUCKET_URI\n",
-        ")"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "debBBljMDqkM"
-      },
-      "source": [
-        "### Initialize Cloud Storage SDK for Python\n",
-        "\n",
-        "Initialize the `Cloud Storage` SDK for Python for your project."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "ZtzmI9tpDr4e"
-      },
-      "outputs": [],
-      "source": [
-        "storage_client = storage.Client(project=PROJECT_ID)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "mvD0BxVXMtJe"
-      },
-      "source": [
-        "## Tutorial\n",
-        "\n",
-        "Now you are ready to start creating an unlabelled `Vertex AI Dataset` text entity extraction dataset from PDF files."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "EurEFM3GBap9"
-      },
-      "source": [
-        "### Convert PDF files to text using Vision API\n",
-        "\n",
-        "First, you make a `Vision API` request to OCR to text the PDFs from the Patent samples stored in the Cloud Storage bucket.\n",
-        "\n",
-        "*Note:* `Visions API` only allows batches of 100 document submissions at a time."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "uXVPOvjTBeK3"
-      },
-      "outputs": [],
-      "source": [
-        "ORIGIN_BUCKET_NAME = \"gcs-public-data--labeled-patents\"\n",
-        "# You can add a path if needed\n",
-        "ORIGIN_BUCKET_PATH = \"\"\n",
-        "\n",
-        "DESTINATION_BUCKET_NAME = BUCKET_NAME\n",
-        "DESTINATION_BUCKET_PATH = \"ocr-output\"\n",
-        "\n",
-        "gcs_destination_uri = f\"gs://{DESTINATION_BUCKET_NAME}/{DESTINATION_BUCKET_PATH}\"\n",
-        "\n",
-        "# Specify the feature for the Vision API processor\n",
-        "feature = vision.Feature(type_=vision.Feature.Type.DOCUMENT_TEXT_DETECTION)\n",
-        "\n",
-        "# Retrieve a list of all files in the bucket and path\n",
-        "blobs = storage_client.list_blobs(\n",
-        "    ORIGIN_BUCKET_NAME, prefix=ORIGIN_BUCKET_PATH, delimiter=\"/\"\n",
-        ")\n",
-        "\n",
-        "# Create a collection of requests. The SDK requires a separate request per each\n",
-        "# file that we want to extract text from\n",
-        "async_requests = []\n",
-        "\n",
-        "# Visions API only supports processing up to 100 documents at a time\n",
-        "# so we will process the first 100 elements only\n",
-        "sliced_blob_list = list(blobs)[:100]\n",
-        "\n",
-        "# Loop through the source bucket and create a request for each file there\n",
-        "for blob in sliced_blob_list:\n",
-        "    # Build input_config\n",
-        "    # Ensure we are only processing PDF files\n",
-        "    if blob.name.endswith(\".pdf\"):\n",
-        "        gcs_source = vision.GcsSource(uri=f\"gs://{ORIGIN_BUCKET_NAME}/{blob.name}\")\n",
-        "        input_config = vision.InputConfig(\n",
-        "            gcs_source=gcs_source, mime_type=\"application/pdf\"\n",
-        "        )\n",
-        "\n",
-        "        # Build output config\n",
-        "        # Get file name\n",
-        "        file_name = os.path.splitext(os.path.basename(blob.name))[0]\n",
-        "        gcs_destination = vision.GcsDestination(\n",
-        "            uri=f\"{gcs_destination_uri}/{file_name}-\"\n",
-        "        )\n",
-        "        output_config = vision.OutputConfig(gcs_destination=gcs_destination)\n",
-        "\n",
-        "        # Build request object and add to the collection\n",
-        "        async_request = vision.AsyncAnnotateFileRequest(\n",
-        "            features=[feature], input_config=input_config, output_config=output_config\n",
-        "        )\n",
-        "\n",
-        "        async_requests.append(async_request)\n",
-        "\n",
-        "print(f\"Created {len(async_requests)} requests\")\n",
-        "\n",
-        "# Submit the batch OCR job\n",
-        "\n",
-        "operation = vision_client.async_batch_annotate_files(requests=async_requests)\n",
-        "print(\"Submitting the batch OCR job\")\n",
-        "\n",
-        "print(\"Waiting for the operation to finish... this will take a short while\")\n",
-        "\n",
-        "response = operation.result(timeout=420)\n",
-        "\n",
-        "print(\"Completed!\")"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "7b15473e1937"
-      },
-      "source": [
-        "#### Quick peek at extracted annotated JSON files\n",
-        "\n",
-        "Next, you take a peek at the contents of one of the extracted JSON annotated files."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "4366442c1373"
-      },
-      "outputs": [],
-      "source": [
-        "json_files = ! gsutil ls {gcs_destination_uri}\n",
-        "\n",
-        "example = json_files[0]\n",
-        "! gsutil cat {example} | head -n 1"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "QWmeHWPIHako"
-      },
-      "source": [
-        "### Process results and build the import file\n",
-        "\n",
-        "The `Vision API` output is in JSON format, and contains detailed text extraction data. You only need the full text output, so you will processs the JSON results, extract the text output, and save it in new text files to be used later in the tutorial."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "WDLtiejKHug6"
-      },
-      "outputs": [],
-      "source": [
-        "import json\n",
-        "\n",
-        "print(\"Extracting text from Vision API output and saving it to text files\")\n",
-        "\n",
-        "ocr_blobs = storage_client.list_blobs(\n",
-        "    DESTINATION_BUCKET_NAME, prefix=DESTINATION_BUCKET_PATH\n",
-        ")\n",
-        "\n",
-        "output_bucket = storage_client.bucket(DESTINATION_BUCKET_NAME)\n",
-        "\n",
-        "# begin building the import file content\n",
-        "import_file_entries = []\n",
-        "\n",
-        "for ocr_blob in ocr_blobs:\n",
-        "    # Only process .json files, in case we previously processed files and had .txt files\n",
-        "    if ocr_blob.name.endswith(\".json\"):\n",
-        "        print(f\"Extracting text from {ocr_blob.name}\")\n",
-        "        # read each blob into a stream\n",
-        "        contents = ocr_blob.download_as_string()\n",
-        "        # load as JSON\n",
-        "        json_object = json.loads(contents)\n",
-        "        # extract text\n",
-        "        full_text = \"\"\n",
-        "        for response in json_object[\"responses\"]:\n",
-        "            if response[\"fullTextAnnotation\"]:\n",
-        "                full_text += response[\"fullTextAnnotation\"][\"text\"] + \"\\r\\n\"\n",
-        "\n",
-        "        # save as a blob\n",
-        "        output_blob_name = f\"{ocr_blob.name}.txt\"\n",
-        "        import_file_blob = output_bucket.blob(output_blob_name)\n",
-        "        import_file_blob.upload_from_string(full_text)\n",
-        "\n",
-        "        # create import file listing\n",
-        "        import_file_entry = {\n",
-        "            \"textGcsUri\": f\"gs://{DESTINATION_BUCKET_NAME}/{output_blob_name}\"\n",
-        "        }\n",
-        "\n",
-        "        import_file_entries.append(import_file_entry)\n",
-        "\n",
-        "print(\"Extraction completed!\")"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "0a5aae0eab44"
-      },
-      "source": [
-        "#### Quick peek at extracted text files\n",
-        "\n",
-        "Next, you take a peek at the contents of one of the extracted text files."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "76ce5f57b1ae"
-      },
-      "outputs": [],
-      "source": [
-        "example = import_file_entries[0][\"textGcsUri\"]\n",
-        "\n",
-        "! gsutil cat {example}"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "hqTLS_AmLWQP"
-      },
-      "source": [
-        "### Generate and save import file to be used in `Vertex AI Dataset` resource\n",
-        "\n",
-        "You will now build the import file that will be used to create the `Vertex AI Dataset` resource."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "_xFvOdQ_LWne"
-      },
-      "outputs": [],
-      "source": [
-        "IMPORT_FILE_PATH = \"import_file\"\n",
-        "\n",
-        "# Convert import file entries to JSON Lines format\n",
-        "import_file_content = \"\"\n",
-        "for entry in import_file_entries:\n",
-        "    import_file_content += json.dumps(entry) + \"\\n\"\n",
-        "\n",
-        "print(f\"Created import file based on {len(import_file_entries)} annotations\")\n",
-        "\n",
-        "# Upload content to GCS to be used in our next step\n",
-        "gcs_annotation_file_name = f\"{IMPORT_FILE_PATH}/import_file_{TIMESTAMP}.jsonl\"\n",
-        "import_file_blob = output_bucket.blob(gcs_annotation_file_name)\n",
-        "import_file_blob.upload_from_string(import_file_content)\n",
-        "\n",
-        "print(f\"Uploaded import file to {output_bucket.name}/{gcs_annotation_file_name}\")"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "6dVjFftOaKdw"
-      },
-      "source": [
-        "### Create an unlabelled `Vertex AI Dataset` resource\n",
-        "\n",
-        "Next, you create the `Dataset` resource using the `create` method for the `TextDataset` class, which takes the following parameters:\n",
-        "\n",
-        "- `display_name`: The human readable name for the `Dataset` resource.\n",
-        "- `gcs_source`: A list of one or more dataset index files to import the data items into the `Dataset` resource.\n",
-        "- `import_schema_uri`: The data labeling schema for the data items.\n",
-        "\n",
-        "This operation may take ten to twenty minutes."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "ciM9HLGCaOTJ"
-      },
-      "outputs": [],
-      "source": [
-        "print(\"Creating dataset ...\")\n",
-        "\n",
-        "dataset = aiplatform.TextDataset.create(\n",
-        "    display_name=\"Text Dataset \" + TIMESTAMP,\n",
-        "    gcs_source=[f\"gs://{output_bucket.name}/{gcs_annotation_file_name}\"],\n",
-        "    import_schema_uri=DATASET_IMPORT_SCHEMA,\n",
-        ")\n",
-        "\n",
-        "print(\"Completed!\")\n",
-        "\n",
-        "print(dataset.resource_name)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "2vagHf5T6Jd4"
-      },
-      "source": [
-        "**Congratulations, your dataset is now ready for annotations!**\n",
-        "\n",
-        "You have two options:\n",
-        "\n",
-        "*   Use Google Cloud Console to manually annotate the dataset in `Vertex AI`. Checkout [this link](https://cloud.google.com/vertex-ai/docs/datasets/label-using-console#entity-extraction) for more details on how to do so.\n",
-        "*   Create a labelling job to request data labelling. Check out [this link](https://cloud.google.com/vertex-ai/docs/datasets/data-labeling-job) and [this notebook](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_with_data_labeling.ipynb) for more details and examples.\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "cleanup:migration,new"
-      },
-      "source": [
-        "# Cleaning up\n",
-        "\n",
-        "To clean up all GCP resources used in this project, you can [delete the GCP\n",
-        "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
-        "\n",
-        "Otherwise, you can delete the individual resources you created in this tutorial.\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "aoJ18d8Y_jAy"
-      },
-      "outputs": [],
-      "source": [
-        "# Set this to true only if you'd like to delete your bucket\n",
-        "delete_bucket = False\n",
-        "\n",
-        "# Delete the dataset using the Vertex AI fully qualified identifier for the dataset\n",
-        "dataset.delete()\n",
-        "\n",
-        "# Delete the bucket created\n",
-        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
-        "    ! gsutil rm -r $BUCKET_URI"
-      ]
-    }
-  ],
-  "metadata": {
-    "accelerator": "GPU",
-    "colab": {
-      "collapsed_sections": [],
-      "name": "get_started_with_visionapi_and_vertex_datasets.ipynb",
-      "toc_visible": true
-    },
-    "kernelspec": {
-      "display_name": "Python 3",
-      "name": "python3"
-    }
-  },
-  "nbformat": 4,
-  "nbformat_minor": 0
-}

notebooks/community/ml_ops/stage1/mlops_data_management.ipynb

@@ -39,14 +39,8 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/mlops_data_management.ipynb\">\n",
-        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\\\" alt=\"Colab logo\"> Run in Colab\n",
-        "        </a>\n",
-        "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage1/mlops_data_management.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
+        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/mlops_data_management.ipynb\">\n",
+        "      Open in Google Cloud Notebooks\n",
         "    </a>\n",
         "  </td>\n",
         "</table>\n",
@@ -139,20 +133,7 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
-        "ONCE_ONLY = True\n",
+        "ONCE_ONLY = False\n",
         "if ONCE_ONLY:\n",
         "    ! pip3 install -U tensorflow==2.5 $USER_FLAG\n",
         "    ! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG\n",
@@ -197,32 +178,6 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "84cd83853240"
-      },
-      "source": [
-        "## Before you begin\n",
-        "\n",
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI, BigQuery, Compute Engine and Cloud Storage APIs](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,bigquery,compute_component,storage_component).\n",
-        "\n",
-        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -299,10 +254,7 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
       ]
     },
     {
@@ -329,67 +281,6 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "77c385f0db59"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
-        "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
-        "\n",
-        "1. **Click Create service account**.\n",
-        "\n",
-        "2. In the **Service account name** field, enter a name, and click **Create**.\n",
-        "\n",
-        "3. In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex AI\" into the filter box, and select **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "4. Click Create. A JSON file that contains your key downloads to your local environment.\n",
-        "\n",
-        "5. Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "535223fa4b84"
-      },
-      "outputs": [],
-      "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = False\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        IS_COLAB = True\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -418,8 +309,7 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -430,9 +320,8 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
-        "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
       ]
     },
     {
@@ -452,7 +341,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_URI"
+        "! gsutil mb -l $REGION $BUCKET_NAME"
       ]
     },
     {
@@ -472,7 +361,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_URI"
+        "! gsutil ls -al $BUCKET_NAME"
       ]
     },
     {
@@ -627,7 +516,7 @@
       },
       "outputs": [],
       "source": [
-        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
+        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_NAME)"
       ]
     },
     {
@@ -868,7 +757,7 @@
         "dataset = aip.TabularDataset.create(\n",
         "    display_name=\"Chicago Taxi\" + \"_\" + TIMESTAMP,\n",
         "    bq_source=[IMPORT_FILE],\n",
-        "    labels={\"user_metadata\": BUCKET_NAME},\n",
+        "    labels={\"user_metadata\": BUCKET_NAME[5:]},\n",
         ")\n",
         "\n",
         "label_column = \"tip_bin\"\n",
@@ -1060,9 +949,9 @@
       },
       "outputs": [],
       "source": [
-        "STATISTICS_SCHEMA = BUCKET_URI + \"/statistics.jsonl\"\n",
+        "STATISTICS_SCHEMA = BUCKET_NAME + \"/statistics.jsonl\"\n",
         "\n",
-        "tfdv.write_stats_text(stats, BUCKET_URI + \"/statistics.jsonl\")\n",
+        "tfdv.write_stats_text(stats, BUCKET_NAME + \"/statistics.jsonl\")\n",
         "\n",
         "with tf.io.gfile.GFile(\n",
         "    \"gs://\" + dataset.labels[\"user_metadata\"] + \"/metadata.jsonl\", \"r\"\n",
@@ -1075,7 +964,7 @@
         ") as f:\n",
         "    json.dump(metadata, f)\n",
         "\n",
-        "! gsutil cat $BUCKET_URI/metadata.jsonl"
+        "!gsutil cat $BUCKET_NAME/metadata.jsonl"
       ]
     },
     {
@@ -1122,7 +1011,7 @@
       },
       "outputs": [],
       "source": [
-        "SCHEMA_LOCATION = BUCKET_URI + \"/schema.txt\"\n",
+        "SCHEMA_LOCATION = BUCKET_NAME + \"/schema.txt\"\n",
         "\n",
         "# When running Apache Beam directly (file is directly accessed)\n",
         "tfdv.write_schema_text(output_path=SCHEMA_LOCATION, schema=schema)\n",
@@ -1160,7 +1049,7 @@
         ") as f:\n",
         "    json.dump(metadata, f)\n",
         "\n",
-        "! gsutil cat $BUCKET_URI/metadata.jsonl"
+        "!gsutil cat $BUCKET_NAME/metadata.jsonl"
       ]
     },
     {
@@ -1483,10 +1372,10 @@
         "            )\n",
         "\n",
         "\n",
-        "EXPORTED_JSONL_PREFIX = os.path.join(BUCKET_URI, \"exported_data/jsonl\")\n",
-        "EXPORTED_TFREC_PREFIX = os.path.join(BUCKET_URI, \"exported_data/tfrec\")\n",
-        "TRANSFORMED_DATA_PREFIX = os.path.join(BUCKET_URI, \"transformed_data\")\n",
-        "TRANSFORM_ARTIFACTS_DIR = os.path.join(BUCKET_URI, \"transformed_artifacts\")\n",
+        "EXPORTED_JSONL_PREFIX = os.path.join(BUCKET_NAME, \"exported_data/jsonl\")\n",
+        "EXPORTED_TFREC_PREFIX = os.path.join(BUCKET_NAME, \"exported_data/tfrec\")\n",
+        "TRANSFORMED_DATA_PREFIX = os.path.join(BUCKET_NAME, \"transformed_data\")\n",
+        "TRANSFORM_ARTIFACTS_DIR = os.path.join(BUCKET_NAME, \"transformed_artifacts\")\n",
         "\n",
         "QUERY_STRING = \"SELECT * FROM {} LIMIT 300000\".format(BQ_TABLE)\n",
         "JOB_NAME = \"chicago\" + TIMESTAMP\n",
@@ -1499,7 +1388,7 @@
         "    \"transform_artifact_dir\": TRANSFORM_ARTIFACTS_DIR,\n",
         "    \"exported_jsonl_prefix\": EXPORTED_JSONL_PREFIX,\n",
         "    \"exported_tfrec_prefix\": EXPORTED_TFREC_PREFIX,\n",
-        "    \"temp_location\": os.path.join(BUCKET_URI, \"temp\"),\n",
+        "    \"temp_location\": os.path.join(BUCKET_NAME, \"temp\"),\n",
         "    \"project\": PROJECT_ID,\n",
         "    \"region\": REGION,\n",
         "    \"setup_file\": \"./setup.py\",\n",
@@ -1570,7 +1459,7 @@
         ") as f:\n",
         "    json.dump(metadata, f)\n",
         "\n",
-        "! gsutil cat $BUCKET_URI/metadata.jsonl"
+        "!gsutil cat $BUCKET_NAME/metadata.jsonl"
       ]
     },
     {
@@ -1584,9 +1473,17 @@
         "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
         "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
         "\n",
-        "Otherwise, you can delete the individual resources you created in this tutorial.\n",
+        "Otherwise, you can delete the individual resources you created in this tutorial:\n",
         "\n",
-        "*Note:* stage2/mlops_experimentation is dependent on the resources created by this stage1 notebook."
+        "- Dataset\n",
+        "- Pipeline\n",
+        "- Model\n",
+        "- Endpoint\n",
+        "- AutoML Training Job\n",
+        "- Batch Job\n",
+        "- Custom Job\n",
+        "- Hyperparameter Tuning Job\n",
+        "- Cloud Storage Bucket"
       ]
     },
     {
@@ -1607,8 +1504,8 @@
         "    except Exception as e:\n",
         "        print(e)\n",
         "\n",
-        "    if \"BUCKET_URI\" in globals():\n",
-        "        ! gsutil rm -r $BUCKET_URI"
+        "    if \"BUCKET_NAME\" in globals():\n",
+        "        ! gsutil rm -r $BUCKET_NAME"
       ]
     }
   ],

notebooks/community/ml_ops/stage2/README.md

@@ -25,46 +25,24 @@ The second stage in MLOps is experimenting in developing one or more baseline mo
 - Use the What-if-Tool (WIT) to explore how the trained model would make predictions in different scenarios.
 
 
-<img src='stage2v3.png'>
-<br/>
-<br/>
-<br/>
-<img src='stage2.2v1.png'>
+<img src='stage2.png'>
 
 ## Notebooks
 
 ### Get Started
 
-[Get Started with Logging](get_started_with_logging.ipynb)
-
-```
-The steps performed include:
-
-- Use Python logging to log training configuration/results locally.
-- Use Google Cloud Logging to log training configuration/results in cloud storage.
-```
-
 [Get Started with Vertex Experiments and Vertex ML Metadata](get_started_vertex_experiments.ipynb)
 
 ```
 The steps performed include:
 
-- Local (notebook) Training
-    - Create an experiment
-    - Create a first run in the experiment
-    - Log parameters and metrics
-    - Create artifact lineage
-    - Visualize the experiment results
-    - Execute a second run
-    - Compare the two runs in the experiment
-- Cloud (`Vertex AI`) Training
-    - Within the training script:
-        - Create an experiment
-        - Log parameters and metrics
-        - Create artifact lineage
-    - Create a `Vertex AI Training` custom job
-    - Execute the custom job
-    - Visualize the experiment results
+- Use Python logging to log training configuration/results locally.
+- Use Google Cloud Logging to log training configuration/results in cloud storage.
+- Create a Vertex AI `Experiment` resource.
+- Instantiate an experiment run.
+- Log parameters for the run.
+- Log metrics for the run.
+- Display the logged experiment run.
 ```
 
 [Get Started with Vertex TensorBoard](get_started_vertex_tensorboard.ipynb)
@@ -137,32 +115,6 @@ The steps performed include:
 - Train a R model using `Vertex AI Trainingh` service with the R-to-Python training package.
 ```
 
-[Get Started with Custom Training Packages (R) and Deployment in R environment](get_started_vertex_training_r_using_r_kernel.ipynb)
-```
-The steps performed include:
-
-- Create a custom R training script
-- Create a custom R serving script
-- Create a custom R deployment (serving) container.
-- Train the model using `Vertex AI` custom training.
-- Create an `Endpoint` resource.
-- Deploy the `Model` resource (trained R model) to the `Endpoint` resource.
-- Make an online prediction.
-```
-
-[Get Started with Custom Training Packages (LightGBM)](get_started_vertex_training_lightgbm.ipynb)
-
-```
-The steps performed include:
-
-- Training using a Python package.
-- Save the model artifacts to Cloud Storage using GCSFuse.
-- Construct a FastAPI prediction server.
-- Construct a Dockerfile deployment image.
-- Test the deployment image locally.
-- Create a `Vertex AI Model` resource.
-```
-
 [Get Started with Distributed Training](get_started_vertex_distributed_training.ipynb)
 
 ```
@@ -262,19 +214,18 @@ The steps performed include:
 - Hyperparameter tuning the Vertex AI TabNet model.
 - Train the model using Vertex AI Training using BigQuery table.
 ```
-
-[Get Started with Vision API and AutoML](get_started_with_visionapi_and_automl.ipynb)
+[Get Started with Vertex AI TabNet builtin algorithm](get_started_with_tabnet.ipynb)
 ```
 The steps performed include:
 
-- Preprocess training files using `Vision AI` APIs to extract the text from PDF files.
-- Create a custom import file that includes annotation data based on the sample `BigQuery` dataset.
-- Create a `Vertex AI Dataset` resource.
-- Train the model.
-- View the model evaluation.
-- Deploy the `Vertex AI Model` resource to a serving `Endpoint` resource.
-- Make a prediction.
-- Undeploy the `Model`.
+- Get the training data.
+- Configure training parameters for the Vertex AI TabNet container.
+- Train the model using Vertex AI Training using CSV data.
+- Upload the model as a Vertex AI Model resource.
+- Deploy the Vertex AI Model resource to a Vertex AI Endpoint resource.
+- Make a prediction with the deployed model.
+- Hyperparameter tuning the Vertex AI TabNet model.
+- Train the model using Vertex AI Training using BigQuery table.
 ```
 
 

notebooks/community/ml_ops/stage2/get_started_automl_training.ipynb

@@ -38,13 +38,13 @@
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
-        "  <td>\n",
+        "    <td>\n",
         "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_automl_training.ipynb\">\n",
-        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\\\" alt=\"Colab logo\"> Run in Colab\n",
         "        </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage2/get_started_automl_training.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_automl_training.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
@@ -173,7 +173,7 @@
       "source": [
         "## Installations\n",
         "\n",
-        "Install the packages required for executing this notebook."
+        "Install the following packages for executing the MLOps notebooks."
       ]
     },
     {
@@ -186,21 +186,16 @@
       "source": [
         "import os\n",
         "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
+        "# The Google Cloud Notebook product has specific requirements\n",
+        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
         "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "# Google Cloud Notebook requires dependencies to be installed with '--user'\n",
         "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "if IS_GOOGLE_CLOUD_NOTEBOOK:\n",
         "    USER_FLAG = \"--user\"\n",
         "\n",
-        "# Install the packages\n",
-        "\n",
-        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG -q\n",
-        "! pip3 install --upgrade google-cloud-storage $USER_FLAG -q"
+        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG\n",
+        "! pip3 install --upgrade google-cloud-storage $USER_FLAG"
       ]
     },
     {
@@ -363,7 +358,7 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
+        "**If you are using Google Cloud Notebooks**, your environment is already authenticated. Skip this step.\n",
         "\n",
         "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
         "\n",
@@ -398,13 +393,9 @@
         "import os\n",
         "import sys\n",
         "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = False\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
+        "# If on Google Cloud Notebook, then don't execute this code\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
         "    if \"google.colab\" in sys.modules:\n",
-        "        IS_COLAB = True\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
         "        google_auth.authenticate_user()\n",
@@ -771,8 +762,9 @@
       },
       "source": [
         "## Review model evaluation scores\n",
+        "After your model training has finished, you can review the evaluation scores for it.\n",
         "\n",
-        "After your model training has finished, you can review the evaluation scores for it using the `list_model_evaluations()` method. This method will return an iterator for each evaluation slice."
+        "First, you need to get a reference to the new model. As with datasets, you can either use the reference to the model variable you created when you deployed the model or you can list all of the models in your project."
       ]
     },
     {
@@ -783,10 +775,20 @@
       },
       "outputs": [],
       "source": [
-        "model_evaluations = model.list_model_evaluations()\n",
+        "# Get model resource ID\n",
+        "models = aiplatform.Model.list(filter=\"display_name=flowers_\" + TIMESTAMP)\n",
         "\n",
-        "for model_evaluation in model_evaluations:\n",
-        "    print(model_evaluation.to_dict())"
+        "# Get a reference to the Model Service client\n",
+        "client_options = {\"api_endpoint\": f\"{REGION}-aiplatform.googleapis.com\"}\n",
+        "model_service_client = aiplatform.gapic.ModelServiceClient(\n",
+        "    client_options=client_options\n",
+        ")\n",
+        "\n",
+        "model_evaluations = model_service_client.list_model_evaluations(\n",
+        "    parent=models[0].resource_name\n",
+        ")\n",
+        "model_evaluation = list(model_evaluations)[0]\n",
+        "print(model_evaluation)"
       ]
     },
     {
@@ -1261,8 +1263,9 @@
       },
       "source": [
         "## Review model evaluation scores\n",
+        "After your model has finished training, you can review the evaluation scores for it.\n",
         "\n",
-        "After your model training has finished, you can review the evaluation scores for it using the `list_model_evaluations()` method. This method will return an iterator for each evaluation slice."
+        "First, you need to get a reference to the new model. As with datasets, you can either use the reference to the model variable you created when you deployed the model or you can list all of the models in your project."
       ]
     },
     {
@@ -1273,10 +1276,20 @@
       },
       "outputs": [],
       "source": [
-        "model_evaluations = model.list_model_evaluations()\n",
+        "# Get model resource ID\n",
+        "models = aiplatform.Model.list(filter=\"display_name=gsod_\" + TIMESTAMP)\n",
         "\n",
-        "for model_evaluation in model_evaluations:\n",
-        "    print(model_evaluation.to_dict())"
+        "# Get a reference to the Model Service client\n",
+        "client_options = {\"api_endpoint\": f\"{REGION}-aiplatform.googleapis.com\"}\n",
+        "model_service_client = aiplatform.gapic.ModelServiceClient(\n",
+        "    client_options=client_options\n",
+        ")\n",
+        "\n",
+        "model_evaluations = model_service_client.list_model_evaluations(\n",
+        "    parent=models[0].resource_name\n",
+        ")\n",
+        "model_evaluation = list(model_evaluations)[0]\n",
+        "print(model_evaluation)"
       ]
     },
     {
@@ -1614,8 +1627,9 @@
       },
       "source": [
         "## Review model evaluation scores\n",
+        "After your model has finished training, you can review the evaluation scores for it.\n",
         "\n",
-        "After your model training has finished, you can review the evaluation scores for it using the `list_model_evaluations()` method. This method will return an iterator for each evaluation slice."
+        "First, you need to get a reference to the new model. As with datasets, you can either use the reference to the model variable you created when you deployed the model or you can list all of the models in your project."
       ]
     },
     {
@@ -1626,10 +1640,20 @@
       },
       "outputs": [],
       "source": [
-        "model_evaluations = model.list_model_evaluations()\n",
+        "# Get model resource ID\n",
+        "models = aiplatform.Model.list(filter=\"display_name=happydb_\" + TIMESTAMP)\n",
         "\n",
-        "for model_evaluation in model_evaluations:\n",
-        "    print(model_evaluation.to_dict())"
+        "# Get a reference to the Model Service client\n",
+        "client_options = {\"api_endpoint\": f\"{REGION}-aiplatform.googleapis.com\"}\n",
+        "model_service_client = aiplatform.gapic.ModelServiceClient(\n",
+        "    client_options=client_options\n",
+        ")\n",
+        "\n",
+        "model_evaluations = model_service_client.list_model_evaluations(\n",
+        "    parent=models[0].resource_name\n",
+        ")\n",
+        "model_evaluation = list(model_evaluations)[0]\n",
+        "print(model_evaluation)"
       ]
     },
     {
@@ -1930,8 +1954,9 @@
       },
       "source": [
         "## Review model evaluation scores\n",
+        "After your model has finished training, you can review the evaluation scores for it.\n",
         "\n",
-        "After your model training has finished, you can review the evaluation scores for it using the `list_model_evaluations()` method. This method will return an iterator for each evaluation slice."
+        "First, you need to get a reference to the new model. As with datasets, you can either use the reference to the model variable you created when you deployed the model or you can list all of the models in your project."
       ]
     },
     {
@@ -1942,10 +1967,20 @@
       },
       "outputs": [],
       "source": [
-        "model_evaluations = model.list_model_evaluations()\n",
+        "# Get model resource ID\n",
+        "models = aiplatform.Model.list(filter=\"display_name=human_motion_\" + TIMESTAMP)\n",
         "\n",
-        "for model_evaluation in model_evaluations:\n",
-        "    print(model_evaluation.to_dict())"
+        "# Get a reference to the Model Service client\n",
+        "client_options = {\"api_endpoint\": f\"{REGION}-aiplatform.googleapis.com\"}\n",
+        "model_service_client = aiplatform.gapic.ModelServiceClient(\n",
+        "    client_options=client_options\n",
+        ")\n",
+        "\n",
+        "model_evaluations = model_service_client.list_model_evaluations(\n",
+        "    parent=models[0].resource_name\n",
+        ")\n",
+        "model_evaluation = list(model_evaluations)[0]\n",
+        "print(model_evaluation)"
       ]
     },
     {

notebooks/community/ml_ops/stage2/get_started_bqml_training.ipynb

@@ -38,13 +38,8 @@
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
-        "    <td>\n",
-        "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_bqml_training.ipynb\">\n",
-        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
-        "        </a>\n",
-        "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_bqml_training.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_bqml_training.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
@@ -133,20 +128,18 @@
       "source": [
         "import os\n",
         "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
+        "# The Google Cloud Notebook product has specific requirements\n",
+        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
         "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "# Google Cloud Notebook requires dependencies to be installed with '--user'\n",
         "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "if IS_GOOGLE_CLOUD_NOTEBOOK:\n",
         "    USER_FLAG = \"--user\"\n",
         "\n",
         "# Install the packages\n",
-        "! pip3 install --upgrade pyarrow \\\n",
-        "                        google-cloud-aiplatform \\\n",
-        "                        google-cloud-bigquery \\\n",
-        "                        google-cloud-bigquery-storage $USER_FLAG -q"
+        "! pip3 install --upgrade pyarrow $USER_FLAG\n",
+        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG\n",
+        "! pip3 install --upgrade google-cloud-bigquery $USER_FLAG"
       ]
     },
     {
@@ -184,23 +177,6 @@
         "id": "project_id"
       },
       "source": [
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI, BigQuery, Compute Engine and Cloud Storage APIs](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,bigquery,compute_component,storage_component).\n",
-        "\n",
-        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands.\n",
-        "\n",
         "#### Set your project ID\n",
         "\n",
         "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
@@ -301,67 +277,6 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "f3bd8c0d0469"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
-        "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
-        "\n",
-        "1. **Click Create service account**.\n",
-        "\n",
-        "2. In the **Service account name** field, enter a name, and click **Create**.\n",
-        "\n",
-        "3. In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex AI\" into the filter box, and select **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "4. Click Create. A JSON file that contains your key downloads to your local environment.\n",
-        "\n",
-        "5. Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "e0953a00668e"
-      },
-      "outputs": [],
-      "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = False\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        IS_COLAB = True\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -385,8 +300,7 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -397,9 +311,8 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
-        "    BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
       ]
     },
     {
@@ -419,7 +332,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_URI"
+        "! gsutil mb -l $REGION $BUCKET_NAME"
       ]
     },
     {
@@ -439,55 +352,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_URI"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "set_service_account"
-      },
-      "source": [
-        "#### Service Account\n",
-        "\n",
-        "You use a service account to create Vertex AI Pipeline jobs. If you do not want to use your project's Compute Engine service account, set `SERVICE_ACCOUNT` to another service account ID."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "set_service_account"
-      },
-      "outputs": [],
-      "source": [
-        "SERVICE_ACCOUNT = \"[your-service-account]\"  # @param {type:\"string\"}"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "autoset_service_account"
-      },
-      "outputs": [],
-      "source": [
-        "if (\n",
-        "    SERVICE_ACCOUNT == \"\"\n",
-        "    or SERVICE_ACCOUNT is None\n",
-        "    or SERVICE_ACCOUNT == \"[your-service-account]\"\n",
-        "):\n",
-        "    # Get your service account from gcloud\n",
-        "    if not IS_COLAB:\n",
-        "        shell_output = !gcloud auth list 2>/dev/null\n",
-        "        SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
-        "\n",
-        "    if IS_COLAB:\n",
-        "        shell_output = ! gcloud projects describe  $PROJECT_ID\n",
-        "        project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n",
-        "        SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n",
-        "\n",
-        "    print(\"Service Account:\", SERVICE_ACCOUNT)"
+        "! gsutil ls -al $BUCKET_NAME"
       ]
     },
     {
@@ -530,7 +395,7 @@
       },
       "outputs": [],
       "source": [
-        "aiplatform.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
+        "aiplatform.init(project=PROJECT_ID, staging_bucket=BUCKET_NAME)"
       ]
     },
     {
@@ -830,10 +695,10 @@
       },
       "outputs": [],
       "source": [
-        "param = f\"{PROJECT_ID}:{BQ_DATASET_NAME}.{MODEL_NAME} {BUCKET_URI}/{MODEL_NAME}\"\n",
+        "param = f\"{PROJECT_ID}:{BQ_DATASET_NAME}.{MODEL_NAME} {BUCKET_NAME}/{MODEL_NAME}\"\n",
         "! bq extract -m $param\n",
         "\n",
-        "MODEL_DIR = f\"{BUCKET_URI}/{BQ_DATASET_NAME}\"\n",
+        "MODEL_DIR = f\"{BUCKET_NAME}/{BQ_DATASET_NAME}\"\n",
         "! gsutil ls $MODEL_DIR"
       ]
     },
@@ -964,7 +829,7 @@
         "id": "model_delete:mbsdk"
       },
       "source": [
-        "#### Delete the `Vertex AI Model` resource\n",
+        "#### Delete the model\n",
         "\n",
         "The method 'delete()' deletes the model."
       ]
@@ -980,32 +845,6 @@
         "model.delete()"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "7890ae6f6410"
-      },
-      "source": [
-        "### Delete the `BigQuery ML` model\n",
-        "\n",
-        "Next, delete the `BigQuery ML` instance of the model."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "f0b6163e70c0"
-      },
-      "outputs": [],
-      "source": [
-        "MODEL_QUERY = f\"\"\"\n",
-        "DROP MODEL `{BQ_DATASET_NAME}.{MODEL_NAME}`\n",
-        "\"\"\"\n",
-        "\n",
-        "job = bqclient.query(MODEL_QUERY)"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -1094,32 +933,6 @@
         "print(results)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "3f3cee1236b1"
-      },
-      "source": [
-        "### Delete the `BigQuery ML` model\n",
-        "\n",
-        "Next, delete the `BigQuery ML` instance of the model."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "957b7d841502"
-      },
-      "outputs": [],
-      "source": [
-        "MODEL_QUERY = f\"\"\"\n",
-        "DROP MODEL `{BQ_DATASET_NAME}.{MODEL_NAME}`\n",
-        "\"\"\"\n",
-        "\n",
-        "job = bqclient.query(MODEL_QUERY)"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -1172,33 +985,7 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "4def8aaf3398"
-      },
-      "source": [
-        "### Delete the `BigQuery ML` model\n",
-        "\n",
-        "Next, delete the `BigQuery ML` instance of the model."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "ff5b32618018"
-      },
-      "outputs": [],
-      "source": [
-        "MODEL_QUERY = f\"\"\"\n",
-        "DROP MODEL `{BQ_DATASET_NAME}.{MODEL_NAME}`\n",
-        "\"\"\"\n",
-        "\n",
-        "job = bqclient.query(MODEL_QUERY)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "2b4498ca6fea"
+        "id": "1bb996026c94"
       },
       "source": [
         "## Model Registry\n",
@@ -1207,27 +994,30 @@
         "\n",
         "### Setting permissions to automatically register the model\n",
         "\n",
-        "You need to set some additional IAM permissions for BigQuery ML to automatically upload and register the model after training. Depending on your service account, the setting of the permissions below may fail. In this case, we recommend executing the permissions in a Cloud Shell.\n",
-        "\n",
-        "Learn more about [Setting permissions for Model Registry](https://cloud.google.com/bigquery-ml/docs/managing-models-vertex)\n"
+        "You need to set some additional IAM permissions for BigQuery ML to automatically upload and register the model after training. Depending on your service account, the setting of the permissions below may fail. In this case, we recommend executing the permissions in a Cloud Shell."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "29229f72d13d"
+        "id": "eaaf24146aad"
       },
       "outputs": [],
       "source": [
         "! gcloud projects add-iam-policy-binding $PROJECT_ID \\\n",
-        "    --member=serviceAccount:$SERVICE_ACCOUNT --role=roles/aiplatform.admin --condition=None"
+        "      --member='serviceAccount:cloud-dataengine@system.gserviceaccount.com' \\\n",
+        "      --role='roles/aiplatform.admin'\n",
+        "\n",
+        "! gcloud projects add-iam-policy-binding $PROJECT_ID \\\n",
+        "      --member='user:cloud-dataengine@prod.google.com' \\\n",
+        "      --role='roles/aiplatform.admin'"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "6c390ee7c11a"
+        "id": "8bfc0b26155f"
       },
       "source": [
         "### Training and registering the model\n",
@@ -1243,7 +1033,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "57db464f4c42"
+        "id": "3ff1d4ef4df2"
       },
       "outputs": [],
       "source": [
@@ -1280,7 +1070,7 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "5b4970272040"
+        "id": "a243d86f9d22"
       },
       "source": [
         "### Find the model in the `Vertex Model Registry`\n",
@@ -1292,7 +1082,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "76c22674ba99"
+        "id": "4fd9a143d900"
       },
       "outputs": [],
       "source": [
@@ -1302,46 +1092,6 @@
         "print(model.gca_resource)"
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "48e6ef5d5ffa"
-      },
-      "outputs": [],
-      "source": [
-        "models = aiplatform.Model.list()\n",
-        "for model in models:\n",
-        "    if model.gca_resource.display_name.startswith(\"bqml\"):\n",
-        "        print(model.gca_resource.display_name)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "ef61354b1a5f"
-      },
-      "source": [
-        "### Delete the `BigQuery ML` model\n",
-        "\n",
-        "Next, delete the `BigQuery ML` instance of the model."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "f6004d1ce59d"
-      },
-      "outputs": [],
-      "source": [
-        "MODEL_QUERY = f\"\"\"\n",
-        "DROP MODEL `{BQ_DATASET_NAME}.{MODEL_NAME}`\n",
-        "\"\"\"\n",
-        "\n",
-        "job = bqclient.query(MODEL_QUERY)"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -1353,9 +1103,12 @@
         "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
         "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
         "\n",
-        "Otherwise, you can delete the individual resources you created in this tutorial.\n",
+        "Otherwise, you can delete the individual resources you created in this tutorial:\n",
         "\n",
-        "Set `delete_storage` to `True` to delete the Cloud Storage bucket used in this notebook."
+        "- Model\n",
+        "- Endpoint\n",
+        "- Dataset\n",
+        "- Cloud Storage Bucket"
       ]
     },
     {
@@ -1376,13 +1129,11 @@
         "except Exception as e:\n",
         "    print(e)\n",
         "\n",
-        "# Delete the created BigQuery dataset\n",
-        "! bq rm -r -f $PROJECT_ID:$BQ_DATASET_NAME\n",
-        "\n",
-        "delete_storage = False\n",
-        "if delete_storage or os.getenv(\"IS_TESTING\"):\n",
+        "if os.getenv(\"IS_TESTING\"):\n",
         "    # Delete the created GCS bucket\n",
-        "    ! gsutil rm -r $BUCKET_URI"
+        "    ! gsutil rm -r $BUCKET_NAME\n",
+        "    # Delete the created BigQuery dataset\n",
+        "    ! bq rm -r -f $PROJECT_ID:$BQ_DATASET_NAME"
       ]
     }
   ],

notebooks/community/ml_ops/stage2/get_started_vertex_distributed_training.ipynb

@@ -33,7 +33,7 @@
         "\n",
         "<table align=\"left\">\n",
         "      <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_distributed_training.ipynb\">\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/ml_ops/stage2/get_started_vertex_distributed_training.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
         "    </a>\n",
         "  </td>\n",
@@ -44,7 +44,7 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage2/get_started_vertex_distributed_training.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_distributed_training.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "Open in Vertex AI Workbench\n",
         "    </a>\n",
@@ -146,7 +146,7 @@
       "source": [
         "### Install additional packages\n",
         "\n",
-        "Install the packages required for executing this notebook."
+        "Install the latest version of Vertex SDK for Python."
       ]
     },
     {
@@ -159,17 +159,23 @@
       "source": [
         "import os\n",
         "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
+        "# The Google Cloud Notebook product has specific requirements\n",
+        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
         "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "# Google Cloud Notebook requires dependencies to be installed with '--user'\n",
         "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
+        "if IS_GOOGLE_CLOUD_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "TjOXHg2VyajN"
+      },
+      "outputs": [],
+      "source": [
         "! pip3 install {USER_FLAG} --upgrade google-cloud-aiplatform"
       ]
     },
@@ -203,32 +209,6 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "84cd83853240"
-      },
-      "source": [
-        "## Before you begin\n",
-        "\n",
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI, BigQuery, Compute Engine and Cloud Storage APIs](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,bigquery,compute_component,storage_component).\n",
-        "\n",
-        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -307,10 +287,7 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
       ]
     },
     {
@@ -345,7 +322,7 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
+        "**If you are using Google Cloud Notebooks**, your environment is already\n",
         "authenticated. Skip this step."
       ]
     },
@@ -387,21 +364,20 @@
       },
       "outputs": [],
       "source": [
+        "import os\n",
+        "import sys\n",
+        "\n",
         "# If you are running this notebook in Colab, run this cell and follow the\n",
         "# instructions to authenticate your GCP account. This provides access to your\n",
         "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
         "# requests.\n",
         "\n",
-        "import os\n",
-        "import sys\n",
+        "# The Google Cloud Notebook product has specific requirements\n",
+        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
         "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = False\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
+        "# If on Google Cloud Notebooks, then don't execute this code\n",
+        "if not IS_GOOGLE_CLOUD_NOTEBOOK:\n",
         "    if \"google.colab\" in sys.modules:\n",
-        "        IS_COLAB = True\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
         "        google_auth.authenticate_user()\n",
@@ -436,8 +412,7 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -448,9 +423,8 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
-        "    BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
       ]
     },
     {
@@ -470,7 +444,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_URI"
+        "! gsutil mb -l $REGION $BUCKET_NAME"
       ]
     },
     {
@@ -490,7 +464,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_URI"
+        "! gsutil ls -al $BUCKET_NAME"
       ]
     },
     {
@@ -535,7 +509,7 @@
       },
       "outputs": [],
       "source": [
-        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
+        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_NAME)"
       ]
     },
     {
@@ -613,7 +587,7 @@
         "if os.getenv(\"IS_TESTING_TF\"):\n",
         "    TF = os.getenv(\"IS_TESTING_TF\")\n",
         "else:\n",
-        "    TF = \"2.5\".replace(\".\", \"-\")\n",
+        "    TF = \"2.1\".replace(\".\", \"-\")\n",
         "\n",
         "if TF[0] == \"2\":\n",
         "    if TRAIN_GPU:\n",
@@ -744,7 +718,7 @@
         "\n",
         "job = aip.CustomPythonPackageTrainingJob(\n",
         "    display_name=DISPLAY_NAME,\n",
-        "    python_package_gcs_uri=f\"{BUCKET_URI}/trainer_boston.tar.gz\",\n",
+        "    python_package_gcs_uri=f\"{BUCKET_NAME}/trainer_boston.tar.gz\",\n",
         "    python_module_name=\"trainer.task\",\n",
         "    container_uri=TRAIN_IMAGE,\n",
         "    model_serving_container_image_uri=DEPLOY_IMAGE,\n",
@@ -1000,7 +974,7 @@
         "! rm -f custom.tar custom.tar.gz\n",
         "! tar cvf custom.tar custom\n",
         "! gzip custom.tar\n",
-        "! gsutil cp custom.tar.gz $BUCKET_URI/trainer_boston.tar.gz"
+        "! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_boston.tar.gz"
       ]
     },
     {
@@ -1022,7 +996,7 @@
       },
       "outputs": [],
       "source": [
-        "MODEL_DIR = BUCKET_URI\n",
+        "MODEL_DIR = BUCKET_NAME\n",
         "\n",
         "CMDARGS = [\"--epochs=5\", \"--batch_size=16\", \"--distribute=mirrored\"]\n",
         "\n",
@@ -1194,7 +1168,7 @@
         "\n",
         "job = aip.CustomPythonPackageTrainingJob(\n",
         "    display_name=DISPLAY_NAME,\n",
-        "    python_package_gcs_uri=f\"{BUCKET_URI}/trainer_boston.tar.gz\",\n",
+        "    python_package_gcs_uri=f\"{BUCKET_NAME}/trainer_boston.tar.gz\",\n",
         "    python_module_name=\"trainer.task\",\n",
         "    container_uri=TRAIN_IMAGE,\n",
         "    model_serving_container_image_uri=DEPLOY_IMAGE,\n",
@@ -1221,7 +1195,7 @@
       },
       "outputs": [],
       "source": [
-        "MODEL_DIR = BUCKET_URI\n",
+        "MODEL_DIR = BUCKET_NAME\n",
         "\n",
         "CMDARGS = [\"--epochs=5\", \"--batch_size=16\", \"--distribute=multiworker\"]\n",
         "\n",
@@ -1364,11 +1338,7 @@
       },
       "outputs": [],
       "source": [
-        "if not IS_COLAB:\n",
-        "    ! docker build custom -t $TRAIN_IMAGE\n",
-        "else:\n",
-        "    # install docker daemon\n",
-        "    ! apt-get -qq install docker.io"
+        "! docker build custom -t $TRAIN_IMAGE"
       ]
     },
     {
@@ -1390,8 +1360,7 @@
       },
       "outputs": [],
       "source": [
-        "if not IS_COLAB:\n",
-        "    ! docker run $TRAIN_IMAGE --epochs=5 --model-dir=./"
+        "! docker run $TRAIN_IMAGE --epochs=5 --model-dir=./"
       ]
     },
     {
@@ -1413,38 +1382,7 @@
       },
       "outputs": [],
       "source": [
-        "if not IS_COLAB:\n",
-        "    ! docker push $TRAIN_IMAGE"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "f50e9c553fb7"
-      },
-      "source": [
-        "*Executes in Colab*"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "a7e8c98f1e56"
-      },
-      "outputs": [],
-      "source": [
-        "%%bash -s $IS_COLAB $TRAIN_IMAGE\n",
-        "if [ $1 == \"False\" ]; then\n",
-        "  exit 0\n",
-        "fi\n",
-        "set -x\n",
-        "dockerd -b none --iptables=0 -l warn &\n",
-        "for i in $(seq 5); do [ ! -S \"/var/run/docker.sock\" ] && sleep 2 || break; done\n",
-        "docker build custom -t $2\n",
-        "docker run $2 --epochs=5 --model-dir=./\n",
-        "docker push $2\n",
-        "kill $(jobs -p)"
+        "! docker push $TRAIN_IMAGE"
       ]
     },
     {
@@ -1470,7 +1408,7 @@
       "source": [
         "PRIMARY_COMPUTE = \"n2-highcpu-64\"\n",
         "\n",
-        "MODEL_DIR = BUCKET_URI\n",
+        "MODEL_DIR = BUCKET_NAME\n",
         "\n",
         "CMDARGS = [\n",
         "    \"--model-dir=\" + MODEL_DIR,\n",

notebooks/community/ml_ops/stage2/get_started_vertex_feature_store.ipynb

@@ -41,12 +41,12 @@
         "    \n",
         "  <td>\n",
         "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_feature_store.ipynb\">\n",
-        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\\\" alt=\"Colab logo\"> Run in Colab\n",
         "        </a>\n",
         "  </td>\n",
         "    \n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage2/get_started_vertex_feature_store.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_feature_store.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
@@ -144,15 +144,12 @@
       "source": [
         "import os\n",
         "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
+        "# The Google Cloud Notebook product has specific requirements\n",
+        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
         "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "# Google Cloud Notebook requires dependencies to be installed with '--user'\n",
         "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "if IS_GOOGLE_CLOUD_NOTEBOOK:\n",
         "    USER_FLAG = \"--user\"\n",
         "\n",
         "# Install the dependecies\n",
@@ -188,30 +185,6 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "project_id"
-      },
-      "source": [
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI, Compute Engine, Cloud Storage and Cloud Logging APIs](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component,storage_component,logging).\n",
-        "\n",
-        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -326,7 +299,7 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
+        "**If you are using Google Cloud Notebooks**, your environment is already authenticated. Skip this step.\n",
         "\n",
         "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
         "\n",
@@ -361,11 +334,8 @@
         "import os\n",
         "import sys\n",
         "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = \"google.colab\" in sys.modules\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
+        "# If on Google Cloud Notebook, then don't execute this code\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
         "    if \"google.colab\" in sys.modules:\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
@@ -489,7 +459,7 @@
       "outputs": [],
       "source": [
         "# Represents featurestore resource path.\n",
-        "FEATURESTORE_NAME = \"movies_\" + TIMESTAMP\n",
+        "FEATURESTORE_NAME = \"movies\"\n",
         "\n",
         "featurestore = aiplatform.Featurestore.create(\n",
         "    featurestore_id=FEATURESTORE_NAME,\n",

notebooks/community/ml_ops/stage2/get_started_vertex_tensorboard.ipynb

@@ -29,7 +29,7 @@
         "id": "title:generic,gcp"
       },
       "source": [
-        "# E2E ML on GCP: MLOps stage 2 : experimentation: get started with Vertex AI Tensorboard\n",
+        "# E2E ML on GCP: MLOps stage 2 : experimentation: get started with Vertex Tensorboard\n",
         "\n",
         "<table align=\"left\">\n",
         "   <td>\n",
@@ -44,7 +44,7 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/notebook_template.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/notebook_template.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
@@ -62,7 +62,7 @@
         "## Overview\n",
         "\n",
         "\n",
-        "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 2 : experimentation: get started with Vertex AI Tensorboard."
+        "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 2 : experimentation: get started with Vertex Tensorboard."
       ]
     },
     {
@@ -110,14 +110,21 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "56cb7f08a9e8"
+        "id": "94a148f11da5"
       },
       "source": [
         "### Set up your local development environment\n",
         "\n",
         "**If you are using Colab or Google Cloud Notebooks**, your environment already meets\n",
-        "all the requirements to run this notebook. You can skip this step.\n",
-        "\n",
+        "all the requirements to run this notebook. You can skip this step."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "56cb7f08a9e8"
+      },
+      "source": [
         "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
         "You need the following:\n",
         "\n",
@@ -180,7 +187,7 @@
       "source": [
         "## Installations\n",
         "\n",
-        "Install the following packages for executing this notebook."
+        "Install *one time* the packages for executing the MLOps notebooks."
       ]
     },
     {
@@ -193,19 +200,25 @@
       "source": [
         "import os\n",
         "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
+        "# The Google Cloud Notebook product has specific requirements\n",
+        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
         "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "# Google Cloud Notebook requires dependencies to be installed with '--user'\n",
         "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
-        "! pip3 install -U tensorflow==2.8 $USER_FLAG -q\n",
-        "! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG -q"
+        "if IS_GOOGLE_CLOUD_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "install_mlops"
+      },
+      "outputs": [],
+      "source": [
+        "! pip3 install -U tensorflow==2.8 $USER_FLAG\n",
+        "! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG"
       ]
     },
     {
@@ -237,14 +250,21 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d6a00c14b087"
+      },
+      "source": [
+        "## Before you begin"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "2721ef0202d9"
       },
       "source": [
-        "## Before you begin\n",
-        "\n",
         "### Set up your Google Cloud project\n",
         "\n",
         "**The following steps are required, regardless of your notebook environment.**\n",
@@ -253,7 +273,7 @@
         "\n",
         "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
         "\n",
-        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com). \n",
+        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com). {TODO: Update the APIs needed for your tutorial. Edit the API names, and update the link to append the API IDs, separating each one with a comma. For example, container.googleapis.com,cloudbuild.googleapis.com}\n",
         "\n",
         "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
         "\n",
@@ -339,10 +359,7 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
       ]
     },
     {
@@ -377,7 +394,7 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
+        "**If you are using Google Cloud Notebooks**, your environment is already\n",
         "authenticated. Skip this step."
       ]
     },
@@ -419,21 +436,20 @@
       },
       "outputs": [],
       "source": [
+        "import os\n",
+        "import sys\n",
+        "\n",
         "# If you are running this notebook in Colab, run this cell and follow the\n",
         "# instructions to authenticate your GCP account. This provides access to your\n",
         "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
         "# requests.\n",
         "\n",
-        "import os\n",
-        "import sys\n",
+        "# The Google Cloud Notebook product has specific requirements\n",
+        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
         "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = False\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
+        "# If on Google Cloud Notebooks, then don't execute this code\n",
+        "if not IS_GOOGLE_CLOUD_NOTEBOOK:\n",
         "    if \"google.colab\" in sys.modules:\n",
-        "        IS_COLAB = True\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
         "        google_auth.authenticate_user()\n",
@@ -558,16 +574,9 @@
         "    or SERVICE_ACCOUNT is None\n",
         "    or SERVICE_ACCOUNT == \"[your-service-account]\"\n",
         "):\n",
-        "    # Get your service account from gcloud\n",
-        "    if not IS_COLAB:\n",
-        "        shell_output = !gcloud auth list 2>/dev/null\n",
-        "        SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
-        "\n",
-        "    if IS_COLAB:\n",
-        "        shell_output = ! gcloud projects describe  $PROJECT_ID\n",
-        "        project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n",
-        "        SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n",
-        "\n",
+        "    # Get your GCP project id from gcloud\n",
+        "    shell_output = !gcloud auth list 2>/dev/null\n",
+        "    SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
         "    print(\"Service Account:\", SERVICE_ACCOUNT)"
       ]
     },

notebooks/community/ml_ops/stage2/get_started_vertex_training.ipynb

@@ -29,7 +29,7 @@
         "id": "title:generic,gcp"
       },
       "source": [
-        "# E2E ML on GCP: MLOps stage 2 : experimentation: get started with Vertex AI Training\n",
+        "# E2E ML on GCP: MLOps stage 2 : experimentation: get started with Vertex Training\n",
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
@@ -39,14 +39,8 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training.ipynb\">\n",
-        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
-        "        </a>\n",
-        "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage2/get_started_vertex_training.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
+        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training.ipynb\">\n",
+        "      Open in Google Cloud Notebooks\n",
         "    </a>\n",
         "  </td>\n",
         "</table>\n",
@@ -133,7 +127,7 @@
       "source": [
         "## Installations\n",
         "\n",
-        "Install the packages required for executing this notebook"
+        "Install *one time* the packages for executing the MLOps notebooks."
       ]
     },
     {
@@ -144,20 +138,20 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
-        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG -q"
+        "ONCE_ONLY = False\n",
+        "if ONCE_ONLY:\n",
+        "    ! pip3 install -U tensorflow==2.5 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-transform==1.2 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-io==0.18 $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-bigquery $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-logging $USER_FLAG\n",
+        "    ! pip3 install --upgrade apache-beam[gcp] $USER_FLAG\n",
+        "    ! pip3 install --upgrade pyarrow $USER_FLAG\n",
+        "    ! pip3 install --upgrade cloudml-hypertune $USER_FLAG\n",
+        "    ! pip3 install --upgrade kfp $USER_FLAG"
       ]
     },
     {
@@ -179,6 +173,8 @@
       },
       "outputs": [],
       "source": [
+        "import os\n",
+        "\n",
         "if not os.getenv(\"IS_TESTING\"):\n",
         "    # Automatically restart kernel after installs\n",
         "    import IPython\n",
@@ -187,36 +183,6 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "before_you_begin"
-      },
-      "source": [
-        "## Before you begin\n",
-        "\n",
-        "### GPU runtime\n",
-        "\n",
-        "*Make sure you're running this notebook in a GPU runtime if you have that option. In Colab, select* **Runtime > Change Runtime Type > GPU**\n",
-        "\n",
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "2. [Make sure that billing is enabled for your project.](https://cloud.google.com/billing/docs/how-to/modify-project)\n",
-        "\n",
-        "3. [Enable the following APIs: Vertex AI APIs, Compute Engine APIs, and Cloud Storage.](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component,storage-component.googleapis.com)\n",
-        "\n",
-        "4. If you are running this notebook locally, you will need to install the [Cloud SDK]((https://cloud.google.com/sdk)).\n",
-        "\n",
-        "5. Enter your project ID in the cell below. Then run the  cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$`."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -293,10 +259,7 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
       ]
     },
     {
@@ -323,67 +286,6 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "gcp_authenticate"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
-        "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
-        "\n",
-        "**Click Create service account**.\n",
-        "\n",
-        "In the **Service account name** field, enter a name, and click **Create**.\n",
-        "\n",
-        "In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex\" into the filter box, and select **Vertex Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "Click Create. A JSON file that contains your key downloads to your local environment.\n",
-        "\n",
-        "Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "gcp_authenticate"
-      },
-      "outputs": [],
-      "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = False\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        IS_COLAB = True\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -407,8 +309,7 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -419,9 +320,8 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
-        "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
       ]
     },
     {
@@ -441,7 +341,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_URI"
+        "! gsutil mb -l $REGION $BUCKET_NAME"
       ]
     },
     {
@@ -461,7 +361,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_URI"
+        "! gsutil ls -al $BUCKET_NAME"
       ]
     },
     {
@@ -484,7 +384,7 @@
       },
       "outputs": [],
       "source": [
-        "import google.cloud.aiplatform as aiplatform"
+        "import google.cloud.aiplatform as aip"
       ]
     },
     {
@@ -506,7 +406,7 @@
       },
       "outputs": [],
       "source": [
-        "aiplatform.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
+        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_NAME)"
       ]
     },
     {
@@ -541,7 +441,7 @@
       "source": [
         "if os.getenv(\"IS_TESTING_TRAIN_GPU\"):\n",
         "    TRAIN_GPU, TRAIN_NGPU = (\n",
-        "        aiplatform.gapic.AcceleratorType.NVIDIA_TESLA_K80,\n",
+        "        aip.gapic.AcceleratorType.NVIDIA_TESLA_K80,\n",
         "        int(os.getenv(\"IS_TESTING_TRAIN_GPU\")),\n",
         "    )\n",
         "else:\n",
@@ -549,7 +449,7 @@
         "\n",
         "if os.getenv(\"IS_TESTING_DEPLOY_GPU\"):\n",
         "    DEPLOY_GPU, DEPLOY_NGPU = (\n",
-        "        aiplatform.gapic.AcceleratorType.NVIDIA_TESLA_K80,\n",
+        "        aip.gapic.AcceleratorType.NVIDIA_TESLA_K80,\n",
         "        int(os.getenv(\"IS_TESTING_DEPLOY_GPU\")),\n",
         "    )\n",
         "else:\n",
@@ -584,7 +484,7 @@
         "if os.getenv(\"IS_TESTING_TF\"):\n",
         "    TF = os.getenv(\"IS_TESTING_TF\")\n",
         "else:\n",
-        "    TF = \"2.5\".replace(\".\", \"-\")\n",
+        "    TF = \"2.1\".replace(\".\", \"-\")\n",
         "\n",
         "if TF[0] == \"2\":\n",
         "    if TRAIN_GPU:\n",
@@ -702,7 +602,7 @@
         "DISPLAY_NAME = \"boston_\" + TIMESTAMP\n",
         "REQUIREMENTS = [\"tensorflow==2.3\"]\n",
         "\n",
-        "job = aiplatform.CustomTrainingJob(\n",
+        "job = aip.CustomTrainingJob(\n",
         "    display_name=DISPLAY_NAME,\n",
         "    script_path=\"task.py\",\n",
         "    requirements=REQUIREMENTS,\n",
@@ -793,12 +693,12 @@
       "outputs": [],
       "source": [
         "CMDARGS = [\n",
-        "    \"--model-dir=\" + BUCKET_URI,\n",
+        "    \"--model-dir=\" + BUCKET_NAME,\n",
         "]\n",
         "\n",
         "job.run(args=CMDARGS, replica_count=1, machine_type=TRAIN_COMPUTE, sync=True)\n",
         "\n",
-        "! gsutil cat {BUCKET_URI}/test.txt"
+        "! gsutil cat {BUCKET_NAME}/test.txt"
       ]
     },
     {
@@ -868,9 +768,9 @@
       "source": [
         "DISPLAY_NAME = \"boston_\" + TIMESTAMP\n",
         "\n",
-        "job = aiplatform.CustomPythonPackageTrainingJob(\n",
+        "job = aip.CustomPythonPackageTrainingJob(\n",
         "    display_name=DISPLAY_NAME,\n",
-        "    python_package_gcs_uri=f\"{BUCKET_URI}/trainer_boston.tar.gz\",\n",
+        "    python_package_gcs_uri=f\"{BUCKET_NAME}/trainer_boston.tar.gz\",\n",
         "    python_module_name=\"trainer.task\",\n",
         "    container_uri=TRAIN_IMAGE,\n",
         ")"
@@ -1000,7 +900,7 @@
         "! rm -f custom.tar custom.tar.gz\n",
         "! tar cvf custom.tar custom\n",
         "! gzip custom.tar\n",
-        "! gsutil cp custom.tar.gz $BUCKET_URI/trainer_boston.tar.gz"
+        "! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_boston.tar.gz"
       ]
     },
     {
@@ -1022,11 +922,11 @@
       },
       "outputs": [],
       "source": [
-        "CMDARGS = [\"--model-dir=\" + BUCKET_URI, \"--epochs=5\"]\n",
+        "CMDARGS = [\"--model-dir=\" + BUCKET_NAME, \"--epochs=5\"]\n",
         "\n",
         "job.run(args=CMDARGS, replica_count=1, machine_type=TRAIN_COMPUTE, sync=True)\n",
         "\n",
-        "! gsutil cat {BUCKET_URI}/test.txt"
+        "! gsutil cat {BUCKET_NAME}/test.txt"
       ]
     },
     {
@@ -1251,11 +1151,7 @@
       },
       "outputs": [],
       "source": [
-        "if not IS_COLAB:\n",
-        "    ! docker build custom -t $TRAIN_IMAGE\n",
-        "else:\n",
-        "    # install docker daemon\n",
-        "    ! apt-get -qq install docker.io"
+        "! docker build custom -t $TRAIN_IMAGE"
       ]
     },
     {
@@ -1277,8 +1173,7 @@
       },
       "outputs": [],
       "source": [
-        "if not IS_COLAB:\n",
-        "    ! docker run $TRAIN_IMAGE --epochs=5 --model-dir=./"
+        "! docker run $TRAIN_IMAGE --epochs=5 --model-dir=./"
       ]
     },
     {
@@ -1300,38 +1195,7 @@
       },
       "outputs": [],
       "source": [
-        "if not IS_COLAB:\n",
-        "    ! docker push $TRAIN_IMAGE"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "f50e9c553fb7"
-      },
-      "source": [
-        "*Executes in Colab*"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "a7e8c98f1e56"
-      },
-      "outputs": [],
-      "source": [
-        "%%bash -s $IS_COLAB $TRAIN_IMAGE\n",
-        "if [ $1 == \"False\" ]; then\n",
-        "  exit 0\n",
-        "fi\n",
-        "set -x\n",
-        "dockerd -b none --iptables=0 -l warn &\n",
-        "for i in $(seq 5); do [ ! -S \"/var/run/docker.sock\" ] && sleep 2 || break; done\n",
-        "docker build custom -t $2\n",
-        "docker run $2 --epochs=5 --model-dir=./\n",
-        "docker push $2\n",
-        "kill $(jobs -p)"
+        "! docker push $TRAIN_IMAGE"
       ]
     },
     {
@@ -1365,7 +1229,7 @@
       },
       "outputs": [],
       "source": [
-        "job = aiplatform.CustomContainerTrainingJob(\n",
+        "job = aip.CustomContainerTrainingJob(\n",
         "    display_name=\"boston_\" + TIMESTAMP,\n",
         "    container_uri=TRAIN_IMAGE,\n",
         "    command=[\"python3\", \"trainer/task.py\"],\n",
@@ -1393,11 +1257,11 @@
       },
       "outputs": [],
       "source": [
-        "CMDARGS = [\"--model-dir=\" + BUCKET_URI, \"--epochs=5\"]\n",
+        "CMDARGS = [\"--model-dir=\" + BUCKET_NAME, \"--epochs=5\"]\n",
         "\n",
         "job.run(args=CMDARGS, replica_count=1, machine_type=TRAIN_COMPUTE, sync=True)\n",
         "\n",
-        "! gsutil cat {BUCKET_URI}/test.txt"
+        "! gsutil cat {BUCKET_NAME}/test.txt"
       ]
     },
     {
@@ -1602,7 +1466,7 @@
         "! rm -f custom.tar custom.tar.gz\n",
         "! tar cvf custom.tar custom\n",
         "! gzip custom.tar\n",
-        "! gsutil cp custom.tar.gz $BUCKET_URI/trainer_boston.tar.gz"
+        "! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_boston.tar.gz"
       ]
     },
     {
@@ -1632,7 +1496,7 @@
         "if os.getenv(\"IS_TESTING_TF\"):\n",
         "    TF = os.getenv(\"IS_TESTING_TF\")\n",
         "else:\n",
-        "    TF = \"2.5\".replace(\".\", \"-\")\n",
+        "    TF = \"2.1\".replace(\".\", \"-\")\n",
         "\n",
         "if TF[0] == \"2\":\n",
         "    if TRAIN_GPU:\n",
@@ -1687,9 +1551,9 @@
       "source": [
         "DISPLAY_NAME = \"boston_\" + TIMESTAMP\n",
         "\n",
-        "job = aiplatform.CustomPythonPackageTrainingJob(\n",
+        "job = aip.CustomPythonPackageTrainingJob(\n",
         "    display_name=DISPLAY_NAME,\n",
-        "    python_package_gcs_uri=f\"{BUCKET_URI}/trainer_boston.tar.gz\",\n",
+        "    python_package_gcs_uri=f\"{BUCKET_NAME}/trainer_boston.tar.gz\",\n",
         "    python_module_name=\"trainer.task\",\n",
         "    container_uri=TRAIN_IMAGE,\n",
         "    model_serving_container_image_uri=DEPLOY_IMAGE,\n",
@@ -1723,12 +1587,12 @@
       },
       "outputs": [],
       "source": [
-        "MODEL_DIR = \"{}/{}\".format(BUCKET_URI, TIMESTAMP)\n",
+        "MODEL_DIR = \"{}/{}\".format(BUCKET_NAME, TIMESTAMP)\n",
         "\n",
         "EPOCHS = 20\n",
         "STEPS = 100\n",
         "\n",
-        "DIRECT = False\n",
+        "DIRECT = True\n",
         "if DIRECT:\n",
         "    CMDARGS = [\n",
         "        \"--model-dir=\" + MODEL_DIR,\n",
@@ -1894,7 +1758,17 @@
         "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
         "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
         "\n",
-        "Otherwise, you can delete the individual resources you created in this tutorial."
+        "Otherwise, you can delete the individual resources you created in this tutorial:\n",
+        "\n",
+        "- Dataset\n",
+        "- Pipeline\n",
+        "- Model\n",
+        "- Endpoint\n",
+        "- AutoML Training Job\n",
+        "- Batch Job\n",
+        "- Custom Job\n",
+        "- Hyperparameter Tuning Job\n",
+        "- Cloud Storage Bucket"
       ]
     },
     {
@@ -1905,24 +1779,61 @@
       },
       "outputs": [],
       "source": [
-        "delete_bucket = False\n",
-        "delete_model = True\n",
-        "delete_job = True\n",
+        "delete_all = True\n",
         "\n",
-        "if delete_model:\n",
+        "if delete_all:\n",
+        "    # Delete the dataset using the Vertex dataset object\n",
         "    try:\n",
-        "        model.delete()\n",
+        "        if \"dataset\" in globals():\n",
+        "            dataset.delete()\n",
         "    except Exception as e:\n",
         "        print(e)\n",
         "\n",
-        "if delete_job:\n",
+        "    # Delete the model using the Vertex model object\n",
         "    try:\n",
-        "        job.delete()\n",
+        "        if \"model\" in globals():\n",
+        "            model.delete()\n",
         "    except Exception as e:\n",
         "        print(e)\n",
         "\n",
-        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
-        "    ! gsutil rm -rf {BUCKET_URI}"
+        "    # Delete the endpoint using the Vertex endpoint object\n",
+        "    try:\n",
+        "        if \"endpoint\" in globals():\n",
+        "            endpoint.undeploy_all()\n",
+        "            endpoint.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the AutoML or Pipeline training job\n",
+        "    try:\n",
+        "        if \"dag\" in globals():\n",
+        "            dag.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the custom training job\n",
+        "    try:\n",
+        "        if \"job\" in globals():\n",
+        "            job.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the batch prediction job using the Vertex batch prediction object\n",
+        "    try:\n",
+        "        if \"batch_predict_job\" in globals():\n",
+        "            batch_predict_job.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the hyperparameter tuning job using the Vertex hyperparameter tuning object\n",
+        "    try:\n",
+        "        if \"hpt_job\" in globals():\n",
+        "            hpt_job.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    if \"BUCKET_NAME\" in globals():\n",
+        "        ! gsutil rm -r $BUCKET_NAME"
       ]
     }
   ],

notebooks/community/ml_ops/stage2/get_started_vertex_training_pytorch.ipynb

@@ -33,18 +33,13 @@
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_pytorch.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
-        "    </a>\n",
-        "  </td>\n",
-        "  <td>\n",
         "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_pytorch.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/notebook_template.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/notebook_template.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
@@ -177,7 +172,7 @@
       "source": [
         "## Installations\n",
         "\n",
-        "Install the following packages to execute this notebook."
+        "Install *one time* the packages for executing the MLOps notebooks."
       ]
     },
     {
@@ -188,22 +183,10 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
-        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG -q\n",
-        "! pip3 install --upgrade cloudml-hypertune $USER_FLAG -q\n",
-        "! pip3 install --upgrade torchvision $USER_FLAG -q"
+        "! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG\n",
+        "! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG\n",
+        "! pip3 install --upgrade cloudml-hypertune $USER_FLAG\n",
+        "! pip3 install --upgrade torchvision $USER_FLAG"
       ]
     },
     {
@@ -235,32 +218,6 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "84cd83853240"
-      },
-      "source": [
-        "## Before you begin\n",
-        "\n",
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI, BigQuery, Compute Engine and Cloud Storage APIs](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,bigquery,compute_component,storage_component).\n",
-        "\n",
-        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -337,10 +294,7 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
       ]
     },
     {
@@ -367,67 +321,6 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "77c385f0db59"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
-        "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
-        "\n",
-        "1. **Click Create service account**.\n",
-        "\n",
-        "2. In the **Service account name** field, enter a name, and click **Create**.\n",
-        "\n",
-        "3. In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex AI\" into the filter box, and select **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "4. Click Create. A JSON file that contains your key downloads to your local environment.\n",
-        "\n",
-        "5. Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "535223fa4b84"
-      },
-      "outputs": [],
-      "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = False\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        IS_COLAB = True\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -1101,52 +994,11 @@
       "source": [
         "APP_NAME = \"cifar10\"\n",
         "DEPLOY_IMAGE = f\"gcr.io/{PROJECT_ID}/pytorch_predict_{APP_NAME}\"\n",
-        "print(DEPLOY_IMAGE)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "85739262f629"
-      },
-      "outputs": [],
-      "source": [
-        "if not IS_COLAB:\n",
-        "    ! docker build --tag=$DEPLOY_IMAGE ./\n",
-        "    ! docker push $DEPLOY_IMAGE\n",
-        "else:\n",
-        "    # install docker daemon\n",
-        "    ! apt-get -qq install docker.io"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "f50e9c553fb7"
-      },
-      "source": [
-        "*Executes in Colab*"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "a7e8c98f1e56"
-      },
-      "outputs": [],
-      "source": [
-        "%%bash -s $IS_COLAB $DEPLOY_IMAGE\n",
-        "if [ $1 == \"False\" ]; then\n",
-        "  exit 0\n",
-        "fi\n",
-        "set -x\n",
-        "dockerd -b none --iptables=0 -l warn &\n",
-        "for i in $(seq 5); do [ ! -S \"/var/run/docker.sock\" ] && sleep 2 || break; done\n",
-        "docker build --tag=$2 ./\n",
-        "docker push $2\n",
-        "kill $(jobs -p)"
+        "print(DEPLOY_IMAGE)\n",
+        "\n",
+        "! docker build --tag=$DEPLOY_IMAGE ./\n",
+        "\n",
+        "! docker push $DEPLOY_IMAGE"
       ]
     },
     {
@@ -1359,6 +1211,7 @@
         "Otherwise, you can delete the individual resources you created in this tutorial:\n",
         "\n",
         "- Model\n",
+        "- Custom Job (Custom job deleted in previous cell)\n",
         "- Cloud Storage Bucket"
       ]
     },
@@ -1373,8 +1226,8 @@
         "# Delete the model using the Vertex model object\n",
         "model.delete()\n",
         "\n",
-        "delete_bucket = False\n",
-        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "\n",
+        "if os.getenv(\"IS_TESTING\"):\n",
         "    ! gsutil rm -r $BUCKET_URI"
       ]
     }

notebooks/community/ml_ops/stage2/get_started_vertex_training_r.ipynb

@@ -39,12 +39,7 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_r.ipynb\">\n",
-        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
-        "        </a>\n",
-        "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_r.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_r.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
@@ -134,7 +129,7 @@
       "source": [
         "## Installations\n",
         "\n",
-        "Install the packages required for executing this notebook."
+        "Install *one time* the packages for executing the MLOps notebooks."
       ]
     },
     {
@@ -147,19 +142,25 @@
       "source": [
         "import os\n",
         "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
+        "# The Google Cloud Notebook product has specific requirements\n",
+        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
         "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "# Google Cloud Notebook requires dependencies to be installed with '--user'\n",
         "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
-        "! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG -q\n",
-        "! pip3 install --upgrade rpy2 $USER_FLAG -q"
+        "if IS_GOOGLE_CLOUD_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "install_mlops"
+      },
+      "outputs": [],
+      "source": [
+        "! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG\n",
+        "! pip3 install --upgrade rpy2 $USER_FLAG"
       ]
     },
     {
@@ -214,7 +215,7 @@
         "\n",
         "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
         "\n",
-        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com). \n",
+        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com). {TODO: Update the APIs needed for your tutorial. Edit the API names, and update the link to append the API IDs, separating each one with a comma. For example, container.googleapis.com,cloudbuild.googleapis.com}\n",
         "\n",
         "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
         "\n",
@@ -331,67 +332,6 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "3ffa6b6c7cdb"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
-        "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
-        "\n",
-        "1. **Click Create service account**.\n",
-        "\n",
-        "2. In the **Service account name** field, enter a name, and click **Create**.\n",
-        "\n",
-        "3. In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex AI\" into the filter box, and select **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "4. Click Create. A JSON file that contains your key downloads to your local environment.\n",
-        "\n",
-        "5. Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "2b72272258fc"
-      },
-      "outputs": [],
-      "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = False\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        IS_COLAB = True\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -1426,9 +1366,7 @@
       },
       "outputs": [],
       "source": [
-        "delete_bucket = False\n",
-        "\n",
-        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "if os.getenv(\"IS_TESTING\"):\n",
         "    ! gsutil rm -r $BUCKET_URI"
       ]
     }

notebooks/community/ml_ops/stage2/get_started_vertex_training_sklearn.ipynb

@@ -33,23 +33,21 @@
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_sklearn.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
-        "    </a>\n",
-        "  </td>\n",
-        "  <td>\n",
         "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_sklearn.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_sklearn.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_sklearn.ipynb\">\n",
         "     <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "Open in Vertex AI Workbench\n",
         "    </a>\n",
         "  </td>\n",
         "</table>\n",
+        "    </a>\n",
+        "  </td>\n",
+        "</table>\n",
         "<br/><br/><br/>"
       ]
     },
@@ -62,7 +60,7 @@
         "## Overview\n",
         "\n",
         "\n",
-        "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 2 : experimentation: get started with Vertex AI Training for scikit-Learn."
+        "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 2 : experimentation: get started with Vertex Training for Scikit-Learn."
       ]
     },
     {
@@ -99,156 +97,15 @@
         "- Create a `Vertex AI Model` resource."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "b132d4ef86d6"
-      },
-      "source": [
-        "### Costs \n",
-        "\n",
-        "This tutorial uses billable components of Google Cloud:\n",
-        "\n",
-        "* Vertex AI\n",
-        "* Cloud Storage\n",
-        "\n",
-        "Learn about [Vertex AI\n",
-        "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
-        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
-        "Calculator](https://cloud.google.com/products/calculator/)\n",
-        "to generate a cost estimate based on your projected usage."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "94a148f11da5"
-      },
-      "source": [
-        "### Set up your local development environment\n",
-        "\n",
-        "**If you are using Colab or Google Cloud Notebooks**, your environment already meets\n",
-        "all the requirements to run this notebook. You can skip this step.\n",
-        "\n",
-        "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
-        "You need the following:\n",
-        "\n",
-        "* The Google Cloud SDK\n",
-        "* Git\n",
-        "* Python 3\n",
-        "* virtualenv\n",
-        "* Jupyter notebook running in a virtual environment with Python 3\n",
-        "\n",
-        "The Google Cloud guide to [Setting up a Python development\n",
-        "environment](https://cloud.google.com/python/setup) and the [Jupyter\n",
-        "installation guide](https://jupyter.org/install) provide detailed instructions\n",
-        "for meeting these requirements. The following steps provide a condensed set of\n",
-        "instructions:\n",
-        "\n",
-        "1. [Install and initialize the Cloud SDK.](https://cloud.google.com/sdk/docs/)\n",
-        "\n",
-        "1. [Install Python 3.](https://cloud.google.com/python/setup#installing_python)\n",
-        "\n",
-        "1. [Install\n",
-        "   virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv)\n",
-        "   and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
-        "\n",
-        "1. To install Jupyter, run `pip3 install jupyter` on the\n",
-        "command-line in a terminal shell.\n",
-        "\n",
-        "1. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
-        "\n",
-        "1. Open this notebook in the Jupyter Notebook Dashboard.\n"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "install_mlops"
       },
       "source": [
-        "### Install additional packages\n",
+        "## Installations\n",
         "\n",
-        "Install the following packages for executing this notebook."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "78168417490e"
-      },
-      "outputs": [],
-      "source": [
-        "import os\n",
-        "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
-        "! pip3 install {USER_FLAG} --upgrade google-cloud-aiplatform -q"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "restart"
-      },
-      "source": [
-        "### Restart the kernel\n",
-        "\n",
-        "Once you've installed the additional packages, you need to restart the notebook kernel so it can find the packages."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "restart"
-      },
-      "outputs": [],
-      "source": [
-        "import os\n",
-        "\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    # Automatically restart kernel after installs\n",
-        "    import IPython\n",
-        "\n",
-        "    app = IPython.Application.instance()\n",
-        "    app.kernel.do_shutdown(True)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "2721ef0202d9"
-      },
-      "source": [
-        "## Before you begin\n",
-        "\n",
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com). \n",
-        "\n",
-        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+        "You will not need special packages for this notebook."
       ]
     },
     {
@@ -327,10 +184,7 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
       ]
     },
     {
@@ -357,67 +211,6 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "77c385f0db59"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
-        "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
-        "\n",
-        "1. **Click Create service account**.\n",
-        "\n",
-        "2. In the **Service account name** field, enter a name, and click **Create**.\n",
-        "\n",
-        "3. In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex AI\" into the filter box, and select **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "4. Click Create. A JSON file that contains your key downloads to your local environment.\n",
-        "\n",
-        "5. Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "535223fa4b84"
-      },
-      "outputs": [],
-      "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = False\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        IS_COLAB = True\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -675,9 +468,9 @@
         "id": "sklearn_intro"
       },
       "source": [
-        "## Introduction to scikit-learn training\n",
+        "## Introduction to Scikit-learn training\n",
         "\n",
-        "Once you have trained a scikit-learn model, you will want to save it at a Cloud Storage location, so it can subsequently be uploaded to a `Vertex AI Model` resource. The Scikit-learn package does not have support to save the model to a Cloud Storage location. Instead, you will do the following steps to save to a Cloud Storage location.\n",
+        "Once you have trained a Scikit-learn model, you will want to save it at a Cloud Storage location, so it can subsequently be uploaded to a `Vertex AI Model` resource. The Scikit-learn package does not have support to save the model to a Cloud Storage location. Instead, you will do the following steps to save to a Cloud Storage location.\n",
         "\n",
         "1. Save the in-memory model to the local filesystem in pickle format (e.g., model.pkl).\n",
         "2. Create a Cloud Storage storage client.\n",
@@ -1162,6 +955,7 @@
         "\n",
         "\n",
         "- Model\n",
+        "- Custom Job (already deleted in previous cell)\n",
         "- Cloud Storage Bucket"
       ]
     },
@@ -1176,8 +970,7 @@
         "# Delete the model using the Vertex model object\n",
         "model.delete()\n",
         "\n",
-        "delete_bucket = False\n",
-        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "if os.getenv(\"IS_TESTING\"):\n",
         "    ! gsutil rm -r $BUCKET_URI"
       ]
     }

notebooks/community/ml_ops/stage2/get_started_vertex_training_xgboost.ipynb

@@ -45,7 +45,7 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_xgboost.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_xgboost.ipynb\">\n",
         "    <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
@@ -122,7 +122,7 @@
       "source": [
         "## Installations\n",
         "\n",
-        "Install the following packages to execute this notebook."
+        "Install *one time* the packages for executing the MLOps notebooks."
       ]
     },
     {
@@ -133,46 +133,20 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
-        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG -q"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "2721ef0202d9"
-      },
-      "source": [
-        "## Before you begin\n",
-        "\n",
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com). \n",
-        "\n",
-        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+        "# ! pip3 install -U tensorflow==2.5 $USER_FLAG\n",
+        "# ! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG\n",
+        "# ! pip3 install -U tensorflow-transform==1.2 $USER_FLAG\n",
+        "# ! pip3 install -U tensorflow-io==0.18 $USER_FLAG\n",
+        "! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG\n",
+        "# ! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG\n",
+        "# ! pip3 install --upgrade google-cloud-bigquery $USER_FLAG\n",
+        "# ! pip3 install --upgrade google-cloud-logging $USER_FLAG\n",
+        "# ! pip3 install --upgrade apache-beam[gcp] $USER_FLAG\n",
+        "# ! pip3 install --upgrade pyarrow $USER_FLAG\n",
+        "# ! pip3 install --upgrade cloudml-hypertune $USER_FLAG\n",
+        "# ! pip3 install --upgrade kfp $USER_FLAG\n",
+        "# ! pip3 install --upgrade torchvision $USER_FLAG\n",
+        "# ! pip3 install --upgrade rpy2 $USER_FLAG"
       ]
     },
     {
@@ -281,33 +255,6 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "77c385f0db59"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
-        "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
-        "\n",
-        "1. **Click Create service account**.\n",
-        "\n",
-        "2. In the **Service account name** field, enter a name, and click **Create**.\n",
-        "\n",
-        "3. In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex AI\" into the filter box, and select **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "4. Click Create. A JSON file that contains your key downloads to your local environment.\n",
-        "\n",
-        "5. Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -316,21 +263,20 @@
       },
       "outputs": [],
       "source": [
+        "import os\n",
+        "import sys\n",
+        "\n",
         "# If you are running this notebook in Colab, run this cell and follow the\n",
         "# instructions to authenticate your GCP account. This provides access to your\n",
         "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
         "# requests.\n",
         "\n",
-        "import os\n",
-        "import sys\n",
+        "# The Google Cloud Notebook product has specific requirements\n",
+        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
         "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = False\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
+        "# If on Google Cloud Notebooks, then don't execute this code\n",
+        "if not IS_GOOGLE_CLOUD_NOTEBOOK:\n",
         "    if \"google.colab\" in sys.modules:\n",
-        "        IS_COLAB = True\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
         "        google_auth.authenticate_user()\n",
@@ -1058,9 +1004,7 @@
       },
       "outputs": [],
       "source": [
-        "delete_bucket = False\n",
-        "\n",
-        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "if os.getenv(\"IS_TESTING\"):\n",
         "    ! gsutil rm -r $BUCKET_URI"
       ]
     }

notebooks/community/ml_ops/stage2/get_started_vertex_vizier.ipynb

@@ -29,7 +29,7 @@
         "id": "title:generic,gcp"
       },
       "source": [
-        "# E2E ML on GCP: MLOps stage 2 : experimentation: get started with Vertex AI Vizier\n",
+        "# E2E ML on GCP: MLOps stage 2 : experimentation: get started with Vertex Vizier\n",
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
@@ -39,12 +39,7 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_vizier.ipynb\">\n",
-        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
-        "        </a>\n",
-        "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage2/get_started_vertex_vizier.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_vizier.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
@@ -142,7 +137,7 @@
       "source": [
         "## Installations\n",
         "\n",
-        "Install the packages required for executing this notebook."
+        "Install *one time* the packages for executing the MLOps notebooks."
       ]
     },
     {
@@ -155,18 +150,25 @@
       "source": [
         "import os\n",
         "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
+        "# The Google Cloud Notebook product has specific requirements\n",
+        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
         "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "# Google Cloud Notebook requires dependencies to be installed with '--user'\n",
         "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
-        "! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG"
+        "if IS_GOOGLE_CLOUD_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "install_mlops"
+      },
+      "outputs": [],
+      "source": [
+        "! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG\n",
+        "    "
       ]
     },
     {
@@ -198,32 +200,6 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "2721ef0202d9"
-      },
-      "source": [
-        "## Before you begin\n",
-        "\n",
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com). \n",
-        "\n",
-        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -329,67 +305,6 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "gcp_authenticate"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
-        "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
-        "\n",
-        "**Click Create service account**.\n",
-        "\n",
-        "In the **Service account name** field, enter a name, and click **Create**.\n",
-        "\n",
-        "In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex\" into the filter box, and select **Vertex Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "Click Create. A JSON file that contains your key downloads to your local environment.\n",
-        "\n",
-        "Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "gcp_authenticate"
-      },
-      "outputs": [],
-      "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = False\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        IS_COLAB = True\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -1683,8 +1598,6 @@
       },
       "outputs": [],
       "source": [
-        "delete_bucket = False\n",
-        "\n",
         "if os.getenv(\"IS_TESTING\"):\n",
         "    ! gsutil rm -r $BUCKET_URI"
       ]

notebooks/community/ml_ops/stage2/get_started_with_cmek_training.ipynb

@@ -38,14 +38,8 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_with_cmek_training.ipynb\">\n",
-        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
-        "        </a>\n",
-        "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage2/get_started_with_cmek_training.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
+        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_with_cmek_training.ipynb\">\n",
+        "      Open in Google Cloud Notebooks\n",
         "    </a>\n",
         "  </td>\n",
         "</table>\n",
@@ -97,26 +91,6 @@
         "- Train an AutoML model with CMEK encryption."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "5e2eba58ad71"
-      },
-      "source": [
-        "### Costs \n",
-        "\n",
-        "This tutorial uses billable components of Google Cloud:\n",
-        "\n",
-        "* Vertex AI\n",
-        "* Cloud Storage\n",
-        "\n",
-        "Learn about [Vertex AI\n",
-        "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
-        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
-        "Calculator](https://cloud.google.com/products/calculator/)\n",
-        "to generate a cost estimate based on your projected usage."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -136,21 +110,10 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
+        "USER_FLAG = \"--user\"\n",
         "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
-        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG -q\n",
-        "! pip3 install --upgrade google-cloud-kms $USER_FLAG -q"
+        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG\n",
+        "! pip3 install --upgrade google-cloud-kms $USER_FLAG"
       ]
     },
     {
@@ -182,39 +145,6 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "013daf3de88e"
-      },
-      "source": [
-        "## Before you begin"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "d1afc945645f"
-      },
-      "source": [
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com). \n",
-        "\n",
-        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -318,82 +248,6 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "d35af059208d"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
-        "authenticated. Skip this step."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "a00567d0660a"
-      },
-      "source": [
-        "**If you are using Colab**, run the cell below and follow the instructions\n",
-        "when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "1. In the Cloud Console, go to the [**Create service account key**\n",
-        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
-        "\n",
-        "2. Click **Create service account**.\n",
-        "\n",
-        "3. In the **Service account name** field, enter a name, and\n",
-        "   click **Create**.\n",
-        "\n",
-        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
-        "into the filter box, and select\n",
-        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
-        "local environment.\n",
-        "\n",
-        "6. Enter the path to your service account key as the\n",
-        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "40160162ea4c"
-      },
-      "outputs": [],
-      "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = False\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        IS_COLAB = True\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -417,7 +271,7 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_URI = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
+        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -428,8 +282,8 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
       ]
     },
     {
@@ -449,7 +303,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_URI"
+        "! gsutil mb -l $REGION $BUCKET_NAME"
       ]
     },
     {
@@ -469,7 +323,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_URI"
+        "! gsutil ls -al $BUCKET_NAME"
       ]
     },
     {
@@ -492,7 +346,7 @@
       },
       "outputs": [],
       "source": [
-        "import google.cloud.aiplatform as aiplatform\n",
+        "import google.cloud.aiplatform as aip\n",
         "from google.cloud import kms"
       ]
     },
@@ -515,7 +369,7 @@
       },
       "outputs": [],
       "source": [
-        "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=BUCKET_URI)"
+        "aip.init(project=PROJECT_ID, location=REGION, staging_bucket=BUCKET_NAME)"
       ]
     },
     {
@@ -530,7 +384,7 @@
         "\n",
         "### Enable KMS API\n",
         "\n",
-        "First, you enable the [Cloud Key Management Service (KMS)](https://console.cloud.google.com/flows/enableapi?apiid=cloudkms.googleapis.com)\n",
+        "First, you enble the [Cloud Key Management Service (KMS)](https://console.cloud.google.com/flows/enableapi?apiid=cloudkms.googleapis.com)\n",
         "\n",
         "Learn more about [Customer managed encryption keys (CMEK)](https://cloud.google.com/vertex-ai/docs/general/cmek)"
       ]
@@ -695,8 +549,6 @@
         "\n",
         "Next, you set permissions for your Vertex AI service account to encrypt and decrypt resources using your key.\n",
         "\n",
-        "Note: Compute Engine default service account which is used by this notebook instance for authentication purposes during Google API calls, should be granted the role of Cloud KMS Admin.\n",
-        "\n",
         "Learn more about [Grant Vertex AI permissions](https://cloud.google.com/vertex-ai/docs/general/cmek#grant_permissions)"
       ]
     },
@@ -786,9 +638,9 @@
       },
       "outputs": [],
       "source": [
-        "aiplatform.init(\n",
+        "aip.init(\n",
         "    project=PROJECT_ID,\n",
-        "    staging_bucket=BUCKET_URI,\n",
+        "    staging_bucket=BUCKET_NAME,\n",
         "    location=REGION,\n",
         "    encryption_spec_key_name=ENCRYPTION_SPEC_KEY_NAME,\n",
         ")"
@@ -837,10 +689,10 @@
       },
       "outputs": [],
       "source": [
-        "dataset = aiplatform.ImageDataset.create(\n",
+        "dataset = aip.ImageDataset.create(\n",
         "    display_name=\"flowers_\" + TIMESTAMP,\n",
         "    gcs_source=[IMPORT_FILE],\n",
-        "    import_schema_uri=aiplatform.schema.dataset.ioformat.image.single_label_classification,\n",
+        "    import_schema_uri=aip.schema.dataset.ioformat.image.single_label_classification,\n",
         ")\n",
         "\n",
         "print(dataset.resource_name)"
@@ -875,7 +727,7 @@
         "\n",
         "# This will take around half an hour to run\n",
         "model = job.run(\n",
-        "    dataset=dataset,\n",
+        "    dataset=ds,\n",
         "    model_display_name=\"flowers_\" + TIMESTAMP,\n",
         "    training_fraction_split=0.6,\n",
         "    validation_fraction_split=0.2,\n",
@@ -975,25 +827,6 @@
         "endpoint.undeploy_all()"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "ba77c4e02355"
-      },
-      "source": [
-        "## Cleaning up\n",
-        "\n",
-        "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
-        "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
-        "\n",
-        "Otherwise, you can delete the individual resources you created in this tutorial:\n",
-        "\n",
-        "- Model\n",
-        "- Dataset\n",
-        "- Cloud Storage Bucket\n",
-        "- Endpoint"
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -1002,54 +835,17 @@
       },
       "outputs": [],
       "source": [
-        "# Delete endpoint resource\n",
+        "# missing\n",
         "endpoint.delete()\n",
-        "\n",
-        "# Delete model resource\n",
         "model.delete()\n",
-        "\n",
-        "# Delete dataset resource\n",
         "dataset.delete()\n",
         "\n",
-        "# Set this to true only if you'd like to delete your bucket\n",
-        "delete_bucket = False\n",
-        "\n",
-        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
-        "    ! gsutil rm -r $BUCKET_URI"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "d88c8a053a90"
-      },
-      "source": [
-        "## Destroying CMEK by providing key-version value and other parameters."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "5d8d2fc34346"
-      },
-      "outputs": [],
-      "source": [
-        "! gcloud kms keys versions destroy 1 \\\n",
-        "  --key {KEY_ID} \\\n",
+        "! gcloud kms keys versions destroy key-version \\\n",
+        "  --key key {KEY_ID} \\\n",
         "  --keyring={KEY_RING_ID} \\\n",
         "  --location={REGION} "
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "f7e42e642ad3"
-      },
-      "source": [
-        "## List of keys "
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,

notebooks/community/ml_ops/stage2/get_started_with_logging.ipynb

@@ -1,684 +0,0 @@
-{
-  "cells": [
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "copyright"
-      },
-      "outputs": [],
-      "source": [
-        "# Copyright 2022 Google LLC\n",
-        "#\n",
-        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
-        "# you may not use this file except in compliance with the License.\n",
-        "# You may obtain a copy of the License at\n",
-        "#\n",
-        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
-        "#\n",
-        "# Unless required by applicable law or agreed to in writing, software\n",
-        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
-        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
-        "# See the License for the specific language governing permissions and\n",
-        "# limitations under the License."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "title:generic,gcp"
-      },
-      "source": [
-        "# E2E ML on GCP: MLOps stage 2 : experimentation: get started with Logging\n",
-        "\n",
-        "<table align=\"left\">\n",
-        "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_with_logging.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
-        "      View on GitHub\n",
-        "    </a>\n",
-        "  </td>\n",
-        "    <td>\n",
-        "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_with_logging.ipynb\">\n",
-        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
-        "        </a>\n",
-        "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage2/get_started_with_logging.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
-        "    </a>\n",
-        "  </td>\n",
-        "</table>\n",
-        "<br/><br/><br/>"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "overview:mlops"
-      },
-      "source": [
-        "## Overview\n",
-        "\n",
-        "\n",
-        "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 2 : experimentation: get started with Logging."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "objective:mlops,stage2,get_started_vertex_experiments"
-      },
-      "source": [
-        "### Objective\n",
-        "\n",
-        "In this tutorial, you learn how to use Python and Cloud logging awhen training with `Vertex AI`.\n",
-        "\n",
-        "This tutorial uses the following Google Cloud ML services:\n",
-        "\n",
-        "- `Cloud Logging`\n",
-        "\n",
-        "The steps performed include:\n",
-        "\n",
-        "- Use Python logging to log training configuration/results locally.\n",
-        "- Use Google Cloud Logging to log training configuration/results in cloud storage."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "recommendation:mlops,stage2,logging"
-      },
-      "source": [
-        "### Recommendations\n",
-        "\n",
-        "When doing E2E MLOps on Google Cloud, the following are some of the best practices for logging data when experimenting or formally training a model.\n",
-        "\n",
-        "#### Python Logging\n",
-        "\n",
-        "Use Python's logging package when doing ad-hoc training locally.\n",
-        "\n",
-        "#### Cloud Logging\n",
-        "\n",
-        "Use `Google Cloud Logging` when doing training on the cloud.\n",
-        "\n",
-        "#### Experiments\n",
-        "\n",
-        "Use Vertex AI Experiments in conjunction with logging when performing experiments to compare results for different experiment configurations.\n",
-        "\n",
-        "### Costs\n",
-        "This tutorial uses billable components of Google Cloud:\n",
-        "\n",
-        "- Vertex AI\n",
-        "\n",
-        "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing) and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "install_mlops"
-      },
-      "source": [
-        "## Installations\n",
-        "\n",
-        "Install the following packages for executing this notebook."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "install_mlops"
-      },
-      "outputs": [],
-      "source": [
-        "import os\n",
-        "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
-        "! pip3 install --upgrade google-cloud-logging $USER_FLAG -q"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "restart"
-      },
-      "source": [
-        "### Restart the kernel\n",
-        "\n",
-        "Once you've installed the additional packages, you need to restart the notebook kernel so it can find the packages."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "restart"
-      },
-      "outputs": [],
-      "source": [
-        "import os\n",
-        "\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    # Automatically restart kernel after installs\n",
-        "    import IPython\n",
-        "\n",
-        "    app = IPython.Application.instance()\n",
-        "    app.kernel.do_shutdown(True)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "project_id"
-      },
-      "source": [
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI, Compute Engine, Cloud Storage and Cloud Logging APIs](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component,storage_component,logging).\n",
-        "\n",
-        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands.\n",
-        "\n",
-        "\n",
-        "#### Set your project ID\n",
-        "\n",
-        "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "set_project_id"
-      },
-      "outputs": [],
-      "source": [
-        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "autoset_project_id"
-      },
-      "outputs": [],
-      "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
-        "    # Get your GCP project id from gcloud\n",
-        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
-        "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID:\", PROJECT_ID)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "set_gcloud_project_id"
-      },
-      "outputs": [],
-      "source": [
-        "! gcloud config set project $PROJECT_ID"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "region"
-      },
-      "source": [
-        "#### Region\n",
-        "\n",
-        "You can also change the `REGION` variable, which is used for operations\n",
-        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
-        "\n",
-        "- Americas: `us-central1`\n",
-        "- Europe: `europe-west4`\n",
-        "- Asia Pacific: `asia-east1`\n",
-        "\n",
-        "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
-        "\n",
-        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "region"
-      },
-      "outputs": [],
-      "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "timestamp"
-      },
-      "source": [
-        "#### Timestamp\n",
-        "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "timestamp"
-      },
-      "outputs": [],
-      "source": [
-        "from datetime import datetime\n",
-        "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "f3bd8c0d0469"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
-        "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
-        "\n",
-        "1. **Click Create service account**.\n",
-        "\n",
-        "2. In the **Service account name** field, enter a name, and click **Create**.\n",
-        "\n",
-        "3. In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex AI\" into the filter box, and select **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "4. Click Create. A JSON file that contains your key downloads to your local environment.\n",
-        "\n",
-        "5. Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "e0953a00668e"
-      },
-      "outputs": [],
-      "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = \"google.colab\" in sys.modules\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "setup_vars"
-      },
-      "source": [
-        "### Set up variables\n",
-        "\n",
-        "Next, set up some variables used throughout the tutorial.\n",
-        "### Import libraries"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "import_aip:mbsdk"
-      },
-      "outputs": [],
-      "source": [
-        "import logging\n",
-        "\n",
-        "import google.cloud.aiplatform as aiplatform"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "init_aip:mbsdk,region"
-      },
-      "source": [
-        "### Initialize Vertex AI SDK for Python\n",
-        "\n",
-        "Initialize the Vertex AI SDK for Python for your project and corresponding bucket."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "init_aip:mbsdk,region"
-      },
-      "outputs": [],
-      "source": [
-        "aiplatform.init(project=PROJECT_ID, location=REGION)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "python_logging"
-      },
-      "source": [
-        "## Python Logging\n",
-        "\n",
-        "The Python logging package is widely used for logging within Python scripts. Commonly used features:\n",
-        "\n",
-        "- Set logging levels.\n",
-        "- Send log output to console.\n",
-        "- Send log output to a file.\n",
-        "\n",
-        "### Logging Levels in Python Logging\n",
-        "\n",
-        "The logging levels in order (from least to highest) and each level inclusive of the previous level are :\n",
-        "\n",
-        "1. Informational\n",
-        "2. Warnings\n",
-        "3. Errors\n",
-        "4. Debugging\n",
-        "\n",
-        "By default, the logging level is set to error level.\n",
-        "\n",
-        "### Logging output to console\n",
-        "\n",
-        "By default, the Python logging package outputs to the console. Note, in the example the debug log message is not outputted since the default logging level is set to error."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "python_logging"
-      },
-      "outputs": [],
-      "source": [
-        "def logging_examples():\n",
-        "    logging.info(\"Model training started...\")\n",
-        "    logging.warning(\"Using older version of package ...\")\n",
-        "    logging.error(\"Training was terminated ...\")\n",
-        "    logging.debug(\"Hyperparameters were ...\")\n",
-        "\n",
-        "\n",
-        "logging_examples()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "python_logging_level"
-      },
-      "source": [
-        "### Setting logging level\n",
-        "\n",
-        "To set the logging level, you get the logging handler using `getLogger()`. You can have multiple logging handles. When `getLogger()` is called without any arguments, it gets the default handler named ROOT. With the handler, you set the logging level with the method `setLevel()`."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "python_logging_level"
-      },
-      "outputs": [],
-      "source": [
-        "logging.getLogger().setLevel(logging.DEBUG)\n",
-        "\n",
-        "logging_examples()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "python_logging_remove"
-      },
-      "source": [
-        "### Clearing handlers\n",
-        "\n",
-        "At times, you may desire to reconfigure your logging. A common practice in this case is to first remove all existing logging handles for a fresh start."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "python_logging_remove"
-      },
-      "outputs": [],
-      "source": [
-        "for handler in logging.root.handlers[:]:\n",
-        "    logging.root.removeHandler(handler)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "python_logging_file"
-      },
-      "source": [
-        "### Output to a local file\n",
-        "\n",
-        "You can preserve your logging output to a file that is local to where the Python script is running with the method `BasicConfig()`, that takes the following parameters:\n",
-        "\n",
-        "- `filename`: The file path to the local file to write the log output to.\n",
-        "- `level`: Sets the level of logging that is written to the logging file.\n",
-        "\n",
-        "*Note:* You cannot use a Cloud Storage bucket as the output file."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "python_logging_file"
-      },
-      "outputs": [],
-      "source": [
-        "logging.basicConfig(filename=\"mylog.log\", level=logging.DEBUG)\n",
-        "\n",
-        "logging_examples()\n",
-        "\n",
-        "! cat mylog.log"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "cloud_logging"
-      },
-      "source": [
-        "## Logging with Google Cloud Logging\n",
-        "\n",
-        "You can preserve and retrieve your logging output to `Google Cloud Logging` service. Commonly used features:\n",
-        "\n",
-        "- Set logging levels.\n",
-        "- Send log output to storage.\n",
-        "- Retrieve log output from storage.\n",
-        "\n",
-        "### Logging Levels in Cloud Logging\n",
-        "\n",
-        "The logging levels in order (from least to highest) are, with each level inclusive of the previous level:\n",
-        "\n",
-        "1. Informational\n",
-        "2. Warnings\n",
-        "3. Errors\n",
-        "4. Debugging\n",
-        "\n",
-        "By default, the logging level is set to warning level.\n",
-        "\n",
-        "### Configurable and storing log data.\n",
-        "\n",
-        "To use the `Google Cloud Logging` service, you do the following steps:\n",
-        "\n",
-        "1. Create a client to the service.\n",
-        "2. Obtain a handler for the service.\n",
-        "3. Create a logger instance and set logging level.\n",
-        "4. Attach logger instance to the service.\n",
-        "\n",
-        "Learn more about [Logging client libraries](https://cloud.google.com/logging/docs/reference/libraries)."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "cloud_logging"
-      },
-      "outputs": [],
-      "source": [
-        "import google.cloud.logging\n",
-        "from google.cloud.logging.handlers import CloudLoggingHandler\n",
-        "\n",
-        "# Connect to the Cloud Logging service\n",
-        "cl_client = google.cloud.logging.Client(project=PROJECT_ID)\n",
-        "handler = CloudLoggingHandler(cl_client, name=\"mylog\")\n",
-        "\n",
-        "# Create a logger instance and logging level\n",
-        "cloud_logger = logging.getLogger(\"cloudLogger\")\n",
-        "cloud_logger.setLevel(logging.INFO)\n",
-        "\n",
-        "# Attach the logger instance to the service.\n",
-        "cloud_logger.addHandler(handler)\n",
-        "\n",
-        "# Log something\n",
-        "cloud_logger.error(\"bad news\")"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "cloud_logging_write"
-      },
-      "source": [
-        "### Logging output\n",
-        "\n",
-        "Logging output at specific levels is identical to Python logging with respect to method and method names. The only difference is that you use your instance of the cloud logger in place of logging."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "cloud_logging_write"
-      },
-      "outputs": [],
-      "source": [
-        "cloud_logger.info(\"Model training started...\")\n",
-        "cloud_logger.warning(\"Using older version of package ...\")\n",
-        "cloud_logger.error(\"Training was terminated ...\")\n",
-        "cloud_logger.debug(\"Hyperparameters were ...\")"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "cloud_logging_list"
-      },
-      "source": [
-        "### Get logging entries\n",
-        "\n",
-        "To get the logged output, you:\n",
-        "\n",
-        "1. Retrieve the log handle to the service.\n",
-        "2. Using the handle, call the method `list_entries()`.\n",
-        "3. Iterate through the entries."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "cloud_logging_list"
-      },
-      "outputs": [],
-      "source": [
-        "logger = cl_client.logger(\"mylog\")\n",
-        "\n",
-        "for entry in logger.list_entries():\n",
-        "    timestamp = entry.timestamp.isoformat()\n",
-        "    print(\"* {}: {}: {}\".format(timestamp, entry.severity, entry.payload))"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "cleanup:mbsdk"
-      },
-      "source": [
-        "# Cleaning up\n",
-        "\n",
-        "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
-        "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
-        "\n",
-        "Otherwise, you can delete the individual resources you created in this tutorial."
-      ]
-    }
-  ],
-  "metadata": {
-    "colab": {
-      "name": "get_started_with_logging.ipynb",
-      "toc_visible": true
-    },
-    "kernelspec": {
-      "display_name": "Python 3",
-      "name": "python3"
-    }
-  },
-  "nbformat": 4,
-  "nbformat_minor": 0
-}

notebooks/community/ml_ops/stage2/get_started_with_tabnet.ipynb

@@ -34,18 +34,18 @@
         "<table align=\"left\">\n",
         "\n",
         "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_with_tabnet.ipynb\">\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/ml_ops/stage2/get_started_with_tabnet.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samplestree/main/notebooks/community/ml_ops/stage2/get_started_with_tabnet.ipynb\">\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samplestree/master/notebooks/community/ml_ops/stage2/get_started_with_tabnet.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage2/get_started_with_tabnet.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/ml_ops/stage2/get_started_with_tabnet.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
@@ -132,16 +132,11 @@
       "source": [
         "import os\n",
         "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "# Google Cloud Notebook\n",
+        "if os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
         "    USER_FLAG = \"--user\"\n",
+        "else:\n",
+        "    USER_FLAG = \"\"\n",
         "\n",
         "! pip3 install {USER_FLAG} --upgrade tensorflow\n",
         "! pip3 install {USER_FLAG} --upgrade google-cloud-aiplatform tensorboard-plugin-profile\n",
@@ -198,7 +193,7 @@
         "\n",
         "2. [Make sure that billing is enabled for your project.](https://cloud.google.com/billing/docs/how-to/modify-project)\n",
         "\n",
-        "3. [Enable the following APIs: Vertex AI APIs, Compute Engine APIs, and Cloud Storage.](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component,storage-component.googleapis.com)\n",
+        "3. [Enable the following APIs: Vertex AI APIs, Compute Engine APIs, and Cloud Storage.](https://console.cloud.google.com/flows/enableapi?apiid=ml.googleapis.com,compute_component,storage-component.googleapis.com)\n",
         "\n",
         "4. If you are running this notebook locally, you will need to install the [Cloud SDK]((https://cloud.google.com/sdk)).\n",
         "\n",
@@ -322,7 +317,7 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
+        "**If you are using Google Cloud Notebooks**, your environment is already\n",
         "authenticated. Skip this step."
       ]
     },
@@ -364,21 +359,20 @@
       },
       "outputs": [],
       "source": [
+        "import os\n",
+        "import sys\n",
+        "\n",
         "# If you are running this notebook in Colab, run this cell and follow the\n",
         "# instructions to authenticate your GCP account. This provides access to your\n",
         "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
         "# requests.\n",
         "\n",
-        "import os\n",
-        "import sys\n",
+        "# The Google Cloud Notebook product has specific requirements\n",
+        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
         "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = False\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
+        "# If on Google Cloud Notebooks, then don't execute this code\n",
+        "if not IS_GOOGLE_CLOUD_NOTEBOOK:\n",
         "    if \"google.colab\" in sys.modules:\n",
-        "        IS_COLAB = True\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
         "        google_auth.authenticate_user()\n",
@@ -488,6 +482,8 @@
       "outputs": [],
       "source": [
         "import os\n",
+        "import re\n",
+        "import time\n",
         "from datetime import datetime\n",
         "\n",
         "import google.cloud.aiplatform as aip\n",
@@ -1316,31 +1312,27 @@
         "print(\"Service Account:\", SERVICE_ACCOUNT)\n",
         "\n",
         "\n",
-        "MODEL_DIR = OUTPUT_DIR\n",
+        "MODEL_DIR = OUTPUT_DIR \n",
         "\n",
         "if TRAIN_GPU:\n",
-        "    model_bq = job.run(\n",
-        "        model_display_name=f\"{DATASET_NAME}_{TIMESTAMP}\",\n",
-        "        args=CMDARGS,\n",
-        "        replica_count=1,\n",
-        "        machine_type=TRAIN_COMPUTE,\n",
-        "        base_output_dir=MODEL_DIR,\n",
-        "        accelerator_type=TRAIN_GPU.name,\n",
-        "        accelerator_count=TRAIN_NGPU,\n",
-        "        service_account=SERVICE_ACCOUNT,\n",
-        "        sync=True,\n",
-        "    )\n",
+        "    model_bq = job.run(model_display_name=f\"{DATASET_NAME}_{TIMESTAMP}\",\n",
+        "            args=CMDARGS, \n",
+        "            replica_count=1, \n",
+        "            machine_type=TRAIN_COMPUTE, \n",
+        "            base_output_dir=MODEL_DIR,\n",
+        "            accelerator_type=TRAIN_GPU.name,\n",
+        "            accelerator_count=TRAIN_NGPU,\n",
+        "            service_account=SERVICE_ACCOUNT,\n",
+        "            sync=True)\n",
         "else:\n",
-        "    model_bq = job.run(\n",
-        "        model_display_name=f\"{DATASET_NAME}_{TIMESTAMP}\",\n",
-        "        args=CMDARGS,\n",
-        "        replica_count=1,\n",
-        "        machine_type=TRAIN_COMPUTE,\n",
-        "        base_output_dir=MODEL_DIR,\n",
-        "        service_account=SERVICE_ACCOUNT,\n",
-        "        sync=True,\n",
-        "    )\n",
-        "\n",
+        "    model_bq = job.run(model_display_name=f\"{DATASET_NAME}_{TIMESTAMP}\",\n",
+        "            args=CMDARGS, \n",
+        "            replica_count=1, \n",
+        "            machine_type=TRAIN_COMPUTE, \n",
+        "            base_output_dir=MODEL_DIR,\n",
+        "            service_account=SERVICE_ACCOUNT,\n",
+        "            sync=True)\n",
+        "    \n",
         "print(model_bq.gca_resource)"
       ]
     },

notebooks/community/ml_ops/stage2/get_started_with_tfhub_models.ipynb

@@ -33,19 +33,18 @@
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_with_tfhub_models.ipynb\">\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_with_tfhub_models.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_with_tfhub_models.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_with_tfhub_models.ipynb\">\n",
-        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
-        "        </a>\n",
-        "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage2/get_started_with_tfhub_models.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_with_tfhub_models.ipynb\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
         "  </td>\n",
@@ -144,20 +143,14 @@
       "source": [
         "import os\n",
         "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "# Google Cloud Notebook\n",
+        "if os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
         "    USER_FLAG = \"--user\"\n",
+        "else:\n",
         "    USER_FLAG = \"\"\n",
         "\n",
-        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG -q\n",
-        "! pip3 install tensorflow-datasets $USER_FLAG -q"
+        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG\n",
+        "! pip3 install tensorflow-datasets $USER_FLAG"
       ]
     },
     {
@@ -229,7 +222,7 @@
         "\n",
         "2. [Make sure that billing is enabled for your project.](https://cloud.google.com/billing/docs/how-to/modify-project)\n",
         "\n",
-        "3. [Enable the following APIs: Vertex AI APIs, Compute Engine APIs, and Cloud Storage.](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component,storage-component.googleapis.com)\n",
+        "3. [Enable the following APIs: Vertex AI APIs, Compute Engine APIs, and Cloud Storage.](https://console.cloud.google.com/flows/enableapi?apiid=ml.googleapis.com,compute_component,storage-component.googleapis.com)\n",
         "\n",
         "4. If you are running this notebook locally, you will need to install the [Cloud SDK]((https://cloud.google.com/sdk)).\n",
         "\n",
@@ -353,7 +346,7 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
+        "**If you are using Google Cloud Notebooks**, your environment is already authenticated. Skip this step.\n",
         "\n",
         "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
         "\n",
@@ -388,13 +381,9 @@
         "import os\n",
         "import sys\n",
         "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = False\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
+        "# If on Google Cloud Notebook, then don't execute this code\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
         "    if \"google.colab\" in sys.modules:\n",
-        "        IS_COLAB = True\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
         "        google_auth.authenticate_user()\n",
@@ -1397,7 +1386,7 @@
       },
       "outputs": [],
       "source": [
-        "delete_bucket = False\n",
+        "delete_bucket = True\n",
         "\n",
         "if delete_bucket and \"BUCKET_URI\" in globals():\n",
         "    ! gsutil rm -r $BUCKET_URI"

notebooks/community/ml_ops/stage2/mlops_experimentation.ipynb

@@ -39,14 +39,8 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/mlops_experimentation.ipynb\">\n",
-        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
-        "        </a>\n",
-        "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage2/mlops_experimentation.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
+        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/mlops_experimentation.ipynb\">\n",
+        "      Open in Google Cloud Notebooks\n",
         "    </a>\n",
         "  </td>\n",
         "</table>\n",
@@ -179,19 +173,6 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
         "ONCE_ONLY = False\n",
         "if ONCE_ONLY:\n",
         "    ! pip3 install -U tensorflow==2.5 $USER_FLAG\n",
@@ -239,32 +220,6 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "2721ef0202d9"
-      },
-      "source": [
-        "## Before you begin\n",
-        "\n",
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com). \n",
-        "\n",
-        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -341,9 +296,7 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"[your-region]\"  # @param {type:\"string\"}\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
       ]
     },
     {
@@ -370,75 +323,6 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "648aa9824ac6"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
-        "authenticated. Skip this step.\n",
-        "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions\n",
-        "when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "1. In the Cloud Console, go to the [**Create service account key**\n",
-        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
-        "\n",
-        "2. Click **Create service account**.\n",
-        "\n",
-        "3. In the **Service account name** field, enter a name, and\n",
-        "   click **Create**.\n",
-        "\n",
-        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
-        "into the filter box, and select\n",
-        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
-        "local environment.\n",
-        "\n",
-        "6. Enter the path to your service account key as the\n",
-        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "535223fa4b84"
-      },
-      "outputs": [],
-      "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = False\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        IS_COLAB = True\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -552,16 +436,9 @@
         "    or SERVICE_ACCOUNT is None\n",
         "    or SERVICE_ACCOUNT == \"[your-service-account]\"\n",
         "):\n",
-        "    # Get your service account from gcloud\n",
-        "    if not IS_COLAB:\n",
-        "        shell_output = !gcloud auth list 2>/dev/null\n",
-        "        SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
-        "\n",
-        "    if IS_COLAB:\n",
-        "        shell_output = ! gcloud projects describe  $PROJECT_ID\n",
-        "        project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n",
-        "        SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n",
-        "\n",
+        "    # Get your GCP project id from gcloud\n",
+        "    shell_output = !gcloud auth list 2>/dev/null\n",
+        "    SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
         "    print(\"Service Account:\", SERVICE_ACCOUNT)"
       ]
     },
@@ -706,8 +583,6 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
         "if os.getenv(\"IS_TESTING_TRAIN_GPU\"):\n",
         "    TRAIN_GPU, TRAIN_NGPU = (\n",
         "        aip.gapic.AcceleratorType.NVIDIA_TESLA_K80,\n",
@@ -1343,7 +1218,7 @@
         "setup_cfg = \"[egg_info]\\n\\ntag_build =\\n\\ntag_date = 0\"\n",
         "! echo \"$setup_cfg\" > custom/setup.cfg\n",
         "\n",
-        "setup_py = \"import setuptools\\n\\nsetuptools.setup(\\n\\n    install_requires=[\\n\\n        'google-cloud-aiplatform',\\n\\n        'cloudml-hypertune',\\n\\n        'tensorflow_datasets==1.3.0',\\n\\n        'tensorflow==2.5',\\n\\n    'tensorflow_data_validation==1.2',\\n\\n    ],\\n\\n    packages=setuptools.find_packages())\"\n",
+        "setup_py = \"import setuptools\\n\\nsetuptools.setup(\\n\\n    install_requires=[\\n\\n        'google-cloud-aiplatform',\\n\\n        'cloudml-hypertune',\\n\\n        'tensorflow_datasets==1.3.0',\\n\\n        'tensorflow_data_validation==1.2',\\n\\n    ],\\n\\n    packages=setuptools.find_packages())\"\n",
         "! echo \"$setup_py\" > custom/setup.py\n",
         "\n",
         "pkg_info = \"Metadata-Version: 1.0\\n\\nName: Chicago Taxi tabular binary classifier\\n\\nVersion: 0.0.0\\n\\nSummary: Demostration training script\\n\\nHome-page: www.google.com\\n\\nAuthor: Google\\n\\nAuthor-email: cdpe@google.com\\n\\nLicense: Public\\n\\nDescription: Demo\\n\\nPlatform: Vertex AI\"\n",
@@ -3136,8 +3011,9 @@
       },
       "source": [
         "## Review model evaluation scores\n",
+        "After your model has finished training, you can review the evaluation scores for it.\n",
         "\n",
-        "After your model training has finished, you can review the evaluation scores for it using the `list_model_evaluations()` method. This method will return an iterator for each evaluation slice."
+        "First, you need to get a reference to the new model. As with datasets, you can either use the reference to the model variable you created when you deployed the model or you can list all of the models in your project."
       ]
     },
     {
@@ -3148,10 +3024,18 @@
       },
       "outputs": [],
       "source": [
-        "model_evaluations = model.list_model_evaluations()\n",
+        "# Get model resource ID\n",
+        "models = aip.Model.list(filter=\"display_name=chicago_\" + TIMESTAMP)\n",
         "\n",
-        "for model_evaluation in model_evaluations:\n",
-        "    print(model_evaluation.to_dict())"
+        "# Get a reference to the Model Service client\n",
+        "client_options = {\"api_endpoint\": f\"{REGION}-aiplatform.googleapis.com\"}\n",
+        "model_service_client = aip.gapic.ModelServiceClient(client_options=client_options)\n",
+        "\n",
+        "model_evaluations = model_service_client.list_model_evaluations(\n",
+        "    parent=models[0].resource_name\n",
+        ")\n",
+        "model_evaluation = list(model_evaluations)[0]\n",
+        "print(model_evaluation)"
       ]
     },
     {

notebooks/community/ml_ops/stage3/README.md

@@ -27,7 +27,7 @@ The third stage in MLOps is formalization to develop an automated pipeline proce
 - Use early stop procedure in training script to detect failure to achieve training objective.
 - Store the results of the trained model evaluation in Vertex AI ML Metadata.
 
-<img src='stage3v3.png'>
+<img src='stage3.png'>
 
 ## Notebooks
 
@@ -144,72 +144,6 @@ The steps performed include:
 - Testing the deployed model infrastructure.
 ```
 
-[Get Started with TFX Pipelines with Vertex AI](get_started_with_tfx_pipeline.ipynb)
-
-```
-The steps performed include:
-
-- Create a TFX e2e pipeline.
-- Execute the pipeline locally.
-- Execute the pipeline on Google Cloud using `Vertex AI Training`
-- Execute the pipeline using `Vertex AI Pipelines`.
-```
-
-[Get Started with machine management](get_started_with_machine_management.ipynb)
-
-```
-The steps performed in this tutorial include:
-
-- Create a custom component with a self-contained training job.
-- Execute pipeline using component-level settings for machine resources
-- Convert the self-contained training componnt into a Vertex AI CustomJob.
-- Execute pipeline using customjob-level settings for machine resources
-```
-
-[Get Started with Apache Airflow and Vertex AI Pipelines](get_started_with_airflow_and_vertex_pipelines.ipynb)
-
-```
-The steps performed in this tutorial include:
-
-- Create Cloud Composer environment.
-- Upload Airflow DAG to Composer environment that performs data processing -- i.e., creates a BigQuery table from a CSV file.
-- Create a Vertex Pipeline that triggers the Airflow DAG.
-- Execute the `Vertex AI Pipeline`.
-```
-
-
-[Get Started with Vertex AI Model Registry](get_started_with_model_registry.ipynb)
-
-```
-The steps performed in this tutorial include:
-
-- Create and register a first version of a model to `Vertex AI Model Registry`
-- Create and register a second version of a model to `Vertex AI Model Registry`
-- List all versions of a `Model` resource.
-- Change the default version of a `Model` resource`
-- Deploy the default version of a `Model` resource.
-- Delete a model version from a `Model` resource.
-- Delete a `Model` resource along with all model versions.
-```
-
-[Get Started with AutoML Tabular Pipeline Workflow](get_started_with_automl_tabular_pipeline_workflow.ipynb)
-
-```
-The steps performed in this tutorial include:
-
-- Define training specification.
-    - Dataset specification
-    - Hyperparameter overide specification
-    - machine specifications
-- Construct tabular workflow pipeline.
-- Compile and execute pipeline.
-- View evaluation metrics artifact.
-- Export AutoML model as an OSS TF model.
-- Create `Endpoint` resource.
-- Deploy exported OSS TF model.
-- Make a prediction.
-```
-
 ### E2E Stage Example
 
 [Stage 3: Formalization](mlops_formalization.ipynb)

notebooks/community/ml_ops/stage3/get_started_with_automl_pipeline_components.ipynb

@@ -40,11 +40,11 @@
         "  </td>\n",
         "    <td>\n",
         "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_automl_pipeline_components.ipynb\">\n",
-        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\\\" alt=\"Colab logo\"> Run in Colab\n",
         "        </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage3/get_started_with_automl_pipeline_components.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_automl_pipeline_components.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
@@ -132,15 +132,12 @@
       "source": [
         "import os\n",
         "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
+        "# The Google Cloud Notebook product has specific requirements\n",
+        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
         "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "# Google Cloud Notebook requires dependencies to be installed with '--user'\n",
         "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "if IS_GOOGLE_CLOUD_NOTEBOOK:\n",
         "    USER_FLAG = \"--user\"\n",
         "    \n",
         "! pip3 install tensorflow-io==0.18 $USER_FLAG -q\n",
@@ -180,30 +177,6 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "BF1j6f9HApxa"
-      },
-      "source": [
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com).\n",
-        "\n",
-        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -318,7 +291,7 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Vertex AI Notebook Notebooks**, your environment is already authenticated. Skip this step.\n",
+        "**If you are using Google Cloud Notebooks**, your environment is already authenticated. Skip this step.\n",
         "\n",
         "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
         "\n",
@@ -353,13 +326,9 @@
         "import os\n",
         "import sys\n",
         "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = False\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
+        "# If on Google Cloud Notebook, then don't execute this code\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
         "    if \"google.colab\" in sys.modules:\n",
-        "        IS_COLAB = True\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
         "        google_auth.authenticate_user()\n",
@@ -482,20 +451,15 @@
       "source": [
         "if (\n",
         "    SERVICE_ACCOUNT == \"\"\n",
-        "    or SERVICE_ACCOUNT is None\n",
         "    or SERVICE_ACCOUNT == \"[your-service-account]\"\n",
+        "    or SERVICE_ACCOUNT is None\n",
         "):\n",
-        "    # Get your service account from gcloud\n",
-        "    if not IS_COLAB:\n",
-        "        shell_output = !gcloud auth list 2>/dev/null\n",
-        "        SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
+        "    shell_output = ! gcloud projects describe $PROJECT_ID | sed -nre 's:.*projectNumber\\: (.*):\\1:p'\n",
+        "    SERVICE_ACCOUNT = (\n",
+        "        shell_output[0].replace(\"'\", \"\") + \"-compute@developer.gserviceaccount.com\"\n",
+        "    )\n",
         "\n",
-        "    if IS_COLAB:\n",
-        "        shell_output = ! gcloud projects describe  $PROJECT_ID\n",
-        "        project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n",
-        "        SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n",
-        "\n",
-        "    print(\"Service Account:\", SERVICE_ACCOUNT)"
+        "print(\"Service Account:\", SERVICE_ACCOUNT)"
       ]
     },
     {

notebooks/community/ml_ops/stage3/get_started_with_bq_tfdv_pipeline_components.ipynb

@@ -44,7 +44,7 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage3/get_started_with_bq_tfdv_pipeline_components.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_bq_tfdv_pipeline_components.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
@@ -129,7 +129,7 @@
       "source": [
         "## Installations\n",
         "\n",
-        "Install the packages for executing this notebook."
+        "Install *one time* the packages for executing the MLOps notebooks."
       ]
     },
     {
@@ -142,20 +142,26 @@
       "source": [
         "import os\n",
         "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
+        "# The Google Cloud Notebook product has specific requirements\n",
+        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
         "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "# Google Cloud Notebook requires dependencies to be installed with '--user'\n",
         "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
-        "! pip3 install -U tensorflow $USER_FLAG -q\n",
-        "! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG -q\n",
-        "! pip3 install --upgrade kfp $USER_FLAG -q"
+        "if IS_GOOGLE_CLOUD_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "install_mlops"
+      },
+      "outputs": [],
+      "source": [
+        "! pip3 install -U tensorflow $USER_FLAG\n",
+        "! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG\n",
+        "! pip3 install --upgrade kfp $USER_FLAG"
       ]
     },
     {
@@ -187,30 +193,6 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "BF1j6f9HApxa"
-      },
-      "source": [
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,dataflow.googleapis.com).\n",
-        "\n",
-        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -326,7 +308,7 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
+        "**If you are using Google Cloud Notebooks**, your environment is already\n",
         "authenticated. Skip this step."
       ]
     },
@@ -368,21 +350,20 @@
       },
       "outputs": [],
       "source": [
+        "import os\n",
+        "import sys\n",
+        "\n",
         "# If you are running this notebook in Colab, run this cell and follow the\n",
         "# instructions to authenticate your GCP account. This provides access to your\n",
         "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
         "# requests.\n",
         "\n",
-        "import os\n",
-        "import sys\n",
+        "# The Google Cloud Notebook product has specific requirements\n",
+        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
         "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = False\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
+        "# If on Google Cloud Notebooks, then don't execute this code\n",
+        "if not IS_GOOGLE_CLOUD_NOTEBOOK:\n",
         "    if \"google.colab\" in sys.modules:\n",
-        "        IS_COLAB = True\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
         "        google_auth.authenticate_user()\n",
@@ -509,16 +490,9 @@
         "    or SERVICE_ACCOUNT is None\n",
         "    or SERVICE_ACCOUNT == \"[your-service-account]\"\n",
         "):\n",
-        "    # Get your service account from gcloud\n",
-        "    if not IS_COLAB:\n",
-        "        shell_output = !gcloud auth list 2>/dev/null\n",
-        "        SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
-        "\n",
-        "    if IS_COLAB:\n",
-        "        shell_output = ! gcloud projects describe  $PROJECT_ID\n",
-        "        project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n",
-        "        SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n",
-        "\n",
+        "    # Get your GCP project id from gcloud\n",
+        "    shell_output = !gcloud auth list 2>/dev/null\n",
+        "    SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
         "    print(\"Service Account:\", SERVICE_ACCOUNT)"
       ]
     },

notebooks/community/ml_ops/stage3/get_started_with_bqml_pipeline_components.ipynb

@@ -44,7 +44,7 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage3/get_started_with_bqml_pipeline_components.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_bqml_pipeline_components.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
@@ -135,7 +135,7 @@
       "source": [
         "## Installations\n",
         "\n",
-        "Install the packages required for executing the notebook."
+        "Install *one time* the packages for executing the MLOps notebooks."
       ]
     },
     {
@@ -148,22 +148,28 @@
       "source": [
         "import os\n",
         "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
+        "# The Google Cloud Notebook product has specific requirements\n",
+        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
         "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "# Google Cloud Notebook requires dependencies to be installed with '--user'\n",
         "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
-        "! pip3 install -U tensorflow $USER_FLAG -q\n",
-        "! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG -q\n",
-        "! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG -q\n",
-        "! pip3 install --upgrade google-cloud-bigquery $USER_FLAG -q\n",
-        "! pip3 install --upgrade kfp $USER_FLAG -q"
+        "if IS_GOOGLE_CLOUD_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "-OxtcyNNJ39g"
+      },
+      "outputs": [],
+      "source": [
+        "! pip3 install -U tensorflow $USER_FLAG\n",
+        "! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG\n",
+        "! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG\n",
+        "! pip3 install --upgrade google-cloud-bigquery $USER_FLAG\n",
+        "! pip3 install --upgrade kfp $USER_FLAG\n"
       ]
     },
     {
@@ -195,30 +201,6 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "BF1j6f9HApxa"
-      },
-      "source": [
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,dataflow.googleapis.com).\n",
-        "\n",
-        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -332,7 +314,10 @@
         "id": "UG2SHSlTJ39k"
       },
       "source": [
-        "### Authenticate your Google Cloud account\n"
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using Google Cloud Notebooks**, your environment is already\n",
+        "authenticated. Skip this step."
       ]
     },
     {
@@ -341,11 +326,8 @@
         "id": "ETQR4H1HJ39k"
       },
       "source": [
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
-        "authenticated. Skip this step.\n",
-        "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
-        "\n",
+        "**If you are using Colab**, run the cell below and follow the instructions\n",
+        "when prompted to authenticate your account via oAuth.\n",
         "\n",
         "**Otherwise**, follow these steps:\n",
         "\n",
@@ -376,21 +358,20 @@
       },
       "outputs": [],
       "source": [
+        "import os\n",
+        "import sys\n",
+        "\n",
         "# If you are running this notebook in Colab, run this cell and follow the\n",
         "# instructions to authenticate your GCP account. This provides access to your\n",
         "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
         "# requests.\n",
         "\n",
-        "import os\n",
-        "import sys\n",
+        "# The Google Cloud Notebook product has specific requirements\n",
+        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
         "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = False\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
+        "# If on Google Cloud Notebooks, then don't execute this code\n",
+        "if not IS_GOOGLE_CLOUD_NOTEBOOK:\n",
         "    if \"google.colab\" in sys.modules:\n",
-        "        IS_COLAB = True\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
         "        google_auth.authenticate_user()\n",
@@ -518,17 +499,9 @@
         "    or SERVICE_ACCOUNT is None\n",
         "    or SERVICE_ACCOUNT == \"[your-service-account]\"\n",
         "):\n",
-        "    # Get your service account from gcloud\n",
-        "    if not IS_COLAB:\n",
-        "        shell_output = !gcloud auth list 2>/dev/null\n",
-        "        SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
-        "\n",
-        "    if IS_COLAB:\n",
-        "        shell_output = ! gcloud projects describe  $PROJECT_ID\n",
-        "        # print(\"shell_output=\", shell_output)\n",
-        "        project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n",
-        "        SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n",
-        "\n",
+        "    # Get your GCP project id from gcloud\n",
+        "    shell_output = !gcloud auth list 2>/dev/null\n",
+        "    SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
         "    print(\"Service Account:\", SERVICE_ACCOUNT)"
       ]
     },

notebooks/community/ml_ops/stage3/get_started_with_custom_training_pipeline_components.ipynb

@@ -44,7 +44,7 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage3/get_started_with_custom_training_pipeline_components.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_custom_training_pipeline_components.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
@@ -139,7 +139,7 @@
       "source": [
         "## Installations\n",
         "\n",
-        "Install the required packages for executing the notebook."
+        "Install *one time* the packages for executing the MLOps notebooks."
       ]
     },
     {
@@ -152,21 +152,27 @@
       "source": [
         "import os\n",
         "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
+        "# The Google Cloud Notebook product has specific requirements\n",
+        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
         "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "# Google Cloud Notebook requires dependencies to be installed with '--user'\n",
         "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
-        "! pip3 install -U tensorflow $USER_FLAG -q\n",
-        "! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG -q\n",
-        "! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG -q\n",
-        "! pip3 install --upgrade kfp $USER_FLAG -q"
+        "if IS_GOOGLE_CLOUD_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "install_mlops"
+      },
+      "outputs": [],
+      "source": [
+        "! pip3 install -U tensorflow $USER_FLAG\n",
+        "! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG\n",
+        "! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG\n",
+        "! pip3 install --upgrade kfp $USER_FLAG\n"
       ]
     },
     {
@@ -198,30 +204,6 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "BF1j6f9HApxa"
-      },
-      "source": [
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,dataflow.googleapis.com).\n",
-        "\n",
-        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -335,7 +317,10 @@
         "id": "648aa9824ac6"
       },
       "source": [
-        "### Authenticate your Google Cloud account\n"
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using Google Cloud Notebooks**, your environment is already\n",
+        "authenticated. Skip this step."
       ]
     },
     {
@@ -344,10 +329,8 @@
         "id": "fc52bba17ee3"
       },
       "source": [
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
-        "authenticated. Skip this step.\n",
-        "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
+        "**If you are using Colab**, run the cell below and follow the instructions\n",
+        "when prompted to authenticate your account via oAuth.\n",
         "\n",
         "**Otherwise**, follow these steps:\n",
         "\n",
@@ -378,21 +361,20 @@
       },
       "outputs": [],
       "source": [
+        "import os\n",
+        "import sys\n",
+        "\n",
         "# If you are running this notebook in Colab, run this cell and follow the\n",
         "# instructions to authenticate your GCP account. This provides access to your\n",
         "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
         "# requests.\n",
         "\n",
-        "import os\n",
-        "import sys\n",
+        "# The Google Cloud Notebook product has specific requirements\n",
+        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
         "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = False\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
+        "# If on Google Cloud Notebooks, then don't execute this code\n",
+        "if not IS_GOOGLE_CLOUD_NOTEBOOK:\n",
         "    if \"google.colab\" in sys.modules:\n",
-        "        IS_COLAB = True\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
         "        google_auth.authenticate_user()\n",
@@ -518,17 +500,9 @@
         "    or SERVICE_ACCOUNT is None\n",
         "    or SERVICE_ACCOUNT == \"[your-service-account]\"\n",
         "):\n",
-        "    # Get your service account from gcloud\n",
-        "    if not IS_COLAB:\n",
-        "        shell_output = !gcloud auth list 2>/dev/null\n",
-        "        SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
-        "\n",
-        "    if IS_COLAB:\n",
-        "        shell_output = ! gcloud projects describe  $PROJECT_ID\n",
-        "        # print(\"shell_output=\", shell_output)\n",
-        "        project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n",
-        "        SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n",
-        "\n",
+        "    # Get your GCP project id from gcloud\n",
+        "    shell_output = !gcloud auth list 2>/dev/null\n",
+        "    SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
         "    print(\"Service Account:\", SERVICE_ACCOUNT)"
       ]
     },

notebooks/community/ml_ops/stage3/get_started_with_dataflow_pipeline_components.ipynb

@@ -40,17 +40,9 @@
         "  </td>\n",
         "  <td>\n",
         "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_dataflow_pipeline_components.ipynb\">\n",
-        "<img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> \n",
-        "        Colab logo Run in Colab\n",
+        "      Open in Google Cloud Notebooks\n",
         "    </a>\n",
         "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage3/get_started_with_dataflow_pipeline_components.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
-        "    </a>\n",
-        "  </td>   \n",
-        "    \n",
         "</table>\n",
         "<br/><br/><br/>"
       ]
@@ -109,7 +101,7 @@
       "source": [
         "## Installations\n",
         "\n",
-        "Install the required packages for executing the notebook."
+        "Install *one time* the packages for executing the MLOps notebooks."
       ]
     },
     {
@@ -120,25 +112,24 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
-        "! pip3 install -U tensorflow $USER_FLAG -q\n",
-        "! pip3 install -U tensorflow-data-validation $USER_FLAG -q\n",
-        "! pip3 install -U tensorflow-transform $USER_FLAG -q\n",
-        "! pip3 install -U tensorflow-io $USER_FLAG -q\n",
-        "! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG -q\n",
-        "! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG -q"
+        "ONCE_ONLY = False\n",
+        "if ONCE_ONLY:\n",
+        "    ! pip3 install -U tensorflow==2.5 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-transform==1.2 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-io==0.18 $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-bigquery $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-logging $USER_FLAG\n",
+        "    ! pip3 install --upgrade apache-beam[gcp] $USER_FLAG\n",
+        "    ! pip3 install --upgrade pyarrow $USER_FLAG\n",
+        "    ! pip3 install --upgrade cloudml-hypertune $USER_FLAG\n",
+        "    ! pip3 install --upgrade kfp $USER_FLAG\n",
+        "    ! pip3 install --upgrade torchvision $USER_FLAG\n",
+        "    ! pip3 install --upgrade rpy2 $USER_FLAG\n",
+        "    ! pip3 install --upgrade python-tabulate $USER_FLAG\n",
+        "    ! pip3 install -U opencv-python-headless==4.5.2.52 $USER_FLAG"
       ]
     },
     {
@@ -170,30 +161,6 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "BF1j6f9HApxa"
-      },
-      "source": [
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI API and Dataflow API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,dataflow.googleapis.com).\n",
-        "\n",
-        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -213,24 +180,7 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
-        "PROJECT_ID = \"\"\n",
-        "\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    # Get your Google Cloud project ID from gcloud\n",
-        "    shell_output = !gcloud config list --format 'value(core.project)' 2>/dev/null\n",
-        "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID: \", PROJECT_ID)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "37c0a68ff20d"
-      },
-      "source": [
-        "Otherwise, set your project ID here."
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -241,8 +191,22 @@
       },
       "outputs": [],
       "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
-        "    PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
+        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
+        "    # Get your GCP project id from gcloud\n",
+        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "    PROJECT_ID = shell_output[0]\n",
+        "    print(\"Project ID:\", PROJECT_ID)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "set_gcloud_project_id"
+      },
+      "outputs": [],
+      "source": [
+        "! gcloud config set project $PROJECT_ID"
       ]
     },
     {
@@ -273,10 +237,7 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
       ]
     },
     {
@@ -303,89 +264,6 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "927085b84a07"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
-        "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
-        "\n",
-        "**Click Create service account**.\n",
-        "\n",
-        "In the **Service account name** field, enter a name, and click **Create**.\n",
-        "\n",
-        "In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex\" into the filter box, and select **Vertex Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "Click Create. A JSON file that contains your key downloads to your local environment.\n",
-        "\n",
-        "Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "89788a802687"
-      },
-      "outputs": [],
-      "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = False\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        IS_COLAB = True\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "40ed98f5cc48"
-      },
-      "source": [
-        "#### If you are using Colab Notebooks, set the project using gcloud config."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "fde1a355f1e9"
-      },
-      "outputs": [],
-      "source": [
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        ! gcloud config set project $PROJECT_ID"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -409,7 +287,7 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_URI = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
+        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -420,8 +298,8 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
       ]
     },
     {
@@ -441,7 +319,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_URI"
+        "! gsutil mb -l $REGION $BUCKET_NAME"
       ]
     },
     {
@@ -461,7 +339,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_URI"
+        "! gsutil ls -al $BUCKET_NAME"
       ]
     },
     {
@@ -499,16 +377,9 @@
         "    or SERVICE_ACCOUNT is None\n",
         "    or SERVICE_ACCOUNT == \"[your-service-account]\"\n",
         "):\n",
-        "    # Get your service account from gcloud\n",
-        "    if not IS_COLAB:\n",
-        "        shell_output = !gcloud auth list 2>/dev/null\n",
-        "        SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
-        "\n",
-        "    if IS_COLAB:\n",
-        "        shell_output = ! gcloud projects describe  $PROJECT_ID\n",
-        "        project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n",
-        "        SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n",
-        "\n",
+        "    # Get your GCP project id from gcloud\n",
+        "    shell_output = !gcloud auth list 2>/dev/null\n",
+        "    SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
         "    print(\"Service Account:\", SERVICE_ACCOUNT)"
       ]
     },
@@ -531,9 +402,9 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
+        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_NAME\n",
         "\n",
-        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
+        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_NAME"
       ]
     },
     {
@@ -556,12 +427,35 @@
       },
       "outputs": [],
       "source": [
-        "import google.cloud.aiplatform as aip\n",
+        "import google.cloud.aiplatform as aip"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "import_kfp"
+      },
+      "outputs": [],
+      "source": [
+        "import json\n",
+        "\n",
+        "from kfp import dsl\n",
+        "from kfp.v2 import compiler\n",
+        "from kfp.v2.dsl import component"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "import_gcpc:dataflow"
+      },
+      "outputs": [],
+      "source": [
         "from google_cloud_pipeline_components.v1.dataflow import DataflowPythonJobOp\n",
         "from google_cloud_pipeline_components.v1.wait_gcp_resources import \\\n",
-        "    WaitGcpResourcesOp\n",
-        "from kfp import dsl\n",
-        "from kfp.v2 import compiler"
+        "    WaitGcpResourcesOp"
       ]
     },
     {
@@ -583,7 +477,7 @@
       },
       "outputs": [],
       "source": [
-        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
+        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_NAME)"
       ]
     },
     {
@@ -779,12 +673,12 @@
       },
       "outputs": [],
       "source": [
-        "GCS_WC_PY = BUCKET_URI + \"/wc.py\"\n",
+        "GCS_WC_PY = BUCKET_NAME + \"/wc.py\"\n",
         "! gsutil cp wc.py $GCS_WC_PY\n",
-        "GCS_REQUIREMENTS_TXT = BUCKET_URI + \"/requirements.txt\"\n",
+        "GCS_REQUIREMENTS_TXT = BUCKET_NAME + \"/requirements.txt\"\n",
         "! gsutil cp requirements.txt $GCS_REQUIREMENTS_TXT\n",
         "\n",
-        "GCS_WC_OUT = BUCKET_URI + \"/wc_out.txt\""
+        "GCS_WC_OUT = BUCKET_NAME + \"/wc_out.txt\""
       ]
     },
     {
@@ -815,7 +709,9 @@
       },
       "outputs": [],
       "source": [
-        "PIPELINE_ROOT = \"{}/pipeline_root/dataflow_wc\".format(BUCKET_URI)\n",
+        "import json\n",
+        "\n",
+        "PIPELINE_ROOT = \"{}/pipeline_root/dataflow_wc\".format(BUCKET_NAME)\n",
         "\n",
         "\n",
         "@dsl.pipeline(name=\"dataflow-wc\", description=\"Dataflow word count component pipeline\")\n",
@@ -837,7 +733,9 @@
         "        args=args,\n",
         "    )\n",
         "\n",
-        "    _ = WaitGcpResourcesOp(gcp_resources=dataflow_python_op.outputs[\"gcp_resources\"])\n",
+        "    dataflow_wait_op = WaitGcpResourcesOp(\n",
+        "        gcp_resources=dataflow_python_op.outputs[\"gcp_resources\"]\n",
+        "    )\n",
         "\n",
         "\n",
         "compiler.Compiler().compile(pipeline_func=pipeline, package_path=\"dataflow_wc.json\")\n",
@@ -915,80 +813,82 @@
         "from apache_beam.options.pipeline_options import SetupOptions\n",
         "\n",
         "def run(argv=None):\n",
-        "    \"\"\"Main entry point; defines and runs the wordcount pipeline.\"\"\"\n",
+        "  \"\"\"Main entry point; defines and runs the wordcount pipeline.\"\"\"\n",
         "\n",
-        "    parser = argparse.ArgumentParser()\n",
-        "    parser.add_argument('--bq_table',\n",
+        "  parser = argparse.ArgumentParser()\n",
+        "  parser.add_argument('--bq_table',\n",
         "                      dest='bq_table')\n",
-        "    parser.add_argument('--bucket',\n",
+        "  parser.add_argument('--bucket',\n",
         "                      dest='bucket')\n",
-        "    args, pipeline_args = parser.parse_known_args(argv)\n",
-        "    logging.info(\"ARGS\")\n",
-        "    logging.info(args)\n",
-        "    logging.info(\"PIPELINE ARGS\")\n",
-        "    logging.info(pipeline_args)\n",
-        "    for i in range(0, len(pipeline_args), 2):\n",
+        "  args, pipeline_args = parser.parse_known_args(argv)\n",
+        "  logging.info(\"ARGS\")\n",
+        "  logging.info(args)\n",
+        "  logging.info(\"PIPELINE ARGS\")\n",
+        "  logging.info(pipeline_args)\n",
+        "  for i in range(0, len(pipeline_args), 2):\n",
         "        if \"--temp_location\" == pipeline_args[i]:\n",
         "            temp_location = pipeline_args[i+1]\n",
         "        elif \"--project\" == pipeline_args[i]:\n",
         "            project = pipeline_args[i+1]\n",
         "\n",
-        "    exported_train = args.bucket + '/exported_data/train'\n",
-        "    exported_eval  = args.bucket + '/exported_data/eval'\n",
+        "  exported_train = args.bucket + '/exported_data/train'\n",
+        "  exported_eval  = args.bucket + '/exported_data/eval'\n",
         "\n",
-        "    pipeline_options = PipelineOptions(pipeline_args)\n",
-        "    pipeline_options.view_as(SetupOptions).save_main_session = True\n",
-        "    with beam.Pipeline(options=pipeline_options) as pipeline:\n",
-        "        with tft_beam.Context(temp_location):\n",
-        "            raw_data_query = \"SELECT {0},{1} FROM {2} LIMIT 500\".format(\"CAST(station_number as STRING) AS station_number,year,month,day\",\"mean_temp\", args.bq_table)\n",
         "\n",
-        "            def parse_bq_record(bq_record):\n",
-        "                \"\"\"Parses a bq_record to a dictionary.\"\"\"\n",
-        "                output = {}\n",
-        "                for key in bq_record:\n",
-        "                    output[key] = [bq_record[key]]\n",
-        "                return output\n",
+        "  pipeline_options = PipelineOptions(pipeline_args)\n",
+        "  pipeline_options.view_as(SetupOptions).save_main_session = True\n",
+        "  with beam.Pipeline(options=pipeline_options) as pipeline:\n",
+        "    with tft_beam.Context(temp_location):\n",
         "\n",
-        "            def split_dataset(bq_row, num_partitions, ratio):\n",
-        "                \"\"\"Returns a partition number for a given bq_row.\"\"\"\n",
-        "                import json\n",
+        "        raw_data_query = \"SELECT {0},{1} FROM {2} LIMIT 500\".format(\"CAST(station_number as STRING) AS station_number,year,month,day\",\"mean_temp\", args.bq_table)\n",
         "\n",
-        "                assert num_partitions == len(ratio)\n",
-        "                bucket = sum(map(ord, json.dumps(bq_row))) % sum(ratio)\n",
-        "                total = 0\n",
-        "                for i, part in enumerate(ratio):\n",
-        "                    total += part\n",
-        "                    if bucket < total:\n",
-        "                        return i\n",
-        "                return len(ratio) - 1\n",
+        "        def parse_bq_record(bq_record):\n",
+        "            \"\"\"Parses a bq_record to a dictionary.\"\"\"\n",
+        "            output = {}\n",
+        "            for key in bq_record:\n",
+        "                output[key] = [bq_record[key]]\n",
+        "            return output\n",
         "\n",
-        "            # Read raw BigQuery data.\n",
-        "            raw_train_data, raw_eval_data = (\n",
-        "                pipeline\n",
-        "                | \"Read Raw Data\"\n",
-        "                >> beam.io.ReadFromBigQuery(\n",
-        "                    query=raw_data_query,\n",
-        "                    project=project,\n",
-        "                    use_standard_sql=True,\n",
-        "                )\n",
-        "                | \"Parse Data\" >> beam.Map(parse_bq_record)\n",
-        "                | \"Split\" >> beam.Partition(split_dataset, 2, ratio=[8, 2])\n",
+        "        def split_dataset(bq_row, num_partitions, ratio):\n",
+        "            \"\"\"Returns a partition number for a given bq_row.\"\"\"\n",
+        "            import json\n",
+        "\n",
+        "            assert num_partitions == len(ratio)\n",
+        "            bucket = sum(map(ord, json.dumps(bq_row))) % sum(ratio)\n",
+        "            total = 0\n",
+        "            for i, part in enumerate(ratio):\n",
+        "                total += part\n",
+        "                if bucket < total:\n",
+        "                    return i\n",
+        "            return len(ratio) - 1\n",
+        "\n",
+        "        # Read raw BigQuery data.\n",
+        "        raw_train_data, raw_eval_data = (\n",
+        "            pipeline\n",
+        "            | \"Read Raw Data\"\n",
+        "            >> beam.io.ReadFromBigQuery(\n",
+        "                query=raw_data_query,\n",
+        "                project=project,\n",
+        "                use_standard_sql=True,\n",
         "            )\n",
+        "            | \"Parse Data\" >> beam.Map(parse_bq_record)\n",
+        "            | \"Split\" >> beam.Partition(split_dataset, 2, ratio=[8, 2])\n",
+        "        )\n",
         "\n",
-        "            # Write raw train data to GCS .\n",
-        "            _ = raw_train_data | \"Write Raw Train Data\" >> beam.io.WriteToText(\n",
-        "                file_path_prefix=exported_train, file_name_suffix=\".csv\"\n",
-        "            )\n",
+        "        # Write raw train data to GCS .\n",
+        "        _ = raw_train_data | \"Write Raw Train Data\" >> beam.io.WriteToText(\n",
+        "            file_path_prefix=exported_train, file_name_suffix=\".csv\"\n",
+        "        )\n",
         "\n",
-        "            # Write raw eval data to GCS .\n",
-        "            _ = raw_eval_data | \"Write Raw Eval Data\" >> beam.io.WriteToText(\n",
-        "                file_path_prefix=exported_eval, file_name_suffix=\".csv\"\n",
-        "            )\n",
+        "        # Write raw eval data to GCS .\n",
+        "        _ = raw_eval_data | \"Write Raw Eval Data\" >> beam.io.WriteToText(\n",
+        "            file_path_prefix=exported_eval, file_name_suffix=\".csv\"\n",
+        "        )\n",
         "\n",
         "\n",
         "if __name__ == '__main__':\n",
-        "    logging.getLogger().setLevel(logging.INFO)\n",
-        "    run()"
+        "  logging.getLogger().setLevel(logging.INFO)\n",
+        "  run()"
       ]
     },
     {
@@ -1075,11 +975,11 @@
       },
       "outputs": [],
       "source": [
-        "GCS_SPLIT_PY = BUCKET_URI + \"/split.py\"\n",
+        "GCS_SPLIT_PY = BUCKET_NAME + \"/split.py\"\n",
         "! gsutil cp split.py $GCS_SPLIT_PY\n",
-        "GCS_REQUIREMENTS_TXT = BUCKET_URI + \"/requirements.txt\"\n",
+        "GCS_REQUIREMENTS_TXT = BUCKET_NAME + \"/requirements.txt\"\n",
         "! gsutil cp requirements.txt $GCS_REQUIREMENTS_TXT\n",
-        "GCS_SETUP_PY = BUCKET_URI + \"/setup.py\"\n",
+        "GCS_SETUP_PY = BUCKET_NAME + \"/setup.py\"\n",
         "! gsutil cp setup.py $GCS_SETUP_PY"
       ]
     },
@@ -1136,7 +1036,7 @@
       },
       "outputs": [],
       "source": [
-        "PIPELINE_ROOT = \"{}/pipeline_root/dataflow_split\".format(BUCKET_URI)\n",
+        "PIPELINE_ROOT = \"{}/pipeline_root/dataflow_split\".format(BUCKET_NAME)\n",
         "\n",
         "\n",
         "@dsl.pipeline(name=\"dataflow-split\", description=\"Dataflow split dataset\")\n",
@@ -1147,7 +1047,7 @@
         "    staging_dir: str = PIPELINE_ROOT,\n",
         "    args: list = [\n",
         "        \"--bucket\",\n",
-        "        BUCKET_URI,\n",
+        "        BUCKET_NAME,\n",
         "        \"--bq_table\",\n",
         "        BQ_TABLE,\n",
         "        \"--runner\",\n",
@@ -1167,7 +1067,9 @@
         "        args=args,\n",
         "    )\n",
         "\n",
-        "    _ = WaitGcpResourcesOp(gcp_resources=dataflow_python_op.outputs[\"gcp_resources\"])\n",
+        "    dataflow_wait_op = WaitGcpResourcesOp(\n",
+        "        gcp_resources=dataflow_python_op.outputs[\"gcp_resources\"]\n",
+        "    )\n",
         "\n",
         "\n",
         "compiler.Compiler().compile(pipeline_func=pipeline, package_path=\"dataflow_split.json\")\n",
@@ -1181,7 +1083,7 @@
         "\n",
         "pipeline.run()\n",
         "\n",
-        "! gsutil ls {BUCKET_URI}/exported_data\n",
+        "! gsutil ls {BUCKET_NAME}/exported_data\n",
         "\n",
         "! rm -f dataflow_split.json split.py requirements.txt"
       ]
@@ -1222,6 +1124,13 @@
         "Otherwise, you can delete the individual resources you created in this tutorial:\n",
         "\n",
         "- Dataset\n",
+        "- Pipeline\n",
+        "- Model\n",
+        "- Endpoint\n",
+        "- AutoML Training Job\n",
+        "- Batch Job\n",
+        "- Custom Job\n",
+        "- Hyperparameter Tuning Job\n",
         "- Cloud Storage Bucket"
       ]
     },
@@ -1233,11 +1142,61 @@
       },
       "outputs": [],
       "source": [
-        "# Warning: Setting this to true will delete everything in your bucket\n",
-        "delete_bucket = False\n",
+        "delete_all = True\n",
         "\n",
-        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
-        "    ! gsutil rm -r $BUCKET_URI"
+        "if delete_all:\n",
+        "    # Delete the dataset using the Vertex dataset object\n",
+        "    try:\n",
+        "        if \"dataset\" in globals():\n",
+        "            dataset.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the model using the Vertex model object\n",
+        "    try:\n",
+        "        if \"model\" in globals():\n",
+        "            model.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the endpoint using the Vertex endpoint object\n",
+        "    try:\n",
+        "        if \"endpoint\" in globals():\n",
+        "            endpoint.undeploy_all()\n",
+        "            endpoint.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the AutoML or Pipeline training job\n",
+        "    try:\n",
+        "        if \"dag\" in globals():\n",
+        "            dag.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the custom training job\n",
+        "    try:\n",
+        "        if \"job\" in globals():\n",
+        "            job.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the batch prediction job using the Vertex batch prediction object\n",
+        "    try:\n",
+        "        if \"batch_predict_job\" in globals():\n",
+        "            batch_predict_job.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the hyperparameter tuning job using the Vertex hyperparameter tuning object\n",
+        "    try:\n",
+        "        if \"hpt_job\" in globals():\n",
+        "            hpt_job.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    if \"BUCKET_NAME\" in globals():\n",
+        "        ! gsutil rm -r $BUCKET_NAME"
       ]
     }
   ],

notebooks/community/ml_ops/stage3/get_started_with_dataproc_serverless_pipeline_components.ipynb

@@ -44,7 +44,7 @@
         "        </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage3/get_started_with_dataproc_serverless_pipeline_components.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_dataproc_serverless_pipeline_components.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
@@ -134,15 +134,12 @@
       "source": [
         "import os\n",
         "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
+        "# The Google Cloud Notebook product has specific requirements\n",
+        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
         "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "# Google Cloud Notebook requires dependencies to be installed with '--user'\n",
         "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "if IS_GOOGLE_CLOUD_NOTEBOOK:\n",
         "    USER_FLAG = \"--user\"\n",
         "\n",
         "# Install the required packages\n",
@@ -179,32 +176,6 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "1df9ff75fa88"
-      },
-      "source": [
-        "## Before you begin\n",
-        "\n",
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com).\n",
-        "\n",
-        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -319,7 +290,7 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
+        "**If you are using Google Cloud Notebooks**, your environment is already authenticated. Skip this step.\n",
         "\n",
         "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
         "\n",
@@ -354,13 +325,9 @@
         "import os\n",
         "import sys\n",
         "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = False\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
+        "# If on Google Cloud Notebook, then don't execute this code\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
         "    if \"google.colab\" in sys.modules:\n",
-        "        IS_COLAB = True\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
         "        google_auth.authenticate_user()\n",
@@ -483,20 +450,15 @@
       "source": [
         "if (\n",
         "    SERVICE_ACCOUNT == \"\"\n",
-        "    or SERVICE_ACCOUNT is None\n",
         "    or SERVICE_ACCOUNT == \"[your-service-account]\"\n",
+        "    or SERVICE_ACCOUNT is None\n",
         "):\n",
-        "    # Get your service account from gcloud\n",
-        "    if not IS_COLAB:\n",
-        "        shell_output = !gcloud auth list 2>/dev/null\n",
-        "        SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
+        "    shell_output = ! gcloud projects describe $PROJECT_ID | sed -nre 's:.*projectNumber\\: (.*):\\1:p'\n",
+        "    SERVICE_ACCOUNT = (\n",
+        "        shell_output[0].replace(\"'\", \"\") + \"-compute@developer.gserviceaccount.com\"\n",
+        "    )\n",
         "\n",
-        "    if IS_COLAB:\n",
-        "        shell_output = ! gcloud projects describe  $PROJECT_ID\n",
-        "        project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n",
-        "        SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n",
-        "\n",
-        "    print(\"Service Account:\", SERVICE_ACCOUNT)"
+        "print(\"Service Account:\", SERVICE_ACCOUNT)"
       ]
     },
     {

notebooks/community/ml_ops/stage3/get_started_with_hpt_pipeline_components.ipynb

@@ -44,7 +44,7 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage3/get_started_with_hpt_pipeline_components.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_hpt_pipeline_components.ipynb\">\n",
         "        <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Google Cloud Notebooks\n",
         "    </a>\n",
@@ -124,7 +124,7 @@
       "source": [
         "## Installations\n",
         "\n",
-        "Install the required packages for executing the notebook."
+        "Install *one time* the packages for executing the MLOps notebooks."
       ]
     },
     {
@@ -135,26 +135,22 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
-        "! pip3 install -U tensorflow==2.5 $USER_FLAG -q\n",
-        "! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG -q\n",
-        "! pip3 install -U tensorflow-transform==1.2 $USER_FLAG -q\n",
-        "! pip3 install -U tensorflow-io==0.18 $USER_FLAG -q\n",
-        "! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG -q\n",
-        "! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG -q\n",
-        "! pip3 install --upgrade kfp $USER_FLAG -q"
+        "! pip3 install -U tensorflow==2.5 $USER_FLAG\n",
+        "! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG\n",
+        "! pip3 install -U tensorflow-transform==1.2 $USER_FLAG\n",
+        "! pip3 install -U tensorflow-io==0.18 $USER_FLAG\n",
+        "! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG\n",
+        "! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG\n",
+        "# ! pip3 install --upgrade google-cloud-bigquery $USER_FLAG\n",
+        "# ! pip3 install --upgrade google-cloud-logging $USER_FLAG\n",
+        "# ! pip3 install --upgrade apache-beam[gcp] $USER_FLAG\n",
+        "# ! pip3 install --upgrade pyarrow $USER_FLAG\n",
+        "# ! pip3 install --upgrade cloudml-hypertune $USER_FLAG\n",
+        "! pip3 install --upgrade kfp $USER_FLAG\n",
+        "# ! pip3 install --upgrade torchvision $USER_FLAG\n",
+        "# ! pip3 install --upgrade rpy2 $USER_FLAG\n",
+        "# ! pip3 install --upgrade python-tabulate $USER_FLAG\n",
+        "# ! pip3 install -U opencv-python-headless==4.5.2.52 $USER_FLAG"
       ]
     },
     {
@@ -186,32 +182,6 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "1df9ff75fa88"
-      },
-      "source": [
-        "## Before you begin\n",
-        "\n",
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com).\n",
-        "\n",
-        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -318,33 +288,6 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "gcp_authenticate"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
-        "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
-        "\n",
-        "**Click Create service account**.\n",
-        "\n",
-        "In the **Service account name** field, enter a name, and click **Create**.\n",
-        "\n",
-        "In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex\" into the filter box, and select **Vertex Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "Click Create. A JSON file that contains your key downloads to your local environment.\n",
-        "\n",
-        "Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -353,21 +296,20 @@
       },
       "outputs": [],
       "source": [
+        "import os\n",
+        "import sys\n",
+        "\n",
         "# If you are running this notebook in Colab, run this cell and follow the\n",
         "# instructions to authenticate your GCP account. This provides access to your\n",
         "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
         "# requests.\n",
         "\n",
-        "import os\n",
-        "import sys\n",
+        "# The Google Cloud Notebook product has specific requirements\n",
+        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
         "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = False\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
+        "# If on Google Cloud Notebooks, then don't execute this code\n",
+        "if not IS_GOOGLE_CLOUD_NOTEBOOK:\n",
         "    if \"google.colab\" in sys.modules:\n",
-        "        IS_COLAB = True\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
         "        google_auth.authenticate_user()\n",
@@ -489,20 +431,15 @@
       "source": [
         "if (\n",
         "    SERVICE_ACCOUNT == \"\"\n",
-        "    or SERVICE_ACCOUNT is None\n",
         "    or SERVICE_ACCOUNT == \"[your-service-account]\"\n",
+        "    or SERVICE_ACCOUNT is None\n",
         "):\n",
-        "    # Get your service account from gcloud\n",
-        "    if not IS_COLAB:\n",
-        "        shell_output = !gcloud auth list 2>/dev/null\n",
-        "        SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
+        "    shell_output = ! gcloud projects describe $PROJECT_ID | sed -nre 's:.*projectNumber\\: (.*):\\1:p'\n",
+        "    SERVICE_ACCOUNT = (\n",
+        "        shell_output[0].replace(\"'\", \"\") + \"-compute@developer.gserviceaccount.com\"\n",
+        "    )\n",
         "\n",
-        "    if IS_COLAB:\n",
-        "        shell_output = ! gcloud projects describe  $PROJECT_ID\n",
-        "        project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n",
-        "        SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n",
-        "\n",
-        "    print(\"Service Account:\", SERVICE_ACCOUNT)"
+        "print(\"Service Account:\", SERVICE_ACCOUNT)"
       ]
     },
     {
@@ -1520,9 +1457,7 @@
       },
       "outputs": [],
       "source": [
-        "delete_bucket = False\n",
-        "\n",
-        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "if os.getenv(\"IS_TESTING\"):\n",
         "    ! gsutil rm -r $BUCKET_URI"
       ]
     }

notebooks/community/ml_ops/stage3/get_started_with_kubeflow_pipelines.ipynb

@@ -45,7 +45,7 @@
         "  </td>\n",
         "    \n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage3/get_started_with_kubeflow_pipelines.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_kubeflow_pipelines.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
@@ -106,7 +106,7 @@
       "source": [
         "## Installations\n",
         "\n",
-        "Install the required packages for executing this MLOps notebook."
+        "Install the following packages for executing this MLOps notebooks."
       ]
     },
     {
@@ -119,13 +119,12 @@
       "source": [
         "import os\n",
         "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
+        "# The Google Cloud Notebook product has specific requirements\n",
+        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
         "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "# Google Cloud Notebook requires dependencies to be installed with '--user'\n",
         "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "if IS_GOOGLE_CLOUD_NOTEBOOK:\n",
         "    USER_FLAG = \"--user\"\n",
         "    \n",
         "! pip3 install tensorflow-io==0.18 $USER_FLAG -q\n",
@@ -163,32 +162,6 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "e56d698e5d52"
-      },
-      "source": [
-        "## Before you begin\n",
-        "\n",
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com).\n",
-        "\n",
-        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -303,7 +276,7 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
+        "**If you are using Google Cloud Notebooks**, your environment is already authenticated. Skip this step.\n",
         "\n",
         "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
         "\n",
@@ -338,13 +311,9 @@
         "import os\n",
         "import sys\n",
         "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = False\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
+        "# If on Google Cloud Notebook, then don't execute this code\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
         "    if \"google.colab\" in sys.modules:\n",
-        "        IS_COLAB = True\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
         "        google_auth.authenticate_user()\n",
@@ -467,20 +436,15 @@
       "source": [
         "if (\n",
         "    SERVICE_ACCOUNT == \"\"\n",
-        "    or SERVICE_ACCOUNT is None\n",
         "    or SERVICE_ACCOUNT == \"[your-service-account]\"\n",
+        "    or SERVICE_ACCOUNT is None\n",
         "):\n",
-        "    # Get your service account from gcloud\n",
-        "    if not IS_COLAB:\n",
-        "        shell_output = !gcloud auth list 2>/dev/null\n",
-        "        SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
+        "    shell_output = ! gcloud projects describe $PROJECT_ID | sed -nre 's:.*projectNumber\\: (.*):\\1:p'\n",
+        "    SERVICE_ACCOUNT = (\n",
+        "        shell_output[0].replace(\"'\", \"\") + \"-compute@developer.gserviceaccount.com\"\n",
+        "    )\n",
         "\n",
-        "    if IS_COLAB:\n",
-        "        shell_output = ! gcloud projects describe  $PROJECT_ID\n",
-        "        project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n",
-        "        SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n",
-        "\n",
-        "    print(\"Service Account:\", SERVICE_ACCOUNT)"
+        "print(\"Service Account:\", SERVICE_ACCOUNT)"
       ]
     },
     {

notebooks/community/ml_ops/stage3/get_started_with_rapid_prototyping_bqml_automl.ipynb

@@ -43,7 +43,7 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage3/get_started_with_rapid_prototyping_bqml_automl.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_rapid_prototyping_bqml_automl.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
@@ -241,7 +241,7 @@
       "source": [
         "## Installation\n",
         "\n",
-        "Install the packages required for executing this notebook."
+        "Install the latest version of Vertex AI SDK for Python."
       ]
     },
     {
@@ -254,20 +254,33 @@
       "source": [
         "import os\n",
         "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "# Google Cloud Notebook\n",
+        "if os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
         "    USER_FLAG = \"--user\"\n",
-        "\n",
-        "! pip3 install --quiet --upgrade google-cloud-aiplatform {USER_FLAG} -q\n",
-        "! pip3 install {USER_FLAG} --quiet -U google-cloud-pipeline-components==1.0 kfp -q\n",
-        "! pip3 install {USER_FLAG} --quiet --upgrade google-cloud-bigquery -q"
+        "else:\n",
+        "    USER_FLAG = \"\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "eeba891a06fc"
+      },
+      "source": [
+        "Install additional packages used in this tutorial."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "739011eb"
+      },
+      "outputs": [],
+      "source": [
+        "! pip3 install --quiet --upgrade google-cloud-aiplatform {USER_FLAG}\n",
+        "! pip3 install {USER_FLAG} --quiet -U google-cloud-pipeline-components==1.0 kfp\n",
+        "! pip3 install {USER_FLAG} --quiet --upgrade google-cloud-bigquery"
       ]
     },
     {
@@ -308,12 +321,10 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
-        "authenticated. Skip this step.\n",
+        "**If you are using Google Cloud Notebooks**, your environment is already authenticated. Skip this step.\n",
         "\n",
         "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
         "\n",
-        "\n",
         "**Otherwise**, follow these steps:\n",
         "\n",
         "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
@@ -345,13 +356,9 @@
         "import os\n",
         "import sys\n",
         "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = \"google.colab\" in sys.modules\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
+        "# If on Google Cloud Notebook, then don't execute this code\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
         "    if \"google.colab\" in sys.modules:\n",
-        "        IS_COLAB = True\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
         "        google_auth.authenticate_user()\n",
@@ -383,7 +390,7 @@
         "\n",
         "2. [Make sure that billing is enabled for your project.](https://cloud.google.com/billing/docs/how-to/modify-project)\n",
         "\n",
-        "3. [Enable the following APIs: Vertex AI APIs, Compute Engine APIs, and Cloud Storage.](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component,storage-component.googleapis.com)\n",
+        "3. [Enable the following APIs: Vertex AI APIs, Compute Engine APIs, and Cloud Storage.](https://console.cloud.google.com/flows/enableapi?apiid=ml.googleapis.com,compute_component,storage-component.googleapis.com)\n",
         "\n",
         "4. If you are running this notebook locally, you will need to install the [Cloud SDK]((https://cloud.google.com/sdk)).\n",
         "\n",
@@ -614,17 +621,9 @@
         "    or SERVICE_ACCOUNT is None\n",
         "    or SERVICE_ACCOUNT == \"[your-service-account]\"\n",
         "):\n",
-        "    # Get your service account from gcloud\n",
-        "    if not IS_COLAB:\n",
-        "        shell_output = !gcloud auth list 2>/dev/null\n",
-        "        SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
-        "\n",
-        "    if IS_COLAB:\n",
-        "        shell_output = ! gcloud projects describe  $PROJECT_ID\n",
-        "        # print(\"shell_output=\", shell_output)\n",
-        "        project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n",
-        "        SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n",
-        "\n",
+        "    # Get your GCP project id from gcloud\n",
+        "    shell_output = !gcloud auth list 2>/dev/null\n",
+        "    SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
         "    print(\"Service Account:\", SERVICE_ACCOUNT)"
       ]
     },

notebooks/community/ml_ops/stage3/mlops_formalization.ipynb

@@ -32,23 +32,17 @@
         "# E2E ML on GCP: MLOps stage 3 : formalization\n",
         "\n",
         "<table align=\"left\">\n",
-        "   <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/mlops_formalization.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
-        "    </a>\n",
-        "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samplestree/main/notebooks/community/ml_ops/stage3/mlops_formalization.ipynb\">\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/mlops_formalization.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage3/mlops_formalization.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
+        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/mlops_formalization.ipynb\">\n",
+        "      Open in Google Cloud Notebooks\n",
         "    </a>\n",
-        "  </td>           \n",
+        "  </td>\n",
         "</table>\n",
         "<br/><br/><br/>"
       ]
@@ -188,19 +182,6 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
         "ONCE_ONLY = False\n",
         "if ONCE_ONLY:\n",
         "    ! pip3 install -U tensorflow==2.5 $USER_FLAG\n",
@@ -250,32 +231,6 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "BF1j6f9HApxa"
-      },
-      "source": [
-        "## Before you begin\n",
-        "\n",
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI API and Dataflow API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,dataflow.googleapis.com).\n",
-        "\n",
-        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -379,67 +334,6 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "gcp_authenticate"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
-        "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
-        "\n",
-        "**Click Create service account**.\n",
-        "\n",
-        "In the **Service account name** field, enter a name, and click **Create**.\n",
-        "\n",
-        "In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex\" into the filter box, and select **Vertex Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "Click Create. A JSON file that contains your key downloads to your local environment.\n",
-        "\n",
-        "Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "gcp_authenticate"
-      },
-      "outputs": [],
-      "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = \"google.colab\" in sys.modules\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        IS_COLAB = True\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {

notebooks/community/ml_ops/stage4/README.md

@@ -36,7 +36,7 @@ This stage may be done entirely by MLOps. We recommend:
 
 
 
-<img src='stage4v3.png'>
+<img src='stage4.png'>
 
 ## Notebooks
 
@@ -75,25 +75,6 @@ The steps performed include:
     - Query your pipeline run metadata.
 ```
 
-[Get started with Vertex ML Metadata and AutoML](get_started_with_vertex_ml_metadata_and_automl.ipynb)
-
-```
-The steps performed include:
-
-- Create a `Dataset` resource.
-- Create a corresponding `google.VertexDataset` artifact.
-- Train a model using `AutoML`.
-- Create a corresponding `google.VertexModel` artifact.
-- Create an `Endpoint` resource.
-- Create a corresponding `google.Endpoint` artifact.
-- Deploy the train model to the `Endpoint`.
-- Create an execution and context for the `AutoML` training job and deployment.
-- Add the corresponding artifacts and context to the execution.
-- Add artifact links (event) to the execution.
-- Display the execution graph.
-```
-
-
 Get started with custom model evaluation
 
 Get started with A/B Testing

notebooks/community/ml_ops/stage4/get_started_with_google_artifact_registry.ipynb

@@ -33,22 +33,16 @@
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage4/get_started_with_google_artifact_registry.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
-        "    </a>\n",
-        "  </td>\n",
-        "  <td>\n",
         "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage4/get_started_with_google_artifact_registry.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage4/get_started_with_google_artifact_registry.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
+        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage4/get_started_with_google_artifact_registry.ipynb\">\n",
+        "      Open in Google Cloud Notebooks\n",
         "    </a>\n",
-        "  </td>                               \n",
+        "  </td>\n",
         "</table>\n",
         "<br/><br/><br/>"
       ]
@@ -96,7 +90,7 @@
       "source": [
         "## Installations\n",
         "\n",
-        "Install the packages required for executing the notebook."
+        "Install *one time* the packages for executing the MLOps notebooks."
       ]
     },
     {
@@ -107,20 +101,22 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
-        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG -q"
+        "ONCE_ONLY = False\n",
+        "if ONCE_ONLY:\n",
+        "    ! pip3 install -U tensorflow==2.5 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-transform==1.2 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-io==0.18 $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-bigquery $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-logging $USER_FLAG\n",
+        "    ! pip3 install --upgrade apache-beam[gcp] $USER_FLAG\n",
+        "    ! pip3 install --upgrade pyarrow $USER_FLAG\n",
+        "    ! pip3 install --upgrade cloudml-hypertune $USER_FLAG\n",
+        "    ! pip3 install --upgrade kfp $USER_FLAG\n",
+        "    ! pip3 install --upgrade torchvision $USER_FLAG\n",
+        "    ! pip3 install --upgrade rpy2 $USER_FLAG"
       ]
     },
     {
@@ -152,36 +148,6 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "before_you_begin"
-      },
-      "source": [
-        "## Before you begin\n",
-        "\n",
-        "### GPU runtime\n",
-        "\n",
-        "*Make sure you're running this notebook in a GPU runtime if you have that option. In Colab, select* **Runtime > Change Runtime Type > GPU**\n",
-        "\n",
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "2. [Make sure that billing is enabled for your project.](https://cloud.google.com/billing/docs/how-to/modify-project)\n",
-        "\n",
-        "3. [Enable the following APIs: Vertex AI APIs, Compute Engine APIs, and Cloud Storage.](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component,storage-component.googleapis.com)\n",
-        "\n",
-        "4. If you are running this notebook locally, you will need to install the [Cloud SDK]((https://cloud.google.com/sdk)).\n",
-        "\n",
-        "5. Enter your project ID in the cell below. Then run the  cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$`."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -258,10 +224,7 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
       ]
     },
     {
@@ -288,67 +251,6 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "gcp_authenticate"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
-        "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
-        "\n",
-        "**Click Create service account**.\n",
-        "\n",
-        "In the **Service account name** field, enter a name, and click **Create**.\n",
-        "\n",
-        "In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex\" into the filter box, and select **Vertex Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "Click Create. A JSON file that contains your key downloads to your local environment.\n",
-        "\n",
-        "Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "gcp_authenticate"
-      },
-      "outputs": [],
-      "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = False\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        IS_COLAB = True\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {

notebooks/community/ml_ops/stage4/get_started_with_vertex_xai.ipynb

@@ -29,7 +29,7 @@
         "id": "title"
       },
       "source": [
-        "# Vertex SDK: E2E ML on GCP: MLOps stage 4 : evaluation: get started with Vertex AI Explanations\n",
+        "# Vertex SDK: E2E ML on GCP: MLOps stage 4 : formalization: get started with Vertex AI Explanations\n",
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
@@ -44,11 +44,10 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage4/get_started_with_vertex_xai.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
+        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage4/get_started_with_vertex_xai.ipynb\">\n",
+        "      Open in Google Cloud Notebooks\n",
         "    </a>\n",
-        "  </td>                \n",
+        "  </td>\n",
         "</table>\n",
         "<br/><br/><br/>"
       ]
@@ -146,7 +145,7 @@
       "source": [
         "## Installations\n",
         "\n",
-        "Install the packages required for executing the notebook."
+        "Install *one time* the packages for executing the MLOps notebooks."
       ]
     },
     {
@@ -157,24 +156,24 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
-        "! pip3 install -U tensorflow==2.5 $USER_FLAG -q\n",
-        "! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG -q\n",
-        "! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG -q\n",
-        "! pip3 install --upgrade python-tabulate $USER_FLAG -q\n",
-        "! pip3 install -U opencv-python-headless==4.5.2.52 $USER_FLAG -q"
+        "ONCE_ONLY = False\n",
+        "if ONCE_ONLY:\n",
+        "    ! pip3 install -U tensorflow==2.5 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-transform==1.2 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-io==0.18 $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-bigquery $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-logging $USER_FLAG\n",
+        "    ! pip3 install --upgrade apache-beam[gcp] $USER_FLAG\n",
+        "    ! pip3 install --upgrade pyarrow $USER_FLAG\n",
+        "    ! pip3 install --upgrade cloudml-hypertune $USER_FLAG\n",
+        "    ! pip3 install --upgrade kfp $USER_FLAG\n",
+        "    ! pip3 install --upgrade torchvision $USER_FLAG\n",
+        "    ! pip3 install --upgrade rpy2 $USER_FLAG\n",
+        "    ! pip3 install --upgrade python-tabulate $USER_FLAG\n",
+        "    ! pip3 install -U opencv-python-headless==4.5.2.52 $USER_FLAG"
       ]
     },
     {
@@ -206,36 +205,6 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "before_you_begin"
-      },
-      "source": [
-        "## Before you begin\n",
-        "\n",
-        "### GPU runtime\n",
-        "\n",
-        "*Make sure you're running this notebook in a GPU runtime if you have that option. In Colab, select* **Runtime > Change Runtime Type > GPU**\n",
-        "\n",
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "2. [Make sure that billing is enabled for your project.](https://cloud.google.com/billing/docs/how-to/modify-project)\n",
-        "\n",
-        "3. [Enable the following APIs: Vertex AI APIs, Compute Engine APIs, and Cloud Storage.](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component,storage-component.googleapis.com)\n",
-        "\n",
-        "4. If you are running this notebook locally, you will need to install the [Cloud SDK]((https://cloud.google.com/sdk)).\n",
-        "\n",
-        "5. Enter your project ID in the cell below. Then run the  cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$`."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -312,10 +281,7 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
       ]
     },
     {
@@ -342,67 +308,6 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "gcp_authenticate"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
-        "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
-        "\n",
-        "**Click Create service account**.\n",
-        "\n",
-        "In the **Service account name** field, enter a name, and click **Create**.\n",
-        "\n",
-        "In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex\" into the filter box, and select **Vertex Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "Click Create. A JSON file that contains your key downloads to your local environment.\n",
-        "\n",
-        "Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "gcp_authenticate"
-      },
-      "outputs": [],
-      "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = False\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        IS_COLAB = True\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -426,8 +331,7 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -438,9 +342,8 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
-        "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
       ]
     },
     {
@@ -460,7 +363,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_URI"
+        "! gsutil mb -l $REGION $BUCKET_NAME"
       ]
     },
     {
@@ -480,7 +383,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_URI"
+        "! gsutil ls -al $BUCKET_NAME"
       ]
     },
     {
@@ -525,7 +428,7 @@
       },
       "outputs": [],
       "source": [
-        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
+        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_NAME)"
       ]
     },
     {
@@ -603,7 +506,7 @@
         "if os.getenv(\"IS_TESTING_TF\"):\n",
         "    TF = os.getenv(\"IS_TESTING_TF\")\n",
         "else:\n",
-        "    TF = \"2-5\".replace(\".\", \"-\")\n",
+        "    TF = \"2-1\".replace(\".\", \"-\")\n",
         "\n",
         "if TF[0] == \"2\":\n",
         "    if TRAIN_GPU:\n",
@@ -757,7 +660,7 @@
         "dataset = aip.TabularDataset.create(\n",
         "    display_name=\"CIFAR10\" + \"_\" + TIMESTAMP,\n",
         "    gcs_source=gcs_source,\n",
-        "    labels={\"user_metadata\": BUCKET_NAME},\n",
+        "    labels={\"user_metadata\": BUCKET_NAME[5:]},\n",
         ")\n",
         "\n",
         "\n",
@@ -906,7 +809,7 @@
         "\n",
         "! cut -d, -f1-16 tmp.csv > batch.csv\n",
         "\n",
-        "gcs_input_uri = BUCKET_URI + \"/test.csv\"\n",
+        "gcs_input_uri = BUCKET_NAME + \"/test.csv\"\n",
         "\n",
         "! gsutil cp batch.csv $gcs_input_uri"
       ]
@@ -941,7 +844,7 @@
         "batch_predict_job = model.batch_predict(\n",
         "    job_display_name=\"cifar10_\" + TIMESTAMP,\n",
         "    gcs_source=gcs_input_uri,\n",
-        "    gcs_destination_prefix=BUCKET_URI,\n",
+        "    gcs_destination_prefix=BUCKET_NAME,\n",
         "    instances_format=\"csv\",\n",
         "    predictions_format=\"csv\",\n",
         "    generate_explanation=True,\n",
@@ -1607,7 +1510,7 @@
         "! rm -f custom.tar custom.tar.gz\n",
         "! tar cvf custom.tar custom\n",
         "! gzip custom.tar\n",
-        "! gsutil cp custom.tar.gz $BUCKET_URI/trainer_cifar10.tar.gz"
+        "! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_cifar10.tar.gz"
       ]
     },
     {
@@ -1675,7 +1578,7 @@
       },
       "outputs": [],
       "source": [
-        "MODEL_DIR = \"{}/{}\".format(BUCKET_URI, TIMESTAMP)\n",
+        "MODEL_DIR = \"{}/{}\".format(BUCKET_NAME, TIMESTAMP)\n",
         "\n",
         "EPOCHS = 20\n",
         "STEPS = 100\n",
@@ -2153,7 +2056,7 @@
       "source": [
         "import json\n",
         "\n",
-        "gcs_input_uri = BUCKET_URI + \"/\" + \"test.jsonl\"\n",
+        "gcs_input_uri = BUCKET_NAME + \"/\" + \"test.jsonl\"\n",
         "with tf.io.gfile.GFile(gcs_input_uri, \"w\") as f:\n",
         "    data = {serving_input: test_item_1.tolist()}\n",
         "    f.write(json.dumps(data) + \"\\n\")\n",
@@ -2194,7 +2097,7 @@
         "batch_predict_job = model.batch_predict(\n",
         "    job_display_name=\"cifar10_\" + TIMESTAMP,\n",
         "    gcs_source=gcs_input_uri,\n",
-        "    gcs_destination_prefix=BUCKET_URI,\n",
+        "    gcs_destination_prefix=BUCKET_NAME,\n",
         "    instances_format=\"jsonl\",\n",
         "    predictions_format=\"jsonl\",\n",
         "    machine_type=DEPLOY_COMPUTE,\n",
@@ -3115,7 +3018,7 @@
         "! rm -f custom.tar custom.tar.gz\n",
         "! tar cvf custom.tar custom\n",
         "! gzip custom.tar\n",
-        "! gsutil cp custom.tar.gz $BUCKET_URI/trainer_cifar10.tar.gz"
+        "! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_cifar10.tar.gz"
       ]
     },
     {
@@ -3183,7 +3086,7 @@
       },
       "outputs": [],
       "source": [
-        "MODEL_DIR = \"{}/{}\".format(BUCKET_URI, TIMESTAMP)\n",
+        "MODEL_DIR = \"{}/{}\".format(BUCKET_NAME, TIMESTAMP)\n",
         "\n",
         "EPOCHS = 20\n",
         "STEPS = 100\n",
@@ -3732,11 +3635,11 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil cp tmp1.jpg $BUCKET_URI/tmp1.jpg\n",
-        "! gsutil cp tmp2.jpg $BUCKET_URI/tmp2.jpg\n",
+        "! gsutil cp tmp1.jpg $BUCKET_NAME/tmp1.jpg\n",
+        "! gsutil cp tmp2.jpg $BUCKET_NAME/tmp2.jpg\n",
         "\n",
-        "test_item_1 = BUCKET_URI + \"/tmp1.jpg\"\n",
-        "test_item_2 = BUCKET_URI + \"/tmp2.jpg\""
+        "test_item_1 = BUCKET_NAME + \"/tmp1.jpg\"\n",
+        "test_item_2 = BUCKET_NAME + \"/tmp2.jpg\""
       ]
     },
     {
@@ -3774,7 +3677,7 @@
         "import base64\n",
         "import json\n",
         "\n",
-        "gcs_input_uri = BUCKET_URI + \"/\" + \"test.jsonl\"\n",
+        "gcs_input_uri = BUCKET_NAME + \"/\" + \"test.jsonl\"\n",
         "with tf.io.gfile.GFile(gcs_input_uri, \"w\") as f:\n",
         "    bytes = tf.io.read_file(test_item_1)\n",
         "    b64str = base64.b64encode(bytes.numpy()).decode(\"utf-8\")\n",
@@ -3819,7 +3722,7 @@
         "batch_predict_job = model.batch_predict(\n",
         "    job_display_name=\"cifar10_\" + TIMESTAMP,\n",
         "    gcs_source=gcs_input_uri,\n",
-        "    gcs_destination_prefix=BUCKET_URI,\n",
+        "    gcs_destination_prefix=BUCKET_NAME,\n",
         "    instances_format=\"jsonl\",\n",
         "    model_parameters=None,\n",
         "    machine_type=DEPLOY_COMPUTE,\n",
@@ -4458,7 +4361,7 @@
         "! rm -f custom.tar custom.tar.gz\n",
         "! tar cvf custom.tar custom\n",
         "! gzip custom.tar\n",
-        "! gsutil cp custom.tar.gz $BUCKET_URI/trainer_cifar10.tar.gz"
+        "! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_cifar10.tar.gz"
       ]
     },
     {
@@ -4498,7 +4401,7 @@
         "\n",
         "job = aip.CustomPythonPackageTrainingJob(\n",
         "    display_name=DISPLAY_NAME,\n",
-        "    python_package_gcs_uri=f\"{BUCKET_URI}/trainer_cifar10.tar.gz\",\n",
+        "    python_package_gcs_uri=f\"{BUCKET_NAME}/trainer_cifar10.tar.gz\",\n",
         "    python_module_name=\"trainer.task\",\n",
         "    container_uri=TRAIN_IMAGE,\n",
         "    model_serving_container_image_uri=DEPLOY_IMAGE,\n",
@@ -4533,7 +4436,7 @@
       },
       "outputs": [],
       "source": [
-        "MODEL_DIR = \"{}/{}\".format(BUCKET_URI, TIMESTAMP)\n",
+        "MODEL_DIR = \"{}/{}\".format(BUCKET_NAME, TIMESTAMP)\n",
         "DATASET_DIR = \"gs://cloud-samples-data/ai-platform/iris\"\n",
         "\n",
         "ROUNDS = 20\n",
@@ -5330,7 +5233,7 @@
         "! rm -f custom.tar custom.tar.gz\n",
         "! tar cvf custom.tar custom\n",
         "! gzip custom.tar\n",
-        "! gsutil cp custom.tar.gz $BUCKET_URI/trainer_cifar10.tar.gz"
+        "! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_cifar10.tar.gz"
       ]
     },
     {
@@ -5370,7 +5273,7 @@
         "\n",
         "job = aip.CustomPythonPackageTrainingJob(\n",
         "    display_name=DISPLAY_NAME,\n",
-        "    python_package_gcs_uri=f\"{BUCKET_URI}/trainer_cifar10.tar.gz\",\n",
+        "    python_package_gcs_uri=f\"{BUCKET_NAME}/trainer_cifar10.tar.gz\",\n",
         "    python_module_name=\"trainer.task\",\n",
         "    container_uri=TRAIN_IMAGE,\n",
         "    model_serving_container_image_uri=DEPLOY_IMAGE,\n",
@@ -5402,7 +5305,7 @@
       },
       "outputs": [],
       "source": [
-        "MODEL_DIR = \"{}/{}\".format(BUCKET_URI, TIMESTAMP)\n",
+        "MODEL_DIR = \"{}/{}\".format(BUCKET_NAME, TIMESTAMP)\n",
         "\n",
         "DIRECT = False\n",
         "if DIRECT:\n",
@@ -5933,7 +5836,17 @@
         "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
         "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
         "\n",
-        "Otherwise, you can delete the individual resources you created in this tutorial."
+        "Otherwise, you can delete the individual resources you created in this tutorial:\n",
+        "\n",
+        "- Dataset\n",
+        "- Pipeline\n",
+        "- Model\n",
+        "- Endpoint\n",
+        "- AutoML Training Job\n",
+        "- Batch Job\n",
+        "- Custom Job\n",
+        "- Hyperparameter Tuning Job\n",
+        "- Cloud Storage Bucket"
       ]
     },
     {
@@ -5944,10 +5857,61 @@
       },
       "outputs": [],
       "source": [
-        "delete_bucket = False\n",
+        "delete_all = True\n",
         "\n",
-        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
-        "    ! gsutil rm -r $BUCKET_URI"
+        "if delete_all:\n",
+        "    # Delete the dataset using the Vertex dataset object\n",
+        "    try:\n",
+        "        if \"dataset\" in globals():\n",
+        "            dataset.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the model using the Vertex model object\n",
+        "    try:\n",
+        "        if \"model\" in globals():\n",
+        "            model.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the endpoint using the Vertex endpoint object\n",
+        "    try:\n",
+        "        if \"endpoint\" in globals():\n",
+        "            endpoint.undeploy_all()\n",
+        "            endpoint.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the AutoML or Pipeline training job\n",
+        "    try:\n",
+        "        if \"dag\" in globals():\n",
+        "            dag.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the custom training job\n",
+        "    try:\n",
+        "        if \"job\" in globals():\n",
+        "            job.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the batch prediction job using the Vertex batch prediction object\n",
+        "    try:\n",
+        "        if \"batch_predict_job\" in globals():\n",
+        "            batch_predict_job.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the hyperparameter tuning job using the Vertex hyperparameter tuning object\n",
+        "    try:\n",
+        "        if \"hpt_job\" in globals():\n",
+        "            hpt_job.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    if \"BUCKET_NAME\" in globals():\n",
+        "        ! gsutil rm -r $BUCKET_NAME"
       ]
     }
   ],