Merge branch 'main' into imkc--single-notebook-execution

* minor changes done

* ran linter test

Co-authored-by: Andrew Ferlitsch <aferlitsch@google.com>

.cloud-build/cleanup/cleanup.py

@@ -1,45 +1,44 @@
 from typing import List
-from resource_cleanup_manager import (
-	ResourceCleanupManager,
-	DatasetResourceCleanupManager,
-	EndpointResourceCleanupManager,
-	ModelResourceCleanupManager,
-)
+
+from resource_cleanup_manager import (DatasetResourceCleanupManager,
+                                      EndpointResourceCleanupManager,
+                                      ModelResourceCleanupManager,
+                                      ResourceCleanupManager)
 
 
 def run_cleanup_managers(managers: List[ResourceCleanupManager], is_dry_run: bool):
-	for manager in managers:
-		type_name = manager.type_name
+    for manager in managers:
+        type_name = manager.type_name
 
-		print(f"Fetching {type_name}'s...")
-		resources = manager.list()
-		print(f"Found {len(resources)} {type_name}'s")
-		for resource in resources:
-			if not manager.is_deletable(resource):
-				continue
+        print(f"Fetching {type_name}'s...")
+        resources = manager.list()
+        print(f"Found {len(resources)} {type_name}'s")
+        for resource in resources:
+            if not manager.is_deletable(resource):
+                continue
 
-			if is_dry_run:
-				resource_name = manager.resource_name(resource)
-				print(f"Will delete '{type_name}': {resource_name}")
-			else:
-				try:
-					manager.delete(resource)
-				except Exception as exception:
-					print(exception)
+            if is_dry_run:
+                resource_name = manager.resource_name(resource)
+                print(f"Will delete '{type_name}': {resource_name}")
+            else:
+                try:
+                    manager.delete(resource)
+                except Exception as exception:
+                    print(exception)
 
-		print("")
+        print("")
 
 
 is_dry_run = False
 
 if is_dry_run:
-	print("Starting cleanup in dry run mode...")
+    print("Starting cleanup in dry run mode...")
 
 # List of all cleanup managers
 managers = [
-	DatasetResourceCleanupManager(),
-	EndpointResourceCleanupManager(),
-	ModelResourceCleanupManager(),
+    DatasetResourceCleanupManager(),
+    EndpointResourceCleanupManager(),
+    ModelResourceCleanupManager(),
 ]
 
 run_cleanup_managers(managers=managers, is_dry_run=is_dry_run)

.cloud-build/cleanup/resource_cleanup_manager.py

@@ -1,8 +1,9 @@
 import abc
-from google.cloud import aiplatform
 from typing import Any
-from proto.datetime_helpers import DatetimeWithNanoseconds
+
+from google.cloud import aiplatform
 from google.cloud.aiplatform import base
+from proto.datetime_helpers import DatetimeWithNanoseconds
 
 # If a resource was updated within this number of seconds, do not delete.
 RESOURCE_UPDATE_BUFFER_IN_SECONDS = 60 * 60 * 8

.cloud-build/execute_changed_notebooks_cli.py

@@ -17,6 +17,7 @@
 
 import argparse
 import pathlib
+
 import execute_changed_notebooks_helper
 
 
@@ -73,6 +74,13 @@ parser.add_argument(
     help="The GCP directory for storing executed notebooks.",
     required=True,
 )
+parser.add_argument(
+    "--timeout",
+    type=int,
+    help="Timeout in seconds",
+    default=86400,
+    required=False,
+)
 parser.add_argument(
     "--private_pool_id",
     type=str,
@@ -102,6 +110,7 @@ execute_changed_notebooks_helper.process_and_execute_notebooks(
     artifacts_bucket=args.artifacts_bucket,
     variable_project_id=args.variable_project_id,
     variable_region=args.variable_region,
-    private_pool_id=args.private_pool_id if not "default" else None,
+    private_pool_id=args.private_pool_id,
     should_parallelize=args.should_parallelize,
+    timeout=args.timeout,
 )

.cloud-build/execute_changed_notebooks_helper.py

@@ -17,18 +17,23 @@ import concurrent
 import dataclasses
 import datetime
 import functools
+import operator
 import os
 import pathlib
-import nbformat
 import re
 import subprocess
 from typing import List, Optional
-from tabulate import tabulate
-import operator
 
+import execute_notebook_helper
 import execute_notebook_remote
-from utils import util, NotebookProcessors
+import nbformat
 from google.cloud.devtools.cloudbuild_v1.types import BuildOperationMetadata
+from ratemate import RateLimit
+from tabulate import tabulate
+from utils import NotebookProcessors, util
+
+# A buffer so that workers finish before the orchestrating job
+WORKER_TIMEOUT_BUFFER_IN_SECONDS: int = 60 * 60
 
 
 def format_timedelta(delta: datetime.timedelta) -> str:
@@ -103,6 +108,9 @@ def _create_tag(filepath: str) -> str:
     return tag
 
 
+rate_limit = RateLimit(max_count=50, per=60, greedy=True)
+
+
 def process_and_execute_notebook(
     container_uri: str,
     staging_bucket: str,
@@ -110,9 +118,12 @@ def process_and_execute_notebook(
     variable_project_id: str,
     variable_region: str,
     private_pool_id: Optional[str],
+    deadline: datetime,
     notebook: str,
     should_get_tail_logs: bool = False,
 ) -> NotebookExecutionResult:
+    rate_limit.wait()  # wait before creating the task
+
     print(f"Running notebook: {notebook}")
 
     # Create paths
@@ -145,14 +156,20 @@ def process_and_execute_notebook(
         # Upload the pre-processed code to a GCS bucket
         code_archive_uri = util.archive_code_and_upload(staging_bucket=staging_bucket)
 
+        # Calculate timeout in seconds
+        timeout_in_seconds = max(
+            int((deadline - datetime.datetime.now()).total_seconds()), 1
+        )
+
         operation = execute_notebook_remote.execute_notebook_remote(
             code_archive_uri=code_archive_uri,
             notebook_uri=notebook,
             notebook_output_uri=notebook_output_uri,
             container_uri=container_uri,
             tag=tag,
-            region=variable_region,
             private_pool_id=private_pool_id,
+            private_pool_region=variable_region,
+            timeout_in_seconds=timeout_in_seconds,
         )
 
         operation_metadata = BuildOperationMetadata(mapping=operation.metadata)
@@ -241,6 +258,7 @@ def process_and_execute_notebooks(
     variable_region: str,
     private_pool_id: Optional[str],
     should_parallelize: bool,
+    timeout: int,
 ):
     """
     Run the notebooks that exist under the folders defined in the test_paths_file.
@@ -267,17 +285,27 @@ def process_and_execute_notebooks(
             Required. The value for REGION to inject into notebooks.
         should_parallelize (bool):
             Required. Should run notebooks in parallel using a thread pool as opposed to in sequence.
+        timeout (str):
+            Required. Timeout string according to https://cloud.google.com/build/docs/build-config-file-schema#timeout.
     """
-    notebook_execution_results: List[NotebookExecutionResult] = []
 
-    if len(notebooks) > 0:
+    # Calculate deadline
+    deadline = datetime.datetime.now() + datetime.timedelta(
+        seconds=max(timeout - WORKER_TIMEOUT_BUFFER_IN_SECONDS, 0)
+    )
+
+    if len(notebooks) > 1:
+        notebook_execution_results: List[NotebookExecutionResult] = []
+
         print(f"Found {len(notebooks)} modified notebooks: {notebooks}")
 
         if should_parallelize and len(notebooks) > 1:
             print(
                 "Running notebooks in parallel, so no logs will be displayed. Please wait..."
             )
-            with concurrent.futures.ThreadPoolExecutor(max_workers=None) as executor:
+            with concurrent.futures.ThreadPoolExecutor(max_workers=100) as executor:
+                print(f"Max workers: {executor._max_workers}")
+
                 notebook_execution_results = list(
                     executor.map(
                         functools.partial(
@@ -288,6 +316,7 @@ def process_and_execute_notebooks(
                             variable_project_id,
                             variable_region,
                             private_pool_id,
+                            deadline,
                         ),
                         notebooks,
                     )
@@ -301,47 +330,61 @@ def process_and_execute_notebooks(
                     variable_project_id=variable_project_id,
                     variable_region=variable_region,
                     private_pool_id=private_pool_id,
+                    deadline=deadline,
                     notebook=notebook,
                 )
                 for notebook in notebooks
             ]
+
+        print("\n=== RESULTS ===\n")
+
+        results_sorted = sorted(
+            notebook_execution_results,
+            key=lambda result: result.is_pass,
+            reverse=True,
+        )
+
+        # Print results
+        print(
+            tabulate(
+                [
+                    [
+                        result.name,
+                        "PASSED" if result.is_pass else "FAILED",
+                        format_timedelta(result.duration),
+                        result.log_url,
+                        result.output_uri,
+                    ]
+                    for result in results_sorted
+                ],
+                headers=["build_tag", "status", "duration", "log_url", "output_url"],
+            )
+        )
+
+        print("\n=== END RESULTS===\n")
+
+        total_notebook_duration = functools.reduce(
+            operator.add,
+            [datetime.timedelta(seconds=0)]
+            + [result.duration for result in results_sorted],
+        )
+
+        print(
+            f"Cumulative notebook duration: {format_timedelta(total_notebook_duration)}"
+        )
+
+        # Raise error if any notebooks failed
+        if not all([result.is_pass for result in results_sorted]):
+            raise RuntimeError("Notebook failures detected. See logs for details")
+
+    elif len(notebooks) == 1:
+        notebook = notebooks[0]
+        execute_notebook_helper.execute_notebook(
+            notebook_source=notebook,
+            output_file_or_uri="/".join(
+                [artifacts_bucket, pathlib.Path(notebook).name]
+            ),
+            should_log_output=True,
+        )
     else:
         print("No notebooks modified in this pull request.")
-
-    print("\n=== RESULTS ===\n")
-
-    results_sorted = sorted(
-        notebook_execution_results,
-        key=lambda result: result.is_pass,
-        reverse=True,
-    )
-
-    # Print results
-    print(
-        tabulate(
-            [
-                [
-                    result.name,
-                    "PASSED" if result.is_pass else "FAILED",
-                    format_timedelta(result.duration),
-                    result.log_url,
-                ]
-                for result in results_sorted
-            ],
-            headers=["build_tag", "status", "duration", "log_url"],
-        )
-    )
-
-    print("\n=== END RESULTS===\n")
-
-    total_notebook_duration = functools.reduce(
-        operator.add,
-        [datetime.timedelta(seconds=0)]
-        + [result.duration for result in results_sorted],
-    )
-
-    print(f"Cumulative notebook duration: {format_timedelta(total_notebook_duration)}")
-
-    # Raise error if any notebooks failed
-    if not all([result.is_pass for result in results_sorted]):
-        raise RuntimeError("Notebook failures detected. See logs for details")

.cloud-build/execute_notebook_cli.py

@@ -16,6 +16,7 @@
 """A CLI to download (optional) and run a single notebook locally"""
 
 import argparse
+
 import execute_notebook_helper
 
 parser = argparse.ArgumentParser(description="Run a single notebook locally.")

.cloud-build/execute_notebook_helper.py

@@ -15,14 +15,14 @@
 
 """Methods to run a notebook locally"""
 
-import sys
-import os
 import errno
-import papermill as pm
+import os
 import shutil
+import sys
 
-from utils import util
+import papermill as pm
 from google.cloud.aiplatform import utils
+from utils import util
 
 # This script is used to execute a notebook and write out the output notebook.
 

.cloud-build/execute_notebook_remote.py

@@ -16,22 +16,18 @@
 """Methods to run a notebook on Google Cloud Build"""
 
 from re import sub
+from typing import Optional
+
+import google.auth
+import yaml
+from google.api_core import client_options, operation
+from google.cloud.aiplatform import utils
+from google.cloud.devtools import cloudbuild_v1
+from google.cloud.devtools.cloudbuild_v1.types import Source, StorageSource
 from google.protobuf import duration_pb2
 from yaml.loader import FullLoader
 
-import google.auth
-from google.cloud.devtools import cloudbuild_v1
-from google.cloud.devtools.cloudbuild_v1.types import Source, StorageSource
-
-from typing import Optional
-import yaml
-
-from google.cloud.aiplatform import utils
-from google.api_core import operation, client_options
-
-
 CLOUD_BUILD_FILEPATH = ".cloud-build/notebook-execution-test-cloudbuild-single.yaml"
-TIMEOUT_IN_SECONDS = 86400
 SERVICE_BASE_PATH = "cloudbuild.googleapis.com"
 
 
@@ -40,12 +36,14 @@ def execute_notebook_remote(
     notebook_uri: str,
     notebook_output_uri: str,
     container_uri: str,
-    region: str,
     private_pool_id: Optional[str],
+    private_pool_region: Optional[str],
     tag: Optional[str],
+    timeout_in_seconds: Optional[int] = None,
 ) -> operation.Operation:
     """Create and execute a single notebook on Google Cloud Build"""
     # Load build steps from YAML
+
     cloudbuild_config = yaml.load(open(CLOUD_BUILD_FILEPATH), Loader=FullLoader)
 
     substitutions = {
@@ -57,13 +55,14 @@ def execute_notebook_remote(
     build = cloudbuild_v1.Build()
 
     options: Optional[client_options.ClientOptions] = None
-    if private_pool_id:
-        substitutions["_PRIVATE_POOL_NAME"] = private_pool_id
-        build.options = cloudbuild_config["options"]
+    if private_pool_id and private_pool_region:
+        # substitutions["_PRIVATE_POOL_NAME"] = private_pool_id
+        build.options = cloudbuild_config.get("options")
+        build.options.pool = {"name": private_pool_id}
 
         # Switch to the regional endpoint of the pool
         options = client_options.ClientOptions(
-            api_endpoint=f"{region}-{SERVICE_BASE_PATH}"
+            api_endpoint=f"{private_pool_region}-{SERVICE_BASE_PATH}"
         )
 
     # Authorize the client with Google defaults
@@ -85,8 +84,8 @@ def execute_notebook_remote(
 
     build.steps = cloudbuild_config["steps"]
     build.substitutions = substitutions
-    build.timeout = duration_pb2.Duration(seconds=TIMEOUT_IN_SECONDS)
-    build.queue_ttl = duration_pb2.Duration(seconds=TIMEOUT_IN_SECONDS)
+    build.timeout = duration_pb2.Duration(seconds=timeout_in_seconds)
+    build.queue_ttl = duration_pb2.Duration(seconds=timeout_in_seconds)
 
     if tag:
         build.tags = [tag]

.cloud-build/notebook-execution-test-cloudbuild-single.yaml

@@ -25,7 +25,4 @@ steps:
     - 'python3 -m pip install -U pip && python3 -m pip freeze && python3 .cloud-build/execute_notebook_cli.py --notebook_source "${_NOTEBOOK_GCS_URI}" --output_file_or_uri "${_NOTEBOOK_OUTPUT_GCS_URI}"'
     env:
     - 'IS_TESTING=1'
-timeout: 86400s
-options:
-  pool:
-    name: ${_PRIVATE_POOL_NAME}
+timeout: 86400s

.cloud-build/requirements.txt

@@ -9,4 +9,4 @@ tabulate
 google-cloud-aiplatform
 google-cloud-storage
 google-cloud-build
-gcloud
+ratemate

.cloud-build/utils/NotebookProcessors.py

@@ -13,8 +13,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from nbconvert.preprocessors import Preprocessor
 from typing import Dict
+
+from nbconvert.preprocessors import Preprocessor
+
 from . import UpdateNotebookVariables as update_notebook_variables
 
 
@@ -60,4 +62,4 @@ class UpdateVariablesPreprocessor(Preprocessor):
 
             executable_cells.append(cell)
         notebook.cells = executable_cells
-        return notebook, resources
+        return notebook, resources

.cloud-build/utils/UpdateNotebookVariables.py

@@ -78,4 +78,4 @@ def test_region():
         variable_name="REGION",
         variable_value="us-central1",
     )
-    assert new_content == 'REGION = "us-central1"  # @param {type:"string"}'
+    assert new_content == 'REGION = "us-central1"  # @param {type:"string"}'

.cloud-build/utils/util.py

@@ -1,13 +1,13 @@
-from datetime import datetime
-from typing import Optional
-from google.cloud import storage
-from google.cloud.aiplatform import utils
-from google.auth import credentials as auth_credentials
 import os
-
 import subprocess
 import tarfile
 import uuid
+from datetime import datetime
+from typing import Optional
+
+from google.auth import credentials as auth_credentials
+from google.cloud import storage
+from google.cloud.aiplatform import utils
 
 
 def download_file(bucket_name: str, blob_name: str, destination_file: str) -> str:
@@ -57,4 +57,4 @@ def archive_code_and_upload(staging_bucket: str):
 
     print(f"Uploaded source code archive to {source_archived_file_gcs}")
 
-    return source_archived_file_gcs
+    return source_archived_file_gcs

.github/PULL_REQUEST_TEMPLATE.md

@@ -2,17 +2,17 @@ If you are opening a PR for `Official Notebooks` under the [notebooks/official](
 - [ ] Use the [notebook template](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/notebook_template.ipynb) as a starting point.
 - [ ] Follow the style and grammar rules outlined in the above notebook template.
 - [ ] Verify the notebook runs successfully in Colab since the automated tests cannot guarantee this even when it passes.
-- [ ] Passes all the required automated checks. You can locally test for formatting and linting with these [instructions](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/docs/contributing.md#code-quality-checks).
+- [ ] Passes all the required automated checks. You can locally test for formatting and linting with these [instructions](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/CONTRIBUTING.md#code-quality-checks).
 - [ ] You have consulted with a tech writer to see if tech writer review is necessary. If so, the notebook has been reviewed by a tech writer, and they have approved it.
-- [ ] This notebook has been added to the [CODEOWNERS](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/docs/CODEOWNERS) file under `# Official Notebooks` section, pointing to the author or the author's team.
+- [ ] This notebook has been added to the [CODEOWNERS](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/CODEOWNERS) file under the `Official Notebooks` section, pointing to the author or the author's team.
 - [ ] The Jupyter notebook cleans up any artifacts it has created (datasets, ML models, endpoints, etc) so as not to eat up unnecessary resources.
 
 
 If you are opening a PR for `Community Notebooks` under the [notebooks/community](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/notebooks/community) folder:
-- [ ] This notebook has been added to the [CODEOWNERS](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/docs/CODEOWNERS) file under the `# Community Notebooks` section, pointing to the author or the author's team.
-- [ ] Passes all the required formatting and linting checks. You can locally test with these [instructions](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/docs/contributing.md#code-quality-checks).
+- [ ] This notebook has been added to the [CODEOWNERS](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/CODEOWNERS) file under the `Community Notebooks` section, pointing to the author or the author's team.
+- [ ] Passes all the required formatting and linting checks. You can locally test with these [instructions](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/CONTRIBUTING.md#code-quality-checks).
 
 If you are opening a PR for `Community Content` under the [community-content](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/community-content) folder:
 - [ ] Make sure your main `Content Directory Name` is descriptive, informative, and includes some of the key products and attributes of your content, so that it is differentiable from other content
-- [ ] The main content directory has been added to the [CODEOWNERS](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/docs/CODEOWNERS) file under the `# Community Content` section, pointing to the author or the author's team.
-- [ ] Passes all the required formatting and linting checks. You can locally test with these [instructions](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/docs/contributing.md#code-quality-checks).
+- [ ] The main content directory has been added to the [CODEOWNERS](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/community-content/CODEOWNERS) file under the `Community Content` section, pointing to the author or the author's team.
+- [ ] Passes all the required formatting and linting checks. You can locally test with these [instructions](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/CONTRIBUTING.md#code-quality-checks).

.github/workflows/linter/requirements.txt

@@ -2,8 +2,9 @@ git+https://github.com/tensorflow/docs
 ipython
 jupyter
 nbconvert
-black==22.1.0
+black==22.3.0
 pyupgrade==2.31.1
 isort==5.10.1
 flake8==4.0.1
 nbqa==1.3.1
+protobuf >= 3.19.0, <4.0.0dev

community-content/CODEOWNERS

@@ -3,3 +3,4 @@
 /pytorch_text_classification_using_vertex_sdk_and_gcloud  @RajeshThallam
 /pytorch_text_classification_using_vertex_sdk_and_gcloud		@RajeshThallam @ultrons
 /sklearn_text_classification_from_script_using_vertex_sdk       @maxhardt
+/pluto_on_workbench @wkharold

community-content/pluto_on_workbench/README.md

@@ -0,0 +1,52 @@
+# Overview
+*Pluto* is a programming environment for Julia, designed to be interactive and helpful. It provides a familiar notebook interface but it is not a Jupyter notebook. The biggest difference is that Pluto notebooks are reactive, changing a variable or function in one cell causes the cells that depend on that variable or function to be reevaluated. Pluto also provides useful interaction mechanisms that allow users to dynamically interact with the notebooks computation state.
+
+The JuliaCon 2020 presentation: [Interactive notebooks ~ Pluto.jl]() provides a good introduction to Pluto. The source is at [fonsp/Pluto.jl]()
+
+# Install Pluto
+
+## Create a Vertex AI JupyterLab Instance
+
+1. From the [GCP console](https://console.cloud.google.com) "hamburger menu"
+
+   select Vertex AI > Workbench
+2. Click NEW NOTEBOOK
+   
+   * Choose Python 3 if you won't be using a GPU
+   * Choose Python 3 (CUDA Toolkit xx.y) if you do want use a GPU
+3. Give the notebook an appropriate name
+4. Edit Notebook properties if you have special requirements otherwise accept the defaults and click CREATE
+5. When the notebook instance is ready click OPEN JUPYTERLAB
+
+## Configure JupyterLab
+
+1. Open a terminal by clicking the Terminal icon.
+1. Install the plutoserver
+pip3 install git+https://github.com/fonsp/pluto-on-jupyterlab.git
+1. In a browser go to [julialang.org/downloads](https://julialang.org/downloads/)
+1. In the Current stable release right click on the `Generic Linux on x86 / 64-bit (glibc)` link
+Select copy link address
+1. Back in the terminal switch to root via
+sudo -i
+1. Download the release to /opt and install julia in /usr/local/bin
+    ```bash
+    cd /opt
+    wget <paste the release link address>
+    tar xf <name of the downloaded tar file>
+    ln -s /opt/<julia-x.y.z>/bin/julia /usr/local/bin
+    ^d
+    ```
+1. Add the Pluto package to Julia
+   ```bash
+    julia
+    julia> ]add Pluto
+    julia> bksp
+    julia> using Pluto
+    julia> ^d
+    ```
+1. From the JupyterLab menu bar select File > Shut Down
+
+# Start Pluto
+1. Click OPEN JUPYTERLAB in the Workbench
+1. In the Notebook section of the Launcher click Pluto.jl
+1. The welcome to Pluto.jl screen should appear

community-content/tf_agents_bandits_movie_recommendation_with_kfp_and_vertex_sdk/step_by_step_sdk_tf_agents_bandits_movie_recommendation/step_by_step_sdk_tf_agents_bandits_movie_recommendation.ipynb

@@ -706,8 +706,8 @@
         "    else:\n",
         "        data_spec = training_data_spec_transformation_fn(\n",
         "            agent.policy.trajectory_spec)\n",
-        "        replay_buffer = trainer.get_replay_buffer(data_spec, environment.batch_size,\n",
-        "                                                  steps_per_loop)\n",
+        "    replay_buffer = trainer.get_replay_buffer(data_spec, environment.batch_size,\n",
+        "                                              steps_per_loop)\n",
         "\n",
         "    # `step_metric` records the number of individual rounds of bandit interaction;\n",
         "    # that is, (number of trajectories) * batch_size.\n",

notebooks/community/CODEOWNERS

@@ -12,6 +12,7 @@
 /managed_notebooks/
 /sdk/SDK_FBProphet_Forecasting_Online.ipynb @brianchunkang
 /pipelines/google_cloud_pipeline_components_TPU_model_train_upload_deploy.ipynb @brianchunkang
+/explainable_ai/SDK_Custom_Container_XAI.ipynb @brianchunkang
 /matching_engine/sdk_matching_engine_for_indexing.ipynb @ivanmkc
 /matching_engine/matching_engine_for_indexing.ipynb @yinghsienwu
 /sdk/pytorch_lightning_custom_container_training.ipynb @brianchunkang
@@ -19,3 +20,5 @@
 /feature_store @nayaknishant @morgandu
 /vertex_endpoints/tf_hub_obj_detection/deploy_tfhub_object_detection_on_vertex_endpoints.ipynb @entrpn
 /vertex_endpoints/nvidia-triton/nvidia-triton-custom-container-prediction.ipynb @RajeshThallam
+/vertex_endpoints/optimized_tensorflow_runtime @vlasenkoalexey
+/notebooks/community/ml_ops/stage2/get_started_with_visionapi_and_automl.ipynb @mansari

notebooks/community/matching_engine/sdk_matching_engine_for_indexing.ipynb

@@ -98,10 +98,10 @@
         "## Before you begin\n",
         "\n",
         "* **Prepare a VPC network**.  To reduce any network overhead that might lead to unnecessary increase in overhead latency, it is best to call the ANN endpoints from your VPC via a direct [VPC Peering](https://cloud.google.com/vertex-ai/docs/general/vpc-peering) connection. The following section describes how to setup a VPC Peering connection if you don't have one. This is a one-time initial setup task. You can also reuse existing VPC network and skip this section.\n",
-        "* **WARNING:** The match service gRPC API (to create online queries against your deployed index) has to be executed in a Google Cloud Notebook instance that is created with the following requirements:\n",
+        "* **WARNING:** The MatchingIndexEndpoint.match method (to create online queries against your deployed index) has to be executed in a Vertex AI Workbench notebook instance that is created with the following requirements:\n",
         "  * **In the same region as where your ANN service is deployed** (for example, if you set `REGION = \"us-central1\"` as same as the tutorial, the notebook instance has to be in `us-central1`).\n",
         "  * **Make sure you select the VPC network you created for ANN service** (instead of using the \"default\" one). That is, you will have to create the VPC network below and then create a new notebook instance that uses that VPC.  \n",
-        "  * If you run it in the colab or a Google Cloud Notebook instance in a different VPC network or region, the gRPC API will fail to peer the network (InactiveRPCError)."
+        "  * If you run it in the colab or a Vertex AI Workbench notebook instance in a different VPC network or region, the gRPC API will fail to peer the network (InactiveRPCError)."
       ]
     },
     {
@@ -114,9 +114,9 @@
       "source": [
         "PROJECT_ID = \"<your_project_id>\"  # @param {type:\"string\"}\n",
         "\n",
-        "NETWORK_NAME = \"ucaip-haystack-vpc-network\"  # @param {type:\"string\"}\n",
+        "NETWORK_NAME = \"my-vpc-network\"  # @param {type:\"string\"}\n",
         "\n",
-        "PEERING_RANGE_NAME = \"ucaip-haystack-range\""
+        "PEERING_RANGE_NAME = \"my-haystack-range\""
       ]
     },
     {
@@ -140,7 +140,7 @@
         "! gcloud compute firewall-rules create {NETWORK_NAME}-allow-ssh --network {NETWORK_NAME} --priority 65534 --project {PROJECT_ID} --allow tcp:22\n",
         "\n",
         "# Reserve IP range\n",
-        "! gcloud compute addresses create {PEERING_RANGE_NAME} --global --prefix-length=16 --network={NETWORK_NAME} --purpose=VPC_PEERING --project={PROJECT_ID} --description=\"peering range for uCAIP Haystack.\"\n",
+        "! gcloud compute addresses create {PEERING_RANGE_NAME} --global --prefix-length=16 --network={NETWORK_NAME} --purpose=VPC_PEERING --project={PROJECT_ID} --description=\"peering range\"\n",
         "\n",
         "# Set up peering with service networking\n",
         "! gcloud services vpc-peerings connect --service=servicenetworking.googleapis.com --network={NETWORK_NAME} --ranges={PEERING_RANGE_NAME} --project={PROJECT_ID}"
@@ -152,7 +152,7 @@
         "id": "d3uj8x73nDX_"
       },
       "source": [
-        "* Authentication: `$ gcloud auth login` rerun this in Google Cloud Notebook terminal when you are logged out and need the credential again."
+        "* Authentication: Rerun the `gcloud auth login` command in the Vertex AI Workbench notebook terminal when you are logged out and need the credential again."
       ]
     },
     {
@@ -163,7 +163,7 @@
       "source": [
         "### Installation\n",
         "\n",
-        "Download and install the latest (preview) version of the Vertex SDK for Python."
+        "Download and install the latest version of the Vertex SDK for Python."
       ]
     },
     {
@@ -174,7 +174,7 @@
       },
       "outputs": [],
       "source": [
-        "! pip install -U git+https://github.com/ivanmkc/python-aiplatform.git@imkc--matching-engine"
+        "! pip install -U google-cloud-aiplatform"
       ]
     },
     {
@@ -333,7 +333,7 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Google Cloud Notebooks**, your environment is already\n",
+        "**If you are using a Vertex AI Workbench notebook**, your environment is already\n",
         "authenticated. Skip this step."
       ]
     },
@@ -383,11 +383,13 @@
         "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
         "# requests.\n",
         "\n",
-        "# The Google Cloud Notebook product has specific requirements\n",
-        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
+        "# The Vertex AI Workbench notebook product has specific requirements\n",
+        "IS_VERTEX_AI_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
         "\n",
-        "# If on Google Cloud Notebooks, then don't execute this code\n",
-        "if not IS_GOOGLE_CLOUD_NOTEBOOK:\n",
+        "# If on a Vertex AI Workbench notebook, then don't execute this code\n",
+        "if not IS_VERTEX_AI_WORKBENCH_NOTEBOOK:\n",
         "    if \"google.colab\" in sys.modules:\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
@@ -539,12 +541,12 @@
         "id": "lR6Wwv-hCCN-"
       },
       "source": [
-        "## Prepare the Data\n",
+        "## Prepare the data\n",
         "\n",
         "The GloVe dataset consists of a set of pre-trained embeddings. The embeddings are split into a \"train\" split, and a \"test\" split.\n",
         "We will create a vector search index from the \"train\" split, and use the embedding vectors in the \"test\" split as query vectors to test the vector search index.\n",
         "\n",
-        "NOTE: While the data split uses the term \"train\", these are pre-trained embeddings and thus are ready to be indexed for search. The terms \"train\" and \"test\" split are used just to be consistent with usual machine learning terminology.\n",
+        "**Note:** While the data split uses the term \"train\", these are pre-trained embeddings and therefore are ready to be indexed for search. The terms \"train\" and \"test\" split are used just to be consistent with machine learning terminology.\n",
         "\n",
         "Download the GloVe dataset.\n"
       ]

notebooks/community/ml_metadata/sdk-metric-parameter-tracking-for-locally-trained-models.ipynb

@@ -1,877 +0,0 @@
-{
-  "cells": [
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "ur8xi4C7S06n"
-      },
-      "outputs": [],
-      "source": [
-        "# Copyright 2021 Google LLC\n",
-        "#\n",
-        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
-        "# you may not use this file except in compliance with the License.\n",
-        "# You may obtain a copy of the License at\n",
-        "#\n",
-        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
-        "#\n",
-        "# Unless required by applicable law or agreed to in writing, software\n",
-        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
-        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
-        "# See the License for the specific language governing permissions and\n",
-        "# limitations under the License."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "JAPoU8Sm5E6e"
-      },
-      "source": [
-        "<table align=\"left\">\n",
-        "\n",
-        "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/ml_metadata/sdk-metric-parameter-tracking-for-locally-trained-models.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
-        "    </a>\n",
-        "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/ml_metadata/sdk-metric-parameter-tracking-for-locally-trained-models.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
-        "      View on GitHub\n",
-        "    </a>\n",
-        "  </td>\n",
-        "</table>"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "WBFL9LagqmwT"
-      },
-      "source": [
-        "#Vertex AI: Track parameters and metrics for locally trained models"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "tvgnzT1CKxrO"
-      },
-      "source": [
-        "## Overview\n",
-        "\n",
-        "This notebook demonstrates how to track metrics and parameters for ML training jobs and analyze this metadata using Vertex SDK for Python.\n",
-        "\n",
-        "### Dataset\n",
-        "\n",
-        "In this notebook, we will train a simple distributed neural network (DNN) model to predict automobile's miles per gallon (MPG) based on automobile information in the [auto-mpg dataset](https://www.kaggle.com/devanshbesain/exploration-and-analysis-auto-mpg).\n",
-        "\n",
-        "### Objective\n",
-        "\n",
-        "In this notebook, you will learn how to use Vertex SDK for Python to:\n",
-        "\n",
-        "    * Track parameters and metrics for a locally trainined model.\n",
-        "    * Extract and perform analysis for all parameters and metrics within an Experiment.\n",
-        "\n",
-        "### Costs \n",
-        "\n",
-        "\n",
-        "This tutorial uses billable components of Google Cloud:\n",
-        "\n",
-        "* Vertex AI\n",
-        "* Cloud Storage\n",
-        "\n",
-        "\n",
-        "Learn about [Vertex AI\n",
-        "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
-        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
-        "Calculator](https://cloud.google.com/products/calculator/)\n",
-        "to generate a cost estimate based on your projected usage."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "ze4-nDLfK4pw"
-      },
-      "source": [
-        "### Set up your local development environment\n",
-        "\n",
-        "**If you are using Colab or Google Cloud Notebooks**, your environment already meets\n",
-        "all the requirements to run this notebook. You can skip this step."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "gCuSR8GkAgzl"
-      },
-      "source": [
-        "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
-        "You need the following:\n",
-        "\n",
-        "* The Google Cloud SDK\n",
-        "* Git\n",
-        "* Python 3\n",
-        "* virtualenv\n",
-        "* Jupyter notebook running in a virtual environment with Python 3\n",
-        "\n",
-        "The Google Cloud guide to [Setting up a Python development\n",
-        "environment](https://cloud.google.com/python/setup) and the [Jupyter\n",
-        "installation guide](https://jupyter.org/install) provide detailed instructions\n",
-        "for meeting these requirements. The following steps provide a condensed set of\n",
-        "instructions:\n",
-        "\n",
-        "1. [Install and initialize the Cloud SDK.](https://cloud.google.com/sdk/docs/)\n",
-        "\n",
-        "1. [Install Python 3.](https://cloud.google.com/python/setup#installing_python)\n",
-        "\n",
-        "1. [Install\n",
-        "   virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv)\n",
-        "   and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
-        "\n",
-        "1. To install Jupyter, run `pip install jupyter` on the\n",
-        "command-line in a terminal shell.\n",
-        "\n",
-        "1. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
-        "\n",
-        "1. Open this notebook in the Jupyter Notebook Dashboard."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "i7EUnXsZhAGF"
-      },
-      "source": [
-        "### Install additional packages\n",
-        "\n",
-        "Run the following commands to install the Vertex SDK for Python."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "IaYsrh0Tc17L"
-      },
-      "outputs": [],
-      "source": [
-        "import sys\n",
-        "\n",
-        "if \"google.colab\" in sys.modules:\n",
-        "    USER_FLAG = \"\"\n",
-        "else:\n",
-        "    USER_FLAG = \"--user\""
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "wyy5Lbnzg5fi"
-      },
-      "outputs": [],
-      "source": [
-        "!python3 -m pip install {USER_FLAG} google-cloud-aiplatform --upgrade"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "hhq5zEbGg0XX"
-      },
-      "source": [
-        "### Restart the kernel\n",
-        "\n",
-        "After you install the additional packages, you need to restart the notebook kernel so it can find the packages."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "EzrelQZ22IZj"
-      },
-      "outputs": [],
-      "source": [
-        "# Automatically restart kernel after installs\n",
-        "import os\n",
-        "\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    # Automatically restart kernel after installs\n",
-        "    import IPython\n",
-        "\n",
-        "    app = IPython.Application.instance()\n",
-        "    app.kernel.do_shutdown(True)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "lWEdiXsJg0XY"
-      },
-      "source": [
-        "## Before you begin\n",
-        "\n",
-        "### Select a GPU runtime\n",
-        "\n",
-        "**Make sure you're running this notebook in a GPU runtime if you have that option. In Colab, select \"Runtime --> Change runtime type > GPU\"**"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "BF1j6f9HApxa"
-      },
-      "source": [
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com).\n",
-        "\n",
-        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "WReHDGG5g0XY"
-      },
-      "source": [
-        "#### Set your project ID\n",
-        "\n",
-        "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "oM1iC_MfAts1"
-      },
-      "outputs": [],
-      "source": [
-        "import os\n",
-        "\n",
-        "PROJECT_ID = \"\"\n",
-        "\n",
-        "# Get your Google Cloud project ID from gcloud\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    shell_output=!gcloud config list --format 'value(core.project)' 2>/dev/null\n",
-        "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID: \", PROJECT_ID)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "qJYoRfYng0XZ"
-      },
-      "source": [
-        "Otherwise, set your project ID here."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "riG_qUokg0XZ"
-      },
-      "outputs": [],
-      "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
-        "    PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "06571eb4063b"
-      },
-      "source": [
-        "#### Timestamp\n",
-        "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append it onto the name of resources you create in this tutorial."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "697568e92bd6"
-      },
-      "outputs": [],
-      "source": [
-        "from datetime import datetime\n",
-        "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "dr--iN2kAylZ"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Google Cloud Notebooks**, your environment is already\n",
-        "authenticated. Skip this step."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "sBCra4QMA2wR"
-      },
-      "source": [
-        "**If you are using Colab**, run the cell below and follow the instructions\n",
-        "when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "1. In the Cloud Console, go to the [**Create service account key**\n",
-        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
-        "\n",
-        "2. Click **Create service account**.\n",
-        "\n",
-        "3. In the **Service account name** field, enter a name, and\n",
-        "   click **Create**.\n",
-        "\n",
-        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
-        "into the filter box, and select\n",
-        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
-        "local environment.\n",
-        "\n",
-        "6. Enter the path to your service account key as the\n",
-        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "PyQmSRbKA8r-"
-      },
-      "outputs": [],
-      "source": [
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "# If on Google Cloud Notebooks, then don't execute this code\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "XoEqT2Y4DJmf"
-      },
-      "source": [
-        "### Import libraries and define constants"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "Y9Uo3tifg1kx"
-      },
-      "source": [
-        "Import required libraries."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "pRUOFELefqf1"
-      },
-      "outputs": [],
-      "source": [
-        "import matplotlib.pyplot as plt\n",
-        "import pandas as pd\n",
-        "from google.cloud import aiplatform\n",
-        "from tensorflow.python.keras import Sequential, layers\n",
-        "from tensorflow.python.keras.utils import data_utils"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "xtXZWmYqJ1bh"
-      },
-      "source": [
-        "Define some constants"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "JIOrI-hoJ46P"
-      },
-      "outputs": [],
-      "source": [
-        "EXPERIMENT_NAME = \"\"  # @param {type:\"string\"}\n",
-        "REGION = \"[your-region]\"  # @param {type:\"string\"}"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "jWQLXXNVN4Lv"
-      },
-      "source": [
-        "If EXEPERIMENT_NAME is not set, set a default one below:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "Q1QInYWOKsmo"
-      },
-      "outputs": [],
-      "source": [
-        "if EXPERIMENT_NAME == \"\" or EXPERIMENT_NAME is None:\n",
-        "    EXPERIMENT_NAME = \"my-experiment-\" + TIMESTAMP"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "Xuny18aMcWDb"
-      },
-      "source": [
-        "## Concepts\n",
-        "\n",
-        "To better understanding how parameters and metrics are stored and organized, we'd like to introduce the following concepts:\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "NThDci5bp0Uw"
-      },
-      "source": [
-        "### Experiment\n",
-        "Experiments describe a context that groups your runs and the artifacts you create into a logical session. For example, in this notebook you create an Experiment and log data to that experiment."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "SAyRR3Ydp4X5"
-      },
-      "source": [
-        "### Run\n",
-        "A run represents a single path/avenue that you executed while performing an experiment. A run includes artifacts that you used as inputs or outputs, and parameters that you used in this execution. An Experiment can contain multiple runs. "
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "l1YW2pgyegFP"
-      },
-      "source": [
-        "## Getting started tracking parameters and metrics\n",
-        "\n",
-        "You can use the Vertex SDK for Python to track metrics and parameters for models trained locally. \n",
-        "\n",
-        "In the following example, you train a simple distributed neural network (DNN) model to predict automobile's miles per gallon (MPG) based on automobile information in the [auto-mpg dataset](https://www.kaggle.com/devanshbesain/exploration-and-analysis-auto-mpg)."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "KPY41M9_AhZU"
-      },
-      "source": [
-        "### Load and process the training dataset"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "bfMQSmRuUuX-"
-      },
-      "source": [
-        "Download and process the dataset."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "RiQuMv4bmpuV"
-      },
-      "outputs": [],
-      "source": [
-        "def read_data(uri):\n",
-        "    dataset_path = data_utils.get_file(\"auto-mpg.data\", uri)\n",
-        "    column_names = [\n",
-        "        \"MPG\",\n",
-        "        \"Cylinders\",\n",
-        "        \"Displacement\",\n",
-        "        \"Horsepower\",\n",
-        "        \"Weight\",\n",
-        "        \"Acceleration\",\n",
-        "        \"Model Year\",\n",
-        "        \"Origin\",\n",
-        "    ]\n",
-        "    raw_dataset = pd.read_csv(\n",
-        "        dataset_path,\n",
-        "        names=column_names,\n",
-        "        na_values=\"?\",\n",
-        "        comment=\"\\t\",\n",
-        "        sep=\" \",\n",
-        "        skipinitialspace=True,\n",
-        "    )\n",
-        "    dataset = raw_dataset.dropna()\n",
-        "    dataset[\"Origin\"] = dataset[\"Origin\"].map(\n",
-        "        lambda x: {1: \"USA\", 2: \"Europe\", 3: \"Japan\"}.get(x)\n",
-        "    )\n",
-        "    dataset = pd.get_dummies(dataset, prefix=\"\", prefix_sep=\"\")\n",
-        "    return dataset\n",
-        "\n",
-        "\n",
-        "dataset = read_data(\n",
-        "    \"http://archive.ics.uci.edu/ml/machine-learning-databases/auto-mpg/auto-mpg.data\"\n",
-        ")"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "Y06J7A7yU21t"
-      },
-      "source": [
-        "Split dataset for training and testing."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "p5JBCBKyH-NC"
-      },
-      "outputs": [],
-      "source": [
-        "def train_test_split(dataset, split_frac=0.8, random_state=0):\n",
-        "    train_dataset = dataset.sample(frac=split_frac, random_state=random_state)\n",
-        "    test_dataset = dataset.drop(train_dataset.index)\n",
-        "    train_labels = train_dataset.pop(\"MPG\")\n",
-        "    test_labels = test_dataset.pop(\"MPG\")\n",
-        "\n",
-        "    return train_dataset, test_dataset, train_labels, test_labels\n",
-        "\n",
-        "\n",
-        "train_dataset, test_dataset, train_labels, test_labels = train_test_split(dataset)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "gaNNTFPaU7KT"
-      },
-      "source": [
-        "Normalize the features in the dataset for better model performance."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "VGq5QCoyIEWJ"
-      },
-      "outputs": [],
-      "source": [
-        "def normalize_dataset(train_dataset, test_dataset):\n",
-        "    train_stats = train_dataset.describe()\n",
-        "    train_stats = train_stats.transpose()\n",
-        "\n",
-        "    def norm(x):\n",
-        "        return (x - train_stats[\"mean\"]) / train_stats[\"std\"]\n",
-        "\n",
-        "    normed_train_data = norm(train_dataset)\n",
-        "    normed_test_data = norm(test_dataset)\n",
-        "\n",
-        "    return normed_train_data, normed_test_data\n",
-        "\n",
-        "\n",
-        "normed_train_data, normed_test_data = normalize_dataset(train_dataset, test_dataset)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "UBXUgxgqA_GB"
-      },
-      "source": [
-        "### Define ML model and training function"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "66odBYKrIN4q"
-      },
-      "outputs": [],
-      "source": [
-        "def train(\n",
-        "    train_data,\n",
-        "    train_labels,\n",
-        "    num_units=64,\n",
-        "    activation=\"relu\",\n",
-        "    dropout_rate=0.0,\n",
-        "    validation_split=0.2,\n",
-        "    epochs=1000,\n",
-        "):\n",
-        "\n",
-        "    model = Sequential(\n",
-        "        [\n",
-        "            layers.Dense(\n",
-        "                num_units,\n",
-        "                activation=activation,\n",
-        "                input_shape=[len(train_dataset.keys())],\n",
-        "            ),\n",
-        "            layers.Dropout(rate=dropout_rate),\n",
-        "            layers.Dense(num_units, activation=activation),\n",
-        "            layers.Dense(1),\n",
-        "        ]\n",
-        "    )\n",
-        "\n",
-        "    model.compile(loss=\"mse\", optimizer=\"adam\", metrics=[\"mae\", \"mse\"])\n",
-        "    print(model.summary())\n",
-        "\n",
-        "    history = model.fit(\n",
-        "        train_data, train_labels, epochs=epochs, validation_split=validation_split\n",
-        "    )\n",
-        "\n",
-        "    return model, history"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "O8XJZB3gR8eL"
-      },
-      "source": [
-        "### Initialize the Vertex AI SDK for Python and create an Experiment\n",
-        "\n",
-        "Initialize the *client* for Vertex AI and create an experiment."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "o_wnT10RJ7-W"
-      },
-      "outputs": [],
-      "source": [
-        "aiplatform.init(project=PROJECT_ID, location=REGION, experiment=EXPERIMENT_NAME)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "u-iTnzt3B6Z_"
-      },
-      "source": [
-        "### Start several model training runs\n",
-        "\n",
-        "Training parameters and metrics are logged for each run."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "i2wnpu8_7JfV"
-      },
-      "outputs": [],
-      "source": [
-        "parameters = [\n",
-        "    {\"num_units\": 16, \"epochs\": 3, \"dropout_rate\": 0.1},\n",
-        "    {\"num_units\": 16, \"epochs\": 10, \"dropout_rate\": 0.1},\n",
-        "    {\"num_units\": 16, \"epochs\": 10, \"dropout_rate\": 0.2},\n",
-        "    {\"num_units\": 32, \"epochs\": 10, \"dropout_rate\": 0.1},\n",
-        "    {\"num_units\": 32, \"epochs\": 10, \"dropout_rate\": 0.2},\n",
-        "]\n",
-        "\n",
-        "for i, params in enumerate(parameters):\n",
-        "    aiplatform.start_run(run=f\"auto-mpg-local-run-{i}\")\n",
-        "    aiplatform.log_params(params)\n",
-        "    model, history = train(\n",
-        "        normed_train_data,\n",
-        "        train_labels,\n",
-        "        num_units=params[\"num_units\"],\n",
-        "        activation=\"relu\",\n",
-        "        epochs=params[\"epochs\"],\n",
-        "        dropout_rate=params[\"dropout_rate\"],\n",
-        "    )\n",
-        "    aiplatform.log_metrics(\n",
-        "        {metric: values[-1] for metric, values in history.history.items()}\n",
-        "    )\n",
-        "\n",
-        "    loss, mae, mse = model.evaluate(normed_test_data, test_labels, verbose=2)\n",
-        "    aiplatform.log_metrics({\"eval_loss\": loss, \"eval_mae\": mae, \"eval_mse\": mse})"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "jZLrJZTfL7tE"
-      },
-      "source": [
-        "### Extract parameters and metrics into a dataframe for analysis"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "A1PqKxlpOZa2"
-      },
-      "source": [
-        "We can also extract all parameters and metrics associated with any Experiment into a dataframe for further analysis."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "jbRf1WoH_vbY"
-      },
-      "outputs": [],
-      "source": [
-        "experiment_df = aiplatform.get_experiment_df()\n",
-        "experiment_df"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "EYuYgqVCMKU1"
-      },
-      "source": [
-        "### Visualizing an experiment's parameters and metrics"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "r8orCj8iJuO1"
-      },
-      "outputs": [],
-      "source": [
-        "plt.rcParams[\"figure.figsize\"] = [15, 5]\n",
-        "\n",
-        "ax = pd.plotting.parallel_coordinates(\n",
-        "    experiment_df.reset_index(level=0),\n",
-        "    \"run_name\",\n",
-        "    cols=[\n",
-        "        \"param.num_units\",\n",
-        "        \"param.dropout_rate\",\n",
-        "        \"param.epochs\",\n",
-        "        \"metric.loss\",\n",
-        "        \"metric.val_loss\",\n",
-        "        \"metric.eval_loss\",\n",
-        "    ],\n",
-        "    color=[\"blue\", \"green\", \"pink\", \"red\"],\n",
-        ")\n",
-        "ax.set_yscale(\"symlog\")\n",
-        "ax.legend(bbox_to_anchor=(1.0, 0.5))"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "WTHvPMweMlP1"
-      },
-      "source": [
-        "## Visualizing experiments in Cloud Console"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "F19_5lw0MqXv"
-      },
-      "source": [
-        "Run the following to get the URL of Vertex AI Experiments for your project.\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "GmN9vE9pqqzt"
-      },
-      "outputs": [],
-      "source": [
-        "print(\"Vertex AI Experiments:\")\n",
-        "print(\n",
-        "    f\"https://console.cloud.google.com/ai/platform/experiments/experiments?folder=&organizationId=&project={PROJECT_ID}\"\n",
-        ")"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "TpV-iwP9qw9c"
-      },
-      "source": [
-        "## Cleaning up\n",
-        "\n",
-        "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
-        "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial."
-      ]
-    }
-  ],
-  "metadata": {
-    "colab": {
-      "collapsed_sections": [],
-      "name": "sdk-metric-parameter-tracking-for-locally-trained-models.ipynb",
-      "toc_visible": true
-    },
-    "kernelspec": {
-      "display_name": "Python 3",
-      "name": "python3"
-    }
-  },
-  "nbformat": 4,
-  "nbformat_minor": 0
-}

notebooks/community/ml_ops/README.md

@@ -12,7 +12,7 @@ The purpose of this set of notebooks and markdown files is to demonstrate Google
 2. [Experimentation](stage2)
 3. [Formalization](stage3)
 4. [Evaluation](stage4)
-5. Deployment
+5. [Deployment](stage5)
 6. [Serving](stage6)
 7. Monitoring
 8. Continuous Training

notebooks/community/ml_ops/stage1/README.md

@@ -22,7 +22,7 @@ The first stage in MLOps is the collection and preparation for the purpose of de
 - Data is preprocessed for training and evaluation using Dataflow.
 - Data augmentation is performed on-the-fly and is coupled with model feeding.
 
-<img src='stage1.jpg'>
+<img src='stage1v2.png'>
 
 ## Notebooks
 
@@ -76,6 +76,18 @@ The steps performed include:
     - image data
 ```
 
+[Get Started with Data Labeling](get_started_data_labeling.ipynb)
+
+```
+The steps performed include:
+
+- Create a Specialist Pool for data labelers.
+- Create a data labeling job.
+- Submit the data labeling job.
+- List data labeling jobs.
+- Cancel a data labeling job.
+```
+
 ### E2E Stage Example
 
 [Stage 1: Data Management](mlops_data_management.ipynb)

notebooks/community/ml_ops/stage1/get_started_bq_datasets.ipynb

@@ -8,7 +8,7 @@
       },
       "outputs": [],
       "source": [
-        "# Copyright 2021 Google LLC\n",
+        "# Copyright 2022 Google LLC\n",
         "#\n",
         "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
         "# you may not use this file except in compliance with the License.\n",
@@ -34,13 +34,18 @@
         "<table align=\"left\">\n",
         "  <td>\n",
         "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_bq_datasets.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_bq_datasets.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_bq_datasets.ipynb\">\n",
-        "      Open in Google Cloud Notebooks\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage1/get_started_bq_datasets.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
         "    </a>\n",
         "  </td>\n",
         "</table>\n",
@@ -67,7 +72,7 @@
       "source": [
         "### Dataset\n",
         "\n",
-        "The dataset used for this tutorial is the GSOD dataset from [BigQuery public datasets](https://cloud.google.com/bigquery/public-data). The version of the dataset you use only the fields year, month and day to predict the value of mean daily temperature (mean_temp)."
+        "The dataset used for this tutorial is the GSOD dataset from [BigQuery public datasets](https://cloud.google.com/bigquery/public-data). In this version of the dataset you consider the fields year, month and day to predict the value of mean daily temperature (mean_temp)."
       ]
     },
     {
@@ -104,7 +109,7 @@
       "source": [
         "### Recommendations\n",
         "\n",
-        "When doing E2E MLOps on Google Cloud, the following best practices with structured (tabular) data in BigQuery:\n",
+        "When doing E2E MLOps on Google Cloud, following are the best practices when dealing with structured (tabular) data in BigQuery:\n",
         "\n",
         "- For AutoML training:\n",
         "  - Create a managed dataset with Vertex AI `TabularDataset`.\n",
@@ -124,7 +129,7 @@
         "        - Within the generator (upstream)\n",
         "        - Within the model (downstream)\n",
         "    - XGBoost model training:\n",
-        "      - Use BigQuery ML builtin XGBoost training.\n",
+        "      - Use BigQuery ML built-in XGBoost training.\n",
         "      - Alternatively, create a DMatrix generator from CSV files extracted from BigQuery table.\n",
         "    - Pytorch model training:\n",
         "        - Extract the BigQuery to a pandas dataframe.\n",
@@ -132,10 +137,19 @@
         "        - Create a DataLoader generator from the pandas dataframe.\n",
         "\n",
         "\n",
-        "- Alternately:\n",
+        "- Alternatively:\n",
         "    - Extract the BigQuery table to CSV files.\n",
         "    - Preprocess the CSV files.\n",
-        "    - Create a tf.data.Dataset generator from the CSV files."
+        "    - Create a tf.data.Dataset generator from the CSV files.\n",
+        "    \n",
+        "### Costs\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "- Vertex AI\n",
+        "- Cloud Storage\n",
+        "- BigQuery\n",
+        "\n",
+        "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing), [Cloud Storage pricing](https://cloud.google.com/storage/pricing) and [BigQuery pricing](https://cloud.google.com/bigquery/pricing) and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
       ]
     },
     {
@@ -146,7 +160,7 @@
       "source": [
         "## Installations\n",
         "\n",
-        "Install *one time* the packages for executing the MLOps notebooks."
+        "Install the following packages to execute this notebook."
       ]
     },
     {
@@ -157,40 +171,26 @@
       },
       "outputs": [],
       "source": [
-        "ONCE_ONLY = False\n",
-        "if ONCE_ONLY:\n",
-        "    ! pip3 install -U tensorflow==2.5 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-transform==1.2 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-io==0.18 $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-bigquery $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-logging $USER_FLAG\n",
-        "    ! pip3 install --upgrade apache-beam[gcp] $USER_FLAG\n",
-        "    ! pip3 install --upgrade pyarrow $USER_FLAG\n",
-        "    ! pip3 install --upgrade cloudml-hypertune $USER_FLAG\n",
-        "    ! pip3 install --upgrade kfp $USER_FLAG"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "install_xgboost"
-      },
-      "source": [
-        "Install the latest GA version of *XGBoost* library as well."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "install_xgboost"
-      },
-      "outputs": [],
-      "source": [
-        "! pip3 install -U xgboost $USER_FLAG"
+        "import os\n",
+        "\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
+        "\n",
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "USER_FLAG = \"\"\n",
+        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\"\n",
+        "\n",
+        "# Install the packages\n",
+        "! pip3 install --upgrade pyarrow $USER_FLAG -q\n",
+        "! pip3 install --upgrade google-cloud-bigquery $USER_FLAG -q\n",
+        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG -q\n",
+        "! pip3 install -U xgboost $USER_FLAG -q\n",
+        "! pip3 install -U tensorflow $USER_FLAG -q\n",
+        "! pip3 install -U tensorflow-io==0.18 $USER_FLAG -q"
       ]
     },
     {
@@ -222,6 +222,32 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "84cd83853240"
+      },
+      "source": [
+        "## Before you begin\n",
+        "\n",
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI, BigQuery, Compute Engine and Cloud Storage APIs](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,bigquery,compute_component,storage_component).\n",
+        "\n",
+        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -298,7 +324,10 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -325,6 +354,67 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "77c385f0db59"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
+        "\n",
+        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
+        "\n",
+        "**Otherwise**, follow these steps:\n",
+        "\n",
+        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
+        "\n",
+        "1. **Click Create service account**.\n",
+        "\n",
+        "2. In the **Service account name** field, enter a name, and click **Create**.\n",
+        "\n",
+        "3. In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex AI\" into the filter box, and select **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "\n",
+        "4. Click Create. A JSON file that contains your key downloads to your local environment.\n",
+        "\n",
+        "5. Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "535223fa4b84"
+      },
+      "outputs": [],
+      "source": [
+        "i  # If you are running this notebook in Colab, run this cell and follow the\n",
+        "# instructions to authenticate your GCP account. This provides access to your\n",
+        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
+        "# requests.\n",
+        "\n",
+        "import os\n",
+        "import sys\n",
+        "\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = False\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
+        "    if \"google.colab\" in sys.modules:\n",
+        "        IS_COLAB = True\n",
+        "        from google.colab import auth as google_auth\n",
+        "\n",
+        "        google_auth.authenticate_user()\n",
+        "\n",
+        "    # If you are running this notebook locally, replace the string below with the\n",
+        "    # path to your service account key and run this cell to authenticate your GCP\n",
+        "    # account.\n",
+        "    elif not os.getenv(\"IS_TESTING\"):\n",
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -335,12 +425,7 @@
         "\n",
         "**The following steps are required, regardless of your notebook environment.**\n",
         "\n",
-        "When you submit a custom training job using the Vertex SDK, you upload a Python package\n",
-        "containing your training code to a Cloud Storage bucket. Vertex AI runs\n",
-        "the code from this package. In this tutorial, Vertex AI also saves the\n",
-        "trained model that results from your job in the same bucket. You can then\n",
-        "create an `Endpoint` resource based on this output in order to serve\n",
-        "online predictions.\n",
+        "When you create a dataset resource using the Vertex SDK, you can provide a Cloud Storage bucket that contains the data. Vertex AI creates the dataset resource from the data. In this tutorial, Vertex AI also creates a dataset resource from your data in the Cloud Storage bucket.\n",
         "\n",
         "Set the name of your Cloud Storage bucket below. Bucket names must be globally unique across all Google Cloud projects, including those outside of your organization."
       ]
@@ -353,7 +438,8 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
+        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
     {
@@ -364,8 +450,9 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
+        "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
       ]
     },
     {
@@ -385,7 +472,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_NAME"
+        "! gsutil mb -l $REGION $BUCKET_URI"
       ]
     },
     {
@@ -405,7 +492,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_NAME"
+        "! gsutil ls -al $BUCKET_URI"
       ]
     },
     {
@@ -414,9 +501,6 @@
         "id": "setup_vars"
       },
       "source": [
-        "### Set up variables\n",
-        "\n",
-        "Next, set up some variables used throughout the tutorial.\n",
         "### Import libraries and define constants"
       ]
     },
@@ -428,75 +512,12 @@
       },
       "outputs": [],
       "source": [
-        "import google.cloud.aiplatform as aip"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "import_bq"
-      },
-      "source": [
-        "#### Import BigQuery\n",
-        "\n",
-        "Import the BigQuery package into your Python environment."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "import_bq"
-      },
-      "outputs": [],
-      "source": [
+        "import google.cloud.aiplatform as aiplatform\n",
+        "import pandas as pd\n",
+        "import xgboost as xgb\n",
         "from google.cloud import bigquery"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "import_xgboost"
-      },
-      "source": [
-        "#### Import XGBoost\n",
-        "\n",
-        "Import the XGBoost package into your Python environment."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "import_xgboost"
-      },
-      "outputs": [],
-      "source": [
-        "import xgboost as xgb"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "import_pandas"
-      },
-      "source": [
-        "#### Import pandas\n",
-        "\n",
-        "Import the pandas package into your Python environment."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "import_pandas"
-      },
-      "outputs": [],
-      "source": [
-        "import pandas as pd"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -516,7 +537,7 @@
       },
       "outputs": [],
       "source": [
-        "aip.init(project=PROJECT_ID, location=REGION)"
+        "aiplatform.init(project=PROJECT_ID, location=REGION)"
       ]
     },
     {
@@ -538,7 +559,7 @@
       },
       "outputs": [],
       "source": [
-        "bqclient = bigquery.Client()"
+        "bqclient = bigquery.Client(project=PROJECT_ID)"
       ]
     },
     {
@@ -549,7 +570,7 @@
       "source": [
         "#### Location of BigQuery training data.\n",
         "\n",
-        "Now set the variable `IMPORT_FILE` to the location of the data table in BigQuery."
+        "Now, set the variable `IMPORT_FILE` to the location of the data table in BigQuery and `BQ_TABLE` with the table id."
       ]
     },
     {
@@ -591,10 +612,10 @@
       },
       "outputs": [],
       "source": [
-        "dataset = aip.TabularDataset.create(\n",
+        "dataset = aiplatform.TabularDataset.create(\n",
         "    display_name=\"NOAA historical weather data\" + \"_\" + TIMESTAMP,\n",
         "    bq_source=[IMPORT_FILE],\n",
-        "    labels={\"user_metadata\": BUCKET_NAME[5:]},\n",
+        "    labels={\"user_metadata\": BUCKET_URI[5:]},\n",
         ")\n",
         "\n",
         "label_column = \"mean_temp\"\n",
@@ -610,7 +631,7 @@
       "source": [
         "### Copy the dataset to Cloud Storage\n",
         "\n",
-        "Next, you make a copy of the BigQuery dataset, as a CSV file, to Cloud Storage using the BigQuery extract command.\n",
+        "Next, you make a copy of the BigQuery table as a CSV file, to Cloud Storage using the BigQuery extract command.\n",
         "\n",
         "Learn more about [BigQuery command line interface](https://cloud.google.com/bigquery/docs/reference/bq-cli-reference)."
       ]
@@ -626,9 +647,9 @@
         "comps = BQ_TABLE.split(\".\")\n",
         "BQ_PROJECT_DATASET_TABLE = comps[0] + \":\" + comps[1] + \".\" + comps[2]\n",
         "\n",
-        "! bq --location=us extract --destination_format CSV $BQ_PROJECT_DATASET_TABLE $BUCKET_NAME/mydata*.csv\n",
+        "! bq --location=us extract --destination_format CSV $BQ_PROJECT_DATASET_TABLE $BUCKET_URI/mydata*.csv\n",
         "\n",
-        "IMPORT_FILES = ! gsutil ls $BUCKET_NAME/mydata*.csv\n",
+        "IMPORT_FILES = ! gsutil ls $BUCKET_URI/mydata*.csv\n",
         "\n",
         "print(IMPORT_FILES)\n",
         "\n",
@@ -664,15 +685,12 @@
       },
       "outputs": [],
       "source": [
-        "if \"IMPORT_FILES\" in globals():\n",
-        "    gcs_source = IMPORT_FILES\n",
-        "else:\n",
-        "    gcs_source = [IMPORT_FILE]\n",
+        "gcs_source = IMPORT_FILES\n",
         "\n",
-        "dataset = aip.TabularDataset.create(\n",
+        "dataset = aiplatform.TabularDataset.create(\n",
         "    display_name=\"NOAA historical weather data\" + \"_\" + TIMESTAMP,\n",
         "    gcs_source=gcs_source,\n",
-        "    labels={\"user_metadata\": BUCKET_NAME[5:]},\n",
+        "    labels={\"user_metadata\": BUCKET_URI[5:]},\n",
         ")\n",
         "\n",
         "\n",
@@ -694,6 +712,30 @@
         "Learn more about [Creating BigQuery views](https://cloud.google.com/bigquery/docs/views)."
       ]
     },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7dc142433e50"
+      },
+      "outputs": [],
+      "source": [
+        "# Set dataset name and view name in BigQuery\n",
+        "BQ_MY_DATASET = \"[your-dataset-name]\"\n",
+        "BQ_MY_TABLE = \"[your-view-name]\"\n",
+        "\n",
+        "# Otherwise, use the default names\n",
+        "if (\n",
+        "    BQ_MY_DATASET == \"\"\n",
+        "    or BQ_MY_DATASET is None\n",
+        "    or BQ_MY_DATASET == \"[your-dataset-name]\"\n",
+        "):\n",
+        "    BQ_MY_DATASET = \"mlops_dataset_\" + TIMESTAMP\n",
+        "\n",
+        "if BQ_MY_TABLE == \"\" or BQ_MY_TABLE is None or BQ_MY_TABLE == \"[your-view-name]\":\n",
+        "    BQ_MY_TABLE = \"mlops_view_\" + TIMESTAMP"
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -702,8 +744,7 @@
       },
       "outputs": [],
       "source": [
-        "BQ_MY_DATASET = 'mydataset'\n",
-        "BQ_MY_TABLE = 'myview'\n",
+        "# Create the resources\n",
         "! bq --location=US mk -d \\\n",
         "$PROJECT_ID:$BQ_MY_DATASET\n",
         "\n",
@@ -744,8 +785,8 @@
       },
       "outputs": [],
       "source": [
-        "# Download a table.\n",
-        "table = bigquery.TableReference.from_string(\"bigquery-public-data.samples.gsod\")\n",
+        "# Download the table.\n",
+        "table = bigquery.TableReference.from_string(BQ_TABLE)\n",
         "\n",
         "rows = bqclient.list_rows(\n",
         "    table,\n",
@@ -1031,22 +1072,6 @@
         "TABLE_ID = \"gsod\"\n",
         "\n",
         "\n",
-        "def create_bigquery_dataset(dataset_id):\n",
-        "    dataset = bigquery.Dataset(\n",
-        "        bigquery.dataset.DatasetReference(PROJECT_ID, dataset_id)\n",
-        "    )\n",
-        "    dataset.location = \"us\"\n",
-        "\n",
-        "    try:\n",
-        "        dataset = bqclient.create_dataset(dataset)  # API request\n",
-        "        return True\n",
-        "    except Exception as err:\n",
-        "        print(err)\n",
-        "        if err.code != 409:  # http_client.CONFLICT\n",
-        "            raise\n",
-        "    return False\n",
-        "\n",
-        "\n",
         "def load_data_into_bigquery(url, dataset_id, table_id):\n",
         "    create_bigquery_dataset(dataset_id)\n",
         "    dataset = bqclient.dataset(dataset_id)\n",
@@ -1079,13 +1104,11 @@
       "source": [
         "### Read BigQuery table into XGboost DMatrix\n",
         "\n",
-        "Currently, there is no direct data feeding connector between BigQuery and the open source XGBoost.\n",
+        "Currently, there is no direct data feeding connector between BigQuery and the open source XGBoost. The BigQuery ML service has a built-in XGBoost training module.\n",
         "\n",
-        "The BigQuery ML service has XGBoost training builtin.\n",
+        "Alernatively, you extract the data either as a pandas dataframe or as CSV files. The extracted data is then given as an input to a `DMatrix` object when training the model.\n",
         "\n",
-        "Alernatively, you extract the data either as a pandas dataframe or as CSV files. The extracted data is then inputted to a `DMatrix` object when training the model.\n",
-        "\n",
-        "Learn more about [Getting started with builtin XGBoost](https://cloud.google.com/ai-platform/training/docs/algorithms/xgboost-start)"
+        "Learn more about [Getting started with built-in XGBoost](https://cloud.google.com/ai-platform/training/docs/algorithms/xgboost-start)."
       ]
     },
     {
@@ -1096,7 +1119,7 @@
       "source": [
         "### Read pandas table into XGboost DMatrix\n",
         "\n",
-        "Next, you load the pandas dataframe into a `DMatrix` object. XGBoost does not support non-numeric inputs. Any column that is categorical will need to be one-hot encoded prior to loading the dataframe."
+        "Next, you load the pandas dataframe into a `DMatrix` object. XGBoost does not support non-numeric inputs. Any column that is categorical need to be one-hot encoded prior to loading the dataframe."
       ]
     },
     {
@@ -1122,7 +1145,7 @@
       "source": [
         "### Read CSV files into XGboost DMatrix\n",
         "\n",
-        "Currently, there is no Cloud Storage support in XGBoost. If you use CSV files for input, you will need to download them locally."
+        "Currently, there is no Cloud Storage support in XGBoost. If you use CSV files for input, you need to download them locally."
       ]
     },
     {
@@ -1144,87 +1167,42 @@
         "id": "cleanup:mbsdk"
       },
       "source": [
-        "# Cleaning up\n",
+        "# Clean up\n",
         "\n",
         "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
         "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
         "\n",
         "Otherwise, you can delete the individual resources you created in this tutorial:\n",
         "\n",
-        "- Dataset\n",
-        "- Pipeline\n",
-        "- Model\n",
-        "- Endpoint\n",
-        "- AutoML Training Job\n",
-        "- Batch Job\n",
-        "- Custom Job\n",
-        "- Hyperparameter Tuning Job\n",
-        "- Cloud Storage Bucket"
+        "- Vertex AI Dataset resource\n",
+        "- Cloud Storage Bucket\n",
+        "- BigQuery Dataset\n",
+        "\n",
+        "Set `delete_storage` to _True_ to delete the storage resources used in this notebook."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "cleanup:mbsdk"
+        "id": "47ad926d84e8"
       },
       "outputs": [],
       "source": [
-        "delete_all = True\n",
+        "import os\n",
         "\n",
-        "if delete_all:\n",
-        "    # Delete the dataset using the Vertex dataset object\n",
-        "    try:\n",
-        "        if \"dataset\" in globals():\n",
-        "            dataset.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
+        "# Delete the dataset using the Vertex dataset object\n",
+        "dataset.delete()\n",
         "\n",
-        "    # Delete the model using the Vertex model object\n",
-        "    try:\n",
-        "        if \"model\" in globals():\n",
-        "            model.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
+        "# Delete the temporary BigQuery dataset\n",
+        "! bq rm -r -f $PROJECT_ID:$DATASET_ID\n",
         "\n",
-        "    # Delete the endpoint using the Vertex endpoint object\n",
-        "    try:\n",
-        "        if \"endpoint\" in globals():\n",
-        "            endpoint.undeploy_all()\n",
-        "            endpoint.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the AutoML or Pipeline training job\n",
-        "    try:\n",
-        "        if \"dag\" in globals():\n",
-        "            dag.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the custom training job\n",
-        "    try:\n",
-        "        if \"job\" in globals():\n",
-        "            job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the batch prediction job using the Vertex batch prediction object\n",
-        "    try:\n",
-        "        if \"batch_predict_job\" in globals():\n",
-        "            batch_predict_job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the hyperparameter tuning job using the Vertex hyperparameter tuning object\n",
-        "    try:\n",
-        "        if \"hpt_job\" in globals():\n",
-        "            hpt_job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    if \"BUCKET_NAME\" in globals():\n",
-        "        ! gsutil rm -r $BUCKET_NAME"
+        "delete_storage = False\n",
+        "if delete_storage or os.getenv(\"IS_TESTING\"):\n",
+        "    # Delete the created GCS bucket\n",
+        "    ! gsutil rm -r $BUCKET_URI\n",
+        "    # Delete the created BigQuery datasets\n",
+        "    ! bq rm -r -f $PROJECT_ID:$BQ_MY_DATASET"
       ]
     }
   ],

notebooks/community/ml_ops/stage1/get_started_dataflow.ipynb

@@ -39,8 +39,14 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_dataflow.ipynb\">\n",
-        "      Open in Google Cloud Notebooks\n",
+        "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_dataflow.ipynb\">\n",
+        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\\\" alt=\"Colab logo\"> Run in Colab\n",
+        "        </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_dataflow.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
         "    </a>\n",
         "  </td>\n",
         "</table>\n",
@@ -139,7 +145,7 @@
       "source": [
         "## Installations\n",
         "\n",
-        "Install *one time* the packages for executing the MLOps notebooks."
+        "Install the following packages to execute this notebook."
       ]
     },
     {
@@ -150,20 +156,26 @@
       },
       "outputs": [],
       "source": [
-        "ONCE_ONLY = False\n",
-        "if ONCE_ONLY:\n",
-        "    ! pip3 install -U tensorflow==2.5 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-transform==1.2 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-io==0.18 $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-bigquery $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-logging $USER_FLAG\n",
-        "    ! pip3 install --upgrade apache-beam[gcp] $USER_FLAG\n",
-        "    ! pip3 install --upgrade pyarrow $USER_FLAG\n",
-        "    ! pip3 install --upgrade cloudml-hypertune $USER_FLAG\n",
-        "    ! pip3 install --upgrade kfp $USER_FLAG"
+        "import os\n",
+        "\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
+        "\n",
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "USER_FLAG = \"\"\n",
+        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\"\n",
+        "\n",
+        "! pip3 install -U tensorflow==2.5 $USER_FLAG -q\n",
+        "! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG -q\n",
+        "! pip3 install -U tensorflow-transform==1.2 $USER_FLAG -q\n",
+        "! pip3 install -U tensorflow-io==0.18 $USER_FLAG -q\n",
+        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG -q\n",
+        "! pip3 install --upgrade google-cloud-bigquery $USER_FLAG -q\n",
+        "! pip3 install --upgrade apache-beam[gcp] $USER_FLAG -q"
       ]
     },
     {
@@ -195,6 +207,32 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "84cd83853240"
+      },
+      "source": [
+        "## Before you begin\n",
+        "\n",
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI, BigQuery, Compute Engine and Cloud Storage APIs](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,bigquery,compute_component,storage_component).\n",
+        "\n",
+        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -271,7 +309,10 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -298,6 +339,67 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "77c385f0db59"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
+        "\n",
+        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
+        "\n",
+        "**Otherwise**, follow these steps:\n",
+        "\n",
+        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
+        "\n",
+        "1. **Click Create service account**.\n",
+        "\n",
+        "2. In the **Service account name** field, enter a name, and click **Create**.\n",
+        "\n",
+        "3. In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex AI\" into the filter box, and select **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "\n",
+        "4. Click Create. A JSON file that contains your key downloads to your local environment.\n",
+        "\n",
+        "5. Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "535223fa4b84"
+      },
+      "outputs": [],
+      "source": [
+        "# If you are running this notebook in Colab, run this cell and follow the\n",
+        "# instructions to authenticate your GCP account. This provides access to your\n",
+        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
+        "# requests.\n",
+        "\n",
+        "import os\n",
+        "import sys\n",
+        "\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = False\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
+        "    if \"google.colab\" in sys.modules:\n",
+        "        IS_COLAB = True\n",
+        "        from google.colab import auth as google_auth\n",
+        "\n",
+        "        google_auth.authenticate_user()\n",
+        "\n",
+        "    # If you are running this notebook locally, replace the string below with the\n",
+        "    # path to your service account key and run this cell to authenticate your GCP\n",
+        "    # account.\n",
+        "    elif not os.getenv(\"IS_TESTING\"):\n",
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -326,7 +428,8 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
+        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
     {
@@ -337,8 +440,9 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
+        "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
       ]
     },
     {
@@ -358,7 +462,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_NAME"
+        "! gsutil mb -l $REGION $BUCKET_URI"
       ]
     },
     {
@@ -378,7 +482,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_NAME"
+        "! gsutil ls -al $BUCKET_URI"
       ]
     },
     {
@@ -401,7 +505,7 @@
       },
       "outputs": [],
       "source": [
-        "import google.cloud.aiplatform as aip"
+        "import google.cloud.aiplatform as aiplatform"
       ]
     },
     {
@@ -555,7 +659,7 @@
       },
       "outputs": [],
       "source": [
-        "aip.init(project=PROJECT_ID, location=REGION)"
+        "aiplatform.init(project=PROJECT_ID, location=REGION)"
       ]
     },
     {
@@ -1004,7 +1108,7 @@
       },
       "outputs": [],
       "source": [
-        "SCHEMA_LOCATION = BUCKET_NAME + \"/schema.txt\"\n",
+        "SCHEMA_LOCATION = BUCKET_URI + \"/schema.txt\"\n",
         "\n",
         "# When running Apache Beam directly (file is directly accessed)\n",
         "tfdv.write_schema_text(output_path=SCHEMA_LOCATION, schema=schema)\n",
@@ -1149,7 +1253,7 @@
         "            )\n",
         "\n",
         "\n",
-        "EXPORTED_DATA_PREFIX = os.path.join(BUCKET_NAME, \"exported_data\")\n",
+        "EXPORTED_DATA_PREFIX = os.path.join(BUCKET_URI, \"exported_data\")\n",
         "\n",
         "QUERY_STRING = \"SELECT {},{} FROM {} LIMIT 500\".format(\n",
         "    \"CAST(station_number as STRING) AS station_number,year,month,day\",\n",
@@ -1162,7 +1266,7 @@
         "    \"runner\": RUNNER,\n",
         "    \"raw_data_query\": QUERY_STRING,\n",
         "    \"exported_data_prefix\": EXPORTED_DATA_PREFIX,\n",
-        "    \"temp_location\": os.path.join(BUCKET_NAME, \"temp\"),\n",
+        "    \"temp_location\": os.path.join(BUCKET_URI, \"temp\"),\n",
         "    \"project\": PROJECT_ID,\n",
         "    \"region\": REGION,\n",
         "    \"setup_file\": \"./setup.py\",\n",
@@ -1187,17 +1291,7 @@
         "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
         "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
         "\n",
-        "Otherwise, you can delete the individual resources you created in this tutorial:\n",
-        "\n",
-        "- Dataset\n",
-        "- Pipeline\n",
-        "- Model\n",
-        "- Endpoint\n",
-        "- AutoML Training Job\n",
-        "- Batch Job\n",
-        "- Custom Job\n",
-        "- Hyperparameter Tuning Job\n",
-        "- Cloud Storage Bucket"
+        "Otherwise, you can delete the individual resources you created in this tutorial."
       ]
     },
     {
@@ -1208,61 +1302,11 @@
       },
       "outputs": [],
       "source": [
-        "delete_all = True\n",
+        "delete_storage = True\n",
         "\n",
-        "if delete_all:\n",
-        "    # Delete the dataset using the Vertex dataset object\n",
-        "    try:\n",
-        "        if \"dataset\" in globals():\n",
-        "            dataset.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the model using the Vertex model object\n",
-        "    try:\n",
-        "        if \"model\" in globals():\n",
-        "            model.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the endpoint using the Vertex endpoint object\n",
-        "    try:\n",
-        "        if \"endpoint\" in globals():\n",
-        "            endpoint.undeploy_all()\n",
-        "            endpoint.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the AutoML or Pipeline training job\n",
-        "    try:\n",
-        "        if \"dag\" in globals():\n",
-        "            dag.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the custom training job\n",
-        "    try:\n",
-        "        if \"job\" in globals():\n",
-        "            job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the batch prediction job using the Vertex batch prediction object\n",
-        "    try:\n",
-        "        if \"batch_predict_job\" in globals():\n",
-        "            batch_predict_job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the hyperparameter tuning job using the Vertex hyperparameter tuning object\n",
-        "    try:\n",
-        "        if \"hpt_job\" in globals():\n",
-        "            hpt_job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    if \"BUCKET_NAME\" in globals():\n",
-        "        ! gsutil rm -r $BUCKET_NAME"
+        "if delete_storage or os.getenv(\"IS_TESTING\"):\n",
+        "    if \"BUCKET_URI\" in globals():\n",
+        "        ! gsutil rm -r $BUCKET_URI"
       ]
     }
   ],

notebooks/community/ml_ops/stage1/get_started_vertex_datasets.ipynb

@@ -8,7 +8,7 @@
       },
       "outputs": [],
       "source": [
-        "# Copyright 2021 Google LLC\n",
+        "# Copyright 2022 Google LLC\n",
         "#\n",
         "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
         "# you may not use this file except in compliance with the License.\n",
@@ -34,15 +34,21 @@
         "<table align=\"left\">\n",
         "  <td>\n",
         "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_vertex_datasets.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
-        "      View on GitHub\n",
+        "     <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
         "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_vertex_datasets.ipynb\">\n",
-        "      Open in Google Cloud Notebooks\n",
+        "<img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
+        "   <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage1/get_started_vertex_datasets.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "  </td>   \n",
         "</table>\n",
         "<br/><br/><br/>"
       ]
@@ -130,7 +136,17 @@
         "            - Create a tf.data.Dataset generator from the CSV index file.\n",
         "        - If text strings are in text files:\n",
         "            - Using the JSON index file, convert the text files and labels to TFRecords.\n",
-        "            - Create a tf.data.Dataset from the TFRecords."
+        "            - Create a tf.data.Dataset from the TFRecords.\n",
+        "\n",
+        "    \n",
+        "### Costs\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "- Vertex AI\n",
+        "- Cloud Storage\n",
+        "- BigQuery\n",
+        "\n",
+        "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing), [Cloud Storage pricing](https://cloud.google.com/storage/pricing) and [BigQuery pricing](https://cloud.google.com/bigquery/pricing) and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
       ]
     },
     {
@@ -141,7 +157,7 @@
       "source": [
         "## Installations\n",
         "\n",
-        "Install *one time* the packages for executing the MLOps notebooks."
+        "Install the packages required for executing this notebook."
       ]
     },
     {
@@ -152,20 +168,27 @@
       },
       "outputs": [],
       "source": [
-        "ONCE_ONLY = False\n",
-        "if ONCE_ONLY:\n",
-        "    ! pip3 install -U tensorflow==2.5 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-transform==1.2 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-io==0.18 $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-bigquery $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-logging $USER_FLAG\n",
-        "    ! pip3 install --upgrade apache-beam[gcp] $USER_FLAG\n",
-        "    ! pip3 install --upgrade pyarrow $USER_FLAG\n",
-        "    ! pip3 install --upgrade cloudml-hypertune $USER_FLAG\n",
-        "    ! pip3 install --upgrade kfp $USER_FLAG"
+        "import os\n",
+        "\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
+        "\n",
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "USER_FLAG = \"\"\n",
+        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\"\n",
+        "\n",
+        "! pip3 install -U tensorflow $USER_FLAG -q\n",
+        "! pip3 install -U tensorflow-data-validation $USER_FLAG -q\n",
+        "! pip3 install -U tensorflow-transform $USER_FLAG -q\n",
+        "! pip3 install -U tensorflow-io $USER_FLAG -q\n",
+        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG -q\n",
+        "! pip3 install --upgrade google-cloud-bigquery $USER_FLAG -q\n",
+        "! pip3 install -U tensorflow-io==0.18 $USER_FLAG -q\n",
+        "! pip3 install --upgrade future $USER_FLAG -q"
       ]
     },
     {
@@ -197,6 +220,30 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "cb082379ed5b"
+      },
+      "source": [
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com). \n",
+        "\n",
+        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -216,7 +263,24 @@
       },
       "outputs": [],
       "source": [
-        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
+        "import os\n",
+        "\n",
+        "PROJECT_ID = \"\"\n",
+        "\n",
+        "if not os.getenv(\"IS_TESTING\"):\n",
+        "    # Get your Google Cloud project ID from gcloud\n",
+        "    shell_output = !gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "    PROJECT_ID = shell_output[0]\n",
+        "    print(\"Project ID: \", PROJECT_ID)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "37c0a68ff20d"
+      },
+      "source": [
+        "Otherwise, set your project ID here."
       ]
     },
     {
@@ -227,18 +291,15 @@
       },
       "outputs": [],
       "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
-        "    # Get your GCP project id from gcloud\n",
-        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
-        "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID:\", PROJECT_ID)"
+        "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
+        "    PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "set_gcloud_project_id"
+        "id": "c021ca495967"
       },
       "outputs": [],
       "source": [
@@ -273,7 +334,10 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -300,6 +364,67 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "927085b84a07"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
+        "\n",
+        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
+        "\n",
+        "**Otherwise**, follow these steps:\n",
+        "\n",
+        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
+        "\n",
+        "**Click Create service account**.\n",
+        "\n",
+        "In the **Service account name** field, enter a name, and click **Create**.\n",
+        "\n",
+        "In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex\" into the filter box, and select **Vertex Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "\n",
+        "Click Create. A JSON file that contains your key downloads to your local environment.\n",
+        "\n",
+        "Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "89788a802687"
+      },
+      "outputs": [],
+      "source": [
+        "# If you are running this notebook in Colab, run this cell and follow the\n",
+        "# instructions to authenticate your GCP account. This provides access to your\n",
+        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
+        "# requests.\n",
+        "\n",
+        "import os\n",
+        "import sys\n",
+        "\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = False\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
+        "    if \"google.colab\" in sys.modules:\n",
+        "        IS_COLAB = True\n",
+        "        from google.colab import auth as google_auth\n",
+        "\n",
+        "        google_auth.authenticate_user()\n",
+        "\n",
+        "    # If you are running this notebook locally, replace the string below with the\n",
+        "    # path to your service account key and run this cell to authenticate your GCP\n",
+        "    # account.\n",
+        "    elif not os.getenv(\"IS_TESTING\"):\n",
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -328,7 +453,7 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
+        "BUCKET_URI = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -339,8 +464,8 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
       ]
     },
     {
@@ -360,7 +485,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_NAME"
+        "! gsutil mb -l $REGION $BUCKET_URI"
       ]
     },
     {
@@ -380,7 +505,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_NAME"
+        "! gsutil ls -al $BUCKET_URI"
       ]
     },
     {
@@ -392,7 +517,11 @@
         "### Set up variables\n",
         "\n",
         "Next, set up some variables used throughout the tutorial.\n",
-        "### Import libraries and define constants"
+        "### Import libraries and define constants\n",
+        "\n",
+        "Import the BigQuery package, TensorFlow Data Validation (TFDV) package and TensorFlow Data Validation package into your Python environment. \n",
+        "\n",
+        "Import TensorFlow Transform (TFT) package and pandas into your Python environment."
       ]
     },
     {
@@ -403,97 +532,13 @@
       },
       "outputs": [],
       "source": [
-        "import google.cloud.aiplatform as aip"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "import_bq"
-      },
-      "source": [
-        "#### Import BigQuery\n",
-        "\n",
-        "Import the BigQuery package into your Python environment."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "import_bq"
-      },
-      "outputs": [],
-      "source": [
+        "import google.cloud.aiplatform as aip\n",
+        "import pandas as pd\n",
+        "import tensorflow_data_validation as tfdv\n",
+        "import tensorflow_transform as tft\n",
         "from google.cloud import bigquery"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "import_tfdv"
-      },
-      "source": [
-        "#### Import TensorFlow Data Validation\n",
-        "\n",
-        "Import the TensorFlow Data Validation (TFDV) package into your Python environment."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "import_tfdv"
-      },
-      "outputs": [],
-      "source": [
-        "import tensorflow_data_validation as tfdv"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "import_tft"
-      },
-      "source": [
-        "#### Import TensorFlow Transform\n",
-        "\n",
-        "Import the TensorFlow Transform (TFT) package into your Python environment."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "import_tft"
-      },
-      "outputs": [],
-      "source": [
-        "import tensorflow_transform as tft"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "import_pandas"
-      },
-      "source": [
-        "#### Import pandas\n",
-        "\n",
-        "Import the pandas package into your Python environment."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "import_pandas"
-      },
-      "outputs": [],
-      "source": [
-        "import pandas as pd"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -1149,9 +1194,9 @@
         "comps = BQ_TABLE.split(\".\")\n",
         "BQ_PROJECT_DATASET_TABLE = comps[0] + \":\" + comps[1] + \".\" + comps[2]\n",
         "\n",
-        "! bq --location=us extract --destination_format CSV $BQ_PROJECT_DATASET_TABLE $BUCKET_NAME/mydata*.csv\n",
+        "! bq --location=us extract --destination_format CSV $BQ_PROJECT_DATASET_TABLE $BUCKET_URI/mydata*.csv\n",
         "\n",
-        "IMPORT_FILES = ! gsutil ls $BUCKET_NAME/mydata*.csv\n",
+        "IMPORT_FILES = ! gsutil ls $BUCKET_URI/mydata*.csv\n",
         "\n",
         "print(IMPORT_FILES)\n",
         "\n",
@@ -1209,6 +1254,38 @@
         "To create a dataframe from multiple CSV sources, you read each CSV file and concatenate the dataframes together."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "bcd2e4e0703b"
+      },
+      "source": [
+        "If you are running this notebook on Colab, run the following cell to install packages fsspec and gcsfs."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "927bd3f92268"
+      },
+      "outputs": [],
+      "source": [
+        "# If you are running this notebook in Colab, run this cell and follow the\n",
+        "# instructions to authenticate your GCP account. This provides access to your\n",
+        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
+        "# requests.\n",
+        "\n",
+        "import os\n",
+        "import sys\n",
+        "\n",
+        "# If on Google Cloud Notebook, then don't execute this code\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
+        "    if \"google.colab\" in sys.modules:\n",
+        "        ! pip3 install fsspec\n",
+        "        ! pip3 install gcsfs"
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -1269,7 +1346,7 @@
       },
       "outputs": [],
       "source": [
-        "EXPORTED_DIR = f\"{BUCKET_NAME}/exported\"\n",
+        "EXPORTED_DIR = f\"{BUCKET_URI}/exported\"\n",
         "exported_files = dataset.export_data(output_dir=EXPORTED_DIR)\n",
         "\n",
         "! gsutil ls $EXPORTED_DIR"
@@ -1498,7 +1575,7 @@
         "    data = f.readlines()\n",
         "\n",
         "# The path to the TFRecord cached file.\n",
-        "GCS_TFRECORD_URI = BUCKET_NAME + \"/flowers.tfrecord\"\n",
+        "GCS_TFRECORD_URI = BUCKET_URI + \"/flowers.tfrecord\"\n",
         "\n",
         "# Create the TFRecord cached file\n",
         "with tf.io.TFRecordWriter(GCS_TFRECORD_URI) as writer:\n",
@@ -1532,14 +1609,7 @@
         "Otherwise, you can delete the individual resources you created in this tutorial:\n",
         "\n",
         "- Dataset\n",
-        "- Pipeline\n",
-        "- Model\n",
-        "- Endpoint\n",
-        "- AutoML Training Job\n",
-        "- Batch Job\n",
-        "- Custom Job\n",
-        "- Hyperparameter Tuning Job\n",
-        "- Cloud Storage Bucket"
+        "- Bucket"
       ]
     },
     {
@@ -1550,61 +1620,12 @@
       },
       "outputs": [],
       "source": [
-        "delete_all = True\n",
+        "# Delete the dataset using the Vertex dataset object\n",
+        "dataset.delete()\n",
         "\n",
-        "if delete_all:\n",
-        "    # Delete the dataset using the Vertex dataset object\n",
-        "    try:\n",
-        "        if \"dataset\" in globals():\n",
-        "            dataset.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the model using the Vertex model object\n",
-        "    try:\n",
-        "        if \"model\" in globals():\n",
-        "            model.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the endpoint using the Vertex endpoint object\n",
-        "    try:\n",
-        "        if \"endpoint\" in globals():\n",
-        "            endpoint.undeploy_all()\n",
-        "            endpoint.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the AutoML or Pipeline training job\n",
-        "    try:\n",
-        "        if \"dag\" in globals():\n",
-        "            dag.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the custom training job\n",
-        "    try:\n",
-        "        if \"job\" in globals():\n",
-        "            job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the batch prediction job using the Vertex batch prediction object\n",
-        "    try:\n",
-        "        if \"batch_predict_job\" in globals():\n",
-        "            batch_predict_job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the hyperparameter tuning job using the Vertex hyperparameter tuning object\n",
-        "    try:\n",
-        "        if \"hpt_job\" in globals():\n",
-        "            hpt_job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    if \"BUCKET_NAME\" in globals():\n",
-        "        ! gsutil rm -r $BUCKET_NAME"
+        "# Delete the bucket\n",
+        "if os.getenv(\"IS_TESTING\"):\n",
+        "    ! gsutil rm -r $BUCKET_URI"
       ]
     }
   ],

notebooks/community/ml_ops/stage1/mlops_data_management.ipynb

@@ -39,8 +39,14 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/mlops_data_management.ipynb\">\n",
-        "      Open in Google Cloud Notebooks\n",
+        "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/mlops_data_management.ipynb\">\n",
+        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\\\" alt=\"Colab logo\"> Run in Colab\n",
+        "        </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage1/mlops_data_management.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
         "    </a>\n",
         "  </td>\n",
         "</table>\n",
@@ -133,7 +139,20 @@
       },
       "outputs": [],
       "source": [
-        "ONCE_ONLY = False\n",
+        "import os\n",
+        "\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
+        "\n",
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "USER_FLAG = \"\"\n",
+        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\"\n",
+        "\n",
+        "ONCE_ONLY = True\n",
         "if ONCE_ONLY:\n",
         "    ! pip3 install -U tensorflow==2.5 $USER_FLAG\n",
         "    ! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG\n",
@@ -178,6 +197,32 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "84cd83853240"
+      },
+      "source": [
+        "## Before you begin\n",
+        "\n",
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI, BigQuery, Compute Engine and Cloud Storage APIs](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,bigquery,compute_component,storage_component).\n",
+        "\n",
+        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -254,7 +299,10 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -281,6 +329,67 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "77c385f0db59"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
+        "\n",
+        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
+        "\n",
+        "**Otherwise**, follow these steps:\n",
+        "\n",
+        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
+        "\n",
+        "1. **Click Create service account**.\n",
+        "\n",
+        "2. In the **Service account name** field, enter a name, and click **Create**.\n",
+        "\n",
+        "3. In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex AI\" into the filter box, and select **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "\n",
+        "4. Click Create. A JSON file that contains your key downloads to your local environment.\n",
+        "\n",
+        "5. Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "535223fa4b84"
+      },
+      "outputs": [],
+      "source": [
+        "# If you are running this notebook in Colab, run this cell and follow the\n",
+        "# instructions to authenticate your GCP account. This provides access to your\n",
+        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
+        "# requests.\n",
+        "\n",
+        "import os\n",
+        "import sys\n",
+        "\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = False\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
+        "    if \"google.colab\" in sys.modules:\n",
+        "        IS_COLAB = True\n",
+        "        from google.colab import auth as google_auth\n",
+        "\n",
+        "        google_auth.authenticate_user()\n",
+        "\n",
+        "    # If you are running this notebook locally, replace the string below with the\n",
+        "    # path to your service account key and run this cell to authenticate your GCP\n",
+        "    # account.\n",
+        "    elif not os.getenv(\"IS_TESTING\"):\n",
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -309,7 +418,8 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
+        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
     {
@@ -320,8 +430,9 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
+        "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
       ]
     },
     {
@@ -341,7 +452,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_NAME"
+        "! gsutil mb -l $REGION $BUCKET_URI"
       ]
     },
     {
@@ -361,7 +472,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_NAME"
+        "! gsutil ls -al $BUCKET_URI"
       ]
     },
     {
@@ -516,7 +627,7 @@
       },
       "outputs": [],
       "source": [
-        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_NAME)"
+        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
       ]
     },
     {
@@ -757,7 +868,7 @@
         "dataset = aip.TabularDataset.create(\n",
         "    display_name=\"Chicago Taxi\" + \"_\" + TIMESTAMP,\n",
         "    bq_source=[IMPORT_FILE],\n",
-        "    labels={\"user_metadata\": BUCKET_NAME[5:]},\n",
+        "    labels={\"user_metadata\": BUCKET_NAME},\n",
         ")\n",
         "\n",
         "label_column = \"tip_bin\"\n",
@@ -949,9 +1060,9 @@
       },
       "outputs": [],
       "source": [
-        "STATISTICS_SCHEMA = BUCKET_NAME + \"/statistics.jsonl\"\n",
+        "STATISTICS_SCHEMA = BUCKET_URI + \"/statistics.jsonl\"\n",
         "\n",
-        "tfdv.write_stats_text(stats, BUCKET_NAME + \"/statistics.jsonl\")\n",
+        "tfdv.write_stats_text(stats, BUCKET_URI + \"/statistics.jsonl\")\n",
         "\n",
         "with tf.io.gfile.GFile(\n",
         "    \"gs://\" + dataset.labels[\"user_metadata\"] + \"/metadata.jsonl\", \"r\"\n",
@@ -964,7 +1075,7 @@
         ") as f:\n",
         "    json.dump(metadata, f)\n",
         "\n",
-        "!gsutil cat $BUCKET_NAME/metadata.jsonl"
+        "! gsutil cat $BUCKET_URI/metadata.jsonl"
       ]
     },
     {
@@ -1011,7 +1122,7 @@
       },
       "outputs": [],
       "source": [
-        "SCHEMA_LOCATION = BUCKET_NAME + \"/schema.txt\"\n",
+        "SCHEMA_LOCATION = BUCKET_URI + \"/schema.txt\"\n",
         "\n",
         "# When running Apache Beam directly (file is directly accessed)\n",
         "tfdv.write_schema_text(output_path=SCHEMA_LOCATION, schema=schema)\n",
@@ -1049,7 +1160,7 @@
         ") as f:\n",
         "    json.dump(metadata, f)\n",
         "\n",
-        "!gsutil cat $BUCKET_NAME/metadata.jsonl"
+        "! gsutil cat $BUCKET_URI/metadata.jsonl"
       ]
     },
     {
@@ -1372,10 +1483,10 @@
         "            )\n",
         "\n",
         "\n",
-        "EXPORTED_JSONL_PREFIX = os.path.join(BUCKET_NAME, \"exported_data/jsonl\")\n",
-        "EXPORTED_TFREC_PREFIX = os.path.join(BUCKET_NAME, \"exported_data/tfrec\")\n",
-        "TRANSFORMED_DATA_PREFIX = os.path.join(BUCKET_NAME, \"transformed_data\")\n",
-        "TRANSFORM_ARTIFACTS_DIR = os.path.join(BUCKET_NAME, \"transformed_artifacts\")\n",
+        "EXPORTED_JSONL_PREFIX = os.path.join(BUCKET_URI, \"exported_data/jsonl\")\n",
+        "EXPORTED_TFREC_PREFIX = os.path.join(BUCKET_URI, \"exported_data/tfrec\")\n",
+        "TRANSFORMED_DATA_PREFIX = os.path.join(BUCKET_URI, \"transformed_data\")\n",
+        "TRANSFORM_ARTIFACTS_DIR = os.path.join(BUCKET_URI, \"transformed_artifacts\")\n",
         "\n",
         "QUERY_STRING = \"SELECT * FROM {} LIMIT 300000\".format(BQ_TABLE)\n",
         "JOB_NAME = \"chicago\" + TIMESTAMP\n",
@@ -1388,7 +1499,7 @@
         "    \"transform_artifact_dir\": TRANSFORM_ARTIFACTS_DIR,\n",
         "    \"exported_jsonl_prefix\": EXPORTED_JSONL_PREFIX,\n",
         "    \"exported_tfrec_prefix\": EXPORTED_TFREC_PREFIX,\n",
-        "    \"temp_location\": os.path.join(BUCKET_NAME, \"temp\"),\n",
+        "    \"temp_location\": os.path.join(BUCKET_URI, \"temp\"),\n",
         "    \"project\": PROJECT_ID,\n",
         "    \"region\": REGION,\n",
         "    \"setup_file\": \"./setup.py\",\n",
@@ -1459,7 +1570,7 @@
         ") as f:\n",
         "    json.dump(metadata, f)\n",
         "\n",
-        "!gsutil cat $BUCKET_NAME/metadata.jsonl"
+        "! gsutil cat $BUCKET_URI/metadata.jsonl"
       ]
     },
     {
@@ -1473,17 +1584,9 @@
         "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
         "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
         "\n",
-        "Otherwise, you can delete the individual resources you created in this tutorial:\n",
+        "Otherwise, you can delete the individual resources you created in this tutorial.\n",
         "\n",
-        "- Dataset\n",
-        "- Pipeline\n",
-        "- Model\n",
-        "- Endpoint\n",
-        "- AutoML Training Job\n",
-        "- Batch Job\n",
-        "- Custom Job\n",
-        "- Hyperparameter Tuning Job\n",
-        "- Cloud Storage Bucket"
+        "*Note:* stage2/mlops_experimentation is dependent on the resources created by this stage1 notebook."
       ]
     },
     {
@@ -1504,8 +1607,8 @@
         "    except Exception as e:\n",
         "        print(e)\n",
         "\n",
-        "    if \"BUCKET_NAME\" in globals():\n",
-        "        ! gsutil rm -r $BUCKET_NAME"
+        "    if \"BUCKET_URI\" in globals():\n",
+        "        ! gsutil rm -r $BUCKET_URI"
       ]
     }
   ],

notebooks/community/ml_ops/stage2/README.md

@@ -25,7 +25,11 @@ The second stage in MLOps is experimenting in developing one or more baseline mo
 - Use the What-if-Tool (WIT) to explore how the trained model would make predictions in different scenarios.
 
 
-<img src='stage2.png'>
+<img src='stage2v3.png'>
+<br/>
+<br/>
+<br/>
+<img src='stage2.2v1.png'>
 
 ## Notebooks
 
@@ -115,6 +119,19 @@ The steps performed include:
 - Train a R model using `Vertex AI Trainingh` service with the R-to-Python training package.
 ```
 
+[Get Started with Custom Training Packages (LightGBM)](get_started_vertex_training_lightgbm.ipynb)
+
+```
+The steps performed include:
+
+- Training using a Python package.
+- Save the model artifacts to Cloud Storage using GCSFuse.
+- Construct a FastAPI prediction server.
+- Construct a Dockerfile deployment image.
+- Test the deployment image locally.
+- Create a `Vertex AI Model` resource.
+```
+
 [Get Started with Distributed Training](get_started_vertex_distributed_training.ipynb)
 
 ```
@@ -153,12 +170,13 @@ The steps performed include:
 ```
 The steps performed include:
 
-- Create a local BQ table in your project.
-- Train a BQML model.
-- Evaluate the BQML model.
-- Export the BQML model as a cloud model.
-- Upload the exported model as a Vertex AI Model resource.
-- Hyperparameter tune a BQML model with Vertex AI Vizier.
+- Create a local BigQuery table in your project
+- Train a BQML model
+- Evaluate the BQML model
+- Export the BQML model as a cloud model
+- Upload the exported model as a `Vertex AI Model` resource
+- Hyperparameter tune a BQML model with `Vertex AI Vizier`
+- Automatically register a BQML model to `Vertex AI Model Registry`
 ```
 
 [Get Started with Vertex Feature Store](get_started_vertex_feature_store.ipynb)
@@ -185,6 +203,62 @@ The steps performed include:
 - Train an AutoML model with CMEK encryption.
 ```
 
+[Get Started with TensorFlow Hub models](get_started_with_tfhub_models.ipynb)
+
+```
+The steps performed include:
+
+- Download a TensorFlow Hub prebuilt model.
+- Add the task component as a classifier for the CIFAR-10 dataset.
+- Fine tune locally the model with transfer learning training.
+- Construct a custom training script:
+    - Get training data from TensorFlow Datasets
+    - Get model architecture from TensorFlow Hub
+    - Train then model
+    - Save model artifacts and upload as Vertex AI Model resource.
+```
+
+[Get Started with Vertex AI TabNet builtin algorithm](get_started_with_tabnet.ipynb)
+```
+The steps performed include:
+
+- Get the training data.
+- Configure training parameters for the Vertex AI TabNet container.
+- Train the model using Vertex AI Training using CSV data.
+- Upload the model as a Vertex AI Model resource.
+- Deploy the Vertex AI Model resource to a Vertex AI Endpoint resource.
+- Make a prediction with the deployed model.
+- Hyperparameter tuning the Vertex AI TabNet model.
+- Train the model using Vertex AI Training using BigQuery table.
+```
+[Get Started with Vertex AI TabNet builtin algorithm](get_started_with_tabnet.ipynb)
+```
+The steps performed include:
+
+- Get the training data.
+- Configure training parameters for the Vertex AI TabNet container.
+- Train the model using Vertex AI Training using CSV data.
+- Upload the model as a Vertex AI Model resource.
+- Deploy the Vertex AI Model resource to a Vertex AI Endpoint resource.
+- Make a prediction with the deployed model.
+- Hyperparameter tuning the Vertex AI TabNet model.
+- Train the model using Vertex AI Training using BigQuery table.
+```
+
+[Get Started with Vision API and AutoML](get_started_with_visionapi_and_automl.ipynb)
+```
+The steps performed include:
+
+- Preprocess training files using `Vision AI` APIs to extract the text from PDF files.
+- Create a custom import file that includes annotation data based on the sample `BigQuery` dataset.
+- Create a `Vertex AI Dataset` resource.
+- Train the model.
+- View the model evaluation.
+- Deploy the `Vertex AI Model` resource to a serving `Endpoint` resource.
+- Make a prediction.
+- Undeploy the `Model`.
+```
+
 
 ### E2E Stage Example
 

notebooks/community/ml_ops/stage2/get_started_automl_training.ipynb

@@ -8,7 +8,7 @@
       },
       "outputs": [],
       "source": [
-        "# Copyright 2021 Google LLC\n",
+        "# Copyright 2022 Google LLC\n",
         "#\n",
         "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
         "# you may not use this file except in compliance with the License.\n",
@@ -39,8 +39,14 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_automl_training.ipynb\">\n",
-        "      Open in Google Cloud Notebooks\n",
+        "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_automl_training.ipynb\">\n",
+        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\\\" alt=\"Colab logo\"> Run in Colab\n",
+        "        </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage2/get_started_automl_training.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
         "    </a>\n",
         "  </td>\n",
         "</table>\n",
@@ -65,9 +71,11 @@
         "id": "dataset:flowers,icn"
       },
       "source": [
-        "### Dataset\n",
+        "### Datasets\n",
         "\n",
-        "The dataset used for this tutorial is the [Flowers dataset](https://www.tensorflow.org/datasets/catalog/tf_flowers) from [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/overview). The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket. The trained model predicts the type of flower an image is from a class of five flowers: daisy, dandelion, rose, sunflower, or tulip."
+        "#### Image\n",
+        "\n",
+        "The image dataset used for this tutorial is the [Flowers dataset](https://www.tensorflow.org/datasets/catalog/tf_flowers) from [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/overview). The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket. The trained model predicts the type of flower in a given image from a class of five flowers: daisy, dandelion, rose, sunflower, or tulip."
       ]
     },
     {
@@ -76,9 +84,9 @@
         "id": "dataset:gsod,lrg"
       },
       "source": [
-        "### Dataset\n",
+        "#### Tabular\n",
         "\n",
-        "The dataset used for this tutorial is the GSOD dataset from [BigQuery public datasets](https://cloud.google.com/bigquery/public-data). The version of the dataset you use only the fields year, month and day to predict the value of mean daily temperature (mean_temp)."
+        "The tabular dataset used for this tutorial is the GSOD dataset from [BigQuery public datasets](https://cloud.google.com/bigquery/public-data). The version of the dataset you use only the fields year, month and day to predict the value of mean daily temperature (mean_temp)."
       ]
     },
     {
@@ -87,9 +95,20 @@
         "id": "dataset:happydb,tcn"
       },
       "source": [
-        "### Dataset\n",
+        "#### Text\n",
         "\n",
-        "The dataset used for this tutorial is the [Happy Moments dataset](https://www.kaggle.com/ritresearch/happydb) from [Kaggle Datasets](https://www.kaggle.com/ritresearch/happydb). The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket."
+        "The text dataset used for this tutorial is the [Happy Moments dataset](https://www.kaggle.com/ritresearch/happydb) from [Kaggle Datasets](https://www.kaggle.com/ritresearch/happydb). The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "98eb93ec6faa"
+      },
+      "source": [
+        "#### Video\n",
+        "\n",
+        "The video dataset used for this tutorial is the golf swing recognition portion of the [Human Motion dataset](https://todo) from [MIT](http://cbcl.mit.edu/publications/ps/Kuehne_etal_iccv11.pdf). The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket. The trained model will predict the start frame where a golf swing begins."
       ]
     },
     {
@@ -108,11 +127,12 @@
         "\n",
         "The steps performed include:\n",
         "\n",
-        "- Train an image model.\n",
-        "- Export the image model as an edge model.\n",
-        "- Train a tabular model.\n",
-        "- Export the tabular model as a cloud model.\n",
-        "- Train a text model."
+        "- Train an image model\n",
+        "- Export the image model as an edge model\n",
+        "- Train a tabular model\n",
+        "- Export the tabular model as a cloud model\n",
+        "- Train a text model\n",
+        "- Train a video model"
       ]
     },
     {
@@ -125,9 +145,24 @@
         "\n",
         "When doing E2E MLOps on Google Cloud, the following are best practices for when to use AutoML:\n",
         "\n",
-        "**You have a limited amount of training data**\n",
+        "* **You have a limited amount of training data**\n",
         "\n",
-        "**You want to establish a baseline metric before experimenting with a custom model**"
+        "* **You want to establish a baseline metric before experimenting with a custom model**"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "fb3451ce8e47"
+      },
+      "source": [
+        "### Costs\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "- Vertex AI\n",
+        "- Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage pricing](https://cloud.google.com/storage/pricing) and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
       ]
     },
     {
@@ -138,7 +173,7 @@
       "source": [
         "## Installations\n",
         "\n",
-        "Install *one time* the packages for executing the MLOps notebooks."
+        "Install the packages required for executing this notebook."
       ]
     },
     {
@@ -149,20 +184,23 @@
       },
       "outputs": [],
       "source": [
-        "ONCE_ONLY = False\n",
-        "if ONCE_ONLY:\n",
-        "    ! pip3 install -U tensorflow==2.5 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-transform==1.2 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-io==0.18 $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-bigquery $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-logging $USER_FLAG\n",
-        "    ! pip3 install --upgrade apache-beam[gcp] $USER_FLAG\n",
-        "    ! pip3 install --upgrade pyarrow $USER_FLAG\n",
-        "    ! pip3 install --upgrade cloudml-hypertune $USER_FLAG\n",
-        "    ! pip3 install --upgrade kfp $USER_FLAG"
+        "import os\n",
+        "\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
+        "\n",
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "USER_FLAG = \"\"\n",
+        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\"\n",
+        "\n",
+        "# Install the packages\n",
+        "\n",
+        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG -q\n",
+        "! pip3 install --upgrade google-cloud-storage $USER_FLAG -q"
       ]
     },
     {
@@ -200,6 +238,23 @@
         "id": "project_id"
       },
       "source": [
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI, Compute Engine and Cloud Storage APIs](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component,storage_component).\n",
+        "\n",
+        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands.\n",
+        "\n",
         "#### Set your project ID\n",
         "\n",
         "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
@@ -270,7 +325,10 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -297,6 +355,67 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3ffa6b6c7cdb"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
+        "\n",
+        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
+        "\n",
+        "**Otherwise**, follow these steps:\n",
+        "\n",
+        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
+        "\n",
+        "1. **Click Create service account**.\n",
+        "\n",
+        "2. In the **Service account name** field, enter a name, and click **Create**.\n",
+        "\n",
+        "3. In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex AI\" into the filter box, and select **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "\n",
+        "4. Click Create. A JSON file that contains your key downloads to your local environment.\n",
+        "\n",
+        "5. Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2b72272258fc"
+      },
+      "outputs": [],
+      "source": [
+        "# If you are running this notebook in Colab, run this cell and follow the\n",
+        "# instructions to authenticate your GCP account. This provides access to your\n",
+        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
+        "# requests.\n",
+        "\n",
+        "import os\n",
+        "import sys\n",
+        "\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = False\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
+        "    if \"google.colab\" in sys.modules:\n",
+        "        IS_COLAB = True\n",
+        "        from google.colab import auth as google_auth\n",
+        "\n",
+        "        google_auth.authenticate_user()\n",
+        "\n",
+        "    # If you are running this notebook locally, replace the string below with the\n",
+        "    # path to your service account key and run this cell to authenticate your GCP\n",
+        "    # account.\n",
+        "    elif not os.getenv(\"IS_TESTING\"):\n",
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -320,7 +439,8 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
+        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
     {
@@ -331,8 +451,9 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
+        "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
       ]
     },
     {
@@ -352,7 +473,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_NAME"
+        "! gsutil mb -l $REGION $BUCKET_URI"
       ]
     },
     {
@@ -372,7 +493,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_NAME"
+        "! gsutil ls -al $BUCKET_URI"
       ]
     },
     {
@@ -395,7 +516,7 @@
       },
       "outputs": [],
       "source": [
-        "import google.cloud.aiplatform as aip"
+        "import google.cloud.aiplatform as aiplatform"
       ]
     },
     {
@@ -417,7 +538,7 @@
       },
       "outputs": [],
       "source": [
-        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_NAME)"
+        "aiplatform.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
       ]
     },
     {
@@ -448,7 +569,7 @@
       "source": [
         "## AutoML image models\n",
         "\n",
-        "AutoML can train the following types of models:\n",
+        "AutoML can train the following types of image models:\n",
         "\n",
         "- classification\n",
         "- objection detection\n",
@@ -504,10 +625,7 @@
       },
       "outputs": [],
       "source": [
-        "if \"IMPORT_FILES\" in globals():\n",
-        "    FILE = IMPORT_FILES[0]\n",
-        "else:\n",
-        "    FILE = IMPORT_FILE\n",
+        "FILE = IMPORT_FILE\n",
         "\n",
         "count = ! gsutil cat $FILE | wc -l\n",
         "print(\"Number of Examples\", int(count[0]))\n",
@@ -545,10 +663,10 @@
       },
       "outputs": [],
       "source": [
-        "dataset = aip.ImageDataset.create(\n",
-        "    display_name=\"Happy Moments\" + \"_\" + TIMESTAMP,\n",
+        "dataset = aiplatform.ImageDataset.create(\n",
+        "    display_name=\"flowers_\" + TIMESTAMP,\n",
         "    gcs_source=[IMPORT_FILE],\n",
-        "    import_schema_uri=aip.schema.dataset.ioformat.image.single_label_classification,\n",
+        "    import_schema_uri=aiplatform.schema.dataset.ioformat.image.single_label_classification,\n",
         ")\n",
         "\n",
         "print(dataset.resource_name)"
@@ -593,8 +711,8 @@
       },
       "outputs": [],
       "source": [
-        "dag = aip.AutoMLImageTrainingJob(\n",
-        "    display_name=\"happydb_\" + TIMESTAMP,\n",
+        "dag = aiplatform.AutoMLImageTrainingJob(\n",
+        "    display_name=\"flowers_\" + TIMESTAMP,\n",
         "    prediction_type=\"classification\",\n",
         "    multi_label=False,\n",
         "    model_type=\"MOBILE_TF_LOW_LATENCY_1\",\n",
@@ -612,14 +730,14 @@
       "source": [
         "#### Run the training pipeline\n",
         "\n",
-        "Next, you run the DAG to start the training job by invoking the method `run`, with the following parameters:\n",
+        "Next, you run the created DAG to start the training job by invoking the method `run`, with the following parameters:\n",
         "\n",
         "- `dataset`: The `Dataset` resource to train the model.\n",
         "- `model_display_name`: The human readable name for the trained model.\n",
         "- `training_fraction_split`: The percentage of the dataset to use for training.\n",
         "- `test_fraction_split`: The percentage of the dataset to use for test (holdout data).\n",
         "- `validation_fraction_split`: The percentage of the dataset to use for validation.\n",
-        "- `budget_milli_node_hours`: (optional) Maximum training time specified in unit of millihours (1000 = hour).\n",
+        "- `budget_milli_node_hours`: (optional) Maximum training time specified in unit of milli node-hours (1000 = node-hour).\n",
         "- `disable_early_stopping`: If `True`, training maybe completed before using the entire budget if the service believes it cannot further improve on the model objective measurements.\n",
         "\n",
         "The `run` method when completed returns the `Model` resource.\n",
@@ -637,7 +755,7 @@
       "source": [
         "model = dag.run(\n",
         "    dataset=dataset,\n",
-        "    model_display_name=\"happydb_\" + TIMESTAMP,\n",
+        "    model_display_name=\"flowers_\" + TIMESTAMP,\n",
         "    training_fraction_split=0.8,\n",
         "    validation_fraction_split=0.1,\n",
         "    test_fraction_split=0.1,\n",
@@ -653,9 +771,8 @@
       },
       "source": [
         "## Review model evaluation scores\n",
-        "After your model has finished training, you can review the evaluation scores for it.\n",
         "\n",
-        "First, you need to get a reference to the new model. As with datasets, you can either use the reference to the model variable you created when you deployed the model or you can list all of the models in your project."
+        "After your model training has finished, you can review the evaluation scores for it using the `list_model_evaluations()` method. This method will return an iterator for each evaluation slice."
       ]
     },
     {
@@ -666,18 +783,10 @@
       },
       "outputs": [],
       "source": [
-        "# Get model resource ID\n",
-        "models = aip.Model.list(filter=\"display_name=happydb_\" + TIMESTAMP)\n",
+        "model_evaluations = model.list_model_evaluations()\n",
         "\n",
-        "# Get a reference to the Model Service client\n",
-        "client_options = {\"api_endpoint\": f\"{REGION}-aiplatform.googleapis.com\"}\n",
-        "model_service_client = aip.gapic.ModelServiceClient(client_options=client_options)\n",
-        "\n",
-        "model_evaluations = model_service_client.list_model_evaluations(\n",
-        "    parent=models[0].resource_name\n",
-        ")\n",
-        "model_evaluation = list(model_evaluations)[0]\n",
-        "print(model_evaluation)"
+        "for model_evaluation in model_evaluations:\n",
+        "    print(model_evaluation.to_dict())"
       ]
     },
     {
@@ -721,7 +830,7 @@
       "source": [
         "### Get test item\n",
         "\n",
-        "You will use an arbitrary example out of the dataset as a test item. Don't be concerned that the example was likely used in training the model -- we just want to demonstrate how to make a prediction."
+        "You will use an arbitrary example out of the dataset as a test item. Don't be concerned that the example was likely used in training the model. You are just looking at how to make a prediction."
       ]
     },
     {
@@ -753,7 +862,7 @@
         "\n",
         "#### Request\n",
         "\n",
-        "Since in this example your test item is in a Cloud Storage bucket, you open and read the contents of the image using `tf.io.gfile.Gfile()`. To pass the test data to the prediction service, you encode the bytes into base64 -- which makes the content safe from modification while transmitting binary data over the network.\n",
+        "Since your test item is in a public Cloud Storage bucket in this example, you copy it to your bucket and read the contents of the image using `Cloud Storage SDK`. To pass the test data to the prediction service, you encode the bytes into base64 which makes the content safe from modification while transmitting binary data over the network.\n",
         "\n",
         "The format of each instance is:\n",
         "\n",
@@ -775,32 +884,66 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "predict_request:mbsdk,icn"
+        "id": "1c1d53e89beb"
       },
       "outputs": [],
       "source": [
         "import base64\n",
         "\n",
-        "import tensorflow as tf\n",
+        "from google.cloud import storage\n",
         "\n",
-        "with tf.io.gfile.GFile(test_item, \"rb\") as f:\n",
-        "    content = f.read()\n",
+        "# Copy the test image to the Cloud storage bucket as \"test.jpg\"\n",
+        "test_image_local = \"{}/test.jpg\".format(BUCKET_URI)\n",
+        "! gsutil cp $test_item $test_image_local\n",
+        "\n",
+        "# Download the test image in bytes format\n",
+        "storage_client = storage.Client(project=PROJECT_ID)\n",
+        "bucket = storage_client.bucket(bucket_name=BUCKET_NAME)\n",
+        "test_content = bucket.get_blob(\"test.jpg\").download_as_bytes()\n",
         "\n",
         "# The format of each instance should conform to the deployed model's prediction input schema.\n",
-        "instances = [{\"content\": base64.b64encode(content).decode(\"utf-8\")}]\n",
+        "instances = [{\"content\": base64.b64encode(test_content).decode(\"utf-8\")}]\n",
         "\n",
         "prediction = endpoint.predict(instances=instances)\n",
         "\n",
         "print(prediction)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3b1b67898533"
+      },
+      "source": [
+        "#### Alternate method using [GFile](https://www.tensorflow.org/api_docs/python/tf/io/gfile/GFile)\n",
+        "\n",
+        "Alternatively, [GFile](https://www.tensorflow.org/api_docs/python/tf/io/gfile/GFile) method from tensorflow-io library can be used to read the data from Cloud storage directly. The following code snippet does the same :\n",
+        "\n",
+        "```\n",
+        "import base64\n",
+        "import tensorflow as tf\n",
+        "\n",
+        "# Read the test file using GFile\n",
+        "with tf.io.gfile.GFile(test_item, \"rb\") as f:\n",
+        "    content = f.read()\n",
+        "\n",
+        "# The format of each instance should conform to the deployed model's prediction input schema.\n",
+        "instances = [{\"content\": base64.b64encode(content).decode(\"utf-8\")}]\n",
+        "\n",
+        "prediction = endpoint.predict(instances=instances)\n",
+        "\n",
+        "print(prediction)\n",
+        "```\n",
+        "Nevertheless, `tf.io.gfile.GFile` supports multiple file system implementations, including local files, Google Cloud Storage (using a gs:// prefix), and HDFS (using an hdfs:// prefix)."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "undeploy_model:mbsdk"
       },
       "source": [
-        "## Undeploy the model\n",
+        "#### Undeploy the model\n",
         "\n",
         "When you are done doing predictions, you undeploy the model from the `Endpoint` resouce. This deprovisions all compute resources and ends billing for the deployed model."
       ]
@@ -846,7 +989,7 @@
       "outputs": [],
       "source": [
         "response = model.export_model(\n",
-        "    artifact_destination=BUCKET_NAME, export_format_id=\"tflite\", sync=True\n",
+        "    artifact_destination=BUCKET_URI, export_format_id=\"tflite\", sync=True\n",
         ")\n",
         "\n",
         "model_package = response[\"artifactOutputUri\"]"
@@ -987,10 +1130,10 @@
       },
       "outputs": [],
       "source": [
-        "dataset = aip.TabularDataset.create(\n",
-        "    display_name=\"Happy Moments\" + \"_\" + TIMESTAMP,\n",
+        "dataset = aiplatform.TabularDataset.create(\n",
+        "    display_name=\"gsod_\" + TIMESTAMP,\n",
         "    bq_source=[IMPORT_FILE],\n",
-        "    labels={\"user_metadata\": BUCKET_NAME[5:]},\n",
+        "    labels={\"user_metadata\": BUCKET_NAME},\n",
         ")\n",
         "\n",
         "label_column = \"mean_temp\"\n",
@@ -1046,9 +1189,7 @@
         "  - regression:\n",
         "    - `minimize-rmse`\n",
         "    - `minimize-mae`\n",
-        "    - `minimize-rmsle`\n",
-        "\n",
-        "The instantiated object is the DAG (directed acyclic graph) for the training pipeline."
+        "    - `minimize-rmsle`"
       ]
     },
     {
@@ -1059,8 +1200,8 @@
       },
       "outputs": [],
       "source": [
-        "dag = aip.AutoMLTabularTrainingJob(\n",
-        "    display_name=\"happydb_\" + TIMESTAMP,\n",
+        "dag = aiplatform.AutoMLTabularTrainingJob(\n",
+        "    display_name=\"gsod_\" + TIMESTAMP,\n",
         "    optimization_prediction_type=\"regression\",\n",
         "    optimization_objective=\"minimize-rmse\",\n",
         "    column_transformations=TRANSFORMATIONS,\n",
@@ -1077,7 +1218,7 @@
       "source": [
         "#### Run the training pipeline\n",
         "\n",
-        "Next, you run the DAG to start the training job by invoking the method `run`, with the following parameters:\n",
+        "Next, you run the created DAG to start the training job by invoking the method `run`, with the following parameters:\n",
         "\n",
         "- `dataset`: The `Dataset` resource to train the model.\n",
         "- `model_display_name`: The human readable name for the trained model.\n",
@@ -1103,7 +1244,7 @@
       "source": [
         "model = dag.run(\n",
         "    dataset=dataset,\n",
-        "    model_display_name=\"happydb_\" + TIMESTAMP,\n",
+        "    model_display_name=\"gsod_\" + TIMESTAMP,\n",
         "    training_fraction_split=0.8,\n",
         "    validation_fraction_split=0.1,\n",
         "    test_fraction_split=0.1,\n",
@@ -1120,9 +1261,8 @@
       },
       "source": [
         "## Review model evaluation scores\n",
-        "After your model has finished training, you can review the evaluation scores for it.\n",
         "\n",
-        "First, you need to get a reference to the new model. As with datasets, you can either use the reference to the model variable you created when you deployed the model or you can list all of the models in your project."
+        "After your model training has finished, you can review the evaluation scores for it using the `list_model_evaluations()` method. This method will return an iterator for each evaluation slice."
       ]
     },
     {
@@ -1133,18 +1273,10 @@
       },
       "outputs": [],
       "source": [
-        "# Get model resource ID\n",
-        "models = aip.Model.list(filter=\"display_name=happydb_\" + TIMESTAMP)\n",
+        "model_evaluations = model.list_model_evaluations()\n",
         "\n",
-        "# Get a reference to the Model Service client\n",
-        "client_options = {\"api_endpoint\": f\"{REGION}-aiplatform.googleapis.com\"}\n",
-        "model_service_client = aip.gapic.ModelServiceClient(client_options=client_options)\n",
-        "\n",
-        "model_evaluations = model_service_client.list_model_evaluations(\n",
-        "    parent=models[0].resource_name\n",
-        ")\n",
-        "model_evaluation = list(model_evaluations)[0]\n",
-        "print(model_evaluation)"
+        "for model_evaluation in model_evaluations:\n",
+        "    print(model_evaluation.to_dict())"
       ]
     },
     {
@@ -1177,7 +1309,7 @@
         "id": "undeploy_model:mbsdk"
       },
       "source": [
-        "## Undeploy the model\n",
+        "#### Undeploy the model\n",
         "\n",
         "When you are done doing predictions, you undeploy the model from the `Endpoint` resouce. This deprovisions all compute resources and ends billing for the deployed model."
       ]
@@ -1218,7 +1350,7 @@
       "outputs": [],
       "source": [
         "response = model.export_model(\n",
-        "    artifact_destination=BUCKET_NAME, export_format_id=\"tf-saved-model\", sync=True\n",
+        "    artifact_destination=BUCKET_URI, export_format_id=\"tf-saved-model\", sync=True\n",
         ")\n",
         "\n",
         "model_package = response[\"artifactOutputUri\"]"
@@ -1350,10 +1482,7 @@
       },
       "outputs": [],
       "source": [
-        "if \"IMPORT_FILES\" in globals():\n",
-        "    FILE = IMPORT_FILES[0]\n",
-        "else:\n",
-        "    FILE = IMPORT_FILE\n",
+        "FILE = IMPORT_FILE\n",
         "\n",
         "count = ! gsutil cat $FILE | wc -l\n",
         "print(\"Number of Examples\", int(count[0]))\n",
@@ -1391,10 +1520,10 @@
       },
       "outputs": [],
       "source": [
-        "dataset = aip.TextDataset.create(\n",
-        "    display_name=\"Happy Moments\" + \"_\" + TIMESTAMP,\n",
+        "dataset = aiplatform.TextDataset.create(\n",
+        "    display_name=\"happydb_\" + TIMESTAMP,\n",
         "    gcs_source=[IMPORT_FILE],\n",
-        "    import_schema_uri=aip.schema.dataset.ioformat.text.single_label_classification,\n",
+        "    import_schema_uri=aiplatform.schema.dataset.ioformat.text.single_label_classification,\n",
         ")\n",
         "\n",
         "print(dataset.resource_name)"
@@ -1420,9 +1549,7 @@
         "  - `sentiment`: A text sentiment analysis model.\n",
         "  - `extraction`: A text entity extraction model.\n",
         "- `multi_label`: If a classification task, whether single (False) or multi-labeled (True).\n",
-        "- `sentiment_max`: If a sentiment analysis task, the maximum sentiment value.\n",
-        "\n",
-        "The instantiated object is the DAG (directed acyclic graph) for the training pipeline."
+        "- `sentiment_max`: If a sentiment analysis task, the maximum sentiment value.\n"
       ]
     },
     {
@@ -1433,7 +1560,7 @@
       },
       "outputs": [],
       "source": [
-        "dag = aip.AutoMLTextTrainingJob(\n",
+        "dag = aiplatform.AutoMLTextTrainingJob(\n",
         "    display_name=\"happydb_\" + TIMESTAMP,\n",
         "    prediction_type=\"classification\",\n",
         "    multi_label=False,\n",
@@ -1450,7 +1577,7 @@
       "source": [
         "#### Run the training pipeline\n",
         "\n",
-        "Next, you run the DAG to start the training job by invoking the method `run`, with the following parameters:\n",
+        "Next, you run the created DAG to start the training job by invoking the method `run`, with the following parameters:\n",
         "\n",
         "- `dataset`: The `Dataset` resource to train the model.\n",
         "- `model_display_name`: The human readable name for the trained model.\n",
@@ -1487,9 +1614,8 @@
       },
       "source": [
         "## Review model evaluation scores\n",
-        "After your model has finished training, you can review the evaluation scores for it.\n",
         "\n",
-        "First, you need to get a reference to the new model. As with datasets, you can either use the reference to the model variable you created when you deployed the model or you can list all of the models in your project."
+        "After your model training has finished, you can review the evaluation scores for it using the `list_model_evaluations()` method. This method will return an iterator for each evaluation slice."
       ]
     },
     {
@@ -1500,18 +1626,10 @@
       },
       "outputs": [],
       "source": [
-        "# Get model resource ID\n",
-        "models = aip.Model.list(filter=\"display_name=happydb_\" + TIMESTAMP)\n",
+        "model_evaluations = model.list_model_evaluations()\n",
         "\n",
-        "# Get a reference to the Model Service client\n",
-        "client_options = {\"api_endpoint\": f\"{REGION}-aiplatform.googleapis.com\"}\n",
-        "model_service_client = aip.gapic.ModelServiceClient(client_options=client_options)\n",
-        "\n",
-        "model_evaluations = model_service_client.list_model_evaluations(\n",
-        "    parent=models[0].resource_name\n",
-        ")\n",
-        "model_evaluation = list(model_evaluations)[0]\n",
-        "print(model_evaluation)"
+        "for model_evaluation in model_evaluations:\n",
+        "    print(model_evaluation.to_dict())"
       ]
     },
     {
@@ -1542,7 +1660,7 @@
         "id": "undeploy_model:mbsdk"
       },
       "source": [
-        "## Undeploy the model\n",
+        "#### Undeploy the model\n",
         "\n",
         "When you are done doing predictions, you undeploy the model from the `Endpoint` resouce. This deprovisions all compute resources and ends billing for the deployed model."
       ]
@@ -1686,10 +1804,7 @@
       },
       "outputs": [],
       "source": [
-        "if \"IMPORT_FILES\" in globals():\n",
-        "    FILE = IMPORT_FILES[0]\n",
-        "else:\n",
-        "    FILE = IMPORT_FILE\n",
+        "FILE = IMPORT_FILE\n",
         "\n",
         "count = ! gsutil cat $FILE | wc -l\n",
         "print(\"Number of Examples\", int(count[0]))\n",
@@ -1726,10 +1841,10 @@
       },
       "outputs": [],
       "source": [
-        "dataset = aip.VideoDataset.create(\n",
-        "    display_name=\"Happy Moments\" + \"_\" + TIMESTAMP,\n",
+        "dataset = aiplatform.VideoDataset.create(\n",
+        "    display_name=\"human_motion_\" + TIMESTAMP,\n",
         "    gcs_source=[IMPORT_FILE],\n",
-        "    import_schema_uri=aip.schema.dataset.ioformat.video.classification,\n",
+        "    import_schema_uri=aiplatform.schema.dataset.ioformat.video.classification,\n",
         ")\n",
         "\n",
         "print(dataset.resource_name)"
@@ -1753,9 +1868,7 @@
         "- `prediction_type`: The type task to train the model for.\n",
         "  - `classification`: A video classification model.\n",
         "  - `object_tracking`: A video object tracking model.\n",
-        "  - `action_recognition`: A video action recognition model.\n",
-        "\n",
-        "The instantiated object is the DAG (directed acyclic graph) for the training pipeline."
+        "  - `action_recognition`: A video action recognition model."
       ]
     },
     {
@@ -1766,8 +1879,8 @@
       },
       "outputs": [],
       "source": [
-        "dag = aip.AutoMLVideoTrainingJob(\n",
-        "    display_name=\"happydb_\" + TIMESTAMP,\n",
+        "dag = aiplatform.AutoMLVideoTrainingJob(\n",
+        "    display_name=\"human_motion_\" + TIMESTAMP,\n",
         "    prediction_type=\"classification\",\n",
         ")\n",
         "\n",
@@ -1782,7 +1895,7 @@
       "source": [
         "#### Run the training pipeline\n",
         "\n",
-        "Next, you run the DAG to start the training job by invoking the method `run`, with the following parameters:\n",
+        "Next, you run the created DAG to start the training job by invoking the method `run`, with the following parameters:\n",
         "\n",
         "- `dataset`: The `Dataset` resource to train the model.\n",
         "- `model_display_name`: The human readable name for the trained model.\n",
@@ -1804,7 +1917,7 @@
       "source": [
         "model = dag.run(\n",
         "    dataset=dataset,\n",
-        "    model_display_name=\"happydb_\" + TIMESTAMP,\n",
+        "    model_display_name=\"human_motion_\" + TIMESTAMP,\n",
         "    training_fraction_split=0.8,\n",
         "    test_fraction_split=0.2,\n",
         ")"
@@ -1817,9 +1930,8 @@
       },
       "source": [
         "## Review model evaluation scores\n",
-        "After your model has finished training, you can review the evaluation scores for it.\n",
         "\n",
-        "First, you need to get a reference to the new model. As with datasets, you can either use the reference to the model variable you created when you deployed the model or you can list all of the models in your project."
+        "After your model training has finished, you can review the evaluation scores for it using the `list_model_evaluations()` method. This method will return an iterator for each evaluation slice."
       ]
     },
     {
@@ -1830,18 +1942,10 @@
       },
       "outputs": [],
       "source": [
-        "# Get model resource ID\n",
-        "models = aip.Model.list(filter=\"display_name=happydb_\" + TIMESTAMP)\n",
+        "model_evaluations = model.list_model_evaluations()\n",
         "\n",
-        "# Get a reference to the Model Service client\n",
-        "client_options = {\"api_endpoint\": f\"{REGION}-aiplatform.googleapis.com\"}\n",
-        "model_service_client = aip.gapic.ModelServiceClient(client_options=client_options)\n",
-        "\n",
-        "model_evaluations = model_service_client.list_model_evaluations(\n",
-        "    parent=models[0].resource_name\n",
-        ")\n",
-        "model_evaluation = list(model_evaluations)[0]\n",
-        "print(model_evaluation)"
+        "for model_evaluation in model_evaluations:\n",
+        "    print(model_evaluation.to_dict())"
       ]
     },
     {
@@ -1899,16 +2003,7 @@
         "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
         "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
         "\n",
-        "Otherwise, you can delete the individual resources you created in this tutorial:\n",
-        "\n",
-        "- Dataset\n",
-        "- Pipeline\n",
-        "- Model\n",
-        "- Endpoint\n",
-        "- Batch Job\n",
-        "- Custom Job\n",
-        "- Hyperparameter Tuning Job\n",
-        "- Cloud Storage Bucket"
+        "Otherwise, you can delete the individual resources you created in this tutorial.\n"
       ]
     },
     {
@@ -1919,66 +2014,11 @@
       },
       "outputs": [],
       "source": [
-        "delete_dataset = True\n",
-        "delete_pipeline = True\n",
-        "delete_model = True\n",
-        "delete_endpoint = True\n",
-        "delete_batchjob = True\n",
-        "delete_customjob = True\n",
-        "delete_hptjob = True\n",
-        "delete_bucket = True\n",
+        "# Set this to true only if you'd like to delete your bucket\n",
+        "delete_bucket = False\n",
         "\n",
-        "# Delete the dataset using the Vertex fully qualified identifier for the dataset\n",
-        "try:\n",
-        "    if delete_dataset and \"dataset_id\" in globals():\n",
-        "        clients[\"dataset\"].delete_dataset(name=dataset_id)\n",
-        "except Exception as e:\n",
-        "    print(e)\n",
-        "\n",
-        "# Delete the training pipeline using the Vertex fully qualified identifier for the pipeline\n",
-        "try:\n",
-        "    if delete_pipeline and \"pipeline_id\" in globals():\n",
-        "        clients[\"pipeline\"].delete_training_pipeline(name=pipeline_id)\n",
-        "except Exception as e:\n",
-        "    print(e)\n",
-        "\n",
-        "# Delete the model using the Vertex fully qualified identifier for the model\n",
-        "try:\n",
-        "    if delete_model and \"model_to_deploy_id\" in globals():\n",
-        "        clients[\"model\"].delete_model(name=model_to_deploy_id)\n",
-        "except Exception as e:\n",
-        "    print(e)\n",
-        "\n",
-        "# Delete the endpoint using the Vertex fully qualified identifier for the endpoint\n",
-        "try:\n",
-        "    if delete_endpoint and \"endpoint_id\" in globals():\n",
-        "        clients[\"endpoint\"].delete_endpoint(name=endpoint_id)\n",
-        "except Exception as e:\n",
-        "    print(e)\n",
-        "\n",
-        "# Delete the batch job using the Vertex fully qualified identifier for the batch job\n",
-        "try:\n",
-        "    if delete_batchjob and \"batch_job_id\" in globals():\n",
-        "        clients[\"job\"].delete_batch_prediction_job(name=batch_job_id)\n",
-        "except Exception as e:\n",
-        "    print(e)\n",
-        "\n",
-        "# Delete the custom job using the Vertex fully qualified identifier for the custom job\n",
-        "try:\n",
-        "    if delete_customjob and \"job_id\" in globals():\n",
-        "        clients[\"job\"].delete_custom_job(name=job_id)\n",
-        "except Exception as e:\n",
-        "    print(e)\n",
-        "\n",
-        "# Delete the hyperparameter tuning job using the Vertex fully qualified identifier for the hyperparameter tuning job\n",
-        "try:\n",
-        "    if delete_hptjob and \"hpt_job_id\" in globals():\n",
-        "        clients[\"job\"].delete_hyperparameter_tuning_job(name=hpt_job_id)\n",
-        "except Exception as e:\n",
-        "    print(e)\n",
-        "\n",
-        "if delete_bucket and \"BUCKET_NAME\" in globals():\n",
-        "    ! gsutil rm -r $BUCKET_NAME"
+        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "    ! gsutil rm -r $BUCKET_URI"
       ]
     }
   ],

notebooks/community/ml_ops/stage2/get_started_vertex_distributed_training.ipynb

@@ -8,7 +8,7 @@
       },
       "outputs": [],
       "source": [
-        "# Copyright 2021 Google LLC\n",
+        "# Copyright 2022 Google LLC\n",
         "#\n",
         "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
         "# you may not use this file except in compliance with the License.\n",
@@ -32,6 +32,11 @@
         "# E2E ML on GCP: MLOps stage 2 : experimentation: get started with Vertex Distributed Training\n",
         "\n",
         "<table align=\"left\">\n",
+        "      <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_distributed_training.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
         "  <td>\n",
         "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_distributed_training.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
@@ -39,8 +44,9 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_distributed_training.ipynb\">\n",
-        "      Open in Google Cloud Notebooks\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage2/get_started_vertex_distributed_training.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "Open in Vertex AI Workbench\n",
         "    </a>\n",
         "  </td>\n",
         "</table>\n",
@@ -56,7 +62,7 @@
         "## Overview\n",
         "\n",
         "\n",
-        "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 2 : experimentation: get started with Vertex Distributed Training."
+        "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 2 : experimentation: get started with Vertex Distributed Training. Please note: There are incompatibilities between Colab and Docker and the Docker section may not work until resolved by the platform."
       ]
     },
     {
@@ -100,6 +106,15 @@
         "id": "recommendation:mlops,stage2,vertex,distributed_training"
       },
       "source": [
+        "### Costs\n",
+        " \n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "Vertex AI\n",
+        "Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage pricing](https://cloud.google.com/storage/pricing), and use the [Pricing Calculator](https://cloud.google.com/products/calculator/),\n",
+        "        to generate a cost estimate based on your projected usage.\n",
         "### Recommendations\n",
         "\n",
         "When doing E2E MLOps on Google Cloud, the following are best practices for when to use Vertex AI Distributed Training:\n",
@@ -126,59 +141,58 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "install_mlops"
+        "id": "XkYpRvOQyVYb"
       },
       "source": [
-        "## Installations\n",
+        "### Install additional packages\n",
         "\n",
-        "Install *one time* the packages for executing the MLOps notebooks."
+        "Install the packages required for executing this notebook."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "install_mlops"
+        "id": "xs_Kt8RcyXTC"
       },
       "outputs": [],
       "source": [
-        "ONCE_ONLY = False\n",
-        "if ONCE_ONLY:\n",
-        "    ! pip3 install -U tensorflow==2.5 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-transform==1.2 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-io==0.18 $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-bigquery $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-logging $USER_FLAG\n",
-        "    ! pip3 install --upgrade apache-beam[gcp] $USER_FLAG\n",
-        "    ! pip3 install --upgrade pyarrow $USER_FLAG\n",
-        "    ! pip3 install --upgrade cloudml-hypertune $USER_FLAG\n",
-        "    ! pip3 install --upgrade kfp $USER_FLAG\n",
-        "    ! pip3 install --upgrade torchvision $USER_FLAG\n",
-        "    ! pip3 install --upgrade rpy2 $USER_FLAG"
+        "import os\n",
+        "\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
+        "\n",
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "USER_FLAG = \"\"\n",
+        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\"\n",
+        "\n",
+        "! pip3 install {USER_FLAG} --upgrade google-cloud-aiplatform"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "restart"
+        "id": "oQhwq1iozAxh"
       },
       "source": [
         "### Restart the kernel\n",
         "\n",
-        "Once you've installed the additional packages, you need to restart the notebook kernel so it can find the packages."
+        "After you install the additional packages, you need to restart the notebook kernel so it can find the packages."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "restart"
+        "id": "zo3YFZXLzCRJ"
       },
       "outputs": [],
       "source": [
+        "# Automatically restart kernel after installs\n",
         "import os\n",
         "\n",
         "if not os.getenv(\"IS_TESTING\"):\n",
@@ -189,6 +203,32 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "84cd83853240"
+      },
+      "source": [
+        "## Before you begin\n",
+        "\n",
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI, BigQuery, Compute Engine and Cloud Storage APIs](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,bigquery,compute_component,storage_component).\n",
+        "\n",
+        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -208,6 +248,8 @@
       },
       "outputs": [],
       "source": [
+        "import os\n",
+        "\n",
         "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
       ]
     },
@@ -261,11 +303,14 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "region"
+        "id": "qohAA9fJulvP"
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -283,7 +328,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "timestamp"
+        "id": "8NKwwe7aulvQ"
       },
       "outputs": [],
       "source": [
@@ -292,6 +337,82 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "poKeKYG8ulvQ"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
+        "authenticated. Skip this step."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MIpJGzF9ulvQ"
+      },
+      "source": [
+        "**If you are using Colab**, run the cell below and follow the instructions\n",
+        "when prompted to authenticate your account via oAuth.\n",
+        "\n",
+        "**Otherwise**, follow these steps:\n",
+        "\n",
+        "1. In the Cloud Console, go to the [**Create service account key**\n",
+        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
+        "\n",
+        "2. Click **Create service account**.\n",
+        "\n",
+        "3. In the **Service account name** field, enter a name, and\n",
+        "   click **Create**.\n",
+        "\n",
+        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
+        "into the filter box, and select\n",
+        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "\n",
+        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
+        "local environment.\n",
+        "\n",
+        "6. Enter the path to your service account key as the\n",
+        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Vh6KDXB5ulvQ"
+      },
+      "outputs": [],
+      "source": [
+        "# If you are running this notebook in Colab, run this cell and follow the\n",
+        "# instructions to authenticate your GCP account. This provides access to your\n",
+        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
+        "# requests.\n",
+        "\n",
+        "import os\n",
+        "import sys\n",
+        "\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = False\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
+        "    if \"google.colab\" in sys.modules:\n",
+        "        IS_COLAB = True\n",
+        "        from google.colab import auth as google_auth\n",
+        "\n",
+        "        google_auth.authenticate_user()\n",
+        "\n",
+        "    # If you are running this notebook locally, replace the string below with the\n",
+        "    # path to your service account key and run this cell to authenticate your GCP\n",
+        "    # account.\n",
+        "    elif not os.getenv(\"IS_TESTING\"):\n",
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -315,7 +436,8 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
+        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
     {
@@ -326,8 +448,9 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
+        "    BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
     {
@@ -343,11 +466,11 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "create_bucket"
+        "id": "Moosy2rOulvR"
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_NAME"
+        "! gsutil mb -l $REGION $BUCKET_URI"
       ]
     },
     {
@@ -363,11 +486,11 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "validate_bucket"
+        "id": "56irx2CvulvS"
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_NAME"
+        "! gsutil ls -al $BUCKET_URI"
       ]
     },
     {
@@ -408,11 +531,11 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "init_aip:mbsdk"
+        "id": "wbvYPSTDulvS"
       },
       "outputs": [],
       "source": [
-        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_NAME)"
+        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
       ]
     },
     {
@@ -441,7 +564,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "accelerators:training,prediction,ngpu,mbsdk"
+        "id": "PryARdnoulvT"
       },
       "outputs": [],
       "source": [
@@ -483,14 +606,14 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "container:training,prediction"
+        "id": "LhhUFw2nulvT"
       },
       "outputs": [],
       "source": [
         "if os.getenv(\"IS_TESTING_TF\"):\n",
         "    TF = os.getenv(\"IS_TESTING_TF\")\n",
         "else:\n",
-        "    TF = \"2.1\".replace(\".\", \"-\")\n",
+        "    TF = \"2.5\".replace(\".\", \"-\")\n",
         "\n",
         "if TF[0] == \"2\":\n",
         "    if TRAIN_GPU:\n",
@@ -551,7 +674,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "machine:training"
+        "id": "vytMaukeulvT"
       },
       "outputs": [],
       "source": [
@@ -613,7 +736,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "create_custom_pp_training_job:mbsdk"
+        "id": "mhw34XoOulvU"
       },
       "outputs": [],
       "source": [
@@ -621,7 +744,7 @@
         "\n",
         "job = aip.CustomPythonPackageTrainingJob(\n",
         "    display_name=DISPLAY_NAME,\n",
-        "    python_package_gcs_uri=f\"{BUCKET_NAME}/trainer_boston.tar.gz\",\n",
+        "    python_package_gcs_uri=f\"{BUCKET_URI}/trainer_boston.tar.gz\",\n",
         "    python_module_name=\"trainer.task\",\n",
         "    container_uri=TRAIN_IMAGE,\n",
         "    model_serving_container_image_uri=DEPLOY_IMAGE,\n",
@@ -662,7 +785,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "examine_training_package"
+        "id": "IAaZpZyyulvU"
       },
       "outputs": [],
       "source": [
@@ -711,7 +834,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "taskpy_contents:mirrored,boston"
+        "id": "zKzddzl6ulvV"
       },
       "outputs": [],
       "source": [
@@ -767,6 +890,13 @@
         "    strategy = tf.distribute.MultiWorkerMirroredStrategy()\n",
         "    logging.info(\"Multi-worker Strategy distributed training\")\n",
         "    logging.info('TF_CONFIG = {}'.format(os.environ.get('TF_CONFIG', 'Not found')))\n",
+        "    # Single Machine, multiple TPU devices\n",
+        "elif args.distribute == 'tpu':\n",
+        "    cluster_resolver = tf.distribute.cluster_resolver.TPUClusterResolver(tpu=\"local\")\n",
+        "    tf.config.experimental_connect_to_cluster(cluster_resolver)\n",
+        "    tf.tpu.experimental.initialize_tpu_system(cluster_resolver)\n",
+        "    strategy = tf.distribute.TPUStrategy(cluster_resolver)\n",
+        "    print(\"All devices: \", tf.config.list_logical_devices('TPU'))\n",
         "\n",
         "logging.info('num_replicas_in_sync = {}'.format(strategy.num_replicas_in_sync))\n",
         "\n",
@@ -825,8 +955,11 @@
         "    else:\n",
         "        task_type, task_id = None, None\n",
         "\n",
+        "    if args.distribute==\"tpu\":\n",
+        "        save_locally = tf.saved_model.SaveOptions(experimental_io_device='/job:localhost')\n",
+        "        model.save(args.model_dir, options=save_locally)\n",
         "    # single, mirrored or primary for multiworker\n",
-        "    if _is_chief(task_type, task_id):\n",
+        "    elif _is_chief(task_type, task_id):\n",
         "        model.save(args.model_dir)\n",
         "    # non-primary workers for multi-workers\n",
         "    else:\n",
@@ -860,14 +993,14 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "tarball_training_script"
+        "id": "LFUHioqTulvV"
       },
       "outputs": [],
       "source": [
         "! rm -f custom.tar custom.tar.gz\n",
         "! tar cvf custom.tar custom\n",
         "! gzip custom.tar\n",
-        "! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_boston.tar.gz"
+        "! gsutil cp custom.tar.gz $BUCKET_URI/trainer_boston.tar.gz"
       ]
     },
     {
@@ -885,11 +1018,11 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "run_custom_pp_training_job:mirrored"
+        "id": "LnUX0UkvulvV"
       },
       "outputs": [],
       "source": [
-        "MODEL_DIR = BUCKET_NAME\n",
+        "MODEL_DIR = BUCKET_URI\n",
         "\n",
         "CMDARGS = [\"--epochs=5\", \"--batch_size=16\", \"--distribute=mirrored\"]\n",
         "\n",
@@ -920,7 +1053,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "delete_job"
+        "id": "iUWHFpPoulvW"
       },
       "outputs": [],
       "source": [
@@ -942,7 +1075,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "model_delete:mbsdk"
+        "id": "-0gqCUTEulvW"
       },
       "outputs": [],
       "source": [
@@ -1027,7 +1160,7 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "create_custom_pp_training_job:mbsdk"
+        "id": "aXvPN8P6ulvX"
       },
       "source": [
         "### Create and run custom training job\n",
@@ -1053,7 +1186,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "create_custom_pp_training_job:mbsdk"
+        "id": "kYcFsVSEulvX"
       },
       "outputs": [],
       "source": [
@@ -1061,7 +1194,7 @@
         "\n",
         "job = aip.CustomPythonPackageTrainingJob(\n",
         "    display_name=DISPLAY_NAME,\n",
-        "    python_package_gcs_uri=f\"{BUCKET_NAME}/trainer_boston.tar.gz\",\n",
+        "    python_package_gcs_uri=f\"{BUCKET_URI}/trainer_boston.tar.gz\",\n",
         "    python_module_name=\"trainer.task\",\n",
         "    container_uri=TRAIN_IMAGE,\n",
         "    model_serving_container_image_uri=DEPLOY_IMAGE,\n",
@@ -1084,11 +1217,11 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "run_custom_pp_training_job:multiworker"
+        "id": "GHRxPU32ulvX"
       },
       "outputs": [],
       "source": [
-        "MODEL_DIR = BUCKET_NAME\n",
+        "MODEL_DIR = BUCKET_URI\n",
         "\n",
         "CMDARGS = [\"--epochs=5\", \"--batch_size=16\", \"--distribute=multiworker\"]\n",
         "\n",
@@ -1111,7 +1244,7 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "delete_job"
+        "id": "92D_hbuVulvX"
       },
       "source": [
         "### Delete a custom training job\n",
@@ -1123,7 +1256,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "delete_job"
+        "id": "CqrfWkB3ulvX"
       },
       "outputs": [],
       "source": [
@@ -1175,14 +1308,13 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "write_docker_file:training,multiworker"
+        "id": "pGI2viDAulvY"
       },
       "outputs": [],
       "source": [
         "%%writefile custom/Dockerfile\n",
         "\n",
         "FROM gcr.io/deeplearning-platform-release/tf2-gpu.2-5\n",
-        "WORKDIR /root\n",
         "\n",
         "WORKDIR /\n",
         "\n",
@@ -1208,7 +1340,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "name_container:training"
+        "id": "7P8cdlFtulvY"
       },
       "outputs": [],
       "source": [
@@ -1228,11 +1360,15 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "build_container:training"
+        "id": "jmw5cakNulvY"
       },
       "outputs": [],
       "source": [
-        "! docker build custom -t $TRAIN_IMAGE"
+        "if not IS_COLAB:\n",
+        "    ! docker build custom -t $TRAIN_IMAGE\n",
+        "else:\n",
+        "    # install docker daemon\n",
+        "    ! apt-get -qq install docker.io"
       ]
     },
     {
@@ -1250,11 +1386,12 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "test_container:training"
+        "id": "jJGLjU-TulvZ"
       },
       "outputs": [],
       "source": [
-        "! docker run $TRAIN_IMAGE --epochs=5 --model-dir=./"
+        "if not IS_COLAB:\n",
+        "    ! docker run $TRAIN_IMAGE --epochs=5 --model-dir=./"
       ]
     },
     {
@@ -1272,11 +1409,42 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "register_container:training"
+        "id": "GAXGjae7ulvZ"
       },
       "outputs": [],
       "source": [
-        "! docker push $TRAIN_IMAGE"
+        "if not IS_COLAB:\n",
+        "    ! docker push $TRAIN_IMAGE"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "f50e9c553fb7"
+      },
+      "source": [
+        "*Executes in Colab*"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "a7e8c98f1e56"
+      },
+      "outputs": [],
+      "source": [
+        "%%bash -s $IS_COLAB $TRAIN_IMAGE\n",
+        "if [ $1 == \"False\" ]; then\n",
+        "  exit 0\n",
+        "fi\n",
+        "set -x\n",
+        "dockerd -b none --iptables=0 -l warn &\n",
+        "for i in $(seq 5); do [ ! -S \"/var/run/docker.sock\" ] && sleep 2 || break; done\n",
+        "docker build custom -t $2\n",
+        "docker run $2 --epochs=5 --model-dir=./\n",
+        "docker push $2\n",
+        "kill $(jobs -p)"
       ]
     },
     {
@@ -1296,13 +1464,13 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "worker_pool_primary"
+        "id": "CEAnXBzCulvZ"
       },
       "outputs": [],
       "source": [
         "PRIMARY_COMPUTE = \"n2-highcpu-64\"\n",
         "\n",
-        "MODEL_DIR = BUCKET_NAME\n",
+        "MODEL_DIR = BUCKET_URI\n",
         "\n",
         "CMDARGS = [\n",
         "    \"--model-dir=\" + MODEL_DIR,\n",
@@ -1339,7 +1507,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "worker_pool_training"
+        "id": "6dchPSfNulvZ"
       },
       "outputs": [],
       "source": [
@@ -1375,7 +1543,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "custom_job:worker_pool"
+        "id": "m2VgmqEOulva"
       },
       "outputs": [],
       "source": [
@@ -1399,7 +1567,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "run_custom_job:multiworker"
+        "id": "hg8vnI_Wulva"
       },
       "outputs": [],
       "source": [
@@ -1413,7 +1581,7 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "delete_job"
+        "id": "WT76Sc-culva"
       },
       "source": [
         "### Delete a custom training job\n",
@@ -1425,7 +1593,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "delete_job"
+        "id": "I_IxVfuDulva"
       },
       "outputs": [],
       "source": [
@@ -1474,7 +1642,7 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "custom_job:worker_pool"
+        "id": "L8Av8ATVulvb"
       },
       "source": [
         "### Create CustomJob with worker pool specifications\n",
@@ -1490,7 +1658,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "custom_job:worker_pool"
+        "id": "TUWEP1Lmulvb"
       },
       "outputs": [],
       "source": [
@@ -1502,7 +1670,7 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "run_custom_job:multiworker"
+        "id": "_95FH8jeulvb"
       },
       "source": [
         "### Run the CustomJob\n",
@@ -1514,7 +1682,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "run_custom_job:multiworker"
+        "id": "IEbrY05Gulvb"
       },
       "outputs": [],
       "source": [
@@ -1528,7 +1696,7 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "delete_job"
+        "id": "8R2Bnmwmulvb"
       },
       "source": [
         "### Delete a custom training job\n",
@@ -1540,7 +1708,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "delete_job"
+        "id": "s1geVE3Lulvb"
       },
       "outputs": [],
       "source": [
@@ -1583,14 +1751,14 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "docker_write:tpu"
+        "id": "nQVPtknpulvb"
       },
       "outputs": [],
       "source": [
         "%%writefile custom/Dockerfile\n",
         "FROM python:3.8\n",
         "\n",
-        "WORKDIR /root\n",
+        "WORKDIR /\n",
         "\n",
         "# Copies the trainer code to the docker image.\n",
         "COPY trainer /trainer\n",
@@ -1622,11 +1790,11 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "docker_push:tpu"
+        "id": "J_d_zEXUulvc"
       },
       "outputs": [],
       "source": [
-        "TRAIN_IMAGE = f\"gcr.io/\" + PROJECT_ID + \"/tpu-train:latest\"\n",
+        "TRAIN_IMAGE = \"gcr.io/\" + PROJECT_ID + \"/tpu-train:latest\"\n",
         "\n",
         "os.chdir(\"custom\")\n",
         "! docker build --quiet --tag={TRAIN_IMAGE} .\n",
@@ -1653,7 +1821,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "worker_pool_tpu"
+        "id": "d514eU7lulvc"
       },
       "outputs": [],
       "source": [
@@ -1701,7 +1869,7 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "custom_job:worker_pool"
+        "id": "RruSqNfrulvc"
       },
       "source": [
         "### Create CustomJob with worker pool specifications\n",
@@ -1717,7 +1885,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "custom_job:worker_pool"
+        "id": "2QvSqbbHulvc"
       },
       "outputs": [],
       "source": [
@@ -1729,7 +1897,7 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "run_custom_job:multiworker"
+        "id": "Iw4L3UIfulvd"
       },
       "source": [
         "### Run the CustomJob\n",
@@ -1741,7 +1909,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "run_custom_job:multiworker"
+        "id": "zmqCNS78ulvd"
       },
       "outputs": [],
       "source": [
@@ -1755,7 +1923,7 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "delete_job"
+        "id": "gWZoH9QKulvd"
       },
       "source": [
         "### Delete a custom training job\n",
@@ -1767,7 +1935,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "delete_job"
+        "id": "Lt8BJ4iBulvd"
       },
       "outputs": [],
       "source": [
@@ -1787,13 +1955,7 @@
         "\n",
         "Otherwise, you can delete the individual resources you created in this tutorial:\n",
         "\n",
-        "- Dataset\n",
-        "- Pipeline\n",
-        "- Model\n",
-        "- Endpoint\n",
-        "- Batch Job\n",
-        "- Custom Job\n",
-        "- Hyperparameter Tuning Job\n",
+        "\n",
         "- Cloud Storage Bucket"
       ]
     },
@@ -1801,70 +1963,15 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "cleanup"
+        "id": "U98Wzc01ulvd"
       },
       "outputs": [],
       "source": [
-        "delete_dataset = True\n",
-        "delete_pipeline = True\n",
-        "delete_model = True\n",
-        "delete_endpoint = True\n",
-        "delete_batchjob = True\n",
-        "delete_customjob = True\n",
-        "delete_hptjob = True\n",
-        "delete_bucket = True\n",
+        "# Set this to true only if you'd like to delete your bucket\n",
+        "delete_bucket = False\n",
         "\n",
-        "# Delete the dataset using the Vertex fully qualified identifier for the dataset\n",
-        "try:\n",
-        "    if delete_dataset and \"dataset_id\" in globals():\n",
-        "        clients[\"dataset\"].delete_dataset(name=dataset_id)\n",
-        "except Exception as e:\n",
-        "    print(e)\n",
-        "\n",
-        "# Delete the training pipeline using the Vertex fully qualified identifier for the pipeline\n",
-        "try:\n",
-        "    if delete_pipeline and \"pipeline_id\" in globals():\n",
-        "        clients[\"pipeline\"].delete_training_pipeline(name=pipeline_id)\n",
-        "except Exception as e:\n",
-        "    print(e)\n",
-        "\n",
-        "# Delete the model using the Vertex fully qualified identifier for the model\n",
-        "try:\n",
-        "    if delete_model and \"model_to_deploy_id\" in globals():\n",
-        "        clients[\"model\"].delete_model(name=model_to_deploy_id)\n",
-        "except Exception as e:\n",
-        "    print(e)\n",
-        "\n",
-        "# Delete the endpoint using the Vertex fully qualified identifier for the endpoint\n",
-        "try:\n",
-        "    if delete_endpoint and \"endpoint_id\" in globals():\n",
-        "        clients[\"endpoint\"].delete_endpoint(name=endpoint_id)\n",
-        "except Exception as e:\n",
-        "    print(e)\n",
-        "\n",
-        "# Delete the batch job using the Vertex fully qualified identifier for the batch job\n",
-        "try:\n",
-        "    if delete_batchjob and \"batch_job_id\" in globals():\n",
-        "        clients[\"job\"].delete_batch_prediction_job(name=batch_job_id)\n",
-        "except Exception as e:\n",
-        "    print(e)\n",
-        "\n",
-        "# Delete the custom job using the Vertex fully qualified identifier for the custom job\n",
-        "try:\n",
-        "    if delete_customjob and \"job_id\" in globals():\n",
-        "        clients[\"job\"].delete_custom_job(name=job_id)\n",
-        "except Exception as e:\n",
-        "    print(e)\n",
-        "\n",
-        "# Delete the hyperparameter tuning job using the Vertex fully qualified identifier for the hyperparameter tuning job\n",
-        "try:\n",
-        "    if delete_hptjob and \"hpt_job_id\" in globals():\n",
-        "        clients[\"job\"].delete_hyperparameter_tuning_job(name=hpt_job_id)\n",
-        "except Exception as e:\n",
-        "    print(e)\n",
-        "\n",
-        "if delete_bucket and \"BUCKET_NAME\" in globals():\n",
-        "    ! gsutil rm -r $BUCKET_NAME"
+        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "    ! gsutil rm -r $BUCKET_URI"
       ]
     }
   ],

notebooks/community/ml_ops/stage2/get_started_vertex_experiments.ipynb

@@ -8,7 +8,7 @@
       },
       "outputs": [],
       "source": [
-        "# Copyright 2021 Google LLC\n",
+        "# Copyright 2022 Google LLC\n",
         "#\n",
         "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
         "# you may not use this file except in compliance with the License.\n",
@@ -29,7 +29,7 @@
         "id": "title:generic,gcp"
       },
       "source": [
-        "# E2E ML on GCP: MLOps stage 2 : experimentation: get started with Logging and Vertex Experiments\n",
+        "# E2E ML on GCP: MLOps stage 2 : experimentation: get started with Logging and Vertex AI Experiments\n",
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
@@ -38,9 +38,15 @@
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
+        "    <td>\n",
+        "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_experiments.ipynb\">\n",
+        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\\\" alt=\"Colab logo\"> Run in Colab\n",
+        "        </a>\n",
+        "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_experiments.ipynb\">\n",
-        "      Open in Google Cloud Notebooks\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage2/get_started_vertex_experiments.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
         "    </a>\n",
         "  </td>\n",
         "</table>\n",
@@ -56,7 +62,7 @@
         "## Overview\n",
         "\n",
         "\n",
-        "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 2 : experimentation: get started with Logging and Vertex Experiments."
+        "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 2 : experimentation: get started with Logging and Vertex AI Experiments."
       ]
     },
     {
@@ -93,7 +99,7 @@
       "source": [
         "### Recommendations\n",
         "\n",
-        "When doing E2E MLOps on Google Cloud, the following best practices for logging data when experimenting or formal training a model.\n",
+        "When doing E2E MLOps on Google Cloud, the following are some of the best practices for logging data when experimenting or formally training a model.\n",
         "\n",
         "#### Python Logging\n",
         "\n",
@@ -105,7 +111,14 @@
         "\n",
         "#### Experiments\n",
         "\n",
-        "Use Vertex AI Experiments in conjunction with logging when doing experiments to compare results for different experiment configurations."
+        "Use Vertex AI Experiments in conjunction with logging when performing experiments to compare results for different experiment configurations.\n",
+        "\n",
+        "### Costs\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "- Vertex AI\n",
+        "\n",
+        "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing) and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
       ]
     },
     {
@@ -116,7 +129,7 @@
       "source": [
         "## Installations\n",
         "\n",
-        "Install *one time* the packages for executing the MLOps notebooks."
+        "Install the following packages for executing this notebook."
       ]
     },
     {
@@ -127,20 +140,20 @@
       },
       "outputs": [],
       "source": [
-        "ONCE_ONLY = False\n",
-        "if ONCE_ONLY:\n",
-        "    ! pip3 install -U tensorflow==2.5 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-transform==1.2 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-io==0.18 $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-bigquery $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-logging $USER_FLAG\n",
-        "    ! pip3 install --upgrade apache-beam[gcp] $USER_FLAG\n",
-        "    ! pip3 install --upgrade pyarrow $USER_FLAG\n",
-        "    ! pip3 install --upgrade cloudml-hypertune $USER_FLAG\n",
-        "    ! pip3 install --upgrade kfp $USER_FLAG"
+        "import os\n",
+        "\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
+        "\n",
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "USER_FLAG = \"\"\n",
+        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\"\n",
+        "\n",
+        "! pip3 install --upgrade google-cloud-logging $USER_FLAG -q"
       ]
     },
     {
@@ -178,6 +191,24 @@
         "id": "project_id"
       },
       "source": [
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI, Compute Engine, Cloud Storage and Cloud Logging APIs](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component,storage_component,logging).\n",
+        "\n",
+        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands.\n",
+        "\n",
+        "\n",
         "#### Set your project ID\n",
         "\n",
         "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
@@ -248,7 +279,10 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -275,6 +309,67 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "f3bd8c0d0469"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
+        "\n",
+        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
+        "\n",
+        "**Otherwise**, follow these steps:\n",
+        "\n",
+        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
+        "\n",
+        "1. **Click Create service account**.\n",
+        "\n",
+        "2. In the **Service account name** field, enter a name, and click **Create**.\n",
+        "\n",
+        "3. In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex AI\" into the filter box, and select **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "\n",
+        "4. Click Create. A JSON file that contains your key downloads to your local environment.\n",
+        "\n",
+        "5. Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "e0953a00668e"
+      },
+      "outputs": [],
+      "source": [
+        "# If you are running this notebook in Colab, run this cell and follow the\n",
+        "# instructions to authenticate your GCP account. This provides access to your\n",
+        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
+        "# requests.\n",
+        "\n",
+        "import os\n",
+        "import sys\n",
+        "\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = False\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
+        "    if \"google.colab\" in sys.modules:\n",
+        "        IS_COLAB = True\n",
+        "        from google.colab import auth as google_auth\n",
+        "\n",
+        "        google_auth.authenticate_user()\n",
+        "\n",
+        "    # If you are running this notebook locally, replace the string below with the\n",
+        "    # path to your service account key and run this cell to authenticate your GCP\n",
+        "    # account.\n",
+        "    elif not os.getenv(\"IS_TESTING\"):\n",
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -284,7 +379,7 @@
         "### Set up variables\n",
         "\n",
         "Next, set up some variables used throughout the tutorial.\n",
-        "### Import libraries and define constants"
+        "### Import libraries"
       ]
     },
     {
@@ -295,29 +390,9 @@
       },
       "outputs": [],
       "source": [
-        "import google.cloud.aiplatform as aip"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "import_logging"
-      },
-      "source": [
-        "#### Import logging\n",
+        "import logging\n",
         "\n",
-        "Import the logging package into your Python environment."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "import_logging"
-      },
-      "outputs": [],
-      "source": [
-        "import logging"
+        "import google.cloud.aiplatform as aiplatform"
       ]
     },
     {
@@ -339,7 +414,7 @@
       },
       "outputs": [],
       "source": [
-        "aip.init(project=PROJECT_ID, location=REGION)"
+        "aiplatform.init(project=PROJECT_ID, location=REGION)"
       ]
     },
     {
@@ -356,9 +431,9 @@
         "- Send log output to console.\n",
         "- Send log output to a file.\n",
         "\n",
-        "### Logging Levels\n",
+        "### Logging Levels in Python Logging\n",
         "\n",
-        "The logging levels in order (from least to highest) are, with each level inclusive of the previous level:\n",
+        "The logging levels in order (from least to highest) and each level inclusive of the previous level are :\n",
         "\n",
         "1. Informational\n",
         "2. Warnings\n",
@@ -398,7 +473,7 @@
       "source": [
         "### Setting logging level\n",
         "\n",
-        "To set the logging level, you get the logging handler using `getLogger()`. You can have multiple logging handles. When `getLogger()` is called w/o arguments it gets the default handler, named ROOT. With the handler, you set the logging level with the method 'setLevel()`."
+        "To set the logging level, you get the logging handler using `getLogger()`. You can have multiple logging handles. When `getLogger()` is called without any arguments, it gets the default handler named ROOT. With the handler, you set the logging level with the method `setLevel()`."
       ]
     },
     {
@@ -445,7 +520,7 @@
       "source": [
         "### Output to a local file\n",
         "\n",
-        "You can preserve your logging output to a file that is local to where the Python script is running with the method `BasicConfig()`, with the following paraneters:\n",
+        "You can preserve your logging output to a file that is local to where the Python script is running with the method `BasicConfig()`, that takes the following parameters:\n",
         "\n",
         "- `filename`: The file path to the local file to write the log output to.\n",
         "- `level`: Sets the level of logging that is written to the logging file.\n",
@@ -482,7 +557,7 @@
         "- Send log output to storage.\n",
         "- Retrieve log output from storage.\n",
         "\n",
-        "### Logging Levels\n",
+        "### Logging Levels in Cloud Logging\n",
         "\n",
         "The logging levels in order (from least to highest) are, with each level inclusive of the previous level:\n",
         "\n",
@@ -517,7 +592,7 @@
         "from google.cloud.logging.handlers import CloudLoggingHandler\n",
         "\n",
         "# Connect to the Cloud Logging service\n",
-        "cl_client = google.cloud.logging.Client()\n",
+        "cl_client = google.cloud.logging.Client(project=PROJECT_ID)\n",
         "handler = CloudLoggingHandler(cl_client, name=\"mylog\")\n",
         "\n",
         "# Create a logger instance and logging level\n",
@@ -539,7 +614,7 @@
       "source": [
         "### Logging output\n",
         "\n",
-        "To log output at specific levels is identical in method, and method names, as in Python logging, except that you use your instance of the cloud logger in place of logging."
+        "Logging output at specific levels is identical to Python logging with respect to method and method names. The only difference is that you use your instance of the cloud logger in place of logging."
       ]
     },
     {
@@ -567,7 +642,7 @@
         "To get the logged output, you:\n",
         "\n",
         "1. Retrieve the log handle to the service.\n",
-        "2. Using the handle call the method `list_entries()`\n",
+        "2. Using the handle, call the method `list_entries()`.\n",
         "3. Iterate through the entries."
       ]
     },
@@ -594,10 +669,10 @@
       "source": [
         "## Logging with Vertex AI Experiments and Vertex AI ML Metadata\n",
         "\n",
-        "You can log results related to training experiments with `Vertex AI Experiments` and `ML Metadata`:\n",
+        "You can log results related to training experiments with `Vertex AI Experiments` and `ML Metadata` including:\n",
         "\n",
         "- Preserve results of an experiment.\n",
-        "- Track multiple runs -- i.e., training runs -- within an experiment.\n",
+        "- Track multiple runs i.e., training runs within an experiment.\n",
         "- Track parameters (configuration) and metrics (results).\n",
         "- Retrieve and display the logged output.\n",
         "\n",
@@ -612,14 +687,29 @@
       "source": [
         "### Create experiment for tracking training related metadata\n",
         "\n",
-        "Setup tracking the parameters (configuration) and metrics (results) for each experiment:\n",
+        "Setup tracking for parameters (configuration) and metrics (results) in each experiment:\n",
         "\n",
-        "- `aip.init()` - Create an experiment instance\n",
-        "- `aip.start_run()` - Track a specific run within the experiment.\n",
+        "- `aiplatform.init()` - Create an experiment instance\n",
+        "- `aiplatform.start_run()` - Track a specific run within the experiment.\n",
         "\n",
         "Learn more about [Introduction to Vertex AI ML Metadata](https://cloud.google.com/vertex-ai/docs/ml-metadata/introduction)."
       ]
     },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "1ed46e349cf2"
+      },
+      "outputs": [],
+      "source": [
+        "# Specify a name for the experiment\n",
+        "EXPERIMENT_NAME = \"[your-experiment-name]\"\n",
+        "\n",
+        "if EXPERIMENT_NAME == \"[your-experiment-name]\":\n",
+        "    EXPERIMENT_NAME = \"example-\" + TIMESTAMP"
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -628,9 +718,9 @@
       },
       "outputs": [],
       "source": [
-        "EXPERIMENT_NAME = \"example-\" + TIMESTAMP\n",
-        "aip.init(experiment=EXPERIMENT_NAME)\n",
-        "aip.start_run(\"run-1\")"
+        "# Create experiment\n",
+        "aiplatform.init(experiment=EXPERIMENT_NAME)\n",
+        "aiplatform.start_run(\"run-1\")"
       ]
     },
     {
@@ -641,14 +731,14 @@
       "source": [
         "### Log parameters for the experiment\n",
         "\n",
-        "Typically, an experiment is associated with a specific dataset and model architecture. Within an experiment, you may have multiple training runs, where each run tries a different configuration. As examples:\n",
+        "Typically, an experiment is associated with a specific dataset and a model architecture. Within an experiment, you may have multiple training runs, where each run tries a different configuration. For example:\n",
         "\n",
         "- Dataset split\n",
         "- Dataset sampling and boosting\n",
         "- Depth and width of layers\n",
         "- Hyperparameters\n",
         "\n",
-        "These configuration settings are referred to as parameters, which you store their key/value pair using the method `log_params()`"
+        "These configuration settings are referred to as parameters, which you store as key-value pairs using the method `log_params()`"
       ]
     },
     {
@@ -663,7 +753,7 @@
         "hyperparams[\"epochs\"] = 100\n",
         "hyperparams[\"batch_size\"] = 32\n",
         "hyperparams[\"learning_rate\"] = 0.01\n",
-        "aip.log_params(hyperparams)"
+        "aiplatform.log_params(hyperparams)"
       ]
     },
     {
@@ -674,14 +764,14 @@
       "source": [
         "### Log metrics for the experiment\n",
         "\n",
-        "At the completion, or termination, of a run within an experiment, you can log results that you use to compare runs. As examples:\n",
+        "At the completion or termination of a run within an experiment, you can log results that you use to compare runs. For example:\n",
         "\n",
         "- Evaluation metrics\n",
         "- Hyperparameter search selection\n",
         "- Time to train the model\n",
         "- Early stop trigger\n",
         "\n",
-        "These results settings are referred to as metrics, which you store their key/value pair using the method `log_metrics()`"
+        "These results are referred to as metrics, which you store as key-value pairs using the method `log_metrics()`"
       ]
     },
     {
@@ -695,7 +785,7 @@
         "metrics = {}\n",
         "metrics[\"test_acc\"] = 98.7\n",
         "metrics[\"train_acc\"] = 99.3\n",
-        "aip.log_metrics(metrics)"
+        "aiplatform.log_metrics(metrics)"
       ]
     },
     {
@@ -717,36 +807,11 @@
       },
       "outputs": [],
       "source": [
-        "EXPERIMENT_NAME = \"example\"\n",
-        "\n",
-        "experiment_df = aip.get_experiment_df()\n",
+        "experiment_df = aiplatform.get_experiment_df()\n",
         "experiment_df = experiment_df[experiment_df.experiment_name == EXPERIMENT_NAME]\n",
         "experiment_df.T"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "delete_experiment"
-      },
-      "source": [
-        "### Delete the experiment\n",
-        "\n",
-        "Next, delete the experiment. You will need to get the context via the metadata to delete it."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "delete_experiment"
-      },
-      "outputs": [],
-      "source": [
-        "c = aiplatform.metadata._Context(EXPERIMENT_NAME)\n",
-        "c.delete()"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -760,15 +825,9 @@
         "\n",
         "Otherwise, you can delete the individual resources you created in this tutorial:\n",
         "\n",
-        "- Dataset\n",
-        "- Pipeline\n",
-        "- Model\n",
-        "- Endpoint\n",
-        "- AutoML Training Job\n",
-        "- Batch Job\n",
-        "- Custom Job\n",
-        "- Hyperparameter Tuning Job\n",
-        "- Cloud Storage Bucket"
+        "### Delete the experiment\n",
+        "\n",
+        "Next, delete the experiment. You will need to get the context via the metadata to delete it."
       ]
     },
     {
@@ -779,61 +838,8 @@
       },
       "outputs": [],
       "source": [
-        "delete_all = True\n",
-        "\n",
-        "if delete_all:\n",
-        "    # Delete the dataset using the Vertex dataset object\n",
-        "    try:\n",
-        "        if \"dataset\" in globals():\n",
-        "            dataset.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the model using the Vertex model object\n",
-        "    try:\n",
-        "        if \"model\" in globals():\n",
-        "            model.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the endpoint using the Vertex endpoint object\n",
-        "    try:\n",
-        "        if \"endpoint\" in globals():\n",
-        "            endpoint.undeploy_all()\n",
-        "            endpoint.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the AutoML or Pipeline training job\n",
-        "    try:\n",
-        "        if \"dag\" in globals():\n",
-        "            dag.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the custom training job\n",
-        "    try:\n",
-        "        if \"job\" in globals():\n",
-        "            job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the batch prediction job using the Vertex batch prediction object\n",
-        "    try:\n",
-        "        if \"batch_predict_job\" in globals():\n",
-        "            batch_predict_job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the hyperparameter tuning job using the Vertex hyperparameter tuning object\n",
-        "    try:\n",
-        "        if \"hpt_job\" in globals():\n",
-        "            hpt_job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    if \"BUCKET_NAME\" in globals():\n",
-        "        ! gsutil rm -r $BUCKET_NAME"
+        "c = aiplatform.metadata._Context(EXPERIMENT_NAME)\n",
+        "c.delete()"
       ]
     }
   ],

notebooks/community/ml_ops/stage2/get_started_vertex_feature_store.ipynb

@@ -38,11 +38,20 @@
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
+        "    \n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_feature_store.ipynb\">\n",
-        "      Open in Google Cloud Notebooks\n",
+        "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_feature_store.ipynb\">\n",
+        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\\\" alt=\"Colab logo\"> Run in Colab\n",
+        "        </a>\n",
+        "  </td>\n",
+        "    \n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage2/get_started_vertex_feature_store.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
         "    </a>\n",
         "  </td>\n",
+        "    \n",
         "</table>\n",
         "<br/><br/><br/>"
       ]
@@ -67,9 +76,9 @@
       "source": [
         "### Dataset\n",
         "\n",
-        "The dataset used for this tutorial is the Movie Recommendations. The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket, in Avro format.\n",
+        "The dataset used for this tutorial is the `Movie Recommendations` dataset. The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket, in Avro format.\n",
         "\n",
-        "The dataset predicts whether a persons will watch a movie."
+        "This dataset is used to predict whether a person will watch a movie or not."
       ]
     },
     {
@@ -80,7 +89,7 @@
       "source": [
         "### Objective\n",
         "\n",
-        "In this tutorial, you learn how to use `Vertex AI Feature Store` for when training and prediction with `Vertex AI`.\n",
+        "In this tutorial, you learn how to use `Vertex AI Feature Store` when training and predicting with `Vertex AI`.\n",
         "\n",
         "This tutorial uses the following Google Cloud ML services:\n",
         "\n",
@@ -98,6 +107,22 @@
         "- Perform batch serving from a `Featurestore` resource."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "81c777b8ad32"
+      },
+      "source": [
+        "### Costs\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "- Vertex AI\n",
+        "- Cloud Storage\n",
+        "- BigQuery\n",
+        "\n",
+        "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing), [Cloud Storage pricing](https://cloud.google.com/storage/pricing) and [BigQuery pricing](https://cloud.google.com/bigquery/pricing) and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -106,7 +131,7 @@
       "source": [
         "## Installations\n",
         "\n",
-        "Install *one time* the packages for executing the MLOps notebooks."
+        "Install the following packages for further running this notebook."
       ]
     },
     {
@@ -117,24 +142,21 @@
       },
       "outputs": [],
       "source": [
-        "ONCE_ONLY = False\n",
-        "if ONCE_ONLY:\n",
-        "    ! pip3 install -U tensorflow==2.5 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-transform==1.2 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-io==0.18 $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-bigquery $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-logging $USER_FLAG\n",
-        "    ! pip3 install --upgrade apache-beam[gcp] $USER_FLAG\n",
-        "    ! pip3 install --upgrade pyarrow $USER_FLAG\n",
-        "    ! pip3 install --upgrade cloudml-hypertune $USER_FLAG\n",
-        "    ! pip3 install --upgrade kfp $USER_FLAG\n",
-        "    ! pip3 install --upgrade torchvision $USER_FLAG\n",
-        "    ! pip3 install --upgrade rpy2 $USER_FLAG\n",
-        "    ! pip3 install --upgrade python-tabulate $USER_FLAG\n",
-        "    ! pip3 install -U opencv-python-headless==4.5.2.52 $USER_FLAG"
+        "import os\n",
+        "\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
+        "\n",
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "USER_FLAG = \"\"\n",
+        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\"\n",
+        "\n",
+        "# Install the dependecies\n",
+        "! pip3 install --upgrade google-cloud-aiplatform google-cloud-bigquery pyarrow avro $USER_FLAG -q"
       ]
     },
     {
@@ -166,6 +188,30 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "project_id"
+      },
+      "source": [
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI, Compute Engine, Cloud Storage and Cloud Logging APIs](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component,storage_component,logging).\n",
+        "\n",
+        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -242,7 +288,10 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -269,15 +318,73 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "29b110b44457"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
+        "\n",
+        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
+        "\n",
+        "**Otherwise**, follow these steps:\n",
+        "\n",
+        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
+        "\n",
+        "1. **Click Create service account**.\n",
+        "\n",
+        "2. In the **Service account name** field, enter a name, and click **Create**.\n",
+        "\n",
+        "3. In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex\" into the filter box, and select **Vertex Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "\n",
+        "4. Click Create. A JSON file that contains your key downloads to your local environment.\n",
+        "\n",
+        "5. Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "89788a802687"
+      },
+      "outputs": [],
+      "source": [
+        "# If you are running this notebook in Colab, run this cell and follow the\n",
+        "# instructions to authenticate your GCP account. This provides access to your\n",
+        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
+        "# requests.\n",
+        "\n",
+        "import os\n",
+        "import sys\n",
+        "\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = False\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
+        "    if \"google.colab\" in sys.modules:\n",
+        "        IS_COLAB = True\n",
+        "        from google.colab import auth as google_auth\n",
+        "\n",
+        "        google_auth.authenticate_user()\n",
+        "\n",
+        "    # If you are running this notebook locally, replace the string below with the\n",
+        "    # path to your service account key and run this cell to authenticate your GCP\n",
+        "    # account.\n",
+        "    elif not os.getenv(\"IS_TESTING\"):\n",
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "setup_vars"
       },
       "source": [
-        "### Set up variables\n",
-        "\n",
-        "Next, set up some variables used throughout the tutorial.\n",
         "### Import libraries and define constants"
       ]
     },
@@ -289,28 +396,7 @@
       },
       "outputs": [],
       "source": [
-        "import google.cloud.aiplatform as aip"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "import_bq"
-      },
-      "source": [
-        "#### Import BigQuery\n",
-        "\n",
-        "Import the BigQuery package into your Python environment."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "import_bq"
-      },
-      "outputs": [],
-      "source": [
+        "import google.cloud.aiplatform as aiplatform\n",
         "from google.cloud import bigquery"
       ]
     },
@@ -320,9 +406,7 @@
         "id": "init_bq"
       },
       "source": [
-        "### Create BigQuery client\n",
-        "\n",
-        "Create the BigQuery client."
+        "Initialize Vertex AI and BigQuery clients."
       ]
     },
     {
@@ -333,7 +417,8 @@
       },
       "outputs": [],
       "source": [
-        "bqclient = bigquery.Client()"
+        "aiplatform.init(project=PROJECT_ID)\n",
+        "bqclient = bigquery.Client(project=PROJECT_ID)"
       ]
     },
     {
@@ -350,11 +435,11 @@
         "\n",
         "Now it's time to do a live prediction. You get a transaction from the cash register, but all it has is the credit card number and this transaction. It does not have the enriched data the model needs. During serving, the credit card number is used as an index to Feature Store to get the enriched data needed for the model.\n",
         "\n",
-        "Next problem. Let's say the enriched data the model was trained on was timestamp June 1. This transaction is June 15. Assume that the user has made other transactions between June 1 and 15, and the enriched data has been continuously updated in Feature Store. But the model was trained on June 1st data. FeatureStore knows the version number and serves the June 1 version to the model (not the current June 15); otherwise, if you used June 15 data you have training-serving skew.\n",
+        "On the other hand, let's say the enriched data the model was trained on was timestamped on June 1st. The current transaction is from June 15th. Assume that the user has made other transactions between June 1st and 15th, and the enriched data has been continuously updated in Feature Store. But the model was trained on June 1st data. FeatureStore knows the version number and serves the June 1st version to the model (not the current June 15th). Otherwise, if you used June 15th data, you would have training-serving skew.\n",
         "\n",
-        "Next problem, data drift. Things change, suddenly one day everybody is buying toilet paper! There is a significant change in the distribution of the current stored enriched data from the distribution that the deployed model was trained on. FeatureStore can detect changes/thresholds in distribution changes and trigger a notification for retraining the model.\n",
+        "Another problem here is the data drift. Things change and suddenly one day, everybody is buying toilet paper! There is a significant change in the distribution of existing enriched data from the distribution that the deployed model was trained on. FeatureStore can detect changes/thresholds in distribution changes and trigger a notification for retraining the model.\n",
         "\n",
-        "Learn more about [Vertex AI Feature Store API](https://cloud.google.com/vertex-ai/docs/featurestore)"
+        "Learn more about [Vertex AI Feature Store API](https://cloud.google.com/vertex-ai/docs/featurestore)."
       ]
     },
     {
@@ -369,9 +454,9 @@
         "\n",
         "        Featurestore -> EntityType -> Feature\n",
         "\n",
-        "- `Featurestore`: the place to store your features\n",
+        "- `Featurestore`: the place to store your features.\n",
         "- `EntityType`: under a `Featurestore`, an `EntityType` describes an object to be modeled, real one or virtual one.\n",
-        "- `Feature`: under an `EntityType`, a `Feature` describes an attribute of the `EntityType`\n",
+        "- `Feature`: under an `EntityType`, a `Feature` describes an attribute of the `EntityType`.\n",
         "\n",
         "Learn more about [Vertex AI Feature Store data model](https://cloud.google.com/vertex-ai/docs/featurestore/concepts).\n",
         "\n",
@@ -407,7 +492,7 @@
         "# Represents featurestore resource path.\n",
         "FEATURESTORE_NAME = \"movies\"\n",
         "\n",
-        "featurestore = aip.Featurestore.create(\n",
+        "featurestore = aiplatform.Featurestore.create(\n",
         "    featurestore_id=FEATURESTORE_NAME,\n",
         "    online_store_fixed_node_count=1,\n",
         "    project=PROJECT_ID,\n",
@@ -436,7 +521,7 @@
       },
       "outputs": [],
       "source": [
-        "for featurestore in aip.Featurestore.list():\n",
+        "for featurestore in aiplatform.Featurestore.list():\n",
         "    print(featurestore)"
       ]
     },
@@ -463,7 +548,7 @@
       },
       "outputs": [],
       "source": [
-        "featurestore = featurestore = aip.Featurestore(\n",
+        "featurestore = featurestore = aiplatform.Featurestore(\n",
         "    featurestore_name=FEATURESTORE_NAME, project=PROJECT_ID, location=REGION\n",
         ")\n",
         "print(featurestore)"
@@ -522,7 +607,7 @@
       "outputs": [],
       "source": [
         "def create_features(featurestore_name, entity_name, features):\n",
-        "    entity_type = aip.EntityType(\n",
+        "    entity_type = aiplatform.EntityType(\n",
         "        entity_type_name=entity_name, featurestore_id=featurestore_name\n",
         "    )\n",
         "\n",
@@ -573,7 +658,7 @@
       },
       "outputs": [],
       "source": [
-        "for featurestore in aip.Featurestore.list():\n",
+        "for featurestore in aiplatform.Featurestore.list():\n",
         "    print(featurestore)"
       ]
     },
@@ -585,7 +670,7 @@
       "source": [
         "### Search `Feature` resources using a filter\n",
         "\n",
-        "You can narrow your search of `Feature` resources using the method `list_features()` and specifying a `filter` filter."
+        "You can narrow your search of `Feature` resources using the method `list_features()` and specifying a `filter` string."
       ]
     },
     {
@@ -637,17 +722,26 @@
       },
       "outputs": [],
       "source": [
-        "features = aip.Feature.search(query=\"value_type=DOUBLE\")\n",
+        "features = aiplatform.Feature.search(query=\"value_type=DOUBLE\")\n",
         "print(\"By data type\")\n",
         "for feature in features:\n",
         "    print(features)\n",
         "\n",
-        "aip.Feature.search(query=\"feature_id=title\")\n",
+        "aiplatform.Feature.search(query=\"feature_id=title\")\n",
         "print(\"By Name\")\n",
         "for feature in features:\n",
         "    print(features)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "277e9884cf37"
+      },
+      "source": [
+        "Define paths to the feature data."
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -677,7 +771,7 @@
         "\n",
         "### Data layout\n",
         "\n",
-        "Each imported `EntityType` resource data must have an ID; also, each `EntityType` resource data item can optionally have a timestamp, sepecifying when the feature values were generated.\n",
+        "Each imported `EntityType` resource data must have an ID. Also, each `EntityType` resource data item can optionally have a timestamp, sepecifying when the feature values were generated.\n",
         "\n",
         "When importing, specify the following in your request:\n",
         "\n",
@@ -685,7 +779,7 @@
         "- Data source URL\n",
         "- Destination: featurestore/entity types/features to be imported\n",
         "\n",
-        "The feature values for the movies dataset are in Avro format. The Avro schemas are as follows:\n",
+        "The feature values for `Movie Recommendations` dataset are in Avro format. The Avro schemas are as follows:\n",
         "\n",
         "**Users entity**:\n",
         "\n",
@@ -797,7 +891,7 @@
       "source": [
         "#### Delete the entity types and corresponding features and feature values\n",
         "\n",
-        "Next, in preparation to repeat importing feature values from a dataframe, you first delete the existing entity types, and corresponding content."
+        "Now, in preparation to repeat the process of importing feature values but from a dataframe this time, you delete the existing entity types, and the corresponding content."
       ]
     },
     {
@@ -822,7 +916,7 @@
       "source": [
         "## Create entity types for your `Featurestore` resource\n",
         "\n",
-        "Next, you create the `EntityType` resources for your `Featurestore` resource using the `create_entity_type()` method, with the following parameters:\n",
+        "Next, you create the `EntityType` resources again for your `Featurestore` resource using the `create_entity_type()` method, with the following parameters:\n",
         "\n",
         "- `entity_type_id`: The name of the `EntityType` resource.\n",
         "- `description`:  A description of the entity type."
@@ -851,7 +945,7 @@
       "source": [
         "### Add `Feature` resources for your `EntityType` resources\n",
         "\n",
-        "Next, you create the `Feature` resources for each of the `EntityType` resources in your `Featurestore` resource using the `create_feature()` method, with the following parameters:\n",
+        "Further, you create the `Feature` resources again for each of the `EntityType` resources in your `Featurestore` resource using the `create_feature()` method, with the following parameters:\n",
         "\n",
         "- `feature_id`: The name of the `Feature` resource.\n",
         "- `description`: A description of the feature.\n",
@@ -867,7 +961,7 @@
       "outputs": [],
       "source": [
         "def create_features(featurestore_name, entity_name, features):\n",
-        "    entity_type = aip.EntityType(\n",
+        "    entity_type = aiplatform.EntityType(\n",
         "        entity_type_name=entity_name, featurestore_id=featurestore_name\n",
         "    )\n",
         "\n",
@@ -899,6 +993,15 @@
         ")"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8715a3f719c8"
+      },
+      "source": [
+        "Now, copy the `users` and `movies` data into avro files."
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -1041,7 +1144,7 @@
       "source": [
         "## Batch Serving\n",
         "\n",
-        "The Vertex AI Feature Store batch serving service is optimized for serving large batches of features in real-time with high-throughput, typically for training a model or batch prediction.\n",
+        "The Vertex AI Feature Store's batch serving service is optimized for serving large batches of features in real-time with high throughput, typically for training a model or batch prediction.\n",
         "\n",
         "One can batch serve to the following destinations:\n",
         "\n",
@@ -1093,7 +1196,7 @@
         "\n",
         "You batch serve entity data items to a BigQuery table using the `read_serve_to_bq()` method, with the following parameters:\n",
         "\n",
-        "- `bq_destination_output_uri`: The destination BigQuery table to serve the features to.\n",
+        "- `bq_destination_output_uri`: The destination BigQuery table to receive the served features.\n",
         "- `serving_feature_ids`: A dictionary of entity type and corresponding features to serve.\n",
         "- `read_instances_uri`: A Cloud Storage location to read the entity data items from.\n",
         "\n",
@@ -1126,6 +1229,7 @@
         "id": "delete_bq_dataset"
       },
       "source": [
+        "## Cleaning up\n",
         "### Delete a BigQuery dataset\n",
         "\n",
         "Use the method `delete_dataset()` to delete a BigQuery dataset along with all its tables, by setting the parameter `delete_contents` to `True`."

notebooks/community/ml_ops/stage2/get_started_vertex_tensorboard.ipynb

@@ -8,7 +8,7 @@
       },
       "outputs": [],
       "source": [
-        "# Copyright 2021 Google LLC\n",
+        "# Copyright 2022 Google LLC\n",
         "#\n",
         "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
         "# you may not use this file except in compliance with the License.\n",
@@ -29,9 +29,14 @@
         "id": "title:generic,gcp"
       },
       "source": [
-        "# E2E ML on GCP: MLOps stage 2 : experimentation: get started with Vertex Tensorboard\n",
+        "# E2E ML on GCP: MLOps stage 2 : experimentation: get started with Vertex AI Tensorboard\n",
         "\n",
         "<table align=\"left\">\n",
+        "   <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_tensorboard.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
         "  <td>\n",
         "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_tensorboard.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
@@ -39,8 +44,9 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_tensorboard.ipynb\">\n",
-        "      Open in Google Cloud Notebooks\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/notebook_template.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
         "    </a>\n",
         "  </td>\n",
         "</table>\n",
@@ -56,7 +62,7 @@
         "## Overview\n",
         "\n",
         "\n",
-        "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 2 : experimentation: get started with Vertex Tensorboard."
+        "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 2 : experimentation: get started with Vertex AI Tensorboard."
       ]
     },
     {
@@ -81,6 +87,68 @@
         "- Using Vertex AI TensorBoard with Vertex AI Training."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "b132d4ef86d6"
+      },
+      "source": [
+        "### Costs \n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI\n",
+        "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
+        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
+        "Calculator](https://cloud.google.com/products/calculator/)\n",
+        "to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "56cb7f08a9e8"
+      },
+      "source": [
+        "### Set up your local development environment\n",
+        "\n",
+        "**If you are using Colab or Google Cloud Notebooks**, your environment already meets\n",
+        "all the requirements to run this notebook. You can skip this step.\n",
+        "\n",
+        "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
+        "You need the following:\n",
+        "\n",
+        "* The Google Cloud SDK\n",
+        "* Git\n",
+        "* Python 3\n",
+        "* virtualenv\n",
+        "* Jupyter notebook running in a virtual environment with Python 3\n",
+        "\n",
+        "The Google Cloud guide to [Setting up a Python development\n",
+        "environment](https://cloud.google.com/python/setup) and the [Jupyter\n",
+        "installation guide](https://jupyter.org/install) provide detailed instructions\n",
+        "for meeting these requirements. The following steps provide a condensed set of\n",
+        "instructions:\n",
+        "\n",
+        "1. [Install and initialize the Cloud SDK.](https://cloud.google.com/sdk/docs/)\n",
+        "\n",
+        "1. [Install Python 3.](https://cloud.google.com/python/setup#installing_python)\n",
+        "\n",
+        "1. [Install\n",
+        "   virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv)\n",
+        "   and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
+        "\n",
+        "1. To install Jupyter, run `pip3 install jupyter` on the\n",
+        "command-line in a terminal shell.\n",
+        "\n",
+        "1. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
+        "\n",
+        "1. Open this notebook in the Jupyter Notebook Dashboard.\n"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -89,7 +157,7 @@
       "source": [
         "### Recommendations\n",
         "\n",
-        "When doing E2E MLOps on Google Cloud, the following best practices for visualizing your training with TensorBoard.\n",
+        "When doing E2E MLOps on Google Cloud, the following are the best practices for visualizing your training with TensorBoard.\n",
         "\n",
         "#### Local TensorBoard\n",
         "\n",
@@ -112,31 +180,32 @@
       "source": [
         "## Installations\n",
         "\n",
-        "Install *one time* the packages for executing the MLOps notebooks."
+        "Install the following packages for executing this notebook."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "install_mlops"
+        "id": "020040f91150"
       },
       "outputs": [],
       "source": [
-        "ONCE_ONLY = False\n",
-        "if ONCE_ONLY:\n",
-        "    ! pip3 install -U tensorflow==2.5 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-transform==1.2 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-io==0.18 $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-bigquery $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-logging $USER_FLAG\n",
-        "    ! pip3 install --upgrade apache-beam[gcp] $USER_FLAG\n",
-        "    ! pip3 install --upgrade pyarrow $USER_FLAG\n",
-        "    ! pip3 install --upgrade cloudml-hypertune $USER_FLAG\n",
-        "    ! pip3 install --upgrade kfp $USER_FLAG"
+        "import os\n",
+        "\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
+        "\n",
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "USER_FLAG = \"\"\n",
+        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\"\n",
+        "\n",
+        "! pip3 install -U tensorflow==2.8 $USER_FLAG -q\n",
+        "! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG -q"
       ]
     },
     {
@@ -168,6 +237,32 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2721ef0202d9"
+      },
+      "source": [
+        "## Before you begin\n",
+        "\n",
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com). \n",
+        "\n",
+        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -244,7 +339,10 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -271,6 +369,82 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2700e693f1b3"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
+        "authenticated. Skip this step."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "885395904904"
+      },
+      "source": [
+        "**If you are using Colab**, run the cell below and follow the instructions\n",
+        "when prompted to authenticate your account via oAuth.\n",
+        "\n",
+        "**Otherwise**, follow these steps:\n",
+        "\n",
+        "1. In the Cloud Console, go to the [**Create service account key**\n",
+        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
+        "\n",
+        "2. Click **Create service account**.\n",
+        "\n",
+        "3. In the **Service account name** field, enter a name, and\n",
+        "   click **Create**.\n",
+        "\n",
+        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
+        "into the filter box, and select\n",
+        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "\n",
+        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
+        "local environment.\n",
+        "\n",
+        "6. Enter the path to your service account key as the\n",
+        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "eff327d0552b"
+      },
+      "outputs": [],
+      "source": [
+        "# If you are running this notebook in Colab, run this cell and follow the\n",
+        "# instructions to authenticate your GCP account. This provides access to your\n",
+        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
+        "# requests.\n",
+        "\n",
+        "import os\n",
+        "import sys\n",
+        "\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = False\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
+        "    if \"google.colab\" in sys.modules:\n",
+        "        IS_COLAB = True\n",
+        "        from google.colab import auth as google_auth\n",
+        "\n",
+        "        google_auth.authenticate_user()\n",
+        "\n",
+        "    # If you are running this notebook locally, replace the string below with the\n",
+        "    # path to your service account key and run this cell to authenticate your GCP\n",
+        "    # account.\n",
+        "    elif not os.getenv(\"IS_TESTING\"):\n",
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -294,7 +468,7 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
+        "BUCKET_URI = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -305,8 +479,8 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
       ]
     },
     {
@@ -326,7 +500,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_NAME"
+        "! gsutil mb -l $REGION $BUCKET_URI"
       ]
     },
     {
@@ -346,7 +520,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_NAME"
+        "! gsutil ls -al $BUCKET_URI"
       ]
     },
     {
@@ -384,9 +558,16 @@
         "    or SERVICE_ACCOUNT is None\n",
         "    or SERVICE_ACCOUNT == \"[your-service-account]\"\n",
         "):\n",
-        "    # Get your GCP project id from gcloud\n",
-        "    shell_output = !gcloud auth list 2>/dev/null\n",
-        "    SERVICE_ACCOUNT = shell_output[2].strip()\n",
+        "    # Get your service account from gcloud\n",
+        "    if not IS_COLAB:\n",
+        "        shell_output = !gcloud auth list 2>/dev/null\n",
+        "        SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
+        "\n",
+        "    if IS_COLAB:\n",
+        "        shell_output = ! gcloud projects describe  $PROJECT_ID\n",
+        "        project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n",
+        "        SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n",
+        "\n",
         "    print(\"Service Account:\", SERVICE_ACCOUNT)"
       ]
     },
@@ -410,7 +591,7 @@
       },
       "outputs": [],
       "source": [
-        "import google.cloud.aiplatform as aip"
+        "import google.cloud.aiplatform as aiplatform"
       ]
     },
     {
@@ -454,7 +635,7 @@
       },
       "outputs": [],
       "source": [
-        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_NAME)"
+        "aiplatform.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
       ]
     },
     {
@@ -484,13 +665,15 @@
       },
       "outputs": [],
       "source": [
+        "import os\n",
+        "\n",
         "if os.getenv(\"IS_TESTING_TRAIN_GPU\"):\n",
         "    TRAIN_GPU, TRAIN_NGPU = (\n",
-        "        aip.gapic.AcceleratorType.NVIDIA_TESLA_K80,\n",
+        "        aiplatform.gapic.AcceleratorType.NVIDIA_TESLA_K80,\n",
         "        int(os.getenv(\"IS_TESTING_TRAIN_GPU\")),\n",
         "    )\n",
         "else:\n",
-        "    TRAIN_GPU, TRAIN_NGPU = (aip.gapic.AcceleratorType.NVIDIA_TESLA_K80, 1)"
+        "    TRAIN_GPU, TRAIN_NGPU = (aiplatform.gapic.AcceleratorType.NVIDIA_TESLA_K80, 1)"
       ]
     },
     {
@@ -663,9 +846,9 @@
         "\n",
         "You can upload your TensorBoard logs and share with others using `tensorboard dev` command. Once uploaded, a URL is returned to open up the TensorBoard instance in a brower for visualizing.\n",
         "\n",
-        "*Note:* Your TensorBoard instance is publicly visable.\n",
+        "*Note:* Your TensorBoard instance is publicly visible.\n",
         "\n",
-        "*Note:* In this example, while running within a notebook, the command will freeze since it is waiting for an interactive yes/no input. You can kill the command with a Ctrl C or kernel interupt.\n",
+        "*Note:* This cell is for demonstration purposes and must be ran in a terminal shell. In this example, while running within a notebook, the command will freeze since it is waiting for an interactive yes/no input. You can kill the command with a Ctrl C or kernel interupt.\n",
         "\n",
         "Learn more about [What is TensorBoard.dev](https://tensorboard.dev/)."
       ]
@@ -678,7 +861,7 @@
       },
       "outputs": [],
       "source": [
-        "! tensorboard dev upload --logdir {LOG_DIR} \\\n",
+        "! tensorboard dev upload --logdir logs \\\n",
         "  --name \"Simple experiment with MNIST\" \\\n",
         "  --description \"Training results\" \\\n",
         "  --one_shot"
@@ -706,7 +889,7 @@
       "outputs": [],
       "source": [
         "TENSORBOARD_DISPLAY_NAME = \"example\"\n",
-        "tensorboard = aip.Tensorboard.create(display_name=TENSORBOARD_DISPLAY_NAME)\n",
+        "tensorboard = aiplatform.Tensorboard.create(display_name=TENSORBOARD_DISPLAY_NAME)\n",
         "tensorboard_resource_name = tensorboard.gca_resource.name\n",
         "print(\"TensorBoard resource name:\", tensorboard_resource_name)"
       ]
@@ -746,9 +929,9 @@
         "\n",
         "url = output[1].split(' ')[-1]\n",
         "\n",
-        "print(url)\n",
+        "#print(url)\n",
         "\n",
-        "from IPython.core.display import display, HTML\n",
+        "from IPython.display import display, HTML\n",
         "display(HTML(\"<a href='\" + url + \"'>click here for TensorBoard instance</a>\"))"
       ]
     },
@@ -953,7 +1136,7 @@
         "! rm -f custom.tar custom.tar.gz\n",
         "! tar cvf custom.tar custom\n",
         "! gzip custom.tar\n",
-        "! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_example.tar.gz"
+        "! gsutil cp custom.tar.gz $BUCKET_URI/trainer_example.tar.gz"
       ]
     },
     {
@@ -985,7 +1168,7 @@
       },
       "outputs": [],
       "source": [
-        "job = aip.CustomTrainingJob(\n",
+        "job = aiplatform.CustomTrainingJob(\n",
         "    display_name=\"example_\" + TIMESTAMP,\n",
         "    script_path=\"custom/trainer/task.py\",\n",
         "    container_uri=TRAIN_IMAGE,\n",
@@ -1021,7 +1204,7 @@
       },
       "outputs": [],
       "source": [
-        "MODEL_DIR = \"{}/{}\".format(BUCKET_NAME, TIMESTAMP)\n",
+        "MODEL_DIR = \"{}/{}\".format(BUCKET_URI, TIMESTAMP)\n",
         "\n",
         "EPOCHS = 20\n",
         "STEPS = 100\n",
@@ -1143,14 +1326,8 @@
         "\n",
         "Otherwise, you can delete the individual resources you created in this tutorial:\n",
         "\n",
-        "- Dataset\n",
-        "- Pipeline\n",
         "- Model\n",
-        "- Endpoint\n",
-        "- AutoML Training Job\n",
-        "- Batch Job\n",
         "- Custom Job\n",
-        "- Hyperparameter Tuning Job\n",
         "- Cloud Storage Bucket"
       ]
     },
@@ -1162,61 +1339,14 @@
       },
       "outputs": [],
       "source": [
-        "delete_all = True\n",
+        "# Delete the custom training job\n",
+        "job.delete()\n",
         "\n",
-        "if delete_all:\n",
-        "    # Delete the dataset using the Vertex dataset object\n",
-        "    try:\n",
-        "        if \"dataset\" in globals():\n",
-        "            dataset.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
+        "# Set this to true only if you'd like to delete your bucket\n",
+        "delete_bucket = False\n",
         "\n",
-        "    # Delete the model using the Vertex model object\n",
-        "    try:\n",
-        "        if \"model\" in globals():\n",
-        "            model.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the endpoint using the Vertex endpoint object\n",
-        "    try:\n",
-        "        if \"endpoint\" in globals():\n",
-        "            endpoint.undeploy_all()\n",
-        "            endpoint.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the AutoML or Pipeline training job\n",
-        "    try:\n",
-        "        if \"dag\" in globals():\n",
-        "            dag.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the custom training job\n",
-        "    try:\n",
-        "        if \"job\" in globals():\n",
-        "            job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the batch prediction job using the Vertex batch prediction object\n",
-        "    try:\n",
-        "        if \"batch_predict_job\" in globals():\n",
-        "            batch_predict_job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the hyperparameter tuning job using the Vertex hyperparameter tuning object\n",
-        "    try:\n",
-        "        if \"hpt_job\" in globals():\n",
-        "            hpt_job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    if \"BUCKET_NAME\" in globals():\n",
-        "        ! gsutil rm -r $BUCKET_NAME"
+        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "    ! gsutil rm -r $BUCKET_URI"
       ]
     }
   ],

notebooks/community/ml_ops/stage2/get_started_vertex_training.ipynb

@@ -29,7 +29,7 @@
         "id": "title:generic,gcp"
       },
       "source": [
-        "# E2E ML on GCP: MLOps stage 2 : experimentation: get started with Vertex Training\n",
+        "# E2E ML on GCP: MLOps stage 2 : experimentation: get started with Vertex AI Training\n",
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
@@ -39,8 +39,14 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training.ipynb\">\n",
-        "      Open in Google Cloud Notebooks\n",
+        "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training.ipynb\">\n",
+        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\\\" alt=\"Colab logo\"> Run in Colab\n",
+        "        </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage2/get_started_vertex_training.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
         "    </a>\n",
         "  </td>\n",
         "</table>\n",
@@ -127,7 +133,7 @@
       "source": [
         "## Installations\n",
         "\n",
-        "Install *one time* the packages for executing the MLOps notebooks."
+        "Install the packages required for executing this notebook"
       ]
     },
     {
@@ -138,20 +144,20 @@
       },
       "outputs": [],
       "source": [
-        "ONCE_ONLY = False\n",
-        "if ONCE_ONLY:\n",
-        "    ! pip3 install -U tensorflow==2.5 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-transform==1.2 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-io==0.18 $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-bigquery $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-logging $USER_FLAG\n",
-        "    ! pip3 install --upgrade apache-beam[gcp] $USER_FLAG\n",
-        "    ! pip3 install --upgrade pyarrow $USER_FLAG\n",
-        "    ! pip3 install --upgrade cloudml-hypertune $USER_FLAG\n",
-        "    ! pip3 install --upgrade kfp $USER_FLAG"
+        "import os\n",
+        "\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
+        "\n",
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "USER_FLAG = \"\"\n",
+        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\"\n",
+        "\n",
+        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG -q"
       ]
     },
     {
@@ -173,8 +179,6 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
         "if not os.getenv(\"IS_TESTING\"):\n",
         "    # Automatically restart kernel after installs\n",
         "    import IPython\n",
@@ -183,6 +187,36 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "before_you_begin"
+      },
+      "source": [
+        "## Before you begin\n",
+        "\n",
+        "### GPU runtime\n",
+        "\n",
+        "*Make sure you're running this notebook in a GPU runtime if you have that option. In Colab, select* **Runtime > Change Runtime Type > GPU**\n",
+        "\n",
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "2. [Make sure that billing is enabled for your project.](https://cloud.google.com/billing/docs/how-to/modify-project)\n",
+        "\n",
+        "3. [Enable the following APIs: Vertex AI APIs, Compute Engine APIs, and Cloud Storage.](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component,storage-component.googleapis.com)\n",
+        "\n",
+        "4. If you are running this notebook locally, you will need to install the [Cloud SDK]((https://cloud.google.com/sdk)).\n",
+        "\n",
+        "5. Enter your project ID in the cell below. Then run the  cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$`."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -259,7 +293,10 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -286,6 +323,67 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gcp_authenticate"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
+        "\n",
+        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
+        "\n",
+        "**Otherwise**, follow these steps:\n",
+        "\n",
+        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
+        "\n",
+        "**Click Create service account**.\n",
+        "\n",
+        "In the **Service account name** field, enter a name, and click **Create**.\n",
+        "\n",
+        "In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex\" into the filter box, and select **Vertex Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "\n",
+        "Click Create. A JSON file that contains your key downloads to your local environment.\n",
+        "\n",
+        "Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "gcp_authenticate"
+      },
+      "outputs": [],
+      "source": [
+        "# If you are running this notebook in Colab, run this cell and follow the\n",
+        "# instructions to authenticate your GCP account. This provides access to your\n",
+        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
+        "# requests.\n",
+        "\n",
+        "import os\n",
+        "import sys\n",
+        "\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = False\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
+        "    if \"google.colab\" in sys.modules:\n",
+        "        IS_COLAB = True\n",
+        "        from google.colab import auth as google_auth\n",
+        "\n",
+        "        google_auth.authenticate_user()\n",
+        "\n",
+        "    # If you are running this notebook locally, replace the string below with the\n",
+        "    # path to your service account key and run this cell to authenticate your GCP\n",
+        "    # account.\n",
+        "    elif not os.getenv(\"IS_TESTING\"):\n",
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -309,7 +407,8 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
+        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
     {
@@ -320,8 +419,9 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
+        "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
       ]
     },
     {
@@ -341,7 +441,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_NAME"
+        "! gsutil mb -l $REGION $BUCKET_URI"
       ]
     },
     {
@@ -361,7 +461,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_NAME"
+        "! gsutil ls -al $BUCKET_URI"
       ]
     },
     {
@@ -384,7 +484,7 @@
       },
       "outputs": [],
       "source": [
-        "import google.cloud.aiplatform as aip"
+        "import google.cloud.aiplatform as aiplatform"
       ]
     },
     {
@@ -406,7 +506,7 @@
       },
       "outputs": [],
       "source": [
-        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_NAME)"
+        "aiplatform.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
       ]
     },
     {
@@ -441,7 +541,7 @@
       "source": [
         "if os.getenv(\"IS_TESTING_TRAIN_GPU\"):\n",
         "    TRAIN_GPU, TRAIN_NGPU = (\n",
-        "        aip.gapic.AcceleratorType.NVIDIA_TESLA_K80,\n",
+        "        aiplatform.gapic.AcceleratorType.NVIDIA_TESLA_K80,\n",
         "        int(os.getenv(\"IS_TESTING_TRAIN_GPU\")),\n",
         "    )\n",
         "else:\n",
@@ -449,7 +549,7 @@
         "\n",
         "if os.getenv(\"IS_TESTING_DEPLOY_GPU\"):\n",
         "    DEPLOY_GPU, DEPLOY_NGPU = (\n",
-        "        aip.gapic.AcceleratorType.NVIDIA_TESLA_K80,\n",
+        "        aiplatform.gapic.AcceleratorType.NVIDIA_TESLA_K80,\n",
         "        int(os.getenv(\"IS_TESTING_DEPLOY_GPU\")),\n",
         "    )\n",
         "else:\n",
@@ -484,7 +584,7 @@
         "if os.getenv(\"IS_TESTING_TF\"):\n",
         "    TF = os.getenv(\"IS_TESTING_TF\")\n",
         "else:\n",
-        "    TF = \"2.1\".replace(\".\", \"-\")\n",
+        "    TF = \"2.5\".replace(\".\", \"-\")\n",
         "\n",
         "if TF[0] == \"2\":\n",
         "    if TRAIN_GPU:\n",
@@ -602,7 +702,7 @@
         "DISPLAY_NAME = \"boston_\" + TIMESTAMP\n",
         "REQUIREMENTS = [\"tensorflow==2.3\"]\n",
         "\n",
-        "job = aip.CustomTrainingJob(\n",
+        "job = aiplatform.CustomTrainingJob(\n",
         "    display_name=DISPLAY_NAME,\n",
         "    script_path=\"task.py\",\n",
         "    requirements=REQUIREMENTS,\n",
@@ -693,12 +793,12 @@
       "outputs": [],
       "source": [
         "CMDARGS = [\n",
-        "    \"--model-dir=\" + BUCKET_NAME,\n",
+        "    \"--model-dir=\" + BUCKET_URI,\n",
         "]\n",
         "\n",
         "job.run(args=CMDARGS, replica_count=1, machine_type=TRAIN_COMPUTE, sync=True)\n",
         "\n",
-        "! gsutil cat {BUCKET_NAME}/test.txt"
+        "! gsutil cat {BUCKET_URI}/test.txt"
       ]
     },
     {
@@ -768,9 +868,9 @@
       "source": [
         "DISPLAY_NAME = \"boston_\" + TIMESTAMP\n",
         "\n",
-        "job = aip.CustomPythonPackageTrainingJob(\n",
+        "job = aiplatform.CustomPythonPackageTrainingJob(\n",
         "    display_name=DISPLAY_NAME,\n",
-        "    python_package_gcs_uri=f\"{BUCKET_NAME}/trainer_boston.tar.gz\",\n",
+        "    python_package_gcs_uri=f\"{BUCKET_URI}/trainer_boston.tar.gz\",\n",
         "    python_module_name=\"trainer.task\",\n",
         "    container_uri=TRAIN_IMAGE,\n",
         ")"
@@ -900,7 +1000,7 @@
         "! rm -f custom.tar custom.tar.gz\n",
         "! tar cvf custom.tar custom\n",
         "! gzip custom.tar\n",
-        "! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_boston.tar.gz"
+        "! gsutil cp custom.tar.gz $BUCKET_URI/trainer_boston.tar.gz"
       ]
     },
     {
@@ -922,11 +1022,11 @@
       },
       "outputs": [],
       "source": [
-        "CMDARGS = [\"--model-dir=\" + BUCKET_NAME, \"--epochs=5\"]\n",
+        "CMDARGS = [\"--model-dir=\" + BUCKET_URI, \"--epochs=5\"]\n",
         "\n",
         "job.run(args=CMDARGS, replica_count=1, machine_type=TRAIN_COMPUTE, sync=True)\n",
         "\n",
-        "! gsutil cat {BUCKET_NAME}/test.txt"
+        "! gsutil cat {BUCKET_URI}/test.txt"
       ]
     },
     {
@@ -1151,7 +1251,11 @@
       },
       "outputs": [],
       "source": [
-        "! docker build custom -t $TRAIN_IMAGE"
+        "if not IS_COLAB:\n",
+        "    ! docker build custom -t $TRAIN_IMAGE\n",
+        "else:\n",
+        "    # install docker daemon\n",
+        "    ! apt-get -qq install docker.io"
       ]
     },
     {
@@ -1173,7 +1277,8 @@
       },
       "outputs": [],
       "source": [
-        "! docker run $TRAIN_IMAGE --epochs=5 --model-dir=./"
+        "if not IS_COLAB:\n",
+        "    ! docker run $TRAIN_IMAGE --epochs=5 --model-dir=./"
       ]
     },
     {
@@ -1195,7 +1300,38 @@
       },
       "outputs": [],
       "source": [
-        "! docker push $TRAIN_IMAGE"
+        "if not IS_COLAB:\n",
+        "    ! docker push $TRAIN_IMAGE"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "f50e9c553fb7"
+      },
+      "source": [
+        "*Executes in Colab*"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "a7e8c98f1e56"
+      },
+      "outputs": [],
+      "source": [
+        "%%bash -s $IS_COLAB $TRAIN_IMAGE\n",
+        "if [ $1 == \"False\" ]; then\n",
+        "  exit 0\n",
+        "fi\n",
+        "set -x\n",
+        "dockerd -b none --iptables=0 -l warn &\n",
+        "for i in $(seq 5); do [ ! -S \"/var/run/docker.sock\" ] && sleep 2 || break; done\n",
+        "docker build custom -t $2\n",
+        "docker run $2 --epochs=5 --model-dir=./\n",
+        "docker push $2\n",
+        "kill $(jobs -p)"
       ]
     },
     {
@@ -1229,7 +1365,7 @@
       },
       "outputs": [],
       "source": [
-        "job = aip.CustomContainerTrainingJob(\n",
+        "job = aiplatform.CustomContainerTrainingJob(\n",
         "    display_name=\"boston_\" + TIMESTAMP,\n",
         "    container_uri=TRAIN_IMAGE,\n",
         "    command=[\"python3\", \"trainer/task.py\"],\n",
@@ -1257,11 +1393,11 @@
       },
       "outputs": [],
       "source": [
-        "CMDARGS = [\"--model-dir=\" + BUCKET_NAME, \"--epochs=5\"]\n",
+        "CMDARGS = [\"--model-dir=\" + BUCKET_URI, \"--epochs=5\"]\n",
         "\n",
         "job.run(args=CMDARGS, replica_count=1, machine_type=TRAIN_COMPUTE, sync=True)\n",
         "\n",
-        "! gsutil cat {BUCKET_NAME}/test.txt"
+        "! gsutil cat {BUCKET_URI}/test.txt"
       ]
     },
     {
@@ -1466,7 +1602,7 @@
         "! rm -f custom.tar custom.tar.gz\n",
         "! tar cvf custom.tar custom\n",
         "! gzip custom.tar\n",
-        "! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_boston.tar.gz"
+        "! gsutil cp custom.tar.gz $BUCKET_URI/trainer_boston.tar.gz"
       ]
     },
     {
@@ -1496,7 +1632,7 @@
         "if os.getenv(\"IS_TESTING_TF\"):\n",
         "    TF = os.getenv(\"IS_TESTING_TF\")\n",
         "else:\n",
-        "    TF = \"2.1\".replace(\".\", \"-\")\n",
+        "    TF = \"2.5\".replace(\".\", \"-\")\n",
         "\n",
         "if TF[0] == \"2\":\n",
         "    if TRAIN_GPU:\n",
@@ -1551,9 +1687,9 @@
       "source": [
         "DISPLAY_NAME = \"boston_\" + TIMESTAMP\n",
         "\n",
-        "job = aip.CustomPythonPackageTrainingJob(\n",
+        "job = aiplatform.CustomPythonPackageTrainingJob(\n",
         "    display_name=DISPLAY_NAME,\n",
-        "    python_package_gcs_uri=f\"{BUCKET_NAME}/trainer_boston.tar.gz\",\n",
+        "    python_package_gcs_uri=f\"{BUCKET_URI}/trainer_boston.tar.gz\",\n",
         "    python_module_name=\"trainer.task\",\n",
         "    container_uri=TRAIN_IMAGE,\n",
         "    model_serving_container_image_uri=DEPLOY_IMAGE,\n",
@@ -1587,12 +1723,12 @@
       },
       "outputs": [],
       "source": [
-        "MODEL_DIR = \"{}/{}\".format(BUCKET_NAME, TIMESTAMP)\n",
+        "MODEL_DIR = \"{}/{}\".format(BUCKET_URI, TIMESTAMP)\n",
         "\n",
         "EPOCHS = 20\n",
         "STEPS = 100\n",
         "\n",
-        "DIRECT = True\n",
+        "DIRECT = False\n",
         "if DIRECT:\n",
         "    CMDARGS = [\n",
         "        \"--model-dir=\" + MODEL_DIR,\n",
@@ -1758,17 +1894,7 @@
         "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
         "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
         "\n",
-        "Otherwise, you can delete the individual resources you created in this tutorial:\n",
-        "\n",
-        "- Dataset\n",
-        "- Pipeline\n",
-        "- Model\n",
-        "- Endpoint\n",
-        "- AutoML Training Job\n",
-        "- Batch Job\n",
-        "- Custom Job\n",
-        "- Hyperparameter Tuning Job\n",
-        "- Cloud Storage Bucket"
+        "Otherwise, you can delete the individual resources you created in this tutorial."
       ]
     },
     {
@@ -1779,61 +1905,24 @@
       },
       "outputs": [],
       "source": [
-        "delete_all = True\n",
+        "delete_bucket = False\n",
+        "delete_model = True\n",
+        "delete_job = True\n",
         "\n",
-        "if delete_all:\n",
-        "    # Delete the dataset using the Vertex dataset object\n",
+        "if delete_model:\n",
         "    try:\n",
-        "        if \"dataset\" in globals():\n",
-        "            dataset.delete()\n",
+        "        model.delete()\n",
         "    except Exception as e:\n",
         "        print(e)\n",
         "\n",
-        "    # Delete the model using the Vertex model object\n",
+        "if delete_job:\n",
         "    try:\n",
-        "        if \"model\" in globals():\n",
-        "            model.delete()\n",
+        "        job.delete()\n",
         "    except Exception as e:\n",
         "        print(e)\n",
         "\n",
-        "    # Delete the endpoint using the Vertex endpoint object\n",
-        "    try:\n",
-        "        if \"endpoint\" in globals():\n",
-        "            endpoint.undeploy_all()\n",
-        "            endpoint.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the AutoML or Pipeline training job\n",
-        "    try:\n",
-        "        if \"dag\" in globals():\n",
-        "            dag.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the custom training job\n",
-        "    try:\n",
-        "        if \"job\" in globals():\n",
-        "            job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the batch prediction job using the Vertex batch prediction object\n",
-        "    try:\n",
-        "        if \"batch_predict_job\" in globals():\n",
-        "            batch_predict_job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the hyperparameter tuning job using the Vertex hyperparameter tuning object\n",
-        "    try:\n",
-        "        if \"hpt_job\" in globals():\n",
-        "            hpt_job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    if \"BUCKET_NAME\" in globals():\n",
-        "        ! gsutil rm -r $BUCKET_NAME"
+        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "    ! gsutil rm -rf {BUCKET_URI}"
       ]
     }
   ],

notebooks/community/ml_ops/stage2/get_started_vertex_training_pytorch.ipynb

@@ -33,14 +33,20 @@
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_pytorch.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
         "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_pytorch.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_pytorch.ipynb\">\n",
-        "      Open in Google Cloud Notebooks\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/notebook_template.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
         "    </a>\n",
         "  </td>\n",
         "</table>\n",
@@ -82,8 +88,9 @@
         "\n",
         "This tutorial uses the following Google Cloud ML services:\n",
         "\n",
-        "- `Vertex AI Training`\n",
-        "- `Vertex AI Model` resource\n",
+        "* `Vertex AI Training`\n",
+        "* `Vertex AI Model` resource\n",
+        "\n",
         "\n",
         "The steps performed include:\n",
         "\n",
@@ -93,6 +100,75 @@
         "- Create a `Vertex AI Model` resource."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "85ee859437ed"
+      },
+      "source": [
+        "## Costs \n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI\n",
+        "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
+        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
+        "Calculator](https://cloud.google.com/products/calculator/)\n",
+        "to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5cd61a5dd9db"
+      },
+      "source": [
+        "### Set up your local development environment\n",
+        "\n",
+        "**If you are using Colab or Google Cloud Notebooks**, your environment already meets\n",
+        "all the requirements to run this notebook. You can skip this step."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "7e689ee0bc3c"
+      },
+      "source": [
+        "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
+        "You need the following:\n",
+        "\n",
+        "* The Google Cloud SDK\n",
+        "* Git\n",
+        "* Python 3\n",
+        "* virtualenv\n",
+        "* Jupyter notebook running in a virtual environment with Python 3\n",
+        "\n",
+        "The Google Cloud guide to [Setting up a Python development\n",
+        "environment](https://cloud.google.com/python/setup) and the [Jupyter\n",
+        "installation guide](https://jupyter.org/install) provide detailed instructions\n",
+        "for meeting these requirements. The following steps provide a condensed set of\n",
+        "instructions:\n",
+        "\n",
+        "1. [Install and initialize the Cloud SDK.](https://cloud.google.com/sdk/docs/)\n",
+        "\n",
+        "1. [Install Python 3.](https://cloud.google.com/python/setup#installing_python)\n",
+        "\n",
+        "1. [Install\n",
+        "   virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv)\n",
+        "   and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
+        "\n",
+        "1. To install Jupyter, run `pip3 install jupyter` on the\n",
+        "command-line in a terminal shell.\n",
+        "\n",
+        "1. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
+        "\n",
+        "1. Open this notebook in the Jupyter Notebook Dashboard."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -101,7 +177,7 @@
       "source": [
         "## Installations\n",
         "\n",
-        "Install *one time* the packages for executing the MLOps notebooks."
+        "Install the following packages to execute this notebook."
       ]
     },
     {
@@ -112,22 +188,22 @@
       },
       "outputs": [],
       "source": [
-        "ONCE_ONLY = False\n",
-        "if ONCE_ONLY:\n",
-        "    ! pip3 install -U tensorflow==2.5 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-transform==1.2 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-io==0.18 $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-bigquery $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-logging $USER_FLAG\n",
-        "    ! pip3 install --upgrade apache-beam[gcp] $USER_FLAG\n",
-        "    ! pip3 install --upgrade pyarrow $USER_FLAG\n",
-        "    ! pip3 install --upgrade cloudml-hypertune $USER_FLAG\n",
-        "    ! pip3 install --upgrade kfp $USER_FLAG\n",
-        "    ! pip3 install --upgrade torchvision $USER_FLAG\n",
-        "    ! pip3 install --upgrade rpy2 $USER_FLAG"
+        "import os\n",
+        "\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
+        "\n",
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "USER_FLAG = \"\"\n",
+        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\"\n",
+        "\n",
+        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG -q\n",
+        "! pip3 install --upgrade cloudml-hypertune $USER_FLAG -q\n",
+        "! pip3 install --upgrade torchvision $USER_FLAG -q"
       ]
     },
     {
@@ -159,6 +235,32 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "84cd83853240"
+      },
+      "source": [
+        "## Before you begin\n",
+        "\n",
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI, BigQuery, Compute Engine and Cloud Storage APIs](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,bigquery,compute_component,storage_component).\n",
+        "\n",
+        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -235,7 +337,10 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -262,6 +367,67 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "77c385f0db59"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
+        "\n",
+        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
+        "\n",
+        "**Otherwise**, follow these steps:\n",
+        "\n",
+        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
+        "\n",
+        "1. **Click Create service account**.\n",
+        "\n",
+        "2. In the **Service account name** field, enter a name, and click **Create**.\n",
+        "\n",
+        "3. In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex AI\" into the filter box, and select **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "\n",
+        "4. Click Create. A JSON file that contains your key downloads to your local environment.\n",
+        "\n",
+        "5. Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "535223fa4b84"
+      },
+      "outputs": [],
+      "source": [
+        "# If you are running this notebook in Colab, run this cell and follow the\n",
+        "# instructions to authenticate your GCP account. This provides access to your\n",
+        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
+        "# requests.\n",
+        "\n",
+        "import os\n",
+        "import sys\n",
+        "\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = False\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
+        "    if \"google.colab\" in sys.modules:\n",
+        "        IS_COLAB = True\n",
+        "        from google.colab import auth as google_auth\n",
+        "\n",
+        "        google_auth.authenticate_user()\n",
+        "\n",
+        "    # If you are running this notebook locally, replace the string below with the\n",
+        "    # path to your service account key and run this cell to authenticate your GCP\n",
+        "    # account.\n",
+        "    elif not os.getenv(\"IS_TESTING\"):\n",
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -285,7 +451,7 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
+        "BUCKET_URI = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -296,8 +462,8 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
       ]
     },
     {
@@ -317,7 +483,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_NAME"
+        "! gsutil mb -l $REGION $BUCKET_URI"
       ]
     },
     {
@@ -337,7 +503,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_NAME"
+        "! gsutil ls -al $BUCKET_URI"
       ]
     },
     {
@@ -360,7 +526,7 @@
       },
       "outputs": [],
       "source": [
-        "import google.cloud.aiplatform as aip"
+        "import google.cloud.aiplatform as aiplatform"
       ]
     },
     {
@@ -382,7 +548,7 @@
       },
       "outputs": [],
       "source": [
-        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_NAME)"
+        "aiplatform.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
       ]
     },
     {
@@ -412,13 +578,15 @@
       },
       "outputs": [],
       "source": [
+        "import os\n",
+        "\n",
         "if os.getenv(\"IS_TESTING_TRAIN_GPU\"):\n",
         "    TRAIN_GPU, TRAIN_NGPU = (\n",
-        "        aip.gapic.AcceleratorType.NVIDIA_TESLA_K80,\n",
+        "        aiplatform.gapic.AcceleratorType.NVIDIA_TESLA_K80,\n",
         "        int(os.getenv(\"IS_TESTING_TRAIN_GPU\")),\n",
         "    )\n",
         "else:\n",
-        "    TRAIN_GPU, TRAIN_NGPU = (aip.gapic.AcceleratorType.NVIDIA_TESLA_K80, 1)"
+        "    TRAIN_GPU, TRAIN_NGPU = (aiplatform.gapic.AcceleratorType.NVIDIA_TESLA_K80, 1)"
       ]
     },
     {
@@ -561,6 +729,7 @@
         "# Add package information\n",
         "! touch custom/README.md\n",
         "\n",
+        "# Instructions for installing package into environment of the docker image\n",
         "setup_cfg = \"[egg_info]\\n\\ntag_build =\\n\\ntag_date = 0\"\n",
         "! echo \"$setup_cfg\" > custom/setup.cfg\n",
         "\n",
@@ -891,7 +1060,7 @@
         "! rm -f custom.tar custom.tar.gz\n",
         "! tar cvf custom.tar custom\n",
         "! gzip custom.tar\n",
-        "! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_cifar10.tar.gz"
+        "! gsutil cp custom.tar.gz $BUCKET_URI/trainer_cifar10.tar.gz"
       ]
     },
     {
@@ -902,7 +1071,7 @@
       "source": [
         "### Make Pytorch container for prediction\n",
         "\n",
-        "Currently, Vertex AI does not have a prefined container for making predictions with a deployed Pytorch model. No problem, you can assemble your own custom container. Typically, one would base the container on the `Torch Server`. For demonstration purpose, you build a placeholder container (not complete) that includes the latest `Torch Server` image, and push it to the `Container Registry`."
+        "Currently, Vertex AI does not have a predefined container for making predictions with a deployed Pytorch model. No problem, you can assemble your own custom container. Typically, one would base the container on the `Torch Server`. For demonstration purpose, you build a placeholder container (not complete) that includes the latest `Torch Server` image, and push it to the `Container Registry`."
       ]
     },
     {
@@ -932,11 +1101,52 @@
       "source": [
         "APP_NAME = \"cifar10\"\n",
         "DEPLOY_IMAGE = f\"gcr.io/{PROJECT_ID}/pytorch_predict_{APP_NAME}\"\n",
-        "print(DEPLOY_IMAGE)\n",
-        "\n",
-        "! docker build --tag=$DEPLOY_IMAGE ./\n",
-        "\n",
-        "! docker push $DEPLOY_IMAGE"
+        "print(DEPLOY_IMAGE)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "85739262f629"
+      },
+      "outputs": [],
+      "source": [
+        "if not IS_COLAB:\n",
+        "    ! docker build --tag=$DEPLOY_IMAGE ./\n",
+        "    ! docker push $DEPLOY_IMAGE\n",
+        "else:\n",
+        "    # install docker daemon\n",
+        "    ! apt-get -qq install docker.io"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "f50e9c553fb7"
+      },
+      "source": [
+        "*Executes in Colab*"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "a7e8c98f1e56"
+      },
+      "outputs": [],
+      "source": [
+        "%%bash -s $IS_COLAB $DEPLOY_IMAGE\n",
+        "if [ $1 == \"False\" ]; then\n",
+        "  exit 0\n",
+        "fi\n",
+        "set -x\n",
+        "dockerd -b none --iptables=0 -l warn &\n",
+        "for i in $(seq 5); do [ ! -S \"/var/run/docker.sock\" ] && sleep 2 || break; done\n",
+        "docker build --tag=$2 ./\n",
+        "docker push $2\n",
+        "kill $(jobs -p)"
       ]
     },
     {
@@ -974,9 +1184,9 @@
       "source": [
         "DISPLAY_NAME = \"cifar10_\" + TIMESTAMP\n",
         "\n",
-        "job = aip.CustomPythonPackageTrainingJob(\n",
+        "job = aiplatform.CustomPythonPackageTrainingJob(\n",
         "    display_name=DISPLAY_NAME,\n",
-        "    python_package_gcs_uri=f\"{BUCKET_NAME}/trainer_cifar10.tar.gz\",\n",
+        "    python_package_gcs_uri=f\"{BUCKET_URI}/trainer_cifar10.tar.gz\",\n",
         "    python_module_name=\"trainer.task\",\n",
         "    container_uri=TRAIN_IMAGE,\n",
         "    model_serving_container_image_uri=DEPLOY_IMAGE,\n",
@@ -1009,7 +1219,7 @@
       },
       "outputs": [],
       "source": [
-        "MODEL_DIR = \"{}/{}\".format(BUCKET_NAME, TIMESTAMP)\n",
+        "MODEL_DIR = \"{}/{}\".format(BUCKET_URI, TIMESTAMP)\n",
         "\n",
         "DIRECT = False\n",
         "if DIRECT:\n",
@@ -1121,7 +1331,7 @@
       "source": [
         "### Delete a custom training job\n",
         "\n",
-        "After a training job is completed, you can delete the training job with the method `delete()`.  Prior to completion, a training job can be canceled with the method `cancel()`."
+        "After a training job is completed, you can delete the training job with the method `delete()`.  Prior to completion, a training job can be cancelled with the method `cancel()`."
       ]
     },
     {
@@ -1148,14 +1358,7 @@
         "\n",
         "Otherwise, you can delete the individual resources you created in this tutorial:\n",
         "\n",
-        "- Dataset\n",
-        "- Pipeline\n",
         "- Model\n",
-        "- Endpoint\n",
-        "- AutoML Training Job\n",
-        "- Batch Job\n",
-        "- Custom Job\n",
-        "- Hyperparameter Tuning Job\n",
         "- Cloud Storage Bucket"
       ]
     },
@@ -1167,61 +1370,12 @@
       },
       "outputs": [],
       "source": [
-        "delete_all = True\n",
+        "# Delete the model using the Vertex model object\n",
+        "model.delete()\n",
         "\n",
-        "if delete_all:\n",
-        "    # Delete the dataset using the Vertex dataset object\n",
-        "    try:\n",
-        "        if \"dataset\" in globals():\n",
-        "            dataset.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the model using the Vertex model object\n",
-        "    try:\n",
-        "        if \"model\" in globals():\n",
-        "            model.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the endpoint using the Vertex endpoint object\n",
-        "    try:\n",
-        "        if \"endpoint\" in globals():\n",
-        "            endpoint.undeploy_all()\n",
-        "            endpoint.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the AutoML or Pipeline training job\n",
-        "    try:\n",
-        "        if \"dag\" in globals():\n",
-        "            dag.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the custom training job\n",
-        "    try:\n",
-        "        if \"job\" in globals():\n",
-        "            job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the batch prediction job using the Vertex batch prediction object\n",
-        "    try:\n",
-        "        if \"batch_predict_job\" in globals():\n",
-        "            batch_predict_job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the hyperparameter tuning job using the Vertex hyperparameter tuning object\n",
-        "    try:\n",
-        "        if \"hpt_job\" in globals():\n",
-        "            hpt_job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    if \"BUCKET_NAME\" in globals():\n",
-        "        ! gsutil rm -r $BUCKET_NAME"
+        "delete_bucket = False\n",
+        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "    ! gsutil rm -r $BUCKET_URI"
       ]
     }
   ],

notebooks/community/ml_ops/stage2/get_started_vertex_training_r.ipynb

@@ -29,7 +29,7 @@
         "id": "title:generic,gcp"
       },
       "source": [
-        "# E2E ML on GCP: MLOps stage 2 : experimentation: get started with Vertex Training for R\n",
+        "# E2E ML on GCP: MLOps stage 2 : experimentation: get started with Vertex AI Training for R\n",
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
@@ -39,8 +39,14 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_r.ipynb\">\n",
-        "      Open in Google Cloud Notebooks\n",
+        "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_r.ipynb\">\n",
+        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\\\" alt=\"Colab logo\"> Run in Colab\n",
+        "        </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_r.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
         "    </a>\n",
         "  </td>\n",
         "</table>\n",
@@ -56,7 +62,7 @@
         "## Overview\n",
         "\n",
         "\n",
-        "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 2 : experimentation: get started with Vertex Training for R."
+        "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 2 : experimentation: get started with Vertex AI Training for R. Please note that this notebook should be ran only in R notebook image (e.g., R4.1)."
       ]
     },
     {
@@ -98,6 +104,28 @@
         "- Train a R model using `Vertex AI Trainingh` service with the R-to-Python training package."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0c997d8d92ce"
+      },
+      "source": [
+        "### Costs \n",
+        "\n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Cloud Storage\n",
+        "\n",
+        "\n",
+        "Learn about [Vertex AI\n",
+        "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
+        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
+        "Calculator](https://cloud.google.com/products/calculator/)\n",
+        "to generate a cost estimate based on your projected usage."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -106,33 +134,32 @@
       "source": [
         "## Installations\n",
         "\n",
-        "Install *one time* the packages for executing the MLOps notebooks."
+        "Install the packages required for executing this notebook."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "install_mlops"
+        "id": "1fd00fa70a2a"
       },
       "outputs": [],
       "source": [
-        "ONCE_ONLY = False\n",
-        "if ONCE_ONLY:\n",
-        "    ! pip3 install -U tensorflow==2.5 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-transform==1.2 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-io==0.18 $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-bigquery $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-logging $USER_FLAG\n",
-        "    ! pip3 install --upgrade apache-beam[gcp] $USER_FLAG\n",
-        "    ! pip3 install --upgrade pyarrow $USER_FLAG\n",
-        "    ! pip3 install --upgrade cloudml-hypertune $USER_FLAG\n",
-        "    ! pip3 install --upgrade kfp $USER_FLAG\n",
-        "    ! pip3 install --upgrade torchvision $USER_FLAG\n",
-        "    ! pip3 install --upgrade rpy2 $USER_FLAG"
+        "import os\n",
+        "\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
+        "\n",
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "USER_FLAG = \"\"\n",
+        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\"\n",
+        "\n",
+        "! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG -q\n",
+        "! pip3 install --upgrade rpy2 $USER_FLAG -q"
       ]
     },
     {
@@ -164,6 +191,39 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0e3cab0cc491"
+      },
+      "source": [
+        "## Before you begin"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "be929e7b4d76"
+      },
+      "source": [
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com). \n",
+        "\n",
+        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -183,6 +243,8 @@
       },
       "outputs": [],
       "source": [
+        "import os\n",
+        "\n",
         "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
       ]
     },
@@ -240,7 +302,9 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+        "REGION = \"[your-region]\"  # @param {type:\"string\"}\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -267,6 +331,67 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3ffa6b6c7cdb"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
+        "\n",
+        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
+        "\n",
+        "**Otherwise**, follow these steps:\n",
+        "\n",
+        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
+        "\n",
+        "1. **Click Create service account**.\n",
+        "\n",
+        "2. In the **Service account name** field, enter a name, and click **Create**.\n",
+        "\n",
+        "3. In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex AI\" into the filter box, and select **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "\n",
+        "4. Click Create. A JSON file that contains your key downloads to your local environment.\n",
+        "\n",
+        "5. Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2b72272258fc"
+      },
+      "outputs": [],
+      "source": [
+        "# If you are running this notebook in Colab, run this cell and follow the\n",
+        "# instructions to authenticate your GCP account. This provides access to your\n",
+        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
+        "# requests.\n",
+        "\n",
+        "import os\n",
+        "import sys\n",
+        "\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = False\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
+        "    if \"google.colab\" in sys.modules:\n",
+        "        IS_COLAB = True\n",
+        "        from google.colab import auth as google_auth\n",
+        "\n",
+        "        google_auth.authenticate_user()\n",
+        "\n",
+        "    # If you are running this notebook locally, replace the string below with the\n",
+        "    # path to your service account key and run this cell to authenticate your GCP\n",
+        "    # account.\n",
+        "    elif not os.getenv(\"IS_TESTING\"):\n",
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -277,7 +402,7 @@
         "\n",
         "**The following steps are required, regardless of your notebook environment.**\n",
         "\n",
-        "When you initialize the Vertex SDK for Python, you specify a Cloud Storage staging bucket. The staging bucket is where all the data associated with your dataset and model resources are retained across sessions.\n",
+        "When you initialize the Vertex AI SDK for Python, you specify a Cloud Storage staging bucket. The staging bucket is where all the data associated with your dataset and model resources are retained across sessions.\n",
         "\n",
         "Set the name of your Cloud Storage bucket below. Bucket names must be globally unique across all Google Cloud projects, including those outside of your organization."
       ]
@@ -290,7 +415,7 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
+        "BUCKET_URI = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -301,8 +426,8 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
       ]
     },
     {
@@ -322,7 +447,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_NAME"
+        "! gsutil mb -l $REGION $BUCKET_URI"
       ]
     },
     {
@@ -342,7 +467,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_NAME"
+        "! gsutil ls -al $BUCKET_URI"
       ]
     },
     {
@@ -365,6 +490,8 @@
       },
       "outputs": [],
       "source": [
+        "import traceback\n",
+        "\n",
         "import google.cloud.aiplatform as aip"
       ]
     },
@@ -387,7 +514,7 @@
       },
       "outputs": [],
       "source": [
-        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_NAME)"
+        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
       ]
     },
     {
@@ -964,14 +1091,17 @@
       },
       "outputs": [],
       "source": [
-        "INSTANCES = [\n",
-        "    {\"sepal_width\": 1, \"sepal_length\": 2, \"petal_width\": 3, \"petal_length\": 1},\n",
-        "    {\"sepal_width\": 4, \"sepal_length\": 2, \"petal_width\": 1, \"petal_length\": 1},\n",
-        "]\n",
+        "try:\n",
+        "    INSTANCES = [\n",
+        "        {\"sepal_width\": 1, \"sepal_length\": 2, \"petal_width\": 3, \"petal_length\": 1},\n",
+        "        {\"sepal_width\": 4, \"sepal_length\": 2, \"petal_width\": 1, \"petal_length\": 1},\n",
+        "    ]\n",
         "\n",
-        "prediction = endpoint.predict(instances=INSTANCES)\n",
+        "    prediction = endpoint.predict(instances=INSTANCES)\n",
         "\n",
-        "print(prediction)"
+        "    print(prediction)\n",
+        "except:\n",
+        "    traceback.print_exc()"
       ]
     },
     {
@@ -1242,7 +1372,7 @@
       },
       "outputs": [],
       "source": [
-        "CMDARGS = [\"--model-dir=\" + BUCKET_NAME]\n",
+        "CMDARGS = [\"--model-dir=\" + BUCKET_URI]\n",
         "\n",
         "job.run(args=CMDARGS, replica_count=1, machine_type=TRAIN_COMPUTE, sync=True)"
       ]
@@ -1282,14 +1412,9 @@
         "\n",
         "Otherwise, you can delete the individual resources you created in this tutorial:\n",
         "\n",
-        "- Dataset\n",
-        "- Pipeline\n",
-        "- Model\n",
-        "- Endpoint\n",
-        "- AutoML Training Job\n",
-        "- Batch Job\n",
-        "- Custom Job\n",
-        "- Hyperparameter Tuning Job\n",
+        "- Model (Already deleted in previous cells)\n",
+        "- Endpoint (Already deleted in previous cells)\n",
+        "- Custom Job (Already deleted in previous cells)\n",
         "- Cloud Storage Bucket"
       ]
     },
@@ -1301,61 +1426,10 @@
       },
       "outputs": [],
       "source": [
-        "delete_all = True\n",
+        "delete_bucket = False\n",
         "\n",
-        "if delete_all:\n",
-        "    # Delete the dataset using the Vertex dataset object\n",
-        "    try:\n",
-        "        if \"dataset\" in globals():\n",
-        "            dataset.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the model using the Vertex model object\n",
-        "    try:\n",
-        "        if \"model\" in globals():\n",
-        "            model.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the endpoint using the Vertex endpoint object\n",
-        "    try:\n",
-        "        if \"endpoint\" in globals():\n",
-        "            endpoint.undeploy_all()\n",
-        "            endpoint.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the AutoML or Pipeline training job\n",
-        "    try:\n",
-        "        if \"dag\" in globals():\n",
-        "            dag.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the custom training job\n",
-        "    try:\n",
-        "        if \"job\" in globals():\n",
-        "            job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the batch prediction job using the Vertex batch prediction object\n",
-        "    try:\n",
-        "        if \"batch_predict_job\" in globals():\n",
-        "            batch_predict_job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the hyperparameter tuning job using the Vertex hyperparameter tuning object\n",
-        "    try:\n",
-        "        if \"hpt_job\" in globals():\n",
-        "            hpt_job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    if \"BUCKET_NAME\" in globals():\n",
-        "        ! gsutil rm -r $BUCKET_NAME"
+        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "    ! gsutil rm -r $BUCKET_URI"
       ]
     }
   ],

notebooks/community/ml_ops/stage2/get_started_vertex_training_sklearn.ipynb

@@ -33,14 +33,23 @@
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_sklearn.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
         "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_sklearn.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_sklearn.ipynb\">\n",
-        "      Open in Google Cloud Notebooks\n",
+        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_sklearn.ipynb\">\n",
+        "     <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "  </td>\n",
+        "</table>\n",
         "    </a>\n",
         "  </td>\n",
         "</table>\n",
@@ -56,7 +65,7 @@
         "## Overview\n",
         "\n",
         "\n",
-        "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 2 : experimentation: get started with Vertex Training for Scikit-Learn."
+        "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 2 : experimentation: get started with Vertex AI Training for scikit-Learn."
       ]
     },
     {
@@ -93,41 +102,101 @@
         "- Create a `Vertex AI Model` resource."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "b132d4ef86d6"
+      },
+      "source": [
+        "### Costs \n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI\n",
+        "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
+        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
+        "Calculator](https://cloud.google.com/products/calculator/)\n",
+        "to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "94a148f11da5"
+      },
+      "source": [
+        "### Set up your local development environment\n",
+        "\n",
+        "**If you are using Colab or Google Cloud Notebooks**, your environment already meets\n",
+        "all the requirements to run this notebook. You can skip this step.\n",
+        "\n",
+        "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
+        "You need the following:\n",
+        "\n",
+        "* The Google Cloud SDK\n",
+        "* Git\n",
+        "* Python 3\n",
+        "* virtualenv\n",
+        "* Jupyter notebook running in a virtual environment with Python 3\n",
+        "\n",
+        "The Google Cloud guide to [Setting up a Python development\n",
+        "environment](https://cloud.google.com/python/setup) and the [Jupyter\n",
+        "installation guide](https://jupyter.org/install) provide detailed instructions\n",
+        "for meeting these requirements. The following steps provide a condensed set of\n",
+        "instructions:\n",
+        "\n",
+        "1. [Install and initialize the Cloud SDK.](https://cloud.google.com/sdk/docs/)\n",
+        "\n",
+        "1. [Install Python 3.](https://cloud.google.com/python/setup#installing_python)\n",
+        "\n",
+        "1. [Install\n",
+        "   virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv)\n",
+        "   and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
+        "\n",
+        "1. To install Jupyter, run `pip3 install jupyter` on the\n",
+        "command-line in a terminal shell.\n",
+        "\n",
+        "1. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
+        "\n",
+        "1. Open this notebook in the Jupyter Notebook Dashboard.\n"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "install_mlops"
       },
       "source": [
-        "## Installations\n",
+        "### Install additional packages\n",
         "\n",
-        "Install *one time* the packages for executing the MLOps notebooks."
+        "Install the following packages for executing this notebook."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "install_mlops"
+        "id": "78168417490e"
       },
       "outputs": [],
       "source": [
-        "ONCE_ONLY = False\n",
-        "if ONCE_ONLY:\n",
-        "    ! pip3 install -U tensorflow==2.5 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-transform==1.2 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-io==0.18 $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-bigquery $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-logging $USER_FLAG\n",
-        "    ! pip3 install --upgrade apache-beam[gcp] $USER_FLAG\n",
-        "    ! pip3 install --upgrade pyarrow $USER_FLAG\n",
-        "    ! pip3 install --upgrade cloudml-hypertune $USER_FLAG\n",
-        "    ! pip3 install --upgrade kfp $USER_FLAG\n",
-        "    ! pip3 install --upgrade torchvision $USER_FLAG\n",
-        "    ! pip3 install --upgrade rpy2 $USER_FLAG"
+        "import os\n",
+        "\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
+        "\n",
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "USER_FLAG = \"\"\n",
+        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\"\n",
+        "\n",
+        "! pip3 install {USER_FLAG} --upgrade google-cloud-aiplatform -q"
       ]
     },
     {
@@ -159,6 +228,32 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2721ef0202d9"
+      },
+      "source": [
+        "## Before you begin\n",
+        "\n",
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com). \n",
+        "\n",
+        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -235,7 +330,10 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -262,6 +360,67 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "77c385f0db59"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
+        "\n",
+        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
+        "\n",
+        "**Otherwise**, follow these steps:\n",
+        "\n",
+        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
+        "\n",
+        "1. **Click Create service account**.\n",
+        "\n",
+        "2. In the **Service account name** field, enter a name, and click **Create**.\n",
+        "\n",
+        "3. In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex AI\" into the filter box, and select **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "\n",
+        "4. Click Create. A JSON file that contains your key downloads to your local environment.\n",
+        "\n",
+        "5. Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "535223fa4b84"
+      },
+      "outputs": [],
+      "source": [
+        "# If you are running this notebook in Colab, run this cell and follow the\n",
+        "# instructions to authenticate your GCP account. This provides access to your\n",
+        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
+        "# requests.\n",
+        "\n",
+        "import os\n",
+        "import sys\n",
+        "\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = False\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
+        "    if \"google.colab\" in sys.modules:\n",
+        "        IS_COLAB = True\n",
+        "        from google.colab import auth as google_auth\n",
+        "\n",
+        "        google_auth.authenticate_user()\n",
+        "\n",
+        "    # If you are running this notebook locally, replace the string below with the\n",
+        "    # path to your service account key and run this cell to authenticate your GCP\n",
+        "    # account.\n",
+        "    elif not os.getenv(\"IS_TESTING\"):\n",
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -285,7 +444,7 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
+        "BUCKET_URI = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -296,8 +455,8 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
       ]
     },
     {
@@ -317,7 +476,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_NAME"
+        "! gsutil mb -l $REGION $BUCKET_URI"
       ]
     },
     {
@@ -337,7 +496,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_NAME"
+        "! gsutil ls -al $BUCKET_URI"
       ]
     },
     {
@@ -382,7 +541,7 @@
       },
       "outputs": [],
       "source": [
-        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_NAME)"
+        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
       ]
     },
     {
@@ -415,6 +574,8 @@
       },
       "outputs": [],
       "source": [
+        "import os\n",
+        "\n",
         "if os.getenv(\"IS_TESTING_TRAIN_GPU\"):\n",
         "    TRAIN_GPU, TRAIN_NGPU = (\n",
         "        aip.gapic.AcceleratorType.NVIDIA_TESLA_K80,\n",
@@ -517,9 +678,9 @@
         "id": "sklearn_intro"
       },
       "source": [
-        "## Introduction to Scikit-learn training\n",
+        "## Introduction to scikit-learn training\n",
         "\n",
-        "Once you have trained a Scikit-learn model, you will want to save it at a Cloud Storage location, so it can subsequently be uploaded to a `Vertex AI Model` resource. The Scikit-learn package does not have support to save the model to a Cloud Storage location. Instead, you will do the following steps to save to a Cloud Storage location.\n",
+        "Once you have trained a scikit-learn model, you will want to save it at a Cloud Storage location, so it can subsequently be uploaded to a `Vertex AI Model` resource. The Scikit-learn package does not have support to save the model to a Cloud Storage location. Instead, you will do the following steps to save to a Cloud Storage location.\n",
         "\n",
         "1. Save the in-memory model to the local filesystem in pickle format (e.g., model.pkl).\n",
         "2. Create a Cloud Storage storage client.\n",
@@ -783,7 +944,7 @@
         "! rm -f custom.tar custom.tar.gz\n",
         "! tar cvf custom.tar custom\n",
         "! gzip custom.tar\n",
-        "! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_newsaggr.tar.gz"
+        "! gsutil cp custom.tar.gz $BUCKET_URI/trainer_newsaggr.tar.gz"
       ]
     },
     {
@@ -823,7 +984,7 @@
         "\n",
         "job = aip.CustomPythonPackageTrainingJob(\n",
         "    display_name=DISPLAY_NAME,\n",
-        "    python_package_gcs_uri=f\"{BUCKET_NAME}/trainer_newsaggr.tar.gz\",\n",
+        "    python_package_gcs_uri=f\"{BUCKET_URI}/trainer_newsaggr.tar.gz\",\n",
         "    python_module_name=\"trainer.task\",\n",
         "    container_uri=TRAIN_IMAGE,\n",
         "    model_serving_container_image_uri=DEPLOY_IMAGE,\n",
@@ -857,7 +1018,7 @@
       },
       "outputs": [],
       "source": [
-        "MODEL_DIR = \"{}/{}\".format(BUCKET_NAME, TIMESTAMP)\n",
+        "MODEL_DIR = \"{}/{}\".format(BUCKET_URI, TIMESTAMP)\n",
         "DATASET_URL = \"https://archive.ics.uci.edu/ml/machine-learning-databases/00359/NewsAggregatorDataset.zip\"\n",
         "\n",
         "DIRECT = False\n",
@@ -1002,14 +1163,8 @@
         "\n",
         "Otherwise, you can delete the individual resources you created in this tutorial:\n",
         "\n",
-        "- Dataset\n",
-        "- Pipeline\n",
+        "\n",
         "- Model\n",
-        "- Endpoint\n",
-        "- AutoML Training Job\n",
-        "- Batch Job\n",
-        "- Custom Job\n",
-        "- Hyperparameter Tuning Job\n",
         "- Cloud Storage Bucket"
       ]
     },
@@ -1017,65 +1172,16 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "cleanup:mbsdk"
+        "id": "b413063dfdcf"
       },
       "outputs": [],
       "source": [
-        "delete_all = True\n",
+        "# Delete the model using the Vertex model object\n",
+        "model.delete()\n",
         "\n",
-        "if delete_all:\n",
-        "    # Delete the dataset using the Vertex dataset object\n",
-        "    try:\n",
-        "        if \"dataset\" in globals():\n",
-        "            dataset.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the model using the Vertex model object\n",
-        "    try:\n",
-        "        if \"model\" in globals():\n",
-        "            model.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the endpoint using the Vertex endpoint object\n",
-        "    try:\n",
-        "        if \"endpoint\" in globals():\n",
-        "            endpoint.undeploy_all()\n",
-        "            endpoint.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the AutoML or Pipeline training job\n",
-        "    try:\n",
-        "        if \"dag\" in globals():\n",
-        "            dag.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the custom training job\n",
-        "    try:\n",
-        "        if \"job\" in globals():\n",
-        "            job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the batch prediction job using the Vertex batch prediction object\n",
-        "    try:\n",
-        "        if \"batch_predict_job\" in globals():\n",
-        "            batch_predict_job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the hyperparameter tuning job using the Vertex hyperparameter tuning object\n",
-        "    try:\n",
-        "        if \"hpt_job\" in globals():\n",
-        "            hpt_job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    if \"BUCKET_NAME\" in globals():\n",
-        "        ! gsutil rm -r $BUCKET_NAME"
+        "delete_bucket = False\n",
+        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "    ! gsutil rm -r $BUCKET_URI"
       ]
     }
   ],

notebooks/community/ml_ops/stage2/get_started_vertex_training_xgboost.ipynb

@@ -29,9 +29,15 @@
         "id": "title:generic,gcp"
       },
       "source": [
-        "# E2E ML on GCP: MLOps stage 2 : experimentation: get started with Vertex Training for XGBoost\n",
+        "# E2E ML on GCP: MLOps stage 2 : experimentation: get started with Vertex AI Training for XGBoost\n",
         "\n",
         "<table align=\"left\">\n",
+        "    <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_xgboost.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "    \n",
         "  <td>\n",
         "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_xgboost.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
@@ -39,8 +45,9 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_xgboost.ipynb\">\n",
-        "      Open in Google Cloud Notebooks\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_xgboost.ipynb\">\n",
+        "    <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
         "    </a>\n",
         "  </td>\n",
         "</table>\n",
@@ -56,7 +63,7 @@
         "## Overview\n",
         "\n",
         "\n",
-        "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 2 : experimentation: get started with Vertex Training for XGBoost."
+        "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 2 : experimentation: get started with Vertex AI Training for XGBoost."
       ]
     },
     {
@@ -90,7 +97,21 @@
         "- Training using a Python package.\n",
         "- Report accuracy when hyperparameter tuning.\n",
         "- Save the model artifacts to Cloud Storage using GCSFuse.\n",
-        "- Create a `Vertex AI Model` resource."
+        "- Create a `Vertex AI Model` resource.\n",
+        "\n",
+        "### Costs \n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Cloud Storage\n",
+        "\n",
+        "\n",
+        "Learn about [Vertex AI\n",
+        "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
+        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
+        "Calculator](https://cloud.google.com/products/calculator/)\n",
+        "to generate a cost estimate based on your projected usage."
       ]
     },
     {
@@ -101,62 +122,57 @@
       "source": [
         "## Installations\n",
         "\n",
-        "Install *one time* the packages for executing the MLOps notebooks."
+        "Install the following packages to execute this notebook."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "install_mlops"
-      },
-      "outputs": [],
-      "source": [
-        "ONCE_ONLY = False\n",
-        "if ONCE_ONLY:\n",
-        "    ! pip3 install -U tensorflow==2.5 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-transform==1.2 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-io==0.18 $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-bigquery $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-logging $USER_FLAG\n",
-        "    ! pip3 install --upgrade apache-beam[gcp] $USER_FLAG\n",
-        "    ! pip3 install --upgrade pyarrow $USER_FLAG\n",
-        "    ! pip3 install --upgrade cloudml-hypertune $USER_FLAG\n",
-        "    ! pip3 install --upgrade kfp $USER_FLAG\n",
-        "    ! pip3 install --upgrade torchvision $USER_FLAG\n",
-        "    ! pip3 install --upgrade rpy2 $USER_FLAG"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "restart"
-      },
-      "source": [
-        "### Restart the kernel\n",
-        "\n",
-        "Once you've installed the additional packages, you need to restart the notebook kernel so it can find the packages."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "restart"
+        "id": "ncRJ_Dfdox9L"
       },
       "outputs": [],
       "source": [
         "import os\n",
         "\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    # Automatically restart kernel after installs\n",
-        "    import IPython\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
         "\n",
-        "    app = IPython.Application.instance()\n",
-        "    app.kernel.do_shutdown(True)"
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "USER_FLAG = \"\"\n",
+        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\"\n",
+        "\n",
+        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG -q"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2721ef0202d9"
+      },
+      "source": [
+        "## Before you begin\n",
+        "\n",
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com). \n",
+        "\n",
+        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
       ]
     },
     {
@@ -231,11 +247,14 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "region"
+        "id": "sKBTnvJpox9P"
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+        "REGION = \"[your-region]\"  # @param {type:\"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\"  # @param {type: \"string\"}"
       ]
     },
     {
@@ -253,7 +272,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "timestamp"
+        "id": "JYtXOocrox9Q"
       },
       "outputs": [],
       "source": [
@@ -262,6 +281,67 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "77c385f0db59"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
+        "\n",
+        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
+        "\n",
+        "**Otherwise**, follow these steps:\n",
+        "\n",
+        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
+        "\n",
+        "1. **Click Create service account**.\n",
+        "\n",
+        "2. In the **Service account name** field, enter a name, and click **Create**.\n",
+        "\n",
+        "3. In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex AI\" into the filter box, and select **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "\n",
+        "4. Click Create. A JSON file that contains your key downloads to your local environment.\n",
+        "\n",
+        "5. Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "NNc5Bf_NpPTq"
+      },
+      "outputs": [],
+      "source": [
+        "# If you are running this notebook in Colab, run this cell and follow the\n",
+        "# instructions to authenticate your GCP account. This provides access to your\n",
+        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
+        "# requests.\n",
+        "\n",
+        "import os\n",
+        "import sys\n",
+        "\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = False\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
+        "    if \"google.colab\" in sys.modules:\n",
+        "        IS_COLAB = True\n",
+        "        from google.colab import auth as google_auth\n",
+        "\n",
+        "        google_auth.authenticate_user()\n",
+        "\n",
+        "    # If you are running this notebook locally, replace the string below with the\n",
+        "    # path to your service account key and run this cell to authenticate your GCP\n",
+        "    # account.\n",
+        "    elif not os.getenv(\"IS_TESTING\"):\n",
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -272,7 +352,7 @@
         "\n",
         "**The following steps are required, regardless of your notebook environment.**\n",
         "\n",
-        "When you initialize the Vertex SDK for Python, you specify a Cloud Storage staging bucket. The staging bucket is where all the data associated with your dataset and model resources are retained across sessions.\n",
+        "When you initialize the Vertex AI SDK for Python, you specify a Cloud Storage staging bucket. The staging bucket is where all the data associated with your dataset and model resources are retained across sessions.\n",
         "\n",
         "Set the name of your Cloud Storage bucket below. Bucket names must be globally unique across all Google Cloud projects, including those outside of your organization."
       ]
@@ -285,7 +365,7 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
+        "BUCKET_URI = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -296,8 +376,8 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
       ]
     },
     {
@@ -313,11 +393,11 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "create_bucket"
+        "id": "aO4sKJfFox9R"
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_NAME"
+        "! gsutil mb -l $REGION $BUCKET_URI"
       ]
     },
     {
@@ -333,11 +413,11 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "validate_bucket"
+        "id": "yWnghzKFox9S"
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_NAME"
+        "! gsutil ls -al $BUCKET_URI"
       ]
     },
     {
@@ -378,11 +458,11 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "init_aip:mbsdk"
+        "id": "JZg2sszQox9T"
       },
       "outputs": [],
       "source": [
-        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_NAME)"
+        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
       ]
     },
     {
@@ -411,10 +491,12 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "accelerators:training,cpu,prediction,cpu,mbsdk"
+        "id": "cQUrG4Mbox9T"
       },
       "outputs": [],
       "source": [
+        "import os\n",
+        "\n",
         "if os.getenv(\"IS_TESTING_TRAIN_GPU\"):\n",
         "    TRAIN_GPU, TRAIN_NGPU = (\n",
         "        aip.gapic.AcceleratorType.NVIDIA_TESLA_K80,\n",
@@ -453,7 +535,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "container:training,prediction,xgboost"
+        "id": "XujRA5ueox9U"
       },
       "outputs": [],
       "source": [
@@ -497,7 +579,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "machine:training"
+        "id": "UMPFgENkox9U"
       },
       "outputs": [],
       "source": [
@@ -561,7 +643,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "examine_training_package:xgboost"
+        "id": "f4wS4eISox9V"
       },
       "outputs": [],
       "source": [
@@ -616,7 +698,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "taskpy_contents:iris,xgboost"
+        "id": "WiSnFuDoox9W"
       },
       "outputs": [],
       "source": [
@@ -731,14 +813,14 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "tarball_training_script"
+        "id": "dnmdycf6ox9X"
       },
       "outputs": [],
       "source": [
         "! rm -f custom.tar custom.tar.gz\n",
         "! tar cvf custom.tar custom\n",
         "! gzip custom.tar\n",
-        "! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_iris.tar.gz"
+        "! gsutil cp custom.tar.gz $BUCKET_URI/trainer_iris.tar.gz"
       ]
     },
     {
@@ -770,7 +852,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "create_custom_pp_training_job:mbsdk"
+        "id": "rVEMz1xqox9X"
       },
       "outputs": [],
       "source": [
@@ -778,7 +860,7 @@
         "\n",
         "job = aip.CustomPythonPackageTrainingJob(\n",
         "    display_name=DISPLAY_NAME,\n",
-        "    python_package_gcs_uri=f\"{BUCKET_NAME}/trainer_iris.tar.gz\",\n",
+        "    python_package_gcs_uri=f\"{BUCKET_URI}/trainer_iris.tar.gz\",\n",
         "    python_module_name=\"trainer.task\",\n",
         "    container_uri=TRAIN_IMAGE,\n",
         "    model_serving_container_image_uri=DEPLOY_IMAGE,\n",
@@ -809,11 +891,11 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "prepare_custom_cmdargs:iris,xgboost"
+        "id": "AoUfpBqVox9Y"
       },
       "outputs": [],
       "source": [
-        "MODEL_DIR = \"{}/{}\".format(BUCKET_NAME, TIMESTAMP)\n",
+        "MODEL_DIR = \"{}/{}\".format(BUCKET_URI, TIMESTAMP)\n",
         "DATASET_DIR = \"gs://cloud-samples-data/ai-platform/iris\"\n",
         "\n",
         "ROUNDS = 20\n",
@@ -858,7 +940,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "run_custom_job:mbsdk"
+        "id": "JCruQq1aox9Y"
       },
       "outputs": [],
       "source": [
@@ -899,7 +981,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "list_job"
+        "id": "KBM_KLMSox9Y"
       },
       "outputs": [],
       "source": [
@@ -922,7 +1004,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "custom_job_wait:mbsdk"
+        "id": "lHPMHbSyox9Z"
       },
       "outputs": [],
       "source": [
@@ -944,7 +1026,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "delete_job"
+        "id": "tlYg7Sp-ox9Z"
       },
       "outputs": [],
       "source": [
@@ -964,14 +1046,7 @@
         "\n",
         "Otherwise, you can delete the individual resources you created in this tutorial:\n",
         "\n",
-        "- Dataset\n",
-        "- Pipeline\n",
-        "- Model\n",
-        "- Endpoint\n",
-        "- AutoML Training Job\n",
-        "- Batch Job\n",
-        "- Custom Job\n",
-        "- Hyperparameter Tuning Job\n",
+        "- Custom Job (Custome Training job is remove in previous step)\n",
         "- Cloud Storage Bucket"
       ]
     },
@@ -979,65 +1054,14 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "cleanup:mbsdk"
+        "id": "JyWy23gDox9a"
       },
       "outputs": [],
       "source": [
-        "delete_all = True\n",
+        "delete_bucket = False\n",
         "\n",
-        "if delete_all:\n",
-        "    # Delete the dataset using the Vertex dataset object\n",
-        "    try:\n",
-        "        if \"dataset\" in globals():\n",
-        "            dataset.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the model using the Vertex model object\n",
-        "    try:\n",
-        "        if \"model\" in globals():\n",
-        "            model.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the endpoint using the Vertex endpoint object\n",
-        "    try:\n",
-        "        if \"endpoint\" in globals():\n",
-        "            endpoint.undeploy_all()\n",
-        "            endpoint.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the AutoML or Pipeline training job\n",
-        "    try:\n",
-        "        if \"dag\" in globals():\n",
-        "            dag.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the custom training job\n",
-        "    try:\n",
-        "        if \"job\" in globals():\n",
-        "            job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the batch prediction job using the Vertex batch prediction object\n",
-        "    try:\n",
-        "        if \"batch_predict_job\" in globals():\n",
-        "            batch_predict_job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the hyperparameter tuning job using the Vertex hyperparameter tuning object\n",
-        "    try:\n",
-        "        if \"hpt_job\" in globals():\n",
-        "            hpt_job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    if \"BUCKET_NAME\" in globals():\n",
-        "        ! gsutil rm -r $BUCKET_NAME"
+        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "    ! gsutil rm -r $BUCKET_URI"
       ]
     }
   ],

notebooks/community/ml_ops/stage2/get_started_vertex_vizier.ipynb

@@ -29,7 +29,7 @@
         "id": "title:generic,gcp"
       },
       "source": [
-        "# E2E ML on GCP: MLOps stage 2 : experimentation: get started with Vertex Vizier\n",
+        "# E2E ML on GCP: MLOps stage 2 : experimentation: get started with Vertex AI Vizier\n",
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
@@ -39,7 +39,12 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_vizier.ipynb\">\n",
+        "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_vizier.ipynb\">\n",
+        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\\\" alt=\"Colab logo\"> Run in Colab\n",
+        "        </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage2/get_started_vertex_vizier.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
@@ -137,7 +142,7 @@
       "source": [
         "## Installations\n",
         "\n",
-        "Install *one time* the packages for executing the MLOps notebooks."
+        "Install the packages required for executing this notebook."
       ]
     },
     {
@@ -150,25 +155,18 @@
       "source": [
         "import os\n",
         "\n",
-        "# The Google Cloud Notebook product has specific requirements\n",
-        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
         "\n",
-        "# Google Cloud Notebook requires dependencies to be installed with '--user'\n",
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
         "USER_FLAG = \"\"\n",
-        "if IS_GOOGLE_CLOUD_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\""
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "install_mlops"
-      },
-      "outputs": [],
-      "source": [
-        "! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG\n",
-        "    "
+        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\"\n",
+        "\n",
+        "! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG"
       ]
     },
     {
@@ -200,6 +198,32 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2721ef0202d9"
+      },
+      "source": [
+        "## Before you begin\n",
+        "\n",
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com). \n",
+        "\n",
+        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -305,6 +329,67 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gcp_authenticate"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
+        "\n",
+        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
+        "\n",
+        "**Otherwise**, follow these steps:\n",
+        "\n",
+        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
+        "\n",
+        "**Click Create service account**.\n",
+        "\n",
+        "In the **Service account name** field, enter a name, and click **Create**.\n",
+        "\n",
+        "In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex\" into the filter box, and select **Vertex Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "\n",
+        "Click Create. A JSON file that contains your key downloads to your local environment.\n",
+        "\n",
+        "Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "gcp_authenticate"
+      },
+      "outputs": [],
+      "source": [
+        "# If you are running this notebook in Colab, run this cell and follow the\n",
+        "# instructions to authenticate your GCP account. This provides access to your\n",
+        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
+        "# requests.\n",
+        "\n",
+        "import os\n",
+        "import sys\n",
+        "\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = False\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
+        "    if \"google.colab\" in sys.modules:\n",
+        "        IS_COLAB = True\n",
+        "        from google.colab import auth as google_auth\n",
+        "\n",
+        "        google_auth.authenticate_user()\n",
+        "\n",
+        "    # If you are running this notebook locally, replace the string below with the\n",
+        "    # path to your service account key and run this cell to authenticate your GCP\n",
+        "    # account.\n",
+        "    elif not os.getenv(\"IS_TESTING\"):\n",
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -1598,6 +1683,8 @@
       },
       "outputs": [],
       "source": [
+        "delete_bucket = False\n",
+        "\n",
         "if os.getenv(\"IS_TESTING\"):\n",
         "    ! gsutil rm -r $BUCKET_URI"
       ]

notebooks/community/ml_ops/stage2/get_started_with_cmek_training.ipynb

@@ -38,8 +38,14 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_with_cmek_training.ipynb\">\n",
-        "      Open in Google Cloud Notebooks\n",
+        "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_with_cmek_training.ipynb\">\n",
+        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\\\" alt=\"Colab logo\"> Run in Colab\n",
+        "        </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage2/get_started_with_cmek_training.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
         "    </a>\n",
         "  </td>\n",
         "</table>\n",
@@ -91,6 +97,26 @@
         "- Train an AutoML model with CMEK encryption."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5e2eba58ad71"
+      },
+      "source": [
+        "### Costs \n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI\n",
+        "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
+        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
+        "Calculator](https://cloud.google.com/products/calculator/)\n",
+        "to generate a cost estimate based on your projected usage."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -110,10 +136,21 @@
       },
       "outputs": [],
       "source": [
-        "USER_FLAG = \"--user\"\n",
+        "import os\n",
         "\n",
-        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG\n",
-        "! pip3 install --upgrade google-cloud-kms $USER_FLAG"
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
+        "\n",
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "USER_FLAG = \"\"\n",
+        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\"\n",
+        "\n",
+        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG -q\n",
+        "! pip3 install --upgrade google-cloud-kms $USER_FLAG -q"
       ]
     },
     {
@@ -145,6 +182,39 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "013daf3de88e"
+      },
+      "source": [
+        "## Before you begin"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d1afc945645f"
+      },
+      "source": [
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com). \n",
+        "\n",
+        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -248,6 +318,82 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d35af059208d"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
+        "authenticated. Skip this step."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "a00567d0660a"
+      },
+      "source": [
+        "**If you are using Colab**, run the cell below and follow the instructions\n",
+        "when prompted to authenticate your account via oAuth.\n",
+        "\n",
+        "**Otherwise**, follow these steps:\n",
+        "\n",
+        "1. In the Cloud Console, go to the [**Create service account key**\n",
+        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
+        "\n",
+        "2. Click **Create service account**.\n",
+        "\n",
+        "3. In the **Service account name** field, enter a name, and\n",
+        "   click **Create**.\n",
+        "\n",
+        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
+        "into the filter box, and select\n",
+        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "\n",
+        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
+        "local environment.\n",
+        "\n",
+        "6. Enter the path to your service account key as the\n",
+        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "40160162ea4c"
+      },
+      "outputs": [],
+      "source": [
+        "# If you are running this notebook in Colab, run this cell and follow the\n",
+        "# instructions to authenticate your GCP account. This provides access to your\n",
+        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
+        "# requests.\n",
+        "\n",
+        "import os\n",
+        "import sys\n",
+        "\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = False\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
+        "    if \"google.colab\" in sys.modules:\n",
+        "        IS_COLAB = True\n",
+        "        from google.colab import auth as google_auth\n",
+        "\n",
+        "        google_auth.authenticate_user()\n",
+        "\n",
+        "    # If you are running this notebook locally, replace the string below with the\n",
+        "    # path to your service account key and run this cell to authenticate your GCP\n",
+        "    # account.\n",
+        "    elif not os.getenv(\"IS_TESTING\"):\n",
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -271,7 +417,7 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
+        "BUCKET_URI = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -282,8 +428,8 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
       ]
     },
     {
@@ -303,7 +449,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_NAME"
+        "! gsutil mb -l $REGION $BUCKET_URI"
       ]
     },
     {
@@ -323,7 +469,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_NAME"
+        "! gsutil ls -al $BUCKET_URI"
       ]
     },
     {
@@ -346,7 +492,7 @@
       },
       "outputs": [],
       "source": [
-        "import google.cloud.aiplatform as aip\n",
+        "import google.cloud.aiplatform as aiplatform\n",
         "from google.cloud import kms"
       ]
     },
@@ -369,7 +515,7 @@
       },
       "outputs": [],
       "source": [
-        "aip.init(project=PROJECT_ID, location=REGION, staging_bucket=BUCKET_NAME)"
+        "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=BUCKET_URI)"
       ]
     },
     {
@@ -384,7 +530,7 @@
         "\n",
         "### Enable KMS API\n",
         "\n",
-        "First, you enble the [Cloud Key Management Service (KMS)](https://console.cloud.google.com/flows/enableapi?apiid=cloudkms.googleapis.com)\n",
+        "First, you enable the [Cloud Key Management Service (KMS)](https://console.cloud.google.com/flows/enableapi?apiid=cloudkms.googleapis.com)\n",
         "\n",
         "Learn more about [Customer managed encryption keys (CMEK)](https://cloud.google.com/vertex-ai/docs/general/cmek)"
       ]
@@ -549,6 +695,8 @@
         "\n",
         "Next, you set permissions for your Vertex AI service account to encrypt and decrypt resources using your key.\n",
         "\n",
+        "Note: Compute Engine default service account which is used by this notebook instance for authentication purposes during Google API calls, should be granted the role of Cloud KMS Admin.\n",
+        "\n",
         "Learn more about [Grant Vertex AI permissions](https://cloud.google.com/vertex-ai/docs/general/cmek#grant_permissions)"
       ]
     },
@@ -638,9 +786,9 @@
       },
       "outputs": [],
       "source": [
-        "aip.init(\n",
+        "aiplatform.init(\n",
         "    project=PROJECT_ID,\n",
-        "    staging_bucket=BUCKET_NAME,\n",
+        "    staging_bucket=BUCKET_URI,\n",
         "    location=REGION,\n",
         "    encryption_spec_key_name=ENCRYPTION_SPEC_KEY_NAME,\n",
         ")"
@@ -689,10 +837,10 @@
       },
       "outputs": [],
       "source": [
-        "dataset = aip.ImageDataset.create(\n",
+        "dataset = aiplatform.ImageDataset.create(\n",
         "    display_name=\"flowers_\" + TIMESTAMP,\n",
         "    gcs_source=[IMPORT_FILE],\n",
-        "    import_schema_uri=aip.schema.dataset.ioformat.image.single_label_classification,\n",
+        "    import_schema_uri=aiplatform.schema.dataset.ioformat.image.single_label_classification,\n",
         ")\n",
         "\n",
         "print(dataset.resource_name)"
@@ -727,7 +875,7 @@
         "\n",
         "# This will take around half an hour to run\n",
         "model = job.run(\n",
-        "    dataset=ds,\n",
+        "    dataset=dataset,\n",
         "    model_display_name=\"flowers_\" + TIMESTAMP,\n",
         "    training_fraction_split=0.6,\n",
         "    validation_fraction_split=0.2,\n",
@@ -827,6 +975,25 @@
         "endpoint.undeploy_all()"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ba77c4e02355"
+      },
+      "source": [
+        "## Cleaning up\n",
+        "\n",
+        "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
+        "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
+        "\n",
+        "Otherwise, you can delete the individual resources you created in this tutorial:\n",
+        "\n",
+        "- Model\n",
+        "- Dataset\n",
+        "- Cloud Storage Bucket\n",
+        "- Endpoint"
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -835,17 +1002,54 @@
       },
       "outputs": [],
       "source": [
-        "# missing\n",
+        "# Delete endpoint resource\n",
         "endpoint.delete()\n",
+        "\n",
+        "# Delete model resource\n",
         "model.delete()\n",
+        "\n",
+        "# Delete dataset resource\n",
         "dataset.delete()\n",
         "\n",
-        "! gcloud kms keys versions destroy key-version \\\n",
-        "  --key key {KEY_ID} \\\n",
+        "# Set this to true only if you'd like to delete your bucket\n",
+        "delete_bucket = False\n",
+        "\n",
+        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "    ! gsutil rm -r $BUCKET_URI"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d88c8a053a90"
+      },
+      "source": [
+        "## Destroying CMEK by providing key-version value and other parameters."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "5d8d2fc34346"
+      },
+      "outputs": [],
+      "source": [
+        "! gcloud kms keys versions destroy 1 \\\n",
+        "  --key {KEY_ID} \\\n",
         "  --keyring={KEY_RING_ID} \\\n",
         "  --location={REGION} "
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "f7e42e642ad3"
+      },
+      "source": [
+        "## List of keys "
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,

notebooks/community/ml_ops/stage2/mlops_experimentation.ipynb

@@ -39,8 +39,14 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/mlops_experimentation.ipynb\">\n",
-        "      Open in Google Cloud Notebooks\n",
+        "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/mlops_experimentation.ipynb\">\n",
+        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\\\" alt=\"Colab logo\"> Run in Colab\n",
+        "        </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage2/mlops_experimentation.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
         "    </a>\n",
         "  </td>\n",
         "</table>\n",
@@ -173,6 +179,19 @@
       },
       "outputs": [],
       "source": [
+        "import os\n",
+        "\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
+        "\n",
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "USER_FLAG = \"\"\n",
+        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\"\n",
+        "\n",
         "ONCE_ONLY = False\n",
         "if ONCE_ONLY:\n",
         "    ! pip3 install -U tensorflow==2.5 $USER_FLAG\n",
@@ -220,6 +239,32 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2721ef0202d9"
+      },
+      "source": [
+        "## Before you begin\n",
+        "\n",
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com). \n",
+        "\n",
+        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -296,7 +341,9 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+        "REGION = \"[your-region]\"  # @param {type:\"string\"}\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -323,6 +370,75 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "648aa9824ac6"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
+        "authenticated. Skip this step.\n",
+        "\n",
+        "**If you are using Colab**, run the cell below and follow the instructions\n",
+        "when prompted to authenticate your account via oAuth.\n",
+        "\n",
+        "**Otherwise**, follow these steps:\n",
+        "\n",
+        "1. In the Cloud Console, go to the [**Create service account key**\n",
+        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
+        "\n",
+        "2. Click **Create service account**.\n",
+        "\n",
+        "3. In the **Service account name** field, enter a name, and\n",
+        "   click **Create**.\n",
+        "\n",
+        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
+        "into the filter box, and select\n",
+        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "\n",
+        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
+        "local environment.\n",
+        "\n",
+        "6. Enter the path to your service account key as the\n",
+        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "535223fa4b84"
+      },
+      "outputs": [],
+      "source": [
+        "# If you are running this notebook in Colab, run this cell and follow the\n",
+        "# instructions to authenticate your GCP account. This provides access to your\n",
+        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
+        "# requests.\n",
+        "\n",
+        "import os\n",
+        "import sys\n",
+        "\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = False\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
+        "    if \"google.colab\" in sys.modules:\n",
+        "        IS_COLAB = True\n",
+        "        from google.colab import auth as google_auth\n",
+        "\n",
+        "        google_auth.authenticate_user()\n",
+        "\n",
+        "    # If you are running this notebook locally, replace the string below with the\n",
+        "    # path to your service account key and run this cell to authenticate your GCP\n",
+        "    # account.\n",
+        "    elif not os.getenv(\"IS_TESTING\"):\n",
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -436,9 +552,16 @@
         "    or SERVICE_ACCOUNT is None\n",
         "    or SERVICE_ACCOUNT == \"[your-service-account]\"\n",
         "):\n",
-        "    # Get your GCP project id from gcloud\n",
-        "    shell_output = !gcloud auth list 2>/dev/null\n",
-        "    SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
+        "    # Get your service account from gcloud\n",
+        "    if not IS_COLAB:\n",
+        "        shell_output = !gcloud auth list 2>/dev/null\n",
+        "        SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
+        "\n",
+        "    if IS_COLAB:\n",
+        "        shell_output = ! gcloud projects describe  $PROJECT_ID\n",
+        "        project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n",
+        "        SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n",
+        "\n",
         "    print(\"Service Account:\", SERVICE_ACCOUNT)"
       ]
     },
@@ -583,6 +706,8 @@
       },
       "outputs": [],
       "source": [
+        "import os\n",
+        "\n",
         "if os.getenv(\"IS_TESTING_TRAIN_GPU\"):\n",
         "    TRAIN_GPU, TRAIN_NGPU = (\n",
         "        aip.gapic.AcceleratorType.NVIDIA_TESLA_K80,\n",
@@ -1218,7 +1343,7 @@
         "setup_cfg = \"[egg_info]\\n\\ntag_build =\\n\\ntag_date = 0\"\n",
         "! echo \"$setup_cfg\" > custom/setup.cfg\n",
         "\n",
-        "setup_py = \"import setuptools\\n\\nsetuptools.setup(\\n\\n    install_requires=[\\n\\n        'google-cloud-aiplatform',\\n\\n        'cloudml-hypertune',\\n\\n        'tensorflow_datasets==1.3.0',\\n\\n        'tensorflow_data_validation==1.2',\\n\\n    ],\\n\\n    packages=setuptools.find_packages())\"\n",
+        "setup_py = \"import setuptools\\n\\nsetuptools.setup(\\n\\n    install_requires=[\\n\\n        'google-cloud-aiplatform',\\n\\n        'cloudml-hypertune',\\n\\n        'tensorflow_datasets==1.3.0',\\n\\n        'tensorflow==2.5',\\n\\n    'tensorflow_data_validation==1.2',\\n\\n    ],\\n\\n    packages=setuptools.find_packages())\"\n",
         "! echo \"$setup_py\" > custom/setup.py\n",
         "\n",
         "pkg_info = \"Metadata-Version: 1.0\\n\\nName: Chicago Taxi tabular binary classifier\\n\\nVersion: 0.0.0\\n\\nSummary: Demostration training script\\n\\nHome-page: www.google.com\\n\\nAuthor: Google\\n\\nAuthor-email: cdpe@google.com\\n\\nLicense: Public\\n\\nDescription: Demo\\n\\nPlatform: Vertex AI\"\n",
@@ -3011,9 +3136,8 @@
       },
       "source": [
         "## Review model evaluation scores\n",
-        "After your model has finished training, you can review the evaluation scores for it.\n",
         "\n",
-        "First, you need to get a reference to the new model. As with datasets, you can either use the reference to the model variable you created when you deployed the model or you can list all of the models in your project."
+        "After your model training has finished, you can review the evaluation scores for it using the `list_model_evaluations()` method. This method will return an iterator for each evaluation slice."
       ]
     },
     {
@@ -3024,18 +3148,10 @@
       },
       "outputs": [],
       "source": [
-        "# Get model resource ID\n",
-        "models = aip.Model.list(filter=\"display_name=chicago_\" + TIMESTAMP)\n",
+        "model_evaluations = model.list_model_evaluations()\n",
         "\n",
-        "# Get a reference to the Model Service client\n",
-        "client_options = {\"api_endpoint\": f\"{REGION}-aiplatform.googleapis.com\"}\n",
-        "model_service_client = aip.gapic.ModelServiceClient(client_options=client_options)\n",
-        "\n",
-        "model_evaluations = model_service_client.list_model_evaluations(\n",
-        "    parent=models[0].resource_name\n",
-        ")\n",
-        "model_evaluation = list(model_evaluations)[0]\n",
-        "print(model_evaluation)"
+        "for model_evaluation in model_evaluations:\n",
+        "    print(model_evaluation.to_dict())"
       ]
     },
     {

notebooks/community/ml_ops/stage3/README.md

@@ -27,7 +27,7 @@ The third stage in MLOps is formalization to develop an automated pipeline proce
 - Use early stop procedure in training script to detect failure to achieve training objective.
 - Store the results of the trained model evaluation in Vertex AI ML Metadata.
 
-<img src='stage3.png'>
+<img src='stage3v3.png'>
 
 ## Notebooks
 
@@ -66,6 +66,17 @@ The steps performed include:
 - Execute a Vertex AI pipeline.
 ```
 
+[Get Started with Dataproc components](get_started_with_dataproc_pipeline_components.ipynb)
+
+```
+The steps performed include:
+
+- DataprocPySparkBatchOp for PySpark batch workloads.
+- DataprocSparkBatchOp for Spark batch workloads.
+- DataprocSparkSqlBatchOp for running Spark SQL batch workloads.
+- DataprocSparkRBatchOp for running SparkR batch workloads.
+```
+
 [Get Started with Vertex AI AutoML components](get_started_with_automl_pipeline_components.ipynb)
 
 ```
@@ -133,6 +144,28 @@ The steps performed include:
 - Testing the deployed model infrastructure.
 ```
 
+[Get Started with TFX Pipelines with Vertex AI](get_started_with_tfx_pipeline.ipynb)
+
+```
+The steps performed include:
+
+- Create a TFX e2e pipeline.
+- Execute the pipeline locally.
+- Execute the pipeline on Google Cloud using `Vertex AI Training`
+- Execute the pipeline using `Vertex AI Pipelines`.
+```
+
+[Get Started with machine management](get_started_with_machine_management.ipynb)
+
+```
+The steps performed in this tutorial include:
+
+- Create a custom component with a self-contained training job.
+- Execute pipeline using component-level settings for machine resources
+- Convert the self-contained training componnt into a Vertex AI CustomJob.
+- Execute pipeline using customjob-level settings for machine resources
+```
+
 ### E2E Stage Example
 
 [Stage 3: Formalization](mlops_formalization.ipynb)

notebooks/community/ml_ops/stage3/get_started_with_automl_pipeline_components.ipynb

@@ -8,7 +8,7 @@
       },
       "outputs": [],
       "source": [
-        "# Copyright 2021 Google LLC\n",
+        "# Copyright 2022 Google LLC\n",
         "#\n",
         "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
         "# you may not use this file except in compliance with the License.\n",
@@ -38,9 +38,15 @@
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
+        "    <td>\n",
+        "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_automl_pipeline_components.ipynb\">\n",
+        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\\\" alt=\"Colab logo\"> Run in Colab\n",
+        "        </a>\n",
+        "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_automl_pipeline_components.ipynb\">\n",
-        "      Open in Google Cloud Notebooks\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage3/get_started_with_automl_pipeline_components.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
         "    </a>\n",
         "  </td>\n",
         "</table>\n",
@@ -67,7 +73,7 @@
       "source": [
         "### Dataset\n",
         "\n",
-        "The dataset used for this tutorial is the [Flowers dataset](https://www.tensorflow.org/datasets/catalog/tf_flowers) from [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/overview). The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket. The trained model predicts the type of flower an image is from a class of five flowers: daisy, dandelion, rose, sunflower, or tulip."
+        "The dataset used for this tutorial is the [Flowers dataset](https://www.tensorflow.org/datasets/catalog/tf_flowers) from [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/overview). The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket. The trained model predicts the type of flower in the given image from the five classes of flowers: daisy, dandelion, rose, sunflower, or tulip."
       ]
     },
     {
@@ -94,7 +100,15 @@
         "    - Training a Vertex AI AutoML trained model.\n",
         "    - Test the serving binary with a batch prediction job.\n",
         "    - Deploying a Vertex AI AutoML trained model.\n",
-        "- Execute a Vertex AI pipeline."
+        "- Execute a Vertex AI pipeline.\n",
+        "\n",
+        "### Costs\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "- Vertex AI\n",
+        "- Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage pricing](https://cloud.google.com/storage/pricing) and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
       ]
     },
     {
@@ -105,7 +119,7 @@
       "source": [
         "## Installations\n",
         "\n",
-        "Install *one time* the packages for executing the MLOps notebooks."
+        "Install the following packages for executing this MLOps notebook."
       ]
     },
     {
@@ -116,24 +130,25 @@
       },
       "outputs": [],
       "source": [
-        "ONCE_ONLY = False\n",
-        "if ONCE_ONLY:\n",
-        "    ! pip3 install -U tensorflow==2.5 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-transform==1.2 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-io==0.18 $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-bigquery $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-logging $USER_FLAG\n",
-        "    ! pip3 install --upgrade apache-beam[gcp] $USER_FLAG\n",
-        "    ! pip3 install --upgrade pyarrow $USER_FLAG\n",
-        "    ! pip3 install --upgrade cloudml-hypertune $USER_FLAG\n",
-        "    ! pip3 install --upgrade kfp $USER_FLAG\n",
-        "    ! pip3 install --upgrade torchvision $USER_FLAG\n",
-        "    ! pip3 install --upgrade rpy2 $USER_FLAG\n",
-        "    ! pip3 install --upgrade python-tabulate $USER_FLAG\n",
-        "    ! pip3 install -U opencv-python-headless==4.5.2.52 $USER_FLAG"
+        "import os\n",
+        "\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
+        "\n",
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "USER_FLAG = \"\"\n",
+        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\"\n",
+        "    \n",
+        "! pip3 install tensorflow-io==0.18 $USER_FLAG -q\n",
+        "! pip3 install --upgrade google-cloud-aiplatform \\\n",
+        "                        google-cloud-pipeline-components \\\n",
+        "                        google-cloud-logging \\\n",
+        "                        pyarrow \\\n",
+        "                        kfp $USER_FLAG -q"
       ]
     },
     {
@@ -144,7 +159,7 @@
       "source": [
         "### Restart the kernel\n",
         "\n",
-        "Once you've installed the additional packages, you need to restart the notebook kernel so it can find the packages."
+        "Once you've installed the additional packages, you need to restart the notebook kernel so that it can find the packages."
       ]
     },
     {
@@ -165,6 +180,30 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "BF1j6f9HApxa"
+      },
+      "source": [
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com).\n",
+        "\n",
+        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -241,7 +280,10 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -268,6 +310,67 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "c38be665ca50"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using Vertex AI Notebook Notebooks**, your environment is already authenticated. Skip this step.\n",
+        "\n",
+        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
+        "\n",
+        "**Otherwise**, follow these steps:\n",
+        "\n",
+        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
+        "\n",
+        "1. **Click Create service account**.\n",
+        "\n",
+        "2. In the **Service account name** field, enter a name, and click **Create**.\n",
+        "\n",
+        "3. In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex\" into the filter box, and select **Vertex Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "\n",
+        "4. Click Create. A JSON file that contains your key downloads to your local environment.\n",
+        "\n",
+        "5. Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "e0953a00668e"
+      },
+      "outputs": [],
+      "source": [
+        "# If you are running this notebook in Colab, run this cell and follow the\n",
+        "# instructions to authenticate your GCP account. This provides access to your\n",
+        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
+        "# requests.\n",
+        "\n",
+        "import os\n",
+        "import sys\n",
+        "\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = False\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
+        "    if \"google.colab\" in sys.modules:\n",
+        "        IS_COLAB = True\n",
+        "        from google.colab import auth as google_auth\n",
+        "\n",
+        "        google_auth.authenticate_user()\n",
+        "\n",
+        "    # If you are running this notebook locally, replace the string below with the\n",
+        "    # path to your service account key and run this cell to authenticate your GCP\n",
+        "    # account.\n",
+        "    elif not os.getenv(\"IS_TESTING\"):\n",
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -291,7 +394,8 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
+        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
     {
@@ -302,8 +406,8 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
       ]
     },
     {
@@ -323,7 +427,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_NAME"
+        "! gsutil mb -l $REGION $BUCKET_URI"
       ]
     },
     {
@@ -343,7 +447,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_NAME"
+        "! gsutil ls -al $BUCKET_URI"
       ]
     },
     {
@@ -354,7 +458,7 @@
       "source": [
         "#### Service Account\n",
         "\n",
-        "**If you don't know your service account**, try to get your service account using `gcloud` command by executing the second cell below."
+        "You use a service account to create Vertex AI Pipeline jobs. If you do not want to use your project's Compute Engine service account, set `SERVICE_ACCOUNT` to another service account ID."
       ]
     },
     {
@@ -381,9 +485,16 @@
         "    or SERVICE_ACCOUNT is None\n",
         "    or SERVICE_ACCOUNT == \"[your-service-account]\"\n",
         "):\n",
-        "    # Get your GCP project id from gcloud\n",
-        "    shell_output = !gcloud auth list 2>/dev/null\n",
-        "    SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
+        "    # Get your service account from gcloud\n",
+        "    if not IS_COLAB:\n",
+        "        shell_output = !gcloud auth list 2>/dev/null\n",
+        "        SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
+        "\n",
+        "    if IS_COLAB:\n",
+        "        shell_output = ! gcloud projects describe  $PROJECT_ID\n",
+        "        project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n",
+        "        SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n",
+        "\n",
         "    print(\"Service Account:\", SERVICE_ACCOUNT)"
       ]
     },
@@ -395,7 +506,7 @@
       "source": [
         "#### Set service account access for Vertex AI Pipelines\n",
         "\n",
-        "Run the following commands to grant your service account access to read and write pipeline artifacts in the bucket that you created in the previous step -- you only need to run these once per service account."
+        "Run the following commands to grant your service account access to read and write pipeline artifacts in the bucket that you created in the previous step. You only need to run this step once per service account."
       ]
     },
     {
@@ -406,9 +517,9 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_NAME\n",
+        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
         "\n",
-        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_NAME"
+        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
       ]
     },
     {
@@ -417,32 +528,7 @@
         "id": "setup_vars"
       },
       "source": [
-        "### Set up variables\n",
-        "\n",
-        "Next, set up some variables used throughout the tutorial.\n",
-        "### Import libraries and define constants"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "import_aip:mbsdk"
-      },
-      "outputs": [],
-      "source": [
-        "import google.cloud.aiplatform as aip"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "import_tf"
-      },
-      "source": [
-        "#### Import TensorFlow\n",
-        "\n",
-        "Import the TensorFlow package into your Python environment."
+        "### Import libraries"
       ]
     },
     {
@@ -453,22 +539,14 @@
       },
       "outputs": [],
       "source": [
-        "import tensorflow as tf"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "import_kfp"
-      },
-      "outputs": [],
-      "source": [
+        "import base64\n",
         "import json\n",
         "\n",
+        "import google.cloud.aiplatform as aiplatform\n",
+        "import tensorflow as tf\n",
         "from kfp import dsl\n",
         "from kfp.v2 import compiler\n",
-        "from kfp.v2.dsl import component"
+        "from kfp.v2.dsl import Artifact, Input, Output, component"
       ]
     },
     {
@@ -490,7 +568,7 @@
       },
       "outputs": [],
       "source": [
-        "aip.init(project=PROJECT_ID, location=REGION, staging_bucket=BUCKET_NAME)"
+        "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=BUCKET_URI)"
       ]
     },
     {
@@ -530,7 +608,7 @@
         "- Takes as input the region and Model artifacts returned from an AutoML training component.\n",
         "- Create a client interface to the Vertex AI Model service (`metadata[\"resource_name\"]).\n",
         "- Construct the resource ID for the model from the model artifact parameter.\n",
-        "- Retrieve the model evaluation\n",
+        "- Retrieve the model evaluation.\n",
         "- Return the model evaluation as a string."
       ]
     },
@@ -542,9 +620,6 @@
       },
       "outputs": [],
       "source": [
-        "from kfp.v2.dsl import Artifact, Input, Model, Output\n",
-        "\n",
-        "\n",
         "@component(packages_to_install=[\"google-cloud-aiplatform\"])\n",
         "def evaluateAutoMLModelOp(\n",
         "    model: Input[Artifact], region: str, model_evaluation: Output[Artifact]\n",
@@ -577,7 +652,7 @@
         "1. Use the prebuilt component `ImageDatasetCreateOp` to create a Vertex AI Dataset resource, where:\n",
         "    - The display name for the dataset is passed into the pipeline.\n",
         "    - The import file for the dataset is passed into the pipeline.\n",
-        "    - The component returns the dataset resource as `outputs[\"dataset\"]`\n",
+        "    - The component returns the dataset resource as `outputs[\"dataset\"]`.\n",
         "\n",
         "\n",
         "2. Use the prebuilt component `AutoMLImageTrainingJobRunOp` to train a Vertex AI AutoML Model resource, where:\n",
@@ -596,12 +671,12 @@
         "    - The component returns the endpoint resource as `outputs[\"endpoint\"]`.\n",
         "\n",
         "\n",
-        "5. Use the prebuilt component `ModelDeployOp` to deploy the trained AutoML model to, where:\n",
+        "5. Use the prebuilt component `ModelDeployOp` to deploy the trained AutoML model where:\n",
         "    - The display name for the dataset is passed into the pipeline.\n",
         "    - The model is the output from the `AutoMLTrainingJobRunOp`.\n",
-        "    - The endpoint is the output from the `EndpointCreateOp`\n",
+        "    - The endpoint is the output from the `EndpointCreateOp`.\n",
         "\n",
-        "*Note:* Since each component is executed as a graph node in its own execution context, you pass the parameter `project` for each component op, in constrast to doing a `aip.init(project=project)` if this was a Python script calling the SDK methods directly within the same execution context."
+        "*Note:* Since each component is executed as a graph node in its own execution context, you pass the parameter `project` for each component op, in constrast to doing a `aiplatform.init(project=project)` if this was a Python script calling the SDK methods directly within the same execution context."
       ]
     },
     {
@@ -612,9 +687,7 @@
       },
       "outputs": [],
       "source": [
-        "from google_cloud_pipeline_components import aiplatform as gcc_aip\n",
-        "\n",
-        "PIPELINE_ROOT = \"{}/pipeline_root/automl_icn_training\".format(BUCKET_NAME)\n",
+        "PIPELINE_ROOT = \"{}/pipeline_root/automl_icn_training\".format(BUCKET_URI)\n",
         "DEPLOY_COMPUTE = \"n1-standard-4\"\n",
         "\n",
         "\n",
@@ -629,12 +702,13 @@
         "    project: str = PROJECT_ID,\n",
         "    region: str = REGION,\n",
         "):\n",
+        "    from google_cloud_pipeline_components import aiplatform as gcc_aip\n",
         "\n",
         "    dataset_op = gcc_aip.ImageDatasetCreateOp(\n",
         "        project=project,\n",
         "        display_name=display_name,\n",
         "        gcs_source=import_file,\n",
-        "        import_schema_uri=aip.schema.dataset.ioformat.image.single_label_classification,\n",
+        "        import_schema_uri=aiplatform.schema.dataset.ioformat.image.single_label_classification,\n",
         "    )\n",
         "\n",
         "    training_op = gcc_aip.AutoMLImageTrainingJobRunOp(\n",
@@ -672,11 +746,12 @@
         "        display_name=display_name,\n",
         "    ).after(batch_op)\n",
         "\n",
-        "    deploy_op = gcc_aip.ModelDeployOp(\n",
+        "    _ = gcc_aip.ModelDeployOp(\n",
         "        model=training_op.outputs[\"model\"],\n",
         "        endpoint=endpoint_op.outputs[\"endpoint\"],\n",
         "        automatic_resources_min_replica_count=1,\n",
         "        automatic_resources_max_replica_count=1,\n",
+        "        traffic_split={\"0\": 100},\n",
         "    )"
       ]
     },
@@ -688,7 +763,7 @@
       "source": [
         "### Get test item(s)\n",
         "\n",
-        "Now do a batch prediction to your Vertex model. You will use arbitrary examples out of the dataset as a test items. Don't be concerned that the examples were likely used in training the model -- we just want to demonstrate how to make a prediction."
+        "In the pipeline, you do a batch prediction on your Vertex model. You will use arbitrary examples from the dataset as test items. Don't be concerned that the examples were likely used while training the model. This step is just to demonstrate how to make a prediction."
       ]
     },
     {
@@ -733,11 +808,11 @@
         "file_1 = test_item_1.split(\"/\")[-1]\n",
         "file_2 = test_item_2.split(\"/\")[-1]\n",
         "\n",
-        "! gsutil cp $test_item_1 $BUCKET_NAME/$file_1\n",
-        "! gsutil cp $test_item_2 $BUCKET_NAME/$file_2\n",
+        "! gsutil cp $test_item_1 $BUCKET_URI/$file_1\n",
+        "! gsutil cp $test_item_2 $BUCKET_URI/$file_2\n",
         "\n",
-        "test_item_1 = BUCKET_NAME + \"/\" + file_1\n",
-        "test_item_2 = BUCKET_NAME + \"/\" + file_2"
+        "test_item_1 = BUCKET_URI + \"/\" + file_1\n",
+        "test_item_2 = BUCKET_URI + \"/\" + file_2"
       ]
     },
     {
@@ -748,14 +823,14 @@
       "source": [
         "### Make the batch input file\n",
         "\n",
-        "Now make a batch input file, which you will store in your local Cloud Storage bucket. The batch input file can only be in JSONL. For JSONL file, you make one dictionary entry per line for each data item (instance). The dictionary contains the key/value pairs:\n",
+        "Now make a batch input file, which you will store in your local Cloud Storage bucket. The batch input file can only be in JSONL format. For JSONL file, you make one dictionary entry per line for each data item (instance). The dictionary contains key/value pairs:\n",
         "\n",
         "- `content`: The Cloud Storage path to the image.\n",
         "- `mime_type`: The content type. In our example, it is a `jpeg` file.\n",
         "\n",
         "For example:\n",
         "\n",
-        "                        {'content': '[your-bucket]/file1.jpg', 'mime_type': 'jpeg'}"
+        "    {'content': '[your-bucket]/file1.jpg', 'mime_type': 'jpeg'}"
       ]
     },
     {
@@ -766,11 +841,7 @@
       },
       "outputs": [],
       "source": [
-        "import json\n",
-        "\n",
-        "import tensorflow as tf\n",
-        "\n",
-        "gcs_input_uri = BUCKET_NAME + \"/test.jsonl\"\n",
+        "gcs_input_uri = BUCKET_URI + \"/test.jsonl\"\n",
         "with tf.io.gfile.GFile(gcs_input_uri, \"w\") as f:\n",
         "    data = {\"content\": test_item_1, \"mime_type\": \"image/jpeg\"}\n",
         "    f.write(json.dumps(data) + \"\\n\")\n",
@@ -810,7 +881,7 @@
         "    pipeline_func=pipeline, package_path=\"automl_icn_training.json\"\n",
         ")\n",
         "\n",
-        "pipeline = aip.PipelineJob(\n",
+        "pipeline = aiplatform.PipelineJob(\n",
         "    display_name=\"automl_icn_training\",\n",
         "    template_path=\"automl_icn_training.json\",\n",
         "    pipeline_root=PIPELINE_ROOT,\n",
@@ -911,9 +982,21 @@
         "print(\"automl-image-training-job\")\n",
         "artifacts = print_pipeline_output(pipeline, \"automl-image-training-job\")\n",
         "print(\"\\n\\n\")\n",
+        "output = !gsutil cat $artifacts\n",
+        "output = json.loads(output[0])\n",
+        "model_id = output[\"artifacts\"][\"model\"][\"artifacts\"][0][\"metadata\"][\"resourceName\"]\n",
+        "print(\"\\n\")\n",
+        "print(model_id)\n",
         "print(\"endpoint-create\")\n",
         "artifacts = print_pipeline_output(pipeline, \"endpoint-create\")\n",
         "print(\"\\n\\n\")\n",
+        "output = !gsutil cat $artifacts\n",
+        "output = json.loads(output[0])\n",
+        "endpoint_id = output[\"artifacts\"][\"endpoint\"][\"artifacts\"][0][\"metadata\"][\n",
+        "    \"resourceName\"\n",
+        "]\n",
+        "print(\"\\n\")\n",
+        "print(endpoint_id)\n",
         "print(\"model-deploy\")\n",
         "artifacts = print_pipeline_output(pipeline, \"model-deploy\")\n",
         "print(\"\\n\\n\")\n",
@@ -929,7 +1012,12 @@
         "    output[\"artifacts\"][\"batchpredictionjob\"][\"artifacts\"][0][\"metadata\"][\n",
         "        \"gcsOutputDirectory\"\n",
         "    ]\n",
-        ")"
+        ")\n",
+        "output = !gsutil cat $artifacts\n",
+        "output = json.loads(output[0])\n",
+        "batch_job_id = output[\"artifacts\"][\"batchpredictionjob\"][\"artifacts\"][0][\"metadata\"][\n",
+        "    \"resourceName\"\n",
+        "]"
       ]
     },
     {
@@ -957,7 +1045,117 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "cleanup:mbsdk"
+        "id": "endpoint_load:mbsdk"
+      },
+      "source": [
+        "#### Load an endpoint\n",
+        "\n",
+        "The 'Endpoint' initializer will load an endpoint from an endpoint identifier."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "endpoint_load:mbsdk"
+      },
+      "outputs": [],
+      "source": [
+        "endpoint = aiplatform.Endpoint(endpoint_id)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "make_prediction"
+      },
+      "source": [
+        "## Send a online prediction request\n",
+        "\n",
+        "Send a online prediction request to your deployed model."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "get_test_item"
+      },
+      "source": [
+        "### Get test item\n",
+        "\n",
+        "You will use an arbitrary example out of the dataset as a test item. Don't be concerned that the example was likely used while training the model. This step is just to demonstrate how to make a prediction."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "get_test_item:automl,icn,csv"
+      },
+      "outputs": [],
+      "source": [
+        "test_item = !gsutil cat $IMPORT_FILE | head -n1\n",
+        "if len(str(test_item[0]).split(\",\")) == 3:\n",
+        "    _, test_item, test_label = str(test_item[0]).split(\",\")\n",
+        "else:\n",
+        "    test_item, test_label = str(test_item[0]).split(\",\")\n",
+        "\n",
+        "print(test_item, test_label)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "predict_request:mbsdk,icn"
+      },
+      "source": [
+        "### Make the prediction\n",
+        "\n",
+        "Now that your `Model` resource is deployed to an `Endpoint` resource, you can do online predictions by sending prediction requests to the Endpoint resource.\n",
+        "\n",
+        "#### Request\n",
+        "\n",
+        "Since in this example your test item is in a Cloud Storage bucket, you open and read the contents of the image using `tf.io.gfile.Gfile()`. To pass the test data to the prediction service, you encode the bytes into base64 which makes the content safe from modification while transmitting binary data over the network.\n",
+        "\n",
+        "The format of each instance is:\n",
+        "\n",
+        "    { 'content': { 'b64': base64_encoded_bytes } }\n",
+        "\n",
+        "Since the `predict()` method can take multiple items (instances), send your single test item as a list of one test item.\n",
+        "\n",
+        "#### Response\n",
+        "\n",
+        "The response from the `predict()` call is a Python dictionary with the following entries:\n",
+        "\n",
+        "- `ids`: The internal assigned unique identifiers for each prediction request.\n",
+        "- `displayNames`: The class names for each class label.\n",
+        "- `confidences`: The predicted confidence, between 0 and 1, per class label.\n",
+        "- `deployed_model_id`: The Vertex AI identifier for the deployed Model resource which did the predictions."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "predict_request:mbsdk,icn"
+      },
+      "outputs": [],
+      "source": [
+        "with tf.io.gfile.GFile(test_item, \"rb\") as f:\n",
+        "    content = f.read()\n",
+        "\n",
+        "# The format of each instance should conform to the deployed model's prediction input schema.\n",
+        "instances = [{\"content\": base64.b64encode(content).decode(\"utf-8\")}]\n",
+        "\n",
+        "prediction = endpoint.predict(instances=instances)\n",
+        "\n",
+        "print(prediction)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9d347472d5ba"
       },
       "source": [
         "# Cleaning up\n",
@@ -965,17 +1163,40 @@
         "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
         "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
         "\n",
-        "Otherwise, you can delete the individual resources you created in this tutorial:\n",
+        "Otherwise, you can delete the individual resources you created in this tutorial.\n",
         "\n",
-        "- Dataset\n",
-        "- Pipeline\n",
-        "- Model\n",
-        "- Endpoint\n",
-        "- AutoML Training Job\n",
-        "- Batch Job\n",
-        "- Custom Job\n",
-        "- Hyperparameter Tuning Job\n",
-        "- Cloud Storage Bucket"
+        "#### Delete the Vertex AI Model, Endpoint and BatchPredictionJob resources\n",
+        "\n",
+        "Undelpoy and delete the Vertex AI Model, Endpoint and BatchPredictionJob resources."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "baa3e1071f7b"
+      },
+      "outputs": [],
+      "source": [
+        "endpoint.undeploy_all()\n",
+        "endpoint.delete()\n",
+        "\n",
+        "model = aiplatform.Model(model_id)\n",
+        "model.delete()\n",
+        "\n",
+        "batch_job = aiplatform.BatchPredictionJob(batch_job_id)\n",
+        "batch_job.delete()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "a802da1f6fa7"
+      },
+      "source": [
+        "#### Delete the Cloud Storage bucket\n",
+        "\n",
+        "Set `delete_bucket` to *True* to delete the Cloud storage bucket used in this notebook."
       ]
     },
     {
@@ -986,61 +1207,10 @@
       },
       "outputs": [],
       "source": [
-        "delete_all = True\n",
+        "delete_bucket = False\n",
         "\n",
-        "if delete_all:\n",
-        "    # Delete the dataset using the Vertex dataset object\n",
-        "    try:\n",
-        "        if \"dataset\" in globals():\n",
-        "            dataset.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the model using the Vertex model object\n",
-        "    try:\n",
-        "        if \"model\" in globals():\n",
-        "            model.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the endpoint using the Vertex endpoint object\n",
-        "    try:\n",
-        "        if \"endpoint\" in globals():\n",
-        "            endpoint.undeploy_all()\n",
-        "            endpoint.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the AutoML or Pipeline training job\n",
-        "    try:\n",
-        "        if \"dag\" in globals():\n",
-        "            dag.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the custom training job\n",
-        "    try:\n",
-        "        if \"job\" in globals():\n",
-        "            job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the batch prediction job using the Vertex batch prediction object\n",
-        "    try:\n",
-        "        if \"batch_predict_job\" in globals():\n",
-        "            batch_predict_job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the hyperparameter tuning job using the Vertex hyperparameter tuning object\n",
-        "    try:\n",
-        "        if \"hpt_job\" in globals():\n",
-        "            hpt_job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    if \"BUCKET_NAME\" in globals():\n",
-        "        ! gsutil rm -r $BUCKET_NAME"
+        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "    ! gsutil rm -r $BUCKET_URI"
       ]
     }
   ],

notebooks/community/ml_ops/stage3/get_started_with_bq_tfdv_pipeline_components.ipynb

@@ -8,7 +8,7 @@
       },
       "outputs": [],
       "source": [
-        "# Copyright 2021 Google LLC\n",
+        "# Copyright 2022 Google LLC\n",
         "#\n",
         "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
         "# you may not use this file except in compliance with the License.\n",
@@ -33,14 +33,20 @@
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_bq_tfdv_pipeline_components.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
         "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_bq_tfdv_pipeline_components.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_bq_tfdv_pipeline_components.ipynb\">\n",
-        "      Open in Google Cloud Notebooks\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage3/get_started_with_bq_tfdv_pipeline_components.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
         "    </a>\n",
         "  </td>\n",
         "</table>\n",
@@ -93,6 +99,28 @@
         "- Execute a Vertex AI pipeline."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0c997d8d92ce"
+      },
+      "source": [
+        "### Costs \n",
+        "\n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Cloud Storage\n",
+        "\n",
+        "\n",
+        "Learn about [Vertex AI\n",
+        "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
+        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
+        "Calculator](https://cloud.google.com/products/calculator/)\n",
+        "to generate a cost estimate based on your projected usage."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -101,31 +129,33 @@
       "source": [
         "## Installations\n",
         "\n",
-        "Install *one time* the packages for executing the MLOps notebooks."
+        "Install the packages for executing this notebook."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "install_mlops"
+        "id": "1fd00fa70a2a"
       },
       "outputs": [],
       "source": [
-        "ONCE_ONLY = False\n",
-        "if ONCE_ONLY:\n",
-        "    ! pip3 install -U tensorflow==2.5 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-transform==1.2 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-io==0.18 $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-bigquery $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-logging $USER_FLAG\n",
-        "    ! pip3 install --upgrade apache-beam[gcp] $USER_FLAG\n",
-        "    ! pip3 install --upgrade pyarrow $USER_FLAG\n",
-        "    ! pip3 install --upgrade cloudml-hypertune $USER_FLAG\n",
-        "    ! pip3 install --upgrade kfp $USER_FLAG"
+        "import os\n",
+        "\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
+        "\n",
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "USER_FLAG = \"\"\n",
+        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\"\n",
+        "\n",
+        "! pip3 install -U tensorflow $USER_FLAG -q\n",
+        "! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG -q\n",
+        "! pip3 install --upgrade kfp $USER_FLAG -q"
       ]
     },
     {
@@ -157,6 +187,30 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "BF1j6f9HApxa"
+      },
+      "source": [
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,dataflow.googleapis.com).\n",
+        "\n",
+        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -176,6 +230,8 @@
       },
       "outputs": [],
       "source": [
+        "import os\n",
+        "\n",
         "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
       ]
     },
@@ -233,7 +289,9 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+        "REGION = \"[your-region]\"  # @param {type:\"string\"}\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -260,6 +318,82 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "648aa9824ac6"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
+        "authenticated. Skip this step."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "fc52bba17ee3"
+      },
+      "source": [
+        "**If you are using Colab**, run the cell below and follow the instructions\n",
+        "when prompted to authenticate your account via oAuth.\n",
+        "\n",
+        "**Otherwise**, follow these steps:\n",
+        "\n",
+        "1. In the Cloud Console, go to the [**Create service account key**\n",
+        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
+        "\n",
+        "2. Click **Create service account**.\n",
+        "\n",
+        "3. In the **Service account name** field, enter a name, and\n",
+        "   click **Create**.\n",
+        "\n",
+        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
+        "into the filter box, and select\n",
+        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "\n",
+        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
+        "local environment.\n",
+        "\n",
+        "6. Enter the path to your service account key as the\n",
+        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "535223fa4b84"
+      },
+      "outputs": [],
+      "source": [
+        "# If you are running this notebook in Colab, run this cell and follow the\n",
+        "# instructions to authenticate your GCP account. This provides access to your\n",
+        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
+        "# requests.\n",
+        "\n",
+        "import os\n",
+        "import sys\n",
+        "\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = False\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
+        "    if \"google.colab\" in sys.modules:\n",
+        "        IS_COLAB = True\n",
+        "        from google.colab import auth as google_auth\n",
+        "\n",
+        "        google_auth.authenticate_user()\n",
+        "\n",
+        "    # If you are running this notebook locally, replace the string below with the\n",
+        "    # path to your service account key and run this cell to authenticate your GCP\n",
+        "    # account.\n",
+        "    elif not os.getenv(\"IS_TESTING\"):\n",
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -283,7 +417,7 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
+        "BUCKET_URI = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -294,8 +428,8 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
       ]
     },
     {
@@ -315,7 +449,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_NAME"
+        "! gsutil mb -l $REGION $BUCKET_URI"
       ]
     },
     {
@@ -335,7 +469,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_NAME"
+        "! gsutil ls -al $BUCKET_URI"
       ]
     },
     {
@@ -346,7 +480,9 @@
       "source": [
         "#### Service Account\n",
         "\n",
-        "**If you don't know your service account**, try to get your service account using `gcloud` command by executing the second cell below."
+        "**If you don't know your service account**, try to get your service account using `gcloud` command by executing the second cell below.\n",
+        "\n",
+        "*Note:* The code for automatically finding your service account works on a user-managed Workbench AI noteboook. If you are using a fully-managed notebook or colab, you will need to manually enter your service account."
       ]
     },
     {
@@ -373,9 +509,16 @@
         "    or SERVICE_ACCOUNT is None\n",
         "    or SERVICE_ACCOUNT == \"[your-service-account]\"\n",
         "):\n",
-        "    # Get your GCP project id from gcloud\n",
-        "    shell_output = !gcloud auth list 2>/dev/null\n",
-        "    SERVICE_ACCOUNT = shell_output[2].strip()\n",
+        "    # Get your service account from gcloud\n",
+        "    if not IS_COLAB:\n",
+        "        shell_output = !gcloud auth list 2>/dev/null\n",
+        "        SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
+        "\n",
+        "    if IS_COLAB:\n",
+        "        shell_output = ! gcloud projects describe  $PROJECT_ID\n",
+        "        project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n",
+        "        SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n",
+        "\n",
         "    print(\"Service Account:\", SERVICE_ACCOUNT)"
       ]
     },
@@ -398,9 +541,9 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_NAME\n",
+        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
         "\n",
-        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_NAME"
+        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
       ]
     },
     {
@@ -482,7 +625,7 @@
       },
       "outputs": [],
       "source": [
-        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_NAME)"
+        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
       ]
     },
     {
@@ -597,7 +740,7 @@
         "    return dataset.column_names\n",
         "\n",
         "\n",
-        "PIPELINE_ROOT = \"{}/pipeline_root/dataset_bq\".format(BUCKET_NAME)\n",
+        "PIPELINE_ROOT = \"{}/pipeline_root/dataset_bq\".format(BUCKET_URI)\n",
         "\n",
         "\n",
         "@dsl.pipeline(\n",
@@ -610,9 +753,9 @@
         "):\n",
         "    create_op = create_dataset_bq(bq_table, display_name, project)\n",
         "\n",
-        "    source_op = get_dataset_source(create_op.output)\n",
+        "    _ = get_dataset_source(create_op.output)\n",
         "\n",
-        "    column_names_op = get_column_names(create_op.output)\n",
+        "    _ = get_column_names(create_op.output)\n",
         "\n",
         "\n",
         "compiler.Compiler().compile(pipeline_func=pipeline, package_path=\"dataset_bq.json\")\n",
@@ -811,7 +954,7 @@
         "    return (stats_file, schema_file)\n",
         "\n",
         "\n",
-        "PIPELINE_ROOT = \"{}/pipeline_root/dataset_stats\".format(BUCKET_NAME)\n",
+        "PIPELINE_ROOT = \"{}/pipeline_root/dataset_stats\".format(BUCKET_URI)\n",
         "\n",
         "\n",
         "@dsl.pipeline(\n",
@@ -819,7 +962,7 @@
         ")\n",
         "def pipeline(dataset_id: str, label: str, bucket: str):\n",
         "\n",
-        "    stats_op = statistics(dataset_id, label, bucket)\n",
+        "    _ = statistics(dataset_id, label, bucket)\n",
         "\n",
         "\n",
         "compiler.Compiler().compile(pipeline_func=pipeline, package_path=\"dataset_stats.json\")\n",
@@ -831,7 +974,7 @@
         "    parameter_values={\n",
         "        \"dataset_id\": dataset_id,\n",
         "        \"label\": \"mean_temp\",\n",
-        "        \"bucket\": BUCKET_NAME,\n",
+        "        \"bucket\": BUCKET_URI,\n",
         "    },\n",
         ")\n",
         "\n",
@@ -901,14 +1044,7 @@
         "\n",
         "Otherwise, you can delete the individual resources you created in this tutorial:\n",
         "\n",
-        "- Dataset\n",
-        "- Pipeline\n",
-        "- Model\n",
-        "- Endpoint\n",
-        "- AutoML Training Job\n",
-        "- Batch Job\n",
-        "- Custom Job\n",
-        "- Hyperparameter Tuning Job\n",
+        "- Vertex AI dataset\n",
         "- Cloud Storage Bucket"
       ]
     },
@@ -920,61 +1056,17 @@
       },
       "outputs": [],
       "source": [
-        "delete_all = True\n",
+        "# Set this to true only if you'd like to delete your bucket\n",
+        "delete_bucket = False\n",
         "\n",
-        "if delete_all:\n",
-        "    # Delete the dataset using the Vertex dataset object\n",
-        "    try:\n",
-        "        if \"dataset\" in globals():\n",
-        "            dataset.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
+        "# Create reference to Vertex AI dataset created in pipeline\n",
+        "dataset = aip.TabularDataset(dataset_id)\n",
         "\n",
-        "    # Delete the model using the Vertex model object\n",
-        "    try:\n",
-        "        if \"model\" in globals():\n",
-        "            model.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
+        "# delete Vertex AI dataset\n",
+        "dataset.delete()\n",
         "\n",
-        "    # Delete the endpoint using the Vertex endpoint object\n",
-        "    try:\n",
-        "        if \"endpoint\" in globals():\n",
-        "            endpoint.undeploy_all()\n",
-        "            endpoint.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the AutoML or Pipeline training job\n",
-        "    try:\n",
-        "        if \"dag\" in globals():\n",
-        "            dag.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the custom training job\n",
-        "    try:\n",
-        "        if \"job\" in globals():\n",
-        "            job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the batch prediction job using the Vertex batch prediction object\n",
-        "    try:\n",
-        "        if \"batch_predict_job\" in globals():\n",
-        "            batch_predict_job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the hyperparameter tuning job using the Vertex hyperparameter tuning object\n",
-        "    try:\n",
-        "        if \"hpt_job\" in globals():\n",
-        "            hpt_job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    if \"BUCKET_NAME\" in globals():\n",
-        "        ! gsutil rm -r $BUCKET_NAME"
+        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "    ! gsutil rm -r $BUCKET_URI"
       ]
     }
   ],

notebooks/community/ml_ops/stage3/get_started_with_bqml_pipeline_components.ipynb

@@ -33,14 +33,20 @@
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_bqml_pipeline_components.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>  \n",
+        "  <td>\n",
         "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_bqml_pipeline_components.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_bqml_pipeline_components.ipynb\">\n",
-        "      Open in Google Cloud Notebooks\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage3/get_started_with_bqml_pipeline_components.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
         "    </a>\n",
         "  </td>\n",
         "</table>\n",
@@ -99,6 +105,28 @@
         "- Make a prediction with the deployed Vertex AI model."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0c997d8d92ce"
+      },
+      "source": [
+        "### Costs \n",
+        "\n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Cloud Storage\n",
+        "\n",
+        "\n",
+        "Learn about [Vertex AI\n",
+        "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
+        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
+        "Calculator](https://cloud.google.com/products/calculator/)\n",
+        "to generate a cost estimate based on your projected usage."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -107,35 +135,35 @@
       "source": [
         "## Installations\n",
         "\n",
-        "Install *one time* the packages for executing the MLOps notebooks."
+        "Install the packages required for executing the notebook."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "install_mlops"
+        "id": "1fd00fa70a2a"
       },
       "outputs": [],
       "source": [
-        "ONCE_ONLY = False\n",
-        "if ONCE_ONLY:\n",
-        "    ! pip3 install -U tensorflow==2.5 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-transform==1.2 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-io==0.18 $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-bigquery $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-logging $USER_FLAG\n",
-        "    ! pip3 install --upgrade apache-beam[gcp] $USER_FLAG\n",
-        "    ! pip3 install --upgrade pyarrow $USER_FLAG\n",
-        "    ! pip3 install --upgrade cloudml-hypertune $USER_FLAG\n",
-        "    ! pip3 install --upgrade kfp $USER_FLAG\n",
-        "    ! pip3 install --upgrade torchvision $USER_FLAG\n",
-        "    ! pip3 install --upgrade rpy2 $USER_FLAG\n",
-        "    ! pip3 install --upgrade python-tabulate $USER_FLAG\n",
-        "    ! pip3 install -U opencv-python-headless==4.5.2.52 $USER_FLAG"
+        "import os\n",
+        "\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
+        "\n",
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "USER_FLAG = \"\"\n",
+        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\"\n",
+        "\n",
+        "! pip3 install -U tensorflow $USER_FLAG -q\n",
+        "! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG -q\n",
+        "! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG -q\n",
+        "! pip3 install --upgrade google-cloud-bigquery $USER_FLAG -q\n",
+        "! pip3 install --upgrade kfp $USER_FLAG -q"
       ]
     },
     {
@@ -153,7 +181,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "restart"
+        "id": "fIuF_ZjxJ39h"
       },
       "outputs": [],
       "source": [
@@ -167,6 +195,30 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "BF1j6f9HApxa"
+      },
+      "source": [
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,dataflow.googleapis.com).\n",
+        "\n",
+        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -186,6 +238,8 @@
       },
       "outputs": [],
       "source": [
+        "import os\n",
+        "\n",
         "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
       ]
     },
@@ -239,11 +293,13 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "region"
+        "id": "c1Rim3ogJ39j"
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+        "REGION = \"[your-region]\"  # @param {type:\"string\"}\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -261,7 +317,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "timestamp"
+        "id": "hdkr5x2jJ39k"
       },
       "outputs": [],
       "source": [
@@ -270,6 +326,82 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "UG2SHSlTJ39k"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ETQR4H1HJ39k"
+      },
+      "source": [
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
+        "authenticated. Skip this step.\n",
+        "\n",
+        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
+        "\n",
+        "\n",
+        "**Otherwise**, follow these steps:\n",
+        "\n",
+        "1. In the Cloud Console, go to the [**Create service account key**\n",
+        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
+        "\n",
+        "2. Click **Create service account**.\n",
+        "\n",
+        "3. In the **Service account name** field, enter a name, and\n",
+        "   click **Create**.\n",
+        "\n",
+        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
+        "into the filter box, and select\n",
+        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "\n",
+        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
+        "local environment.\n",
+        "\n",
+        "6. Enter the path to your service account key as the\n",
+        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9M66jv07J39l"
+      },
+      "outputs": [],
+      "source": [
+        "# If you are running this notebook in Colab, run this cell and follow the\n",
+        "# instructions to authenticate your GCP account. This provides access to your\n",
+        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
+        "# requests.\n",
+        "\n",
+        "import os\n",
+        "import sys\n",
+        "\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = False\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
+        "    if \"google.colab\" in sys.modules:\n",
+        "        IS_COLAB = True\n",
+        "        from google.colab import auth as google_auth\n",
+        "\n",
+        "        google_auth.authenticate_user()\n",
+        "\n",
+        "    # If you are running this notebook locally, replace the string below with the\n",
+        "    # path to your service account key and run this cell to authenticate your GCP\n",
+        "    # account.\n",
+        "    elif not os.getenv(\"IS_TESTING\"):\n",
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -294,7 +426,7 @@
       "outputs": [],
       "source": [
         "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "BUCKET_URI = f\"gs://{BUCKET_NAME}"
+        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
     {
@@ -322,7 +454,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "create_bucket"
+        "id": "V97jQQuiJ39m"
       },
       "outputs": [],
       "source": [
@@ -342,7 +474,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "validate_bucket"
+        "id": "7PN6kSQtJ39m"
       },
       "outputs": [],
       "source": [
@@ -366,7 +498,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "set_service_account"
+        "id": "M4WZi4CDJ39n"
       },
       "outputs": [],
       "source": [
@@ -386,9 +518,17 @@
         "    or SERVICE_ACCOUNT is None\n",
         "    or SERVICE_ACCOUNT == \"[your-service-account]\"\n",
         "):\n",
-        "    # Get your GCP project id from gcloud\n",
-        "    shell_output = !gcloud auth list 2>/dev/null\n",
-        "    SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
+        "    # Get your service account from gcloud\n",
+        "    if not IS_COLAB:\n",
+        "        shell_output = !gcloud auth list 2>/dev/null\n",
+        "        SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
+        "\n",
+        "    if IS_COLAB:\n",
+        "        shell_output = ! gcloud projects describe  $PROJECT_ID\n",
+        "        # print(\"shell_output=\", shell_output)\n",
+        "        project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n",
+        "        SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n",
+        "\n",
         "    print(\"Service Account:\", SERVICE_ACCOUNT)"
       ]
     },
@@ -407,7 +547,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "set_service_account:pipelines"
+        "id": "mI3IJONMJ39n"
       },
       "outputs": [],
       "source": [
@@ -450,8 +590,7 @@
         "import json\n",
         "\n",
         "from kfp import dsl\n",
-        "from kfp.v2 import compiler\n",
-        "from kfp.v2.dsl import component"
+        "from kfp.v2 import compiler"
       ]
     },
     {
@@ -469,7 +608,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "import_bq"
+        "id": "r7p4Iv8_J39o"
       },
       "outputs": [],
       "source": [
@@ -491,7 +630,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "import_tf"
+        "id": "_K5tP8oJJ39p"
       },
       "outputs": [],
       "source": [
@@ -513,11 +652,11 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "init_aip:mbsdk,all"
+        "id": "uAnLpS9cJ39p"
       },
       "outputs": [],
       "source": [
-        "aip.init(project=PROJECT_ID, location=REGION, staging_bucket=BUCKET_NAME)"
+        "aip.init(project=PROJECT_ID, location=REGION, staging_bucket=BUCKET_URI)"
       ]
     },
     {
@@ -535,7 +674,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "init_bq"
+        "id": "I9RloZo9J39p"
       },
       "outputs": [],
       "source": [
@@ -565,7 +704,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "accelerators:prediction,mbsdk"
+        "id": "1-mE_7kXJ39p"
       },
       "outputs": [],
       "source": [
@@ -598,7 +737,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "container:prediction"
+        "id": "AmHM8whxJ39q"
       },
       "outputs": [],
       "source": [
@@ -671,7 +810,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "create_bqml_pipeline:tabular"
+        "id": "LvND7iTpJ39r"
       },
       "outputs": [],
       "source": [
@@ -717,11 +856,11 @@
         "        query=f\"CREATE OR REPLACE MODEL {dataset}.{model} OPTIONS (model_type='dnn_classifier', labels=['{label}'], num_trials={num_trials}) AS SELECT * FROM `{bq_table}` WHERE body_mass_g IS NOT NULL AND sex IS NOT NULL\",\n",
         "    ).after(bq_dataset)\n",
         "\n",
-        "    bq_eval = BigqueryEvaluateModelJobOp(\n",
+        "    _ = BigqueryEvaluateModelJobOp(\n",
         "        project=PROJECT_ID, location=\"US\", model=bq_model.outputs[\"model\"]\n",
         "    ).after(bq_model)\n",
         "\n",
-        "    bq_predict = BigqueryPredictModelJobOp(\n",
+        "    _ = BigqueryPredictModelJobOp(\n",
         "        project=project,\n",
         "        location=location,\n",
         "        model=bq_model.outputs[\"model\"],\n",
@@ -765,7 +904,7 @@
         "        display_name=display_name,\n",
         "    ).after(model_upload)\n",
         "\n",
-        "    deploy_model = ModelDeployOp(\n",
+        "    _ = ModelDeployOp(\n",
         "        model=model_upload.outputs[\"model\"],\n",
         "        endpoint=endpoint.outputs[\"endpoint\"],\n",
         "        dedicated_resources_min_replica_count=min_replica_count,\n",
@@ -807,11 +946,23 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "run_pipeline:bqml"
+        "id": "l2FMs74-J39r"
       },
       "outputs": [],
       "source": [
-        "MODEL_DIR = BUCKET_NAME + \"/bqmodel\"\n",
+        "# If DEPLOY_GPU is None, keeping gpu as no accelerator and accelerator_count as 0\n",
+        "accelerator_count = 0\n",
+        "if DEPLOY_GPU:\n",
+        "    gpu = DEPLOY_GPU.name\n",
+        "    accelerator_count = 1\n",
+        "else:\n",
+        "    gpu = \"ACCELERATOR_TYPE_UNSPECIFIED\"  # Unspecified accelerator type, which means no accelerator.\n",
+        "    accelerator_count = 0\n",
+        "\n",
+        "print(\"gpu=\", gpu)\n",
+        "print(\"accelerator_count=\", accelerator_count)\n",
+        "\n",
+        "MODEL_DIR = BUCKET_URI + \"/bqmodel\"\n",
         "\n",
         "compiler.Compiler().compile(pipeline_func=pipeline, package_path=\"bqml.json\")\n",
         "\n",
@@ -831,8 +982,8 @@
         "        \"machine_type\": \"n1-standard-4\",\n",
         "        \"min_replica_count\": 1,\n",
         "        \"max_replica_count\": 1,\n",
-        "        \"accelerator_type\": DEPLOY_GPU.name,\n",
-        "        \"accelerator_count\": DEPLOY_NGPU,\n",
+        "        \"accelerator_type\": gpu,\n",
+        "        \"accelerator_count\": accelerator_count,\n",
         "        \"project\": PROJECT_ID,\n",
         "        \"location\": \"US\",\n",
         "    },\n",
@@ -857,7 +1008,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "view_pipleline_results:bqml"
+        "id": "2OM8zzJXJ39s"
       },
       "outputs": [],
       "source": [
@@ -936,6 +1087,9 @@
         "print(\"\\n\\n\")\n",
         "print(\"model-upload\")\n",
         "artifacts = print_pipeline_output(pipeline, \"model-upload\")\n",
+        "output = !gsutil cat $artifacts\n",
+        "output = json.loads(output[0])\n",
+        "model_id = output[\"artifacts\"][\"model\"][\"artifacts\"][0][\"metadata\"][\"resourceName\"]\n",
         "print(\"\\n\\n\")\n",
         "print(\"endpoint-create\")\n",
         "artifacts = print_pipeline_output(pipeline, \"endpoint-create\")\n",
@@ -968,7 +1122,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "delete_pipeline"
+        "id": "1UTEiNi9J39s"
       },
       "outputs": [],
       "source": [
@@ -990,7 +1144,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "endpoint_load:mbsdk"
+        "id": "gPEt5GMAJ39s"
       },
       "outputs": [],
       "source": [
@@ -1037,7 +1191,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "make_test_items:bqml,penguins"
+        "id": "sesK_MSdJ39t"
       },
       "outputs": [],
       "source": [
@@ -1083,7 +1237,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "endpoint_predict:mbsdk"
+        "id": "u5_cgdKQJ39t"
       },
       "outputs": [],
       "source": [
@@ -1111,15 +1265,41 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "delete:bqml,penguins"
+        "id": "W0rhdoHmJ39t"
       },
       "outputs": [],
       "source": [
         "try:\n",
-        "    job = bqclient.delete_model(\"bqml_tutorial.penguins_model\")\n",
+        "    job = bqclient.delete_model(f\"{PROJECT_ID}.bqml_tutorial.penguins_model\")\n",
         "except:\n",
         "    pass\n",
-        "job = bqclient.delete_dataset(\"bqml_tutorial\", delete_contents=True)"
+        "job = bqclient.delete_dataset(f\"{PROJECT_ID}.bqml_tutorial\", delete_contents=True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e776f9a3bdc4"
+      },
+      "source": [
+        "#### Delete the Vertex AI Model and Endpoint resources\n",
+        "\n",
+        "Next, undelpoy and delete the Vertex AI Model and Endpoint resources."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "63462e0480f0"
+      },
+      "outputs": [],
+      "source": [
+        "endpoint.undeploy_all()\n",
+        "endpoint.delete()\n",
+        "\n",
+        "model = aip.Model(model_id)\n",
+        "model.delete()"
       ]
     },
     {
@@ -1140,17 +1320,15 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "cleanup:mbsdk"
+        "id": "ufWUEbnZJ39u"
       },
       "outputs": [],
       "source": [
-        "delete_all = True\n",
+        "# Set this to true only if you'd like to delete your bucket\n",
+        "delete_bucket = False\n",
         "\n",
-        "if delete_all:\n",
-        "    # (DEVELOPER TODO) Find generated resources from pipeline and delete\n",
-        "\n",
-        "    if \"BUCKET_URI\" in globals():\n",
-        "        ! gsutil rm -r $BUCKET_NAME"
+        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "    ! gsutil rm -r $BUCKET_URI"
       ]
     }
   ],

notebooks/community/ml_ops/stage3/get_started_with_custom_training_pipeline_components.ipynb

@@ -8,7 +8,7 @@
       },
       "outputs": [],
       "source": [
-        "# Copyright 2021 Google LLC\n",
+        "# Copyright 2022 Google LLC\n",
         "#\n",
         "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
         "# you may not use this file except in compliance with the License.\n",
@@ -33,14 +33,20 @@
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_custom_training_pipeline_components.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>  \n",
+        "  <td>\n",
         "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_custom_training_pipeline_components.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_custom_training_pipeline_components.ipynb\">\n",
-        "      Open in Google Cloud Notebooks\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage3/get_started_with_custom_training_pipeline_components.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
         "    </a>\n",
         "  </td>\n",
         "</table>\n",
@@ -94,9 +100,37 @@
         "    - Training a Vertex AI custom trained model.\n",
         "    - Test the serving binary with a batch prediction job.\n",
         "    - Deploying a Vertex AI custom trained model.\n",
+        "- Execute a Vertex AI pipeline.\n",
+        "- Construct a pipeline for:\n",
+        "     - Construct a custom training component.\n",
+        "     - Convert custom training component to CustomTrainingJobOp.\n",
+        "     - Training a Vertex AI custom trained model using the converted component.\n",
+        "    - Deploying a Vertex AI custom trained model.\n",
         "- Execute a Vertex AI pipeline."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0c997d8d92ce"
+      },
+      "source": [
+        "### Costs \n",
+        "\n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Cloud Storage\n",
+        "\n",
+        "\n",
+        "Learn about [Vertex AI\n",
+        "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
+        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
+        "Calculator](https://cloud.google.com/products/calculator/)\n",
+        "to generate a cost estimate based on your projected usage."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -105,35 +139,34 @@
       "source": [
         "## Installations\n",
         "\n",
-        "Install *one time* the packages for executing the MLOps notebooks."
+        "Install the required packages for executing the notebook."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "install_mlops"
+        "id": "1fd00fa70a2a"
       },
       "outputs": [],
       "source": [
-        "ONCE_ONLY = False\n",
-        "if ONCE_ONLY:\n",
-        "    ! pip3 install -U tensorflow==2.5 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-transform==1.2 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-io==0.18 $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-bigquery $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-logging $USER_FLAG\n",
-        "    ! pip3 install --upgrade apache-beam[gcp] $USER_FLAG\n",
-        "    ! pip3 install --upgrade pyarrow $USER_FLAG\n",
-        "    ! pip3 install --upgrade cloudml-hypertune $USER_FLAG\n",
-        "    ! pip3 install --upgrade kfp $USER_FLAG\n",
-        "    ! pip3 install --upgrade torchvision $USER_FLAG\n",
-        "    ! pip3 install --upgrade rpy2 $USER_FLAG\n",
-        "    ! pip3 install --upgrade python-tabulate $USER_FLAG\n",
-        "    ! pip3 install -U opencv-python-headless==4.5.2.52 $USER_FLAG"
+        "import os\n",
+        "\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
+        "\n",
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "USER_FLAG = \"\"\n",
+        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\"\n",
+        "\n",
+        "! pip3 install -U tensorflow $USER_FLAG -q\n",
+        "! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG -q\n",
+        "! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG -q\n",
+        "! pip3 install --upgrade kfp $USER_FLAG -q"
       ]
     },
     {
@@ -165,6 +198,30 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "BF1j6f9HApxa"
+      },
+      "source": [
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,dataflow.googleapis.com).\n",
+        "\n",
+        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -184,6 +241,8 @@
       },
       "outputs": [],
       "source": [
+        "import os\n",
+        "\n",
         "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
       ]
     },
@@ -241,7 +300,9 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+        "REGION = \"[your-region]\"  # @param {type:\"string\"}\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -268,6 +329,81 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "648aa9824ac6"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "fc52bba17ee3"
+      },
+      "source": [
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
+        "authenticated. Skip this step.\n",
+        "\n",
+        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
+        "\n",
+        "**Otherwise**, follow these steps:\n",
+        "\n",
+        "1. In the Cloud Console, go to the [**Create service account key**\n",
+        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
+        "\n",
+        "2. Click **Create service account**.\n",
+        "\n",
+        "3. In the **Service account name** field, enter a name, and\n",
+        "   click **Create**.\n",
+        "\n",
+        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
+        "into the filter box, and select\n",
+        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "\n",
+        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
+        "local environment.\n",
+        "\n",
+        "6. Enter the path to your service account key as the\n",
+        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "535223fa4b84"
+      },
+      "outputs": [],
+      "source": [
+        "# If you are running this notebook in Colab, run this cell and follow the\n",
+        "# instructions to authenticate your GCP account. This provides access to your\n",
+        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
+        "# requests.\n",
+        "\n",
+        "import os\n",
+        "import sys\n",
+        "\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = False\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
+        "    if \"google.colab\" in sys.modules:\n",
+        "        IS_COLAB = True\n",
+        "        from google.colab import auth as google_auth\n",
+        "\n",
+        "        google_auth.authenticate_user()\n",
+        "\n",
+        "    # If you are running this notebook locally, replace the string below with the\n",
+        "    # path to your service account key and run this cell to authenticate your GCP\n",
+        "    # account.\n",
+        "    elif not os.getenv(\"IS_TESTING\"):\n",
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -291,7 +427,8 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
+        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
     {
@@ -302,8 +439,8 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
       ]
     },
     {
@@ -323,7 +460,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_NAME"
+        "! gsutil mb -l $REGION $BUCKET_URI"
       ]
     },
     {
@@ -343,7 +480,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_NAME"
+        "! gsutil ls -al $BUCKET_URI"
       ]
     },
     {
@@ -381,9 +518,17 @@
         "    or SERVICE_ACCOUNT is None\n",
         "    or SERVICE_ACCOUNT == \"[your-service-account]\"\n",
         "):\n",
-        "    # Get your GCP project id from gcloud\n",
-        "    shell_output = !gcloud auth list 2>/dev/null\n",
-        "    SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
+        "    # Get your service account from gcloud\n",
+        "    if not IS_COLAB:\n",
+        "        shell_output = !gcloud auth list 2>/dev/null\n",
+        "        SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
+        "\n",
+        "    if IS_COLAB:\n",
+        "        shell_output = ! gcloud projects describe  $PROJECT_ID\n",
+        "        # print(\"shell_output=\", shell_output)\n",
+        "        project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n",
+        "        SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n",
+        "\n",
         "    print(\"Service Account:\", SERVICE_ACCOUNT)"
       ]
     },
@@ -406,9 +551,9 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_NAME\n",
+        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
         "\n",
-        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_NAME"
+        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
       ]
     },
     {
@@ -490,7 +635,7 @@
       },
       "outputs": [],
       "source": [
-        "aip.init(project=PROJECT_ID, location=REGION, staging_bucket=BUCKET_NAME)"
+        "aip.init(project=PROJECT_ID, location=REGION, staging_bucket=BUCKET_URI)"
       ]
     },
     {
@@ -529,7 +674,7 @@
         "        int(os.getenv(\"IS_TESTING_TRAIN_GPU\")),\n",
         "    )\n",
         "else:\n",
-        "    TRAIN_GPU, TRAIN_NGPU = (aip.gapic.AcceleratorType.NVIDIA_TESLA_K80, 1)\n",
+        "    TRAIN_GPU, TRAIN_NGPU = (None, None)\n",
         "\n",
         "if os.getenv(\"IS_TESTING_DEPLOY_GPU\"):\n",
         "    DEPLOY_GPU, DEPLOY_NGPU = (\n",
@@ -1017,7 +1162,7 @@
         "! rm -f custom.tar custom.tar.gz\n",
         "! tar cvf custom.tar custom\n",
         "! gzip custom.tar\n",
-        "! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_flowers.tar.gz"
+        "! gsutil cp custom.tar.gz $BUCKET_URI/trainer_flowers.tar.gz"
       ]
     },
     {
@@ -1077,9 +1222,20 @@
       },
       "outputs": [],
       "source": [
-        "PIPELINE_ROOT = \"{}/pipeline_root/custom_icn_training\".format(BUCKET_NAME)\n",
+        "PIPELINE_ROOT = \"{}/pipeline_root/custom_icn_training\".format(BUCKET_URI)\n",
         "DEPLOY_COMPUTE = \"n1-standard-4\"\n",
         "\n",
+        "# If TRAIN_GPU is None, keeping gpu as no accelerator and accelerator_count as 0\n",
+        "gpu = \"ACCELERATOR_TYPE_UNSPECIFIED\"\n",
+        "accelerator_count = 0\n",
+        "\n",
+        "if TRAIN_GPU:\n",
+        "    gpu = TRAIN_GPU.name\n",
+        "    accelerator_count = 1\n",
+        "else:\n",
+        "    gpu = \"ACCELERATOR_TYPE_UNSPECIFIED\"  # Unspecified accelerator type, which means no accelerator.\n",
+        "    accelerator_count = 0\n",
+        "\n",
         "\n",
         "@dsl.pipeline(\n",
         "    name=\"flowers-custom-training\",\n",
@@ -1121,8 +1277,8 @@
         "        args=[\"--epochs\", \"50\", \"--image-width\", \"32\", \"--image-height\", \"32\"],\n",
         "        replica_count=1,\n",
         "        machine_type=TRAIN_COMPUTE,\n",
-        "        accelerator_type=TRAIN_GPU.name,\n",
-        "        accelerator_count=TRAIN_NGPU,\n",
+        "        accelerator_type=gpu,\n",
+        "        accelerator_count=accelerator_count,\n",
         "        # Serving - As part of this operation, the model is registered to Vertex AI\n",
         "        model_serving_container_image_uri=DEPLOY_IMAGE,\n",
         "        model_display_name=display_name,\n",
@@ -1148,7 +1304,7 @@
         "        display_name=display_name,\n",
         "    ).after(batch_op)\n",
         "\n",
-        "    deploy_op = ModelDeployOp(\n",
+        "    _ = ModelDeployOp(\n",
         "        model=training_op.outputs[\"model\"],\n",
         "        endpoint=endpoint_op.outputs[\"endpoint\"],\n",
         "        dedicated_resources_min_replica_count=1,\n",
@@ -1210,11 +1366,11 @@
         "file_1 = test_item_1.split(\"/\")[-1]\n",
         "file_2 = test_item_2.split(\"/\")[-1]\n",
         "\n",
-        "! gsutil cp $test_item_1 $BUCKET_NAME/$file_1\n",
-        "! gsutil cp $test_item_2 $BUCKET_NAME/$file_2\n",
+        "! gsutil cp $test_item_1 $BUCKET_URI/$file_1\n",
+        "! gsutil cp $test_item_2 $BUCKET_URI/$file_2\n",
         "\n",
-        "test_item_1 = BUCKET_NAME + \"/\" + file_1\n",
-        "test_item_2 = BUCKET_NAME + \"/\" + file_2"
+        "test_item_1 = BUCKET_URI + \"/\" + file_1\n",
+        "test_item_2 = BUCKET_URI + \"/\" + file_2"
       ]
     },
     {
@@ -1243,11 +1399,7 @@
       },
       "outputs": [],
       "source": [
-        "import json\n",
-        "\n",
-        "import tensorflow as tf\n",
-        "\n",
-        "gcs_input_uri = BUCKET_NAME + \"/test.jsonl\"\n",
+        "gcs_input_uri = BUCKET_URI + \"/test.jsonl\"\n",
         "with tf.io.gfile.GFile(gcs_input_uri, \"w\") as f:\n",
         "    data = {\"content\": test_item_1, \"mime_type\": \"image/jpeg\"}\n",
         "    f.write(json.dumps(data) + \"\\n\")\n",
@@ -1297,7 +1449,7 @@
         "        \"import_file\": IMPORT_FILE,\n",
         "        \"batch_files\": [gcs_input_uri],\n",
         "        \"display_name\": \"flowers\" + TIMESTAMP,\n",
-        "        \"python_package\": f\"{BUCKET_NAME}/trainer_flowers.tar.gz\",\n",
+        "        \"python_package\": f\"{BUCKET_URI}/trainer_flowers.tar.gz\",\n",
         "        \"python_module\": \"trainer.task\",\n",
         "        \"project\": PROJECT_ID,\n",
         "        \"region\": REGION,\n",
@@ -1361,25 +1513,53 @@
         "            + str(TASK_ID)\n",
         "            + \"/gcp_resources\"\n",
         "        )\n",
+        "        EVAL_METRICS = (\n",
+        "            PIPELINE_ROOT\n",
+        "            + \"/\"\n",
+        "            + PROJECT_NUMBER\n",
+        "            + \"/\"\n",
+        "            + JOB_ID\n",
+        "            + \"/\"\n",
+        "            + output_task_name\n",
+        "            + \"_\"\n",
+        "            + str(TASK_ID)\n",
+        "            + \"/evaluation_metrics\"\n",
+        "        )\n",
         "        if tf.io.gfile.exists(EXECUTE_OUTPUT):\n",
         "            ! gsutil cat $EXECUTE_OUTPUT\n",
-        "            break\n",
+        "            return EXECUTE_OUTPUT\n",
         "        elif tf.io.gfile.exists(GCP_RESOURCES):\n",
         "            ! gsutil cat $GCP_RESOURCES\n",
-        "            break\n",
+        "            return GCP_RESOURCES\n",
+        "        elif tf.io.gfile.exists(EVAL_METRICS):\n",
+        "            ! gsutil cat $EVAL_METRICS\n",
+        "            return EVAL_METRICS\n",
         "\n",
-        "    return EXECUTE_OUTPUT\n",
+        "    return None\n",
         "\n",
         "\n",
         "print(\"image-dataset-create\")\n",
         "artifacts = print_pipeline_output(pipeline, \"image-dataset-create\")\n",
         "print(\"\\n\\n\")\n",
+        "output = !gsutil cat $artifacts\n",
+        "output = json.loads(output[0])\n",
+        "dataset_id = output[\"artifacts\"][\"dataset\"][\"artifacts\"][0][\"metadata\"][\"resourceName\"]\n",
+        "print(\"\\n\\n\")\n",
         "print(\"custompythonpackagetrainingjob-run\")\n",
         "artifacts = print_pipeline_output(pipeline, \"custompythonpackagetrainingjob-run\")\n",
         "print(\"\\n\\n\")\n",
+        "output = !gsutil cat $artifacts\n",
+        "output = json.loads(output[0])\n",
+        "model_id = output[\"artifacts\"][\"model\"][\"artifacts\"][0][\"metadata\"][\"resourceName\"]\n",
+        "print(\"\\n\\n\")\n",
         "print(\"endpoint-create\")\n",
         "artifacts = print_pipeline_output(pipeline, \"endpoint-create\")\n",
         "print(\"\\n\\n\")\n",
+        "output = !gsutil cat $artifacts\n",
+        "output = json.loads(output[0])\n",
+        "endpoint_id = output[\"artifacts\"][\"endpoint\"][\"artifacts\"][0][\"metadata\"][\n",
+        "    \"resourceName\"\n",
+        "]\n",
         "print(\"model-deploy\")\n",
         "artifacts = print_pipeline_output(pipeline, \"model-deploy\")\n",
         "print(\"\\n\\n\")\n",
@@ -1392,7 +1572,13 @@
         "    output[\"artifacts\"][\"batchpredictionjob\"][\"artifacts\"][0][\"metadata\"][\n",
         "        \"gcsOutputDirectory\"\n",
         "    ]\n",
-        ")"
+        ")\n",
+        "output = !gsutil cat $artifacts\n",
+        "output = json.loads(output[0])\n",
+        "batch_job_id = output[\"artifacts\"][\"batchpredictionjob\"][\"artifacts\"][0][\"metadata\"][\n",
+        "    \"resourceName\"\n",
+        "]\n",
+        "print(\"\\n\\n\")"
       ]
     },
     {
@@ -1417,6 +1603,49 @@
         "pipeline.delete()"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d614c239d74c"
+      },
+      "source": [
+        "#### Delete the Vertex AI Model, Endpoint and BatchPredictionJob resources\n",
+        "\n",
+        "Next, delete the daatset, undelpoy and delete the Vertex AI Model, Endpoint and BathPredictionJob resources."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "417791a1a7e2"
+      },
+      "outputs": [],
+      "source": [
+        "dataset = aip.ImageDataset(dataset_id)\n",
+        "try:\n",
+        "    dataset.delete()\n",
+        "except:\n",
+        "    pass\n",
+        "\n",
+        "\n",
+        "endpoint = aip.Endpoint(endpoint_id)\n",
+        "endpoint.undeploy_all()\n",
+        "try:\n",
+        "    endpoint.delete()\n",
+        "except:\n",
+        "    pass\n",
+        "\n",
+        "model = aip.Model(model_id)\n",
+        "try:\n",
+        "    model.delete()\n",
+        "except:\n",
+        "    pass\n",
+        "\n",
+        "batch_job = aip.BatchPredictionJob(batch_job_id)\n",
+        "batch_job.delete()"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -1459,7 +1688,6 @@
       "outputs": [],
       "source": [
         "from google_cloud_pipeline_components.v1.custom_job import utils\n",
-        "from kfp.v2.dsl import Artifact\n",
         "\n",
         "\n",
         "@component(\n",
@@ -1616,9 +1844,7 @@
       },
       "outputs": [],
       "source": [
-        "from google_cloud_pipeline_components import aiplatform as gcc_aip\n",
-        "\n",
-        "PIPELINE_ROOT = \"{}/pipeline_root/custom_cifar10_training\".format(BUCKET_NAME)\n",
+        "PIPELINE_ROOT = \"{}/pipeline_root/custom_cifar10_training\".format(BUCKET_URI)\n",
         "\n",
         "\n",
         "@dsl.pipeline(name=\"custom-model-training-sample-pipeline\")\n",
@@ -1632,6 +1858,8 @@
         "    location: str = REGION,\n",
         "    deploy_image: str = \"us-docker.pkg.dev/cloud-aiplatform/prediction/tf2-cpu.2-3:latest\",\n",
         "):\n",
+        "    from google_cloud_pipeline_components import aiplatform as gcc_aip\n",
+        "\n",
         "    custom_job_op = custom_job_training_op(\n",
         "        model_dir=model_dir,\n",
         "        lr=lr,\n",
@@ -1657,7 +1885,7 @@
         "        display_name=display_name,\n",
         "    ).after(model_upload_op)\n",
         "\n",
-        "    deploy_op = gcc_aip.ModelDeployOp(\n",
+        "    _ = gcc_aip.ModelDeployOp(\n",
         "        model=model_upload_op.outputs[\"model\"],\n",
         "        endpoint=endpoint_op.outputs[\"endpoint\"],\n",
         "        dedicated_resources_min_replica_count=1,\n",
@@ -1735,46 +1963,6 @@
         "PROJECT_NUMBER = pipeline.gca_resource.name.split(\"/\")[1]\n",
         "print(PROJECT_NUMBER)\n",
         "\n",
-        "\n",
-        "def print_pipeline_output(job, output_task_name):\n",
-        "    JOB_ID = job.name\n",
-        "    print(JOB_ID)\n",
-        "    for _ in range(len(job.gca_resource.job_detail.task_details)):\n",
-        "        TASK_ID = job.gca_resource.job_detail.task_details[_].task_id\n",
-        "        EXECUTE_OUTPUT = (\n",
-        "            PIPELINE_ROOT\n",
-        "            + \"/\"\n",
-        "            + PROJECT_NUMBER\n",
-        "            + \"/\"\n",
-        "            + JOB_ID\n",
-        "            + \"/\"\n",
-        "            + output_task_name\n",
-        "            + \"_\"\n",
-        "            + str(TASK_ID)\n",
-        "            + \"/executor_output.json\"\n",
-        "        )\n",
-        "        GCP_RESOURCES = (\n",
-        "            PIPELINE_ROOT\n",
-        "            + \"/\"\n",
-        "            + PROJECT_NUMBER\n",
-        "            + \"/\"\n",
-        "            + JOB_ID\n",
-        "            + \"/\"\n",
-        "            + output_task_name\n",
-        "            + \"_\"\n",
-        "            + str(TASK_ID)\n",
-        "            + \"/gcp_resources\"\n",
-        "        )\n",
-        "        if tf.io.gfile.exists(EXECUTE_OUTPUT):\n",
-        "            ! gsutil cat $EXECUTE_OUTPUT\n",
-        "            break\n",
-        "        elif tf.io.gfile.exists(GCP_RESOURCES):\n",
-        "            ! gsutil cat $GCP_RESOURCES\n",
-        "            break\n",
-        "\n",
-        "    return EXECUTE_OUTPUT\n",
-        "\n",
-        "\n",
         "print(\"custom-train-model\")\n",
         "artifacts = print_pipeline_output(pipeline, \"custom-train-model\")\n",
         "print(\"\\n\\n\")\n",
@@ -1784,9 +1972,17 @@
         "print(\"model-upload\")\n",
         "artifacts = print_pipeline_output(pipeline, \"model-upload\")\n",
         "print(\"\\n\\n\")\n",
+        "output = !gsutil cat $artifacts\n",
+        "output = json.loads(output[0])\n",
+        "model_id = output[\"artifacts\"][\"model\"][\"artifacts\"][0][\"metadata\"][\"resourceName\"]\n",
         "print(\"endpoint-create\")\n",
         "artifacts = print_pipeline_output(pipeline, \"endpoint-create\")\n",
         "print(\"\\n\\n\")\n",
+        "output = !gsutil cat $artifacts\n",
+        "output = json.loads(output[0])\n",
+        "endpoint_id = output[\"artifacts\"][\"endpoint\"][\"artifacts\"][0][\"metadata\"][\n",
+        "    \"resourceName\"\n",
+        "]\n",
         "print(\"model-deploy\")\n",
         "artifacts = print_pipeline_output(pipeline, \"model-deploy\")\n",
         "print(\"\\n\\n\")"
@@ -1814,6 +2010,49 @@
         "pipeline.delete()"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "93e69fc8b9e3"
+      },
+      "source": [
+        "#### Delete the Vertex AI Model and Endpoint resource\n",
+        "\n",
+        "Next, undelpoy and delete the Vertex AI Model and Endpoint resources."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "c9d18ae084b1"
+      },
+      "source": [
+        "#### Delete the Vertex model and endpoint\n",
+        "\n",
+        "Next, undelpoy and delete the Vertex Model and Endpoint resource."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "dccf71121d0b"
+      },
+      "outputs": [],
+      "source": [
+        "endpoint.undeploy_all()\n",
+        "try:\n",
+        "    endpoint.delete()\n",
+        "except:\n",
+        "    pass\n",
+        "\n",
+        "model = aip.Model(model_id)\n",
+        "try:\n",
+        "    model.delete()\n",
+        "except:\n",
+        "    pass"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -1825,17 +2064,7 @@
         "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
         "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
         "\n",
-        "Otherwise, you can delete the individual resources you created in this tutorial:\n",
-        "\n",
-        "- Dataset\n",
-        "- Pipeline\n",
-        "- Model\n",
-        "- Endpoint\n",
-        "- AutoML Training Job\n",
-        "- Batch Job\n",
-        "- Custom Job\n",
-        "- Hyperparameter Tuning Job\n",
-        "- Cloud Storage Bucket"
+        "Otherwise, you can delete the individual resources you created in this tutorial."
       ]
     },
     {
@@ -1846,61 +2075,10 @@
       },
       "outputs": [],
       "source": [
-        "delete_all = True\n",
+        "delete_bucket = False\n",
         "\n",
-        "if delete_all:\n",
-        "    # Delete the dataset using the Vertex dataset object\n",
-        "    try:\n",
-        "        if \"dataset\" in globals():\n",
-        "            dataset.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the model using the Vertex model object\n",
-        "    try:\n",
-        "        if \"model\" in globals():\n",
-        "            model.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the endpoint using the Vertex endpoint object\n",
-        "    try:\n",
-        "        if \"endpoint\" in globals():\n",
-        "            endpoint.undeploy_all()\n",
-        "            endpoint.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the AutoML or Pipeline training job\n",
-        "    try:\n",
-        "        if \"dag\" in globals():\n",
-        "            dag.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the custom training job\n",
-        "    try:\n",
-        "        if \"job\" in globals():\n",
-        "            job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the batch prediction job using the Vertex batch prediction object\n",
-        "    try:\n",
-        "        if \"batch_predict_job\" in globals():\n",
-        "            batch_predict_job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the hyperparameter tuning job using the Vertex hyperparameter tuning object\n",
-        "    try:\n",
-        "        if \"hpt_job\" in globals():\n",
-        "            hpt_job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    if \"BUCKET_NAME\" in globals():\n",
-        "        ! gsutil rm -r $BUCKET_NAME"
+        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "    ! gsutil rm -r $BUCKET_URI"
       ]
     }
   ],

notebooks/community/ml_ops/stage3/get_started_with_dataflow_pipeline_components.ipynb

@@ -40,9 +40,17 @@
         "  </td>\n",
         "  <td>\n",
         "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_dataflow_pipeline_components.ipynb\">\n",
-        "      Open in Google Cloud Notebooks\n",
+        "<img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> \n",
+        "        Colab logo Run in Colab\n",
         "    </a>\n",
         "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage3/get_started_with_dataflow_pipeline_components.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "  </td>   \n",
+        "    \n",
         "</table>\n",
         "<br/><br/><br/>"
       ]
@@ -101,7 +109,7 @@
       "source": [
         "## Installations\n",
         "\n",
-        "Install *one time* the packages for executing the MLOps notebooks."
+        "Install the required packages for executing the notebook."
       ]
     },
     {
@@ -112,24 +120,25 @@
       },
       "outputs": [],
       "source": [
-        "ONCE_ONLY = False\n",
-        "if ONCE_ONLY:\n",
-        "    ! pip3 install -U tensorflow==2.5 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-transform==1.2 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-io==0.18 $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-bigquery $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-logging $USER_FLAG\n",
-        "    ! pip3 install --upgrade apache-beam[gcp] $USER_FLAG\n",
-        "    ! pip3 install --upgrade pyarrow $USER_FLAG\n",
-        "    ! pip3 install --upgrade cloudml-hypertune $USER_FLAG\n",
-        "    ! pip3 install --upgrade kfp $USER_FLAG\n",
-        "    ! pip3 install --upgrade torchvision $USER_FLAG\n",
-        "    ! pip3 install --upgrade rpy2 $USER_FLAG\n",
-        "    ! pip3 install --upgrade python-tabulate $USER_FLAG\n",
-        "    ! pip3 install -U opencv-python-headless==4.5.2.52 $USER_FLAG"
+        "import os\n",
+        "\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
+        "\n",
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "USER_FLAG = \"\"\n",
+        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\"\n",
+        "\n",
+        "! pip3 install -U tensorflow $USER_FLAG -q\n",
+        "! pip3 install -U tensorflow-data-validation $USER_FLAG -q\n",
+        "! pip3 install -U tensorflow-transform $USER_FLAG -q\n",
+        "! pip3 install -U tensorflow-io $USER_FLAG -q\n",
+        "! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG -q\n",
+        "! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG -q"
       ]
     },
     {
@@ -161,6 +170,30 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "BF1j6f9HApxa"
+      },
+      "source": [
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI API and Dataflow API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,dataflow.googleapis.com).\n",
+        "\n",
+        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -180,7 +213,24 @@
       },
       "outputs": [],
       "source": [
-        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
+        "import os\n",
+        "\n",
+        "PROJECT_ID = \"\"\n",
+        "\n",
+        "if not os.getenv(\"IS_TESTING\"):\n",
+        "    # Get your Google Cloud project ID from gcloud\n",
+        "    shell_output = !gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "    PROJECT_ID = shell_output[0]\n",
+        "    print(\"Project ID: \", PROJECT_ID)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "37c0a68ff20d"
+      },
+      "source": [
+        "Otherwise, set your project ID here."
       ]
     },
     {
@@ -191,22 +241,8 @@
       },
       "outputs": [],
       "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
-        "    # Get your GCP project id from gcloud\n",
-        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
-        "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID:\", PROJECT_ID)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "set_gcloud_project_id"
-      },
-      "outputs": [],
-      "source": [
-        "! gcloud config set project $PROJECT_ID"
+        "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
+        "    PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -237,7 +273,10 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -264,6 +303,89 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "927085b84a07"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
+        "\n",
+        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
+        "\n",
+        "**Otherwise**, follow these steps:\n",
+        "\n",
+        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
+        "\n",
+        "**Click Create service account**.\n",
+        "\n",
+        "In the **Service account name** field, enter a name, and click **Create**.\n",
+        "\n",
+        "In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex\" into the filter box, and select **Vertex Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "\n",
+        "Click Create. A JSON file that contains your key downloads to your local environment.\n",
+        "\n",
+        "Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "89788a802687"
+      },
+      "outputs": [],
+      "source": [
+        "# If you are running this notebook in Colab, run this cell and follow the\n",
+        "# instructions to authenticate your GCP account. This provides access to your\n",
+        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
+        "# requests.\n",
+        "\n",
+        "import os\n",
+        "import sys\n",
+        "\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = False\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
+        "    if \"google.colab\" in sys.modules:\n",
+        "        IS_COLAB = True\n",
+        "        from google.colab import auth as google_auth\n",
+        "\n",
+        "        google_auth.authenticate_user()\n",
+        "\n",
+        "    # If you are running this notebook locally, replace the string below with the\n",
+        "    # path to your service account key and run this cell to authenticate your GCP\n",
+        "    # account.\n",
+        "    elif not os.getenv(\"IS_TESTING\"):\n",
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "40ed98f5cc48"
+      },
+      "source": [
+        "#### If you are using Colab Notebooks, set the project using gcloud config."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "fde1a355f1e9"
+      },
+      "outputs": [],
+      "source": [
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
+        "    if \"google.colab\" in sys.modules:\n",
+        "        ! gcloud config set project $PROJECT_ID"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -287,7 +409,7 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
+        "BUCKET_URI = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -298,8 +420,8 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
       ]
     },
     {
@@ -319,7 +441,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_NAME"
+        "! gsutil mb -l $REGION $BUCKET_URI"
       ]
     },
     {
@@ -339,7 +461,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_NAME"
+        "! gsutil ls -al $BUCKET_URI"
       ]
     },
     {
@@ -377,9 +499,16 @@
         "    or SERVICE_ACCOUNT is None\n",
         "    or SERVICE_ACCOUNT == \"[your-service-account]\"\n",
         "):\n",
-        "    # Get your GCP project id from gcloud\n",
-        "    shell_output = !gcloud auth list 2>/dev/null\n",
-        "    SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
+        "    # Get your service account from gcloud\n",
+        "    if not IS_COLAB:\n",
+        "        shell_output = !gcloud auth list 2>/dev/null\n",
+        "        SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
+        "\n",
+        "    if IS_COLAB:\n",
+        "        shell_output = ! gcloud projects describe  $PROJECT_ID\n",
+        "        project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n",
+        "        SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n",
+        "\n",
         "    print(\"Service Account:\", SERVICE_ACCOUNT)"
       ]
     },
@@ -402,9 +531,9 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_NAME\n",
+        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
         "\n",
-        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_NAME"
+        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
       ]
     },
     {
@@ -427,35 +556,12 @@
       },
       "outputs": [],
       "source": [
-        "import google.cloud.aiplatform as aip"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "import_kfp"
-      },
-      "outputs": [],
-      "source": [
-        "import json\n",
-        "\n",
-        "from kfp import dsl\n",
-        "from kfp.v2 import compiler\n",
-        "from kfp.v2.dsl import component"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "import_gcpc:dataflow"
-      },
-      "outputs": [],
-      "source": [
+        "import google.cloud.aiplatform as aip\n",
         "from google_cloud_pipeline_components.v1.dataflow import DataflowPythonJobOp\n",
         "from google_cloud_pipeline_components.v1.wait_gcp_resources import \\\n",
-        "    WaitGcpResourcesOp"
+        "    WaitGcpResourcesOp\n",
+        "from kfp import dsl\n",
+        "from kfp.v2 import compiler"
       ]
     },
     {
@@ -477,7 +583,7 @@
       },
       "outputs": [],
       "source": [
-        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_NAME)"
+        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
       ]
     },
     {
@@ -673,12 +779,12 @@
       },
       "outputs": [],
       "source": [
-        "GCS_WC_PY = BUCKET_NAME + \"/wc.py\"\n",
+        "GCS_WC_PY = BUCKET_URI + \"/wc.py\"\n",
         "! gsutil cp wc.py $GCS_WC_PY\n",
-        "GCS_REQUIREMENTS_TXT = BUCKET_NAME + \"/requirements.txt\"\n",
+        "GCS_REQUIREMENTS_TXT = BUCKET_URI + \"/requirements.txt\"\n",
         "! gsutil cp requirements.txt $GCS_REQUIREMENTS_TXT\n",
         "\n",
-        "GCS_WC_OUT = BUCKET_NAME + \"/wc_out.txt\""
+        "GCS_WC_OUT = BUCKET_URI + \"/wc_out.txt\""
       ]
     },
     {
@@ -709,9 +815,7 @@
       },
       "outputs": [],
       "source": [
-        "import json\n",
-        "\n",
-        "PIPELINE_ROOT = \"{}/pipeline_root/dataflow_wc\".format(BUCKET_NAME)\n",
+        "PIPELINE_ROOT = \"{}/pipeline_root/dataflow_wc\".format(BUCKET_URI)\n",
         "\n",
         "\n",
         "@dsl.pipeline(name=\"dataflow-wc\", description=\"Dataflow word count component pipeline\")\n",
@@ -733,9 +837,7 @@
         "        args=args,\n",
         "    )\n",
         "\n",
-        "    dataflow_wait_op = WaitGcpResourcesOp(\n",
-        "        gcp_resources=dataflow_python_op.outputs[\"gcp_resources\"]\n",
-        "    )\n",
+        "    _ = WaitGcpResourcesOp(gcp_resources=dataflow_python_op.outputs[\"gcp_resources\"])\n",
         "\n",
         "\n",
         "compiler.Compiler().compile(pipeline_func=pipeline, package_path=\"dataflow_wc.json\")\n",
@@ -813,82 +915,80 @@
         "from apache_beam.options.pipeline_options import SetupOptions\n",
         "\n",
         "def run(argv=None):\n",
-        "  \"\"\"Main entry point; defines and runs the wordcount pipeline.\"\"\"\n",
+        "    \"\"\"Main entry point; defines and runs the wordcount pipeline.\"\"\"\n",
         "\n",
-        "  parser = argparse.ArgumentParser()\n",
-        "  parser.add_argument('--bq_table',\n",
+        "    parser = argparse.ArgumentParser()\n",
+        "    parser.add_argument('--bq_table',\n",
         "                      dest='bq_table')\n",
-        "  parser.add_argument('--bucket',\n",
+        "    parser.add_argument('--bucket',\n",
         "                      dest='bucket')\n",
-        "  args, pipeline_args = parser.parse_known_args(argv)\n",
-        "  logging.info(\"ARGS\")\n",
-        "  logging.info(args)\n",
-        "  logging.info(\"PIPELINE ARGS\")\n",
-        "  logging.info(pipeline_args)\n",
-        "  for i in range(0, len(pipeline_args), 2):\n",
+        "    args, pipeline_args = parser.parse_known_args(argv)\n",
+        "    logging.info(\"ARGS\")\n",
+        "    logging.info(args)\n",
+        "    logging.info(\"PIPELINE ARGS\")\n",
+        "    logging.info(pipeline_args)\n",
+        "    for i in range(0, len(pipeline_args), 2):\n",
         "        if \"--temp_location\" == pipeline_args[i]:\n",
         "            temp_location = pipeline_args[i+1]\n",
         "        elif \"--project\" == pipeline_args[i]:\n",
         "            project = pipeline_args[i+1]\n",
         "\n",
-        "  exported_train = args.bucket + '/exported_data/train'\n",
-        "  exported_eval  = args.bucket + '/exported_data/eval'\n",
+        "    exported_train = args.bucket + '/exported_data/train'\n",
+        "    exported_eval  = args.bucket + '/exported_data/eval'\n",
         "\n",
+        "    pipeline_options = PipelineOptions(pipeline_args)\n",
+        "    pipeline_options.view_as(SetupOptions).save_main_session = True\n",
+        "    with beam.Pipeline(options=pipeline_options) as pipeline:\n",
+        "        with tft_beam.Context(temp_location):\n",
+        "            raw_data_query = \"SELECT {0},{1} FROM {2} LIMIT 500\".format(\"CAST(station_number as STRING) AS station_number,year,month,day\",\"mean_temp\", args.bq_table)\n",
         "\n",
-        "  pipeline_options = PipelineOptions(pipeline_args)\n",
-        "  pipeline_options.view_as(SetupOptions).save_main_session = True\n",
-        "  with beam.Pipeline(options=pipeline_options) as pipeline:\n",
-        "    with tft_beam.Context(temp_location):\n",
+        "            def parse_bq_record(bq_record):\n",
+        "                \"\"\"Parses a bq_record to a dictionary.\"\"\"\n",
+        "                output = {}\n",
+        "                for key in bq_record:\n",
+        "                    output[key] = [bq_record[key]]\n",
+        "                return output\n",
         "\n",
-        "        raw_data_query = \"SELECT {0},{1} FROM {2} LIMIT 500\".format(\"CAST(station_number as STRING) AS station_number,year,month,day\",\"mean_temp\", args.bq_table)\n",
+        "            def split_dataset(bq_row, num_partitions, ratio):\n",
+        "                \"\"\"Returns a partition number for a given bq_row.\"\"\"\n",
+        "                import json\n",
         "\n",
-        "        def parse_bq_record(bq_record):\n",
-        "            \"\"\"Parses a bq_record to a dictionary.\"\"\"\n",
-        "            output = {}\n",
-        "            for key in bq_record:\n",
-        "                output[key] = [bq_record[key]]\n",
-        "            return output\n",
+        "                assert num_partitions == len(ratio)\n",
+        "                bucket = sum(map(ord, json.dumps(bq_row))) % sum(ratio)\n",
+        "                total = 0\n",
+        "                for i, part in enumerate(ratio):\n",
+        "                    total += part\n",
+        "                    if bucket < total:\n",
+        "                        return i\n",
+        "                return len(ratio) - 1\n",
         "\n",
-        "        def split_dataset(bq_row, num_partitions, ratio):\n",
-        "            \"\"\"Returns a partition number for a given bq_row.\"\"\"\n",
-        "            import json\n",
-        "\n",
-        "            assert num_partitions == len(ratio)\n",
-        "            bucket = sum(map(ord, json.dumps(bq_row))) % sum(ratio)\n",
-        "            total = 0\n",
-        "            for i, part in enumerate(ratio):\n",
-        "                total += part\n",
-        "                if bucket < total:\n",
-        "                    return i\n",
-        "            return len(ratio) - 1\n",
-        "\n",
-        "        # Read raw BigQuery data.\n",
-        "        raw_train_data, raw_eval_data = (\n",
-        "            pipeline\n",
-        "            | \"Read Raw Data\"\n",
-        "            >> beam.io.ReadFromBigQuery(\n",
-        "                query=raw_data_query,\n",
-        "                project=project,\n",
-        "                use_standard_sql=True,\n",
+        "            # Read raw BigQuery data.\n",
+        "            raw_train_data, raw_eval_data = (\n",
+        "                pipeline\n",
+        "                | \"Read Raw Data\"\n",
+        "                >> beam.io.ReadFromBigQuery(\n",
+        "                    query=raw_data_query,\n",
+        "                    project=project,\n",
+        "                    use_standard_sql=True,\n",
+        "                )\n",
+        "                | \"Parse Data\" >> beam.Map(parse_bq_record)\n",
+        "                | \"Split\" >> beam.Partition(split_dataset, 2, ratio=[8, 2])\n",
         "            )\n",
-        "            | \"Parse Data\" >> beam.Map(parse_bq_record)\n",
-        "            | \"Split\" >> beam.Partition(split_dataset, 2, ratio=[8, 2])\n",
-        "        )\n",
         "\n",
-        "        # Write raw train data to GCS .\n",
-        "        _ = raw_train_data | \"Write Raw Train Data\" >> beam.io.WriteToText(\n",
-        "            file_path_prefix=exported_train, file_name_suffix=\".csv\"\n",
-        "        )\n",
+        "            # Write raw train data to GCS .\n",
+        "            _ = raw_train_data | \"Write Raw Train Data\" >> beam.io.WriteToText(\n",
+        "                file_path_prefix=exported_train, file_name_suffix=\".csv\"\n",
+        "            )\n",
         "\n",
-        "        # Write raw eval data to GCS .\n",
-        "        _ = raw_eval_data | \"Write Raw Eval Data\" >> beam.io.WriteToText(\n",
-        "            file_path_prefix=exported_eval, file_name_suffix=\".csv\"\n",
-        "        )\n",
+        "            # Write raw eval data to GCS .\n",
+        "            _ = raw_eval_data | \"Write Raw Eval Data\" >> beam.io.WriteToText(\n",
+        "                file_path_prefix=exported_eval, file_name_suffix=\".csv\"\n",
+        "            )\n",
         "\n",
         "\n",
         "if __name__ == '__main__':\n",
-        "  logging.getLogger().setLevel(logging.INFO)\n",
-        "  run()"
+        "    logging.getLogger().setLevel(logging.INFO)\n",
+        "    run()"
       ]
     },
     {
@@ -975,11 +1075,11 @@
       },
       "outputs": [],
       "source": [
-        "GCS_SPLIT_PY = BUCKET_NAME + \"/split.py\"\n",
+        "GCS_SPLIT_PY = BUCKET_URI + \"/split.py\"\n",
         "! gsutil cp split.py $GCS_SPLIT_PY\n",
-        "GCS_REQUIREMENTS_TXT = BUCKET_NAME + \"/requirements.txt\"\n",
+        "GCS_REQUIREMENTS_TXT = BUCKET_URI + \"/requirements.txt\"\n",
         "! gsutil cp requirements.txt $GCS_REQUIREMENTS_TXT\n",
-        "GCS_SETUP_PY = BUCKET_NAME + \"/setup.py\"\n",
+        "GCS_SETUP_PY = BUCKET_URI + \"/setup.py\"\n",
         "! gsutil cp setup.py $GCS_SETUP_PY"
       ]
     },
@@ -1036,7 +1136,7 @@
       },
       "outputs": [],
       "source": [
-        "PIPELINE_ROOT = \"{}/pipeline_root/dataflow_split\".format(BUCKET_NAME)\n",
+        "PIPELINE_ROOT = \"{}/pipeline_root/dataflow_split\".format(BUCKET_URI)\n",
         "\n",
         "\n",
         "@dsl.pipeline(name=\"dataflow-split\", description=\"Dataflow split dataset\")\n",
@@ -1047,7 +1147,7 @@
         "    staging_dir: str = PIPELINE_ROOT,\n",
         "    args: list = [\n",
         "        \"--bucket\",\n",
-        "        BUCKET_NAME,\n",
+        "        BUCKET_URI,\n",
         "        \"--bq_table\",\n",
         "        BQ_TABLE,\n",
         "        \"--runner\",\n",
@@ -1067,9 +1167,7 @@
         "        args=args,\n",
         "    )\n",
         "\n",
-        "    dataflow_wait_op = WaitGcpResourcesOp(\n",
-        "        gcp_resources=dataflow_python_op.outputs[\"gcp_resources\"]\n",
-        "    )\n",
+        "    _ = WaitGcpResourcesOp(gcp_resources=dataflow_python_op.outputs[\"gcp_resources\"])\n",
         "\n",
         "\n",
         "compiler.Compiler().compile(pipeline_func=pipeline, package_path=\"dataflow_split.json\")\n",
@@ -1083,7 +1181,7 @@
         "\n",
         "pipeline.run()\n",
         "\n",
-        "! gsutil ls {BUCKET_NAME}/exported_data\n",
+        "! gsutil ls {BUCKET_URI}/exported_data\n",
         "\n",
         "! rm -f dataflow_split.json split.py requirements.txt"
       ]
@@ -1124,13 +1222,6 @@
         "Otherwise, you can delete the individual resources you created in this tutorial:\n",
         "\n",
         "- Dataset\n",
-        "- Pipeline\n",
-        "- Model\n",
-        "- Endpoint\n",
-        "- AutoML Training Job\n",
-        "- Batch Job\n",
-        "- Custom Job\n",
-        "- Hyperparameter Tuning Job\n",
         "- Cloud Storage Bucket"
       ]
     },
@@ -1142,61 +1233,11 @@
       },
       "outputs": [],
       "source": [
-        "delete_all = True\n",
+        "# Warning: Setting this to true will delete everything in your bucket\n",
+        "delete_bucket = False\n",
         "\n",
-        "if delete_all:\n",
-        "    # Delete the dataset using the Vertex dataset object\n",
-        "    try:\n",
-        "        if \"dataset\" in globals():\n",
-        "            dataset.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the model using the Vertex model object\n",
-        "    try:\n",
-        "        if \"model\" in globals():\n",
-        "            model.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the endpoint using the Vertex endpoint object\n",
-        "    try:\n",
-        "        if \"endpoint\" in globals():\n",
-        "            endpoint.undeploy_all()\n",
-        "            endpoint.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the AutoML or Pipeline training job\n",
-        "    try:\n",
-        "        if \"dag\" in globals():\n",
-        "            dag.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the custom training job\n",
-        "    try:\n",
-        "        if \"job\" in globals():\n",
-        "            job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the batch prediction job using the Vertex batch prediction object\n",
-        "    try:\n",
-        "        if \"batch_predict_job\" in globals():\n",
-        "            batch_predict_job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the hyperparameter tuning job using the Vertex hyperparameter tuning object\n",
-        "    try:\n",
-        "        if \"hpt_job\" in globals():\n",
-        "            hpt_job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    if \"BUCKET_NAME\" in globals():\n",
-        "        ! gsutil rm -r $BUCKET_NAME"
+        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "    ! gsutil rm -r $BUCKET_URI"
       ]
     }
   ],

notebooks/community/ml_ops/stage3/get_started_with_hpt_pipeline_components.ipynb

@@ -32,6 +32,11 @@
         "# E2E ML on GCP: MLOps stage 3 : formalization: get started with Hyperparameter Tuning pipeline components\n",
         "\n",
         "<table align=\"left\">\n",
+        "    <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_hpt_pipeline_components.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
         "  <td>\n",
         "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_hpt_pipeline_components.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
@@ -39,7 +44,8 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_hpt_pipeline_components.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage3/get_started_with_hpt_pipeline_components.ipynb\">\n",
+        "        <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Google Cloud Notebooks\n",
         "    </a>\n",
         "  </td>\n",
@@ -94,7 +100,20 @@
         "    - If the metrics exceed a specified threshold.\n",
         "      - Get the location of the model artifacts for the best tuned model.\n",
         "      - Upload the model artifacts to a `Vertex AI Model` resource.\n",
-        "- Execute a Vertex AI pipeline."
+        "- Execute a Vertex AI pipeline.\n",
+        "\n",
+        "### Costs \n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI\n",
+        "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
+        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
+        "Calculator](https://cloud.google.com/products/calculator/)\n",
+        "to generate a cost estimate based on your projected usage."
       ]
     },
     {
@@ -105,35 +124,37 @@
       "source": [
         "## Installations\n",
         "\n",
-        "Install *one time* the packages for executing the MLOps notebooks."
+        "Install the required packages for executing the notebook."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "install_mlops"
+        "id": "LR9HQnyiMoT5"
       },
       "outputs": [],
       "source": [
-        "ONCE_ONLY = False\n",
-        "if ONCE_ONLY:\n",
-        "    ! pip3 install -U tensorflow==2.5 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-transform==1.2 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-io==0.18 $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-bigquery $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-logging $USER_FLAG\n",
-        "    ! pip3 install --upgrade apache-beam[gcp] $USER_FLAG\n",
-        "    ! pip3 install --upgrade pyarrow $USER_FLAG\n",
-        "    ! pip3 install --upgrade cloudml-hypertune $USER_FLAG\n",
-        "    ! pip3 install --upgrade kfp $USER_FLAG\n",
-        "    ! pip3 install --upgrade torchvision $USER_FLAG\n",
-        "    ! pip3 install --upgrade rpy2 $USER_FLAG\n",
-        "    ! pip3 install --upgrade python-tabulate $USER_FLAG\n",
-        "    ! pip3 install -U opencv-python-headless==4.5.2.52 $USER_FLAG"
+        "import os\n",
+        "\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
+        "\n",
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "USER_FLAG = \"\"\n",
+        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\"\n",
+        "\n",
+        "! pip3 install -U tensorflow==2.5 $USER_FLAG -q\n",
+        "! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG -q\n",
+        "! pip3 install -U tensorflow-transform==1.2 $USER_FLAG -q\n",
+        "! pip3 install -U tensorflow-io==0.18 $USER_FLAG -q\n",
+        "! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG -q\n",
+        "! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG -q\n",
+        "! pip3 install --upgrade kfp $USER_FLAG -q"
       ]
     },
     {
@@ -151,7 +172,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "restart"
+        "id": "VeBfL2pmMoT7"
       },
       "outputs": [],
       "source": [
@@ -165,6 +186,32 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "1df9ff75fa88"
+      },
+      "source": [
+        "## Before you begin\n",
+        "\n",
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com).\n",
+        "\n",
+        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -237,11 +284,14 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "region"
+        "id": "7iewOt9NMoT8"
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+        "REGION = \"[your-region]\"  # @param {type:\"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\"  # @param {type: \"string\"}"
       ]
     },
     {
@@ -259,7 +309,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "timestamp"
+        "id": "Y-vhpfibMoT9"
       },
       "outputs": [],
       "source": [
@@ -268,6 +318,67 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gcp_authenticate"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
+        "\n",
+        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
+        "\n",
+        "**Otherwise**, follow these steps:\n",
+        "\n",
+        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
+        "\n",
+        "**Click Create service account**.\n",
+        "\n",
+        "In the **Service account name** field, enter a name, and click **Create**.\n",
+        "\n",
+        "In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex\" into the filter box, and select **Vertex Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "\n",
+        "Click Create. A JSON file that contains your key downloads to your local environment.\n",
+        "\n",
+        "Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "-V_6SvMUNUa1"
+      },
+      "outputs": [],
+      "source": [
+        "# If you are running this notebook in Colab, run this cell and follow the\n",
+        "# instructions to authenticate your GCP account. This provides access to your\n",
+        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
+        "# requests.\n",
+        "\n",
+        "import os\n",
+        "import sys\n",
+        "\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = False\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
+        "    if \"google.colab\" in sys.modules:\n",
+        "        IS_COLAB = True\n",
+        "        from google.colab import auth as google_auth\n",
+        "\n",
+        "        google_auth.authenticate_user()\n",
+        "\n",
+        "    # If you are running this notebook locally, replace the string below with the\n",
+        "    # path to your service account key and run this cell to authenticate your GCP\n",
+        "    # account.\n",
+        "    elif not os.getenv(\"IS_TESTING\"):\n",
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -291,7 +402,7 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
+        "BUCKET_URI = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -302,8 +413,8 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
       ]
     },
     {
@@ -319,11 +430,11 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "create_bucket"
+        "id": "2smRgc53MoT-"
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_NAME"
+        "! gsutil mb -l $REGION $BUCKET_URI"
       ]
     },
     {
@@ -339,11 +450,11 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "validate_bucket"
+        "id": "ME1Tr9j_MoT-"
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_NAME"
+        "! gsutil ls -al $BUCKET_URI"
       ]
     },
     {
@@ -361,7 +472,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "set_service_account"
+        "id": "EIivrR-3MoT-"
       },
       "outputs": [],
       "source": [
@@ -381,9 +492,16 @@
         "    or SERVICE_ACCOUNT is None\n",
         "    or SERVICE_ACCOUNT == \"[your-service-account]\"\n",
         "):\n",
-        "    # Get your GCP project id from gcloud\n",
-        "    shell_output = !gcloud auth list 2>/dev/null\n",
-        "    SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
+        "    # Get your service account from gcloud\n",
+        "    if not IS_COLAB:\n",
+        "        shell_output = !gcloud auth list 2>/dev/null\n",
+        "        SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
+        "\n",
+        "    if IS_COLAB:\n",
+        "        shell_output = ! gcloud projects describe  $PROJECT_ID\n",
+        "        project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n",
+        "        SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n",
+        "\n",
         "    print(\"Service Account:\", SERVICE_ACCOUNT)"
       ]
     },
@@ -402,13 +520,13 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "set_service_account:pipelines"
+        "id": "mtwsjYnIMoT_"
       },
       "outputs": [],
       "source": [
-        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_NAME\n",
+        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
         "\n",
-        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_NAME"
+        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
       ]
     },
     {
@@ -464,7 +582,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "import_tf"
+        "id": "DtTIHh_KMoUA"
       },
       "outputs": [],
       "source": [
@@ -486,11 +604,11 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "init_aip:mbsdk"
+        "id": "sUFJPDW0MoUA"
       },
       "outputs": [],
       "source": [
-        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_NAME)"
+        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
       ]
     },
     {
@@ -519,17 +637,19 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "accelerators:training,prediction,ngpu,mbsdk"
+        "id": "A6dzi4cXMoUA"
       },
       "outputs": [],
       "source": [
+        "import os\n",
+        "\n",
         "if os.getenv(\"IS_TESTING_TRAIN_GPU\"):\n",
         "    TRAIN_GPU, TRAIN_NGPU = (\n",
         "        aip.gapic.AcceleratorType.NVIDIA_TESLA_K80,\n",
         "        int(os.getenv(\"IS_TESTING_TRAIN_GPU\")),\n",
         "    )\n",
         "else:\n",
-        "    TRAIN_GPU, TRAIN_NGPU = (aip.gapic.AcceleratorType.NVIDIA_TESLA_K80, 1)\n",
+        "    TRAIN_GPU, TRAIN_NGPU = (None, None)\n",
         "\n",
         "if os.getenv(\"IS_TESTING_DEPLOY_GPU\"):\n",
         "    DEPLOY_GPU, DEPLOY_NGPU = (\n",
@@ -560,7 +680,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "container:prediction"
+        "id": "gxai072KMoUB"
       },
       "outputs": [],
       "source": [
@@ -616,7 +736,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "machine:training,prediction"
+        "id": "LEYjL1ojMoUB"
       },
       "outputs": [],
       "source": [
@@ -672,7 +792,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "examine_training_package"
+        "id": "UIoZdxPqMoUC"
       },
       "outputs": [],
       "source": [
@@ -735,7 +855,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "taskpy_contents:dataset,horses_or_humans"
+        "id": "uk4MAGErMoUC"
       },
       "outputs": [],
       "source": [
@@ -872,14 +992,14 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "tarball_training_script"
+        "id": "bHcLQLGkMoUD"
       },
       "outputs": [],
       "source": [
         "! rm -f custom.tar custom.tar.gz\n",
         "! tar cvf custom.tar custom\n",
         "! gzip custom.tar\n",
-        "! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_horses_or_humans.tar.gz"
+        "! gsutil cp custom.tar.gz $BUCKET_URI/trainer_horses_or_humans.tar.gz"
       ]
     },
     {
@@ -912,7 +1032,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "write_docker_file:training,tf-dlvm"
+        "id": "MaMK4AoBMoUE"
       },
       "outputs": [],
       "source": [
@@ -945,7 +1065,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "name_container:training"
+        "id": "9qVJGXT2MoUE"
       },
       "outputs": [],
       "source": [
@@ -965,7 +1085,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "build_container:training"
+        "id": "7-dkQP8hMoUE"
       },
       "outputs": [],
       "source": [
@@ -987,7 +1107,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "register_container:training"
+        "id": "7-kh6QBLMoUF"
       },
       "outputs": [],
       "source": [
@@ -1017,11 +1137,11 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "create_hpt_pipeline:icn"
+        "id": "Epzlh8M-MoUF"
       },
       "outputs": [],
       "source": [
-        "PIPELINE_ROOT = \"{}/pipeline_root/custom_icn_tuning\".format(BUCKET_NAME)\n",
+        "PIPELINE_ROOT = \"{}/pipeline_root/custom_icn_tuning\".format(BUCKET_URI)\n",
         "\n",
         "\n",
         "@component(packages_to_install=[\"google-cloud-aiplatform\"])\n",
@@ -1089,9 +1209,7 @@
         "        threshold_op.output == \"true\",\n",
         "        name=\"deploy_decision\",\n",
         "    ):\n",
-        "        best_hyperparameters_op = hyperparameter_tuning_job.GetHyperparametersOp(\n",
-        "            trial=best_trial_op.output\n",
-        "        )\n",
+        "        _ = hyperparameter_tuning_job.GetHyperparametersOp(trial=best_trial_op.output)\n",
         "\n",
         "        model_dir_op = model_dir(base_output_directory, best_trial_op.output)\n",
         "\n",
@@ -1105,7 +1223,7 @@
         "            },\n",
         "        ).after(model_dir_op)\n",
         "\n",
-        "        model_upload_op = ModelUploadOp(\n",
+        "        _ = ModelUploadOp(\n",
         "            project=project,\n",
         "            display_name=display_name,\n",
         "            unmanaged_container_model=import_unmanaged_model_op.outputs[\"artifact\"],\n",
@@ -1139,13 +1257,26 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "create_hpt_specs"
+        "id": "aEm7RuwMMoUG"
       },
       "outputs": [],
       "source": [
         "from google_cloud_pipeline_components.experimental import \\\n",
         "    hyperparameter_tuning_job\n",
         "\n",
+        "gpu = \"ACCELERATOR_TYPE_UNSPECIFIED\"\n",
+        "accelerator_count = 0\n",
+        "\n",
+        "if TRAIN_GPU:\n",
+        "    gpu = TRAIN_GPU.name\n",
+        "    accelerator_count = 1\n",
+        "\n",
+        "else:\n",
+        "    gpu = \"ACCELERATOR_TYPE_UNSPECIFIED\"\n",
+        "    accelerator_count = (\n",
+        "        0  # same problem with accelerator_count, if we keep is as \"None\" its not\n",
+        "    )\n",
+        "\n",
         "CMDARGS = [\n",
         "    \"--epochs=10\",\n",
         "]\n",
@@ -1155,8 +1286,8 @@
         "    {\n",
         "        \"machine_spec\": {\n",
         "            \"machine_type\": TRAIN_COMPUTE,\n",
-        "            \"accelerator_type\": TRAIN_GPU.name,\n",
-        "            \"accelerator_count\": TRAIN_NGPU,\n",
+        "            \"accelerator_type\": gpu,\n",
+        "            \"accelerator_count\": accelerator_count,\n",
         "        },\n",
         "        \"replica_count\": 1,\n",
         "        \"container_spec\": {\"image_uri\": TRAIN_IMAGE, \"args\": CMDARGS},\n",
@@ -1205,7 +1336,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "run_pipeline:hpt"
+        "id": "42YNp9Y9MoUG"
       },
       "outputs": [],
       "source": [
@@ -1246,7 +1377,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "view_pipleline_results:hpt,horses_or_humans"
+        "id": "4X3jrdX1MoUH"
       },
       "outputs": [],
       "source": [
@@ -1358,7 +1489,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "delete_pipeline"
+        "id": "hS53o3FcMoUH"
       },
       "outputs": [],
       "source": [
@@ -1378,14 +1509,6 @@
         "\n",
         "Otherwise, you can delete the individual resources you created in this tutorial:\n",
         "\n",
-        "- Dataset\n",
-        "- Pipeline\n",
-        "- Model\n",
-        "- Endpoint\n",
-        "- AutoML Training Job\n",
-        "- Batch Job\n",
-        "- Custom Job\n",
-        "- Hyperparameter Tuning Job\n",
         "- Cloud Storage Bucket"
       ]
     },
@@ -1393,70 +1516,20 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "cleanup:mbsdk"
+        "id": "laAQFM4aoBm3"
       },
       "outputs": [],
       "source": [
-        "delete_all = True\n",
+        "delete_bucket = False\n",
         "\n",
-        "if delete_all:\n",
-        "    # Delete the dataset using the Vertex dataset object\n",
-        "    try:\n",
-        "        if \"dataset\" in globals():\n",
-        "            dataset.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the model using the Vertex model object\n",
-        "    try:\n",
-        "        if \"model\" in globals():\n",
-        "            model.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the endpoint using the Vertex endpoint object\n",
-        "    try:\n",
-        "        if \"endpoint\" in globals():\n",
-        "            endpoint.undeploy_all()\n",
-        "            endpoint.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the AutoML or Pipeline training job\n",
-        "    try:\n",
-        "        if \"dag\" in globals():\n",
-        "            dag.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the custom training job\n",
-        "    try:\n",
-        "        if \"job\" in globals():\n",
-        "            job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the batch prediction job using the Vertex batch prediction object\n",
-        "    try:\n",
-        "        if \"batch_predict_job\" in globals():\n",
-        "            batch_predict_job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the hyperparameter tuning job using the Vertex hyperparameter tuning object\n",
-        "    try:\n",
-        "        if \"hpt_job\" in globals():\n",
-        "            hpt_job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    if \"BUCKET_NAME\" in globals():\n",
-        "        ! gsutil rm -r $BUCKET_NAME"
+        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "    ! gsutil rm -r $BUCKET_URI"
       ]
     }
   ],
   "metadata": {
     "colab": {
+      "collapsed_sections": [],
       "name": "get_started_with_hpt_pipeline_components.ipynb",
       "toc_visible": true
     },

notebooks/community/ml_ops/stage3/get_started_with_kubeflow_pipelines.ipynb

@@ -8,7 +8,7 @@
       },
       "outputs": [],
       "source": [
-        "# Copyright 2021 Google LLC\n",
+        "# Copyright 2022 Google LLC\n",
         "#\n",
         "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
         "# you may not use this file except in compliance with the License.\n",
@@ -38,9 +38,16 @@
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
+        "    <td>\n",
+        "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_kubeflow_pipelines.ipynb\">\n",
+        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\\\" alt=\"Colab logo\"> Run in Colab\n",
+        "        </a>\n",
+        "  </td>\n",
+        "    \n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_kubeflow_pipelines.ipynb\">\n",
-        "      Open in Google Cloud Notebooks\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage3/get_started_with_kubeflow_pipelines.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
         "    </a>\n",
         "  </td>\n",
         "</table>\n",
@@ -67,7 +74,7 @@
       "source": [
         "### Objective\n",
         "\n",
-        "In this tutorial, you learn how to use `Kubeflow Pipelines`.\n",
+        "In this tutorial, you learn how to use `Kubeflow Pipelines`(KFP).\n",
         "\n",
         "This tutorial uses the following Google Cloud ML services:\n",
         "\n",
@@ -78,8 +85,17 @@
         "- Building KFP lightweight Python function components.\n",
         "- Assembling and compiling KFP components into a pipeline.\n",
         "- Executing a KFP pipeline using Vertex AI Pipelines.\n",
+        "- Loading component and pipeline definitions from a source code repository.\n",
         "- Building sequential, parallel, multiple output components.\n",
-        "- Building control flow into pipelines."
+        "- Building control flow into pipelines.\n",
+        "\n",
+        "### Costs\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "- Vertex AI\n",
+        "- Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage pricing](https://cloud.google.com/storage/pricing) and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
       ]
     },
     {
@@ -90,7 +106,7 @@
       "source": [
         "## Installations\n",
         "\n",
-        "Install *one time* the packages for executing the MLOps notebooks."
+        "Install the required packages for executing this MLOps notebook."
       ]
     },
     {
@@ -101,20 +117,21 @@
       },
       "outputs": [],
       "source": [
-        "ONCE_ONLY = False\n",
-        "if ONCE_ONLY:\n",
-        "    ! pip3 install -U tensorflow==2.5 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-transform==1.2 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-io==0.18 $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-bigquery $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-logging $USER_FLAG\n",
-        "    ! pip3 install --upgrade apache-beam[gcp] $USER_FLAG\n",
-        "    ! pip3 install --upgrade pyarrow $USER_FLAG\n",
-        "    ! pip3 install --upgrade cloudml-hypertune $USER_FLAG\n",
-        "    ! pip3 install --upgrade kfp $USER_FLAG"
+        "import os\n",
+        "\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
+        "\n",
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "USER_FLAG = \"\"\n",
+        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\"\n",
+        "    \n",
+        "! pip3 install tensorflow-io==0.18 $USER_FLAG -q\n",
+        "! pip3 install --upgrade google-cloud-aiplatform \\\n",
+        "                        pyarrow \\\n",
+        "                        kfp $USER_FLAG -q"
       ]
     },
     {
@@ -146,6 +163,32 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e56d698e5d52"
+      },
+      "source": [
+        "## Before you begin\n",
+        "\n",
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com).\n",
+        "\n",
+        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -222,7 +265,10 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -249,6 +295,67 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "b5627478895e"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
+        "\n",
+        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
+        "\n",
+        "**Otherwise**, follow these steps:\n",
+        "\n",
+        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
+        "\n",
+        "1. **Click Create service account**.\n",
+        "\n",
+        "2. In the **Service account name** field, enter a name, and click **Create**.\n",
+        "\n",
+        "3. In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex\" into the filter box, and select **Vertex Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "\n",
+        "4. Click Create. A JSON file that contains your key downloads to your local environment.\n",
+        "\n",
+        "5. Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "49ee8894d674"
+      },
+      "outputs": [],
+      "source": [
+        "# If you are running this notebook in Colab, run this cell and follow the\n",
+        "# instructions to authenticate your GCP account. This provides access to your\n",
+        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
+        "# requests.\n",
+        "\n",
+        "import os\n",
+        "import sys\n",
+        "\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = False\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
+        "    if \"google.colab\" in sys.modules:\n",
+        "        IS_COLAB = True\n",
+        "        from google.colab import auth as google_auth\n",
+        "\n",
+        "        google_auth.authenticate_user()\n",
+        "\n",
+        "    # If you are running this notebook locally, replace the string below with the\n",
+        "    # path to your service account key and run this cell to authenticate your GCP\n",
+        "    # account.\n",
+        "    elif not os.getenv(\"IS_TESTING\"):\n",
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -272,7 +379,8 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
+        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "BUCKET_URI = \"gs://{}\".format(BUCKET_NAME)"
       ]
     },
     {
@@ -283,8 +391,8 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
       ]
     },
     {
@@ -304,7 +412,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_NAME"
+        "! gsutil mb -l $REGION $BUCKET_URI"
       ]
     },
     {
@@ -324,7 +432,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_NAME"
+        "! gsutil ls -al $BUCKET_URI"
       ]
     },
     {
@@ -335,7 +443,7 @@
       "source": [
         "#### Service Account\n",
         "\n",
-        "**If you don't know your service account**, try to get your service account using `gcloud` command by executing the second cell below."
+        "You use a service account to create Vertex AI Pipeline jobs. If you do not want to use your project's Compute Engine service account, set `SERVICE_ACCOUNT` to another service account ID."
       ]
     },
     {
@@ -362,9 +470,16 @@
         "    or SERVICE_ACCOUNT is None\n",
         "    or SERVICE_ACCOUNT == \"[your-service-account]\"\n",
         "):\n",
-        "    # Get your GCP project id from gcloud\n",
-        "    shell_output = !gcloud auth list 2>/dev/null\n",
-        "    SERVICE_ACCOUNT = shell_output[2].strip()\n",
+        "    # Get your service account from gcloud\n",
+        "    if not IS_COLAB:\n",
+        "        shell_output = !gcloud auth list 2>/dev/null\n",
+        "        SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
+        "\n",
+        "    if IS_COLAB:\n",
+        "        shell_output = ! gcloud projects describe  $PROJECT_ID\n",
+        "        project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n",
+        "        SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n",
+        "\n",
         "    print(\"Service Account:\", SERVICE_ACCOUNT)"
       ]
     },
@@ -376,7 +491,7 @@
       "source": [
         "#### Set service account access for Vertex AI Pipelines\n",
         "\n",
-        "Run the following commands to grant your service account access to read and write pipeline artifacts in the bucket that you created in the previous step -- you only need to run these once per service account."
+        "Run the following commands to grant your service account access to read and write pipeline artifacts in the bucket that you created in the previous step. You only need to run this step once per service account."
       ]
     },
     {
@@ -387,9 +502,9 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_NAME\n",
+        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
         "\n",
-        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_NAME"
+        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
       ]
     },
     {
@@ -398,10 +513,7 @@
         "id": "setup_vars"
       },
       "source": [
-        "### Set up variables\n",
-        "\n",
-        "Next, set up some variables used throughout the tutorial.\n",
-        "### Import libraries and define constants"
+        "### Import libraries"
       ]
     },
     {
@@ -411,42 +523,11 @@
         "id": "import_aip:mbsdk"
       },
       "outputs": [],
-      "source": [
-        "import google.cloud.aiplatform as aip"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "import_tf"
-      },
-      "source": [
-        "#### Import TensorFlow\n",
-        "\n",
-        "Import the TensorFlow package into your Python environment."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "import_tf"
-      },
-      "outputs": [],
-      "source": [
-        "import tensorflow as tf"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "import_kfp:namedtuple"
-      },
-      "outputs": [],
       "source": [
         "from typing import NamedTuple\n",
         "\n",
+        "import google.cloud.aiplatform as aiplatform\n",
+        "import tensorflow as tf\n",
         "from kfp import dsl\n",
         "from kfp.v2 import compiler\n",
         "from kfp.v2.dsl import component"
@@ -471,7 +552,7 @@
       },
       "outputs": [],
       "source": [
-        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_NAME)"
+        "aiplatform.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
       ]
     },
     {
@@ -486,14 +567,14 @@
         "\n",
         "    1. Design the pipeline workflow.\n",
         "    2. Compile the pipeline.\n",
-        "    3. Schedule execution (or run now) the pipeline.\n",
+        "    3. Schedule pipeline execution (or run now).\n",
         "    4. Get the pipeline results.\n",
         "\n",
-        "Pipelines are designed using language specific domain specific language (DSL). Vertex AI Pipelines support both KFP DSL and TFX DSL for designing pipelines.\n",
+        "Pipelines are designed using domain specific language (DSL). Vertex AI Pipelines support both KFP DSL and TFX DSL for designing pipelines.\n",
         "\n",
         "In addition to designing components, you can use a wide variety of pre-built Google Cloud Pipeline Components for Vertex AI services.\n",
         "\n",
-        "Learn more about [Building a pipeline](https://cloud.google.com/vertex-ai/docs/pipelines/build-pipeline)"
+        "Learn more about [Building a pipeline](https://cloud.google.com/vertex-ai/docs/pipelines/build-pipeline)."
       ]
     },
     {
@@ -502,9 +583,9 @@
         "id": "pipelines_intro:helloworld"
       },
       "source": [
-        "## Basic pipeline introduction\n",
+        "## Basic pipeline\n",
         "\n",
-        "This demonstrates the basics of constructing and executing a pipeline. You do the following:\n",
+        "This step demonstrates the basics of constructing and executing a pipeline. You do the following:\n",
         "\n",
         "1. Design a simple Python function based component to output the input string.\n",
         "2. Construct a pipeline that uses the component.\n",
@@ -522,8 +603,8 @@
         "\n",
         "To create a KFP component from a Python function, you add the KFP DSL decorator `@component` to the function. In this example, the decorator takes the following parameters:\n",
         "\n",
-        "- `output_component_file`: (optional) write the component description to a YAML file such that the component is portable.\n",
-        "- `base_image`: (optional): The interpreter for executing the Python function. By default it is Python 3.7"
+        "- `output_component_file`(optional): write the component description to a YAML file such that the component is portable.\n",
+        "- `base_image`(optional): The interpreter for executing the Python function. By default it is Python 3.7"
       ]
     },
     {
@@ -566,7 +647,7 @@
       },
       "outputs": [],
       "source": [
-        "PIPELINE_ROOT = \"{}/pipeline_root/hello_world\".format(BUCKET_NAME)\n",
+        "PIPELINE_ROOT = \"{}/pipeline_root/hello_world\".format(BUCKET_URI)\n",
         "\n",
         "\n",
         "@dsl.pipeline(\n",
@@ -575,7 +656,8 @@
         "    pipeline_root=PIPELINE_ROOT,\n",
         ")\n",
         "def pipeline(text: str = \"hi there\"):\n",
-        "    hello_world_task = hello_world(text)"
+        "    hello_world_task = hello_world(text)\n",
+        "    return hello_world_task"
       ]
     },
     {
@@ -588,7 +670,7 @@
         "\n",
         "Once the design of the pipeline is completed, the next step is to compile it. The pipeline definition is compiled into a JSON formatted file, which is transportable and can be interpreted by both KFP and Vertex AI Pipelines.\n",
         "\n",
-        "You compile the pipeline with the method Compiler().compile(), with the following parameters:\n",
+        "Compile the pipeline with the Compiler().compile() method using the following parameters:\n",
         "\n",
         "- `pipeline_func`: The corresponding DSL function that defines the pipeline.\n",
         "- `package_path`: The JSON file to write the transportable compiled pipeline to."
@@ -615,14 +697,14 @@
       "source": [
         "### Execute the hello world pipeline\n",
         "\n",
-        "Now that the pipeline is compiled, you can execute by:\n",
+        "Now that the pipeline is compiled, you can execute it by:\n",
         "\n",
-        "- Create a Vertex AI PipelineJob, with the following parameters:\n",
+        "- Creating a Vertex AI PipelineJob with the following parameters:\n",
         "    - `display_name`: The human readable name for the job.\n",
-        "    - `template_path`: Thee compiled JSON pipeline definition.\n",
+        "    - `template_path`: The compiled JSON pipeline definition.\n",
         "    - `pipeline_root`: Where to write output artifacts to.\n",
         "\n",
-        "Click on the generated link below `INFO:google.cloud.aiplatform.pipeline_jobs:View Pipeline Job:` to see your run in the Cloud Console."
+        "Click on the generated link below `INFO:google.cloud.aiplatform.pipeline_jobs:View Pipeline Job:` to see your job run in the Cloud Console."
       ]
     },
     {
@@ -633,7 +715,7 @@
       },
       "outputs": [],
       "source": [
-        "pipeline = aip.PipelineJob(\n",
+        "pipeline = aiplatform.PipelineJob(\n",
         "    display_name=\"hello_world\",\n",
         "    template_path=\"hello_world.json\",\n",
         "    pipeline_root=PIPELINE_ROOT,\n",
@@ -694,14 +776,29 @@
         "            + str(TASK_ID)\n",
         "            + \"/gcp_resources\"\n",
         "        )\n",
+        "        EVAL_METRICS = (\n",
+        "            PIPELINE_ROOT\n",
+        "            + \"/\"\n",
+        "            + PROJECT_NUMBER\n",
+        "            + \"/\"\n",
+        "            + JOB_ID\n",
+        "            + \"/\"\n",
+        "            + output_task_name\n",
+        "            + \"_\"\n",
+        "            + str(TASK_ID)\n",
+        "            + \"/evaluation_metrics\"\n",
+        "        )\n",
         "        if tf.io.gfile.exists(EXECUTE_OUTPUT):\n",
         "            ! gsutil cat $EXECUTE_OUTPUT\n",
-        "            break\n",
+        "            return EXECUTE_OUTPUT\n",
         "        elif tf.io.gfile.exists(GCP_RESOURCES):\n",
         "            ! gsutil cat $GCP_RESOURCES\n",
-        "            break\n",
+        "            return GCP_RESOURCES\n",
+        "        elif tf.io.gfile.exists(EVAL_METRICS):\n",
+        "            ! gsutil cat $EVAL_METRICS\n",
+        "            return EVAL_METRICS\n",
         "\n",
-        "    return EXECUTE_OUTPUT\n",
+        "    return None\n",
         "\n",
         "\n",
         "print_pipeline_output(pipeline, \"hello-world\")"
@@ -715,7 +812,7 @@
       "source": [
         "### Delete a pipeline job\n",
         "\n",
-        "After a pipeline job is completed, you can delete the pipeline job with the method `delete()`.  Prior to completion, a pipeline job can be canceled with the method `cancel()`."
+        "After a pipeline job is completed, you can delete the pipeline job with the `delete()` method. Prior to completion, a pipeline job can be canceled with the method `cancel()`."
       ]
     },
     {
@@ -741,7 +838,7 @@
         "\n",
         "    hello_world_op = components.load_component_from_file('./hello_world.yaml').\n",
         "\n",
-        "You can also use the load_component_from_url method, if your component YAML file is stored online, such as if in a git repo."
+        "You can also use the `load_component_from_url` method, if your component YAML file is stored online, such as in a git repository."
       ]
     },
     {
@@ -754,7 +851,7 @@
       "source": [
         "from kfp import components\n",
         "\n",
-        "PIPELINE_ROOT = \"{}/pipeline_root/hello_world-v2\".format(BUCKET_NAME)\n",
+        "PIPELINE_ROOT = \"{}/pipeline_root/hello_world-v2\".format(BUCKET_URI)\n",
         "\n",
         "hello_world_op = components.load_component_from_file(\"./hello_world.yaml\")\n",
         "\n",
@@ -765,12 +862,13 @@
         "    pipeline_root=PIPELINE_ROOT,\n",
         ")\n",
         "def pipeline(text: str = \"hi there\"):\n",
-        "    hellow_world_task = hello_world_op(text)\n",
+        "    hello_world_task = hello_world_op(text)\n",
+        "    return hello_world_task\n",
         "\n",
         "\n",
         "compiler.Compiler().compile(pipeline_func=pipeline, package_path=\"hello_world-v2.json\")\n",
         "\n",
-        "pipeline = aip.PipelineJob(\n",
+        "pipeline = aiplatform.PipelineJob(\n",
         "    display_name=\"hello_world-v2\",\n",
         "    template_path=\"hello_world-v2.json\",\n",
         "    pipeline_root=PIPELINE_ROOT,\n",
@@ -789,7 +887,74 @@
       "source": [
         "### Delete a pipeline job\n",
         "\n",
-        "After a pipeline job is completed, you can delete the pipeline job with the method `delete()`.  Prior to completion, a pipeline job can be canceled with the method `cancel()`."
+        "After a pipeline job is completed, you can delete the pipeline job with the `delete()` method. Prior to completion, a pipeline job can be canceled with the method `cancel()`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "delete_pipeline"
+      },
+      "outputs": [],
+      "source": [
+        "pipeline.delete()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "load_component_pipeline_git:helloworld"
+      },
+      "source": [
+        "### Loading components and pipeline YAML definitions from source control\n",
+        "\n",
+        "By storing the component and pipeline definitions in a source repository, like Github, you can version control your components and pipelines, as follows:\n",
+        "\n",
+        "- Use the method `load_component_from_url()`.\n",
+        "\n",
+        "- Pull the raw file format version from the repo. For github, that will be in the form of:\n",
+        "\n",
+        "    https://raw.githubusercontent.com/....\n",
+        "\n",
+        "- Specify the version of the component/pipeline. For github, that will be the branch."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "load_component_pipeline_git:helloworld"
+      },
+      "outputs": [],
+      "source": [
+        "VERSION = \"main\"\n",
+        "hello_world_op = components.load_component_from_url(\n",
+        "    f\"https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/{VERSION}/notebooks/community/ml_ops/stage3/src/hello_world.yaml\"\n",
+        ")\n",
+        "\n",
+        "! wget https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/{VERSION}/notebooks/community/ml_ops/stage3/src/hello_world.json -O hello_git_example.json\n",
+        "\n",
+        "pipeline = aiplatform.PipelineJob(\n",
+        "    display_name=\"hello_world-git\",\n",
+        "    template_path=\"hello_git_example.json\",\n",
+        "    pipeline_root=PIPELINE_ROOT,\n",
+        ")\n",
+        "\n",
+        "pipeline.run()\n",
+        "\n",
+        "! rm -f hello_git_example.json"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "delete_pipeline"
+      },
+      "source": [
+        "### Delete a pipeline job\n",
+        "\n",
+        "After a pipeline job is completed, you can delete the pipeline job with the `delete()` method. Prior to completion, a pipeline job can be canceled with the method `cancel()`."
       ]
     },
     {
@@ -829,7 +994,7 @@
         "    return np.mean(values)\n",
         "\n",
         "\n",
-        "PIPELINE_ROOT = \"{}/pipeline_root/numpy_mean\".format(BUCKET_NAME)\n",
+        "PIPELINE_ROOT = \"{}/pipeline_root/numpy_mean\".format(BUCKET_URI)\n",
         "\n",
         "\n",
         "@dsl.pipeline(\n",
@@ -837,11 +1002,12 @@
         ")\n",
         "def pipeline(values: list = [2, 3]):\n",
         "    numpy_task = numpy_mean(values)\n",
+        "    return numpy_task\n",
         "\n",
         "\n",
         "compiler.Compiler().compile(pipeline_func=pipeline, package_path=\"numpy_mean.json\")\n",
         "\n",
-        "pipeline = aip.PipelineJob(\n",
+        "pipeline = aiplatform.PipelineJob(\n",
         "    display_name=\"numpy_mean\",\n",
         "    template_path=\"numpy_mean.json\",\n",
         "    pipeline_root=PIPELINE_ROOT,\n",
@@ -862,7 +1028,7 @@
       "source": [
         "### Delete a pipeline job\n",
         "\n",
-        "After a pipeline job is completed, you can delete the pipeline job with the method `delete()`.  Prior to completion, a pipeline job can be canceled with the method `cancel()`."
+        "After a pipeline job is completed, you can delete the pipeline job with the `delete()` method. Prior to completion, a pipeline job can be canceled with the method `cancel()`."
       ]
     },
     {
@@ -897,7 +1063,7 @@
       },
       "outputs": [],
       "source": [
-        "PIPELINE_ROOT = \"{}/pipeline_root/add_div2\".format(BUCKET_NAME)\n",
+        "PIPELINE_ROOT = \"{}/pipeline_root/add_div2\".format(BUCKET_URI)\n",
         "\n",
         "\n",
         "@component(output_component_file=\"add.yaml\", base_image=\"python:3.9\")\n",
@@ -916,11 +1082,12 @@
         "def pipeline(v1: int = 4, v2: int = 5):\n",
         "    add_task = add(v1, v2)\n",
         "    div2_task = div_by_2(add_task.output)\n",
+        "    return div2_task\n",
         "\n",
         "\n",
         "compiler.Compiler().compile(pipeline_func=pipeline, package_path=\"add_div2.json\")\n",
         "\n",
-        "pipeline = aip.PipelineJob(\n",
+        "pipeline = aiplatform.PipelineJob(\n",
         "    display_name=\"add_div2\",\n",
         "    template_path=\"add_div2.json\",\n",
         "    pipeline_root=PIPELINE_ROOT,\n",
@@ -941,7 +1108,7 @@
       "source": [
         "### Delete a pipeline job\n",
         "\n",
-        "After a pipeline job is completed, you can delete the pipeline job with the method `delete()`.  Prior to completion, a pipeline job can be canceled with the method `cancel()`."
+        "After a pipeline job is completed, you can delete the pipeline job with the `delete()` method. Prior to completion, a pipeline job can be canceled with the method `cancel()`."
       ]
     },
     {
@@ -963,7 +1130,7 @@
       "source": [
         "### Multiple output pipeline\n",
         "\n",
-        "Next, you design and execute a pipeline where a first component has multiple outputs, which are then used as inputs to the next component. To distinquish between the outputs, when used as inputs to the next component, you do:\n",
+        "Next, you design and execute a pipeline where a first component has multiple outputs, which are then used as inputs to the next component. To distinguish between the outputs, when used as inputs to the next component, you follow:\n",
         "\n",
         "1. Set the function return type to `NamedTuple`.\n",
         "2. In NamedTuple, specify a name and type for each output, in the specified order.\n",
@@ -978,7 +1145,7 @@
       },
       "outputs": [],
       "source": [
-        "PIPELINE_ROOT = \"{}/pipeline_root/multi_output\".format(BUCKET_NAME)\n",
+        "PIPELINE_ROOT = \"{}/pipeline_root/multi_output\".format(BUCKET_URI)\n",
         "\n",
         "\n",
         "@component()\n",
@@ -1012,11 +1179,12 @@
         "        multi_output_task.outputs[\"output_1\"],\n",
         "        multi_output_task.outputs[\"output_2\"],\n",
         "    )\n",
+        "    return concat_task\n",
         "\n",
         "\n",
         "compiler.Compiler().compile(pipeline_func=pipeline, package_path=\"multi_output.json\")\n",
         "\n",
-        "pipeline = aip.PipelineJob(\n",
+        "pipeline = aiplatform.PipelineJob(\n",
         "    display_name=\"multi-output\",\n",
         "    template_path=\"multi_output.json\",\n",
         "    pipeline_root=PIPELINE_ROOT,\n",
@@ -1037,7 +1205,7 @@
       "source": [
         "### Delete a pipeline job\n",
         "\n",
-        "After a pipeline job is completed, you can delete the pipeline job with the method `delete()`.  Prior to completion, a pipeline job can be canceled with the method `cancel()`."
+        "After a pipeline job is completed, you can delete the pipeline job with the `delete()` method. Prior to completion, a pipeline job can be canceled with the method `cancel()`."
       ]
     },
     {
@@ -1059,9 +1227,9 @@
       "source": [
         "## Parallel tasks in component\n",
         "\n",
-        "Next, you design and execute a pipeline with parallel tasks. In this example, one parallel task adds up a list of integers and another substracts them. Note that the compiler knows these two tasks can be ran in parallel, because their input is not dependent on the output of the other task.\n",
+        "Next, you design and execute a pipeline with parallel tasks. In this example, one parallel task adds up a list of integers and another substracts them. Note that the compiler knows these two tasks can be run in parallel, because their input is not dependent on the output of the other task.\n",
         "\n",
-        "Finally, the add task waits on the two parallel tasks to complete, and then adds together the two outputs."
+        "Finally, the `add_int` task waits on the two parallel tasks to complete, and then adds together the two outputs."
       ]
     },
     {
@@ -1072,14 +1240,14 @@
       },
       "outputs": [],
       "source": [
-        "PIPELINE_ROOT = \"{}/pipeline_root/parallel\".format(BUCKET_NAME)\n",
+        "PIPELINE_ROOT = \"{}/pipeline_root/parallel\".format(BUCKET_URI)\n",
         "\n",
         "\n",
         "@component()\n",
         "def add_list(values: list) -> int:\n",
         "    ret = 0\n",
         "    for value in values:\n",
-        "        ret += 1\n",
+        "        ret = value + ret\n",
         "    return ret\n",
         "\n",
         "\n",
@@ -1087,12 +1255,12 @@
         "def sub_list(values: list) -> int:\n",
         "    ret = 0\n",
         "    for value in values:\n",
-        "        ret -= 1\n",
+        "        ret = value - ret\n",
         "    return ret\n",
         "\n",
         "\n",
         "@component()\n",
-        "def add(value1: int, value2: int) -> int:\n",
+        "def add_int(value1: int, value2: int) -> int:\n",
         "    return value1 + value2\n",
         "\n",
         "\n",
@@ -1102,12 +1270,13 @@
         "def pipeline(values: list = [1, 2, 3]):\n",
         "    add_list_task = add_list(values)\n",
         "    sub_list_task = sub_list(values)\n",
-        "    add_task = add(add_list_task.output, sub_list_task.output)\n",
+        "    add_task = add_int(add_list_task.output, sub_list_task.output)\n",
+        "    return add_task\n",
         "\n",
         "\n",
         "compiler.Compiler().compile(pipeline_func=pipeline, package_path=\"parallel.json\")\n",
         "\n",
-        "pipeline = aip.PipelineJob(\n",
+        "pipeline = aiplatform.PipelineJob(\n",
         "    display_name=\"parallel\",\n",
         "    template_path=\"parallel.json\",\n",
         "    pipeline_root=PIPELINE_ROOT,\n",
@@ -1128,7 +1297,7 @@
       "source": [
         "### Delete a pipeline job\n",
         "\n",
-        "After a pipeline job is completed, you can delete the pipeline job with the method `delete()`.  Prior to completion, a pipeline job can be canceled with the method `cancel()`."
+        "After a pipeline job is completed, you can delete the pipeline job with the `delete()` method. Prior to completion, a pipeline job can be canceled with the method `cancel()`."
       ]
     },
     {
@@ -1150,7 +1319,7 @@
       "source": [
         "## Control flow in pipeline\n",
         "\n",
-        "While Python control statements, e.g., if/else, for, can be used in a component, they cannot be used in the pipeline function. Each task in the pipeline function runs as a node in a graph. Thus a control flow statement also has to run as a graph node. To support this, KFP provides a set of DSL statements that implement control flow as a graph node."
+        "While Python control statements(e.g., if/else, for) can be used in a component, they cannot be used in a pipeline function. Each task in a pipeline function runs as a node in a graph. Thus a control flow statement also has to run as a graph node. To support this, KFP provides a set of DSL statements that implement control flow as a graph node."
       ]
     },
     {
@@ -1161,7 +1330,7 @@
       "source": [
         "### dsl.ParallelFor\n",
         "\n",
-        "The statement `dsl.ParallelFor()` implements a for loop, where each iteration in the for loop runs in parallel."
+        "The statement `dsl.ParallelFor()` implements a `for` loop, where each iteration in the `for` loop runs in parallel."
       ]
     },
     {
@@ -1172,7 +1341,7 @@
       },
       "outputs": [],
       "source": [
-        "PIPELINE_ROOT = \"{}/pipeline_root/parallel_for\".format(BUCKET_NAME)\n",
+        "PIPELINE_ROOT = \"{}/pipeline_root/parallel_for\".format(BUCKET_URI)\n",
         "\n",
         "\n",
         "@component()\n",
@@ -1194,11 +1363,12 @@
         "    with dsl.ParallelFor(values) as item:\n",
         "        output = double(item).output\n",
         "        echo_task = echo(output)\n",
+        "    return echo_task\n",
         "\n",
         "\n",
         "compiler.Compiler().compile(pipeline_func=pipeline, package_path=\"parallel_for.json\")\n",
         "\n",
-        "pipeline = aip.PipelineJob(\n",
+        "pipeline = aiplatform.PipelineJob(\n",
         "    display_name=\"parallel-for\",\n",
         "    template_path=\"parallel_for.json\",\n",
         "    pipeline_root=PIPELINE_ROOT,\n",
@@ -1219,7 +1389,7 @@
       "source": [
         "### Delete a pipeline job\n",
         "\n",
-        "After a pipeline job is completed, you can delete the pipeline job with the method `delete()`.  Prior to completion, a pipeline job can be canceled with the method `cancel()`."
+        "After a pipeline job is completed, you can delete the pipeline job with the `delete()` method. Prior to completion, a pipeline job can be canceled with the method `cancel()`."
       ]
     },
     {
@@ -1241,7 +1411,15 @@
       "source": [
         "### dsl.Condition\n",
         "\n",
-        "The statement `dsl.Condition()` implements an `if` statement. There is no support for an `else` or `elif` statement. You use a separate `dsl.Condition()` for each value you want to test for. For example, if the output from a task is `True` or `False`, you will have two `dsl.Condition()` statements, one for True and one for False."
+        "The statement `dsl.Condition()` implements an `if` statement. There is no support for an `else` or `elif` statement. You use a separate `dsl.Condition()` for each value you want to test for. For example, if the output from a task is `1` or `0`, you will have two `dsl.Condition()` statements, one for 1 and one for 0.\n",
+        "\n",
+        "The condition in `dsl.Condition()` is evaluated at run-time, not compile time. As such it is not Python code anymore.  The condition is of type `ConditionOperator`. This operator has three parts:\n",
+        "\n",
+        "1. PipelineParam or task output\n",
+        "2. == or !=\n",
+        "3. string or integer value\n",
+        "\n",
+        "A `dsl.Condition()` can be named using the `name` parameter while defining the condition."
       ]
     },
     {
@@ -1280,11 +1458,12 @@
         "        task = heads()\n",
         "    with dsl.Condition(flip_task.output == 0, name=\"false_clause\"):\n",
         "        task = tails()\n",
+        "    return task\n",
         "\n",
         "\n",
         "compiler.Compiler().compile(pipeline_func=pipeline, package_path=\"condition.json\")\n",
         "\n",
-        "pipeline = aip.PipelineJob(\n",
+        "pipeline = aiplatform.PipelineJob(\n",
         "    display_name=\"condition\",\n",
         "    template_path=\"condition.json\",\n",
         "    pipeline_root=PIPELINE_ROOT,\n",
@@ -1305,7 +1484,7 @@
       "source": [
         "### Delete a pipeline job\n",
         "\n",
-        "After a pipeline job is completed, you can delete the pipeline job with the method `delete()`.  Prior to completion, a pipeline job can be canceled with the method `cancel()`."
+        "After a pipeline job is completed, you can delete the pipeline job with the `delete()` method. Prior to completion, a pipeline job can be canceled with the method `cancel()`."
       ]
     },
     {
@@ -1325,16 +1504,14 @@
         "id": "pipeline_errata"
       },
       "source": [
-        "## Errata\n",
-        "\n",
         "### Caching in pipeline components\n",
         "\n",
-        "When running a pipeline with Vertex AI Pipelines, the outcome state of each task is cached. With caching, if the pipeline is ran again, and the compiled definition of the task and state has not changed, the cached output will be used instead of running the task again.\n",
+        "When running a pipeline with Vertex AI Pipelines, the outcome state of each task is cached. With caching, if the pipeline is run again, and the compiled definition of the task and state has not changed, the cached output will be used instead of running the task again.\n",
         "\n",
-        "To override caching, i.e., forceable run the task, you set the parameter `enable_caching` to `False` when creating the Vertex AI Pipeline job.\n",
+        "To override caching, i.e., force run the task, you set the parameter `enable_caching` to `False` when creating the Vertex AI Pipeline job.\n",
         "\n",
         "```\n",
-        "pipeline = aip.PipelineJob(\n",
+        "pipeline = aiplatform.PipelineJob(\n",
         "    display_name=\"example\",\n",
         "    template_path=\"example.json\",\n",
         "    pipeline_root=PIPELINE_ROOT,\n",
@@ -1344,11 +1521,11 @@
         "\n",
         "### Asynchronous execution of pipeline\n",
         "\n",
-        "When running a pipeline with the method `run()`, the pipeline is ran synchronously. To run asynchronously, you use the method `submit()`. Once the job has started, your Python script can continue to execute. Then when you need to block execution using the method `wait()`.\n",
+        "When running a pipeline with the method `run()`, the pipeline is run synchronously. To run asynchronously, you use the method `submit()`. Once the job has started, your Python script can continue to execute. To block execution, you can use the method `wait()`.\n",
         "\n",
         "### Setting machine resources for pipeline steps\n",
         "\n",
-        "By default, Vertex AI Pipelines will automatically find the best matching machine type to run the component. You can override and specify the machine resources on a per component basis, when you invoke the component in a pipeline, as follows:\n",
+        "By default, Vertex AI Pipelines automatically finds the best matching machine type to run the component. You can override and specify the machine resources on a per component basis, when you invoke the component in a pipeline, as follows:\n",
         "\n",
         "```\n",
         "@dsl.pipeline(name='my-pipeline')\n",
@@ -1376,15 +1553,9 @@
         "\n",
         "Otherwise, you can delete the individual resources you created in this tutorial:\n",
         "\n",
-        "- Dataset\n",
-        "- Pipeline\n",
-        "- Model\n",
-        "- Endpoint\n",
-        "- AutoML Training Job\n",
-        "- Batch Job\n",
-        "- Custom Job\n",
-        "- Hyperparameter Tuning Job\n",
-        "- Cloud Storage Bucket"
+        "### Cloud Storage Bucket\n",
+        "\n",
+        "Set `delete_bucket` to True to delete the Cloud storage bucket used in this notebook."
       ]
     },
     {
@@ -1395,61 +1566,10 @@
       },
       "outputs": [],
       "source": [
-        "delete_all = True\n",
+        "delete_bucket = False\n",
         "\n",
-        "if delete_all:\n",
-        "    # Delete the dataset using the Vertex dataset object\n",
-        "    try:\n",
-        "        if \"dataset\" in globals():\n",
-        "            dataset.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the model using the Vertex model object\n",
-        "    try:\n",
-        "        if \"model\" in globals():\n",
-        "            model.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the endpoint using the Vertex endpoint object\n",
-        "    try:\n",
-        "        if \"endpoint\" in globals():\n",
-        "            endpoint.undeploy_all()\n",
-        "            endpoint.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the AutoML or Pipeline training job\n",
-        "    try:\n",
-        "        if \"dag\" in globals():\n",
-        "            dag.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the custom training job\n",
-        "    try:\n",
-        "        if \"job\" in globals():\n",
-        "            job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the batch prediction job using the Vertex batch prediction object\n",
-        "    try:\n",
-        "        if \"batch_predict_job\" in globals():\n",
-        "            batch_predict_job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the hyperparameter tuning job using the Vertex hyperparameter tuning object\n",
-        "    try:\n",
-        "        if \"hpt_job\" in globals():\n",
-        "            hpt_job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    if \"BUCKET_NAME\" in globals():\n",
-        "        ! gsutil rm -r $BUCKET_NAME"
+        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "    ! gsutil rm -r $BUCKET_URI"
       ]
     }
   ],

notebooks/community/ml_ops/stage3/get_started_with_rapid_prototyping_bqml_automl.ipynb

@@ -32,14 +32,20 @@
         "# E2E ML on GCP: MLOps stage 3 : Get started with rapid prototyping with AutoML and BQML\n",
         "<table align=\"left\">\n",
         "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage6/get_started_with_rapid_prototyping.ipynb\">\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_rapid_prototyping_bqml_automl.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_rapid_prototyping_bqml_automl.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage6/get_started_with_rapid_prototyping.ipynb\">\n",
-        "      Open in Vertex Workbench\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage3/get_started_with_rapid_prototyping_bqml_automl.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
         "    </a>\n",
         "  </td>\n",
         "</table>\n",
@@ -235,7 +241,7 @@
       "source": [
         "## Installation\n",
         "\n",
-        "Install the latest version of Vertex AI SDK for Python."
+        "Install the packages required for executing this notebook."
       ]
     },
     {
@@ -248,34 +254,20 @@
       "source": [
         "import os\n",
         "\n",
-        "# Google Cloud Notebook\n",
-        "if os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
-        "    USER_FLAG = \"--user\"\n",
-        "else:\n",
-        "    USER_FLAG = \"\"\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
         "\n",
-        "! pip3 install --quiet --upgrade google-cloud-aiplatform {USER_FLAG}"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "eeba891a06fc"
-      },
-      "source": [
-        "Install additional packages used in this tutorial."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "739011eb"
-      },
-      "outputs": [],
-      "source": [
-        "! pip3 install {USER_FLAG} --quiet -U google-cloud-pipeline-components==1.0 kfp\n",
-        "! pip3 install {USER_FLAG} --quiet --upgrade google-cloud-bigquery"
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "USER_FLAG = \"\"\n",
+        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\"\n",
+        "\n",
+        "! pip3 install --quiet --upgrade google-cloud-aiplatform {USER_FLAG} -q\n",
+        "! pip3 install {USER_FLAG} --quiet -U google-cloud-pipeline-components==1.0 kfp -q\n",
+        "! pip3 install {USER_FLAG} --quiet --upgrade google-cloud-bigquery -q"
       ]
     },
     {
@@ -316,10 +308,12 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Google Cloud Notebooks**, your environment is already authenticated. Skip this step.\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
+        "authenticated. Skip this step.\n",
         "\n",
         "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
         "\n",
+        "\n",
         "**Otherwise**, follow these steps:\n",
         "\n",
         "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
@@ -351,9 +345,13 @@
         "import os\n",
         "import sys\n",
         "\n",
-        "# If on Google Cloud Notebook, then don't execute this code\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = \"google.colab\" in sys.modules\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
         "    if \"google.colab\" in sys.modules:\n",
+        "        IS_COLAB = True\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
         "        google_auth.authenticate_user()\n",
@@ -385,7 +383,7 @@
         "\n",
         "2. [Make sure that billing is enabled for your project.](https://cloud.google.com/billing/docs/how-to/modify-project)\n",
         "\n",
-        "3. [Enable the following APIs: Vertex AI APIs, Compute Engine APIs, and Cloud Storage.](https://console.cloud.google.com/flows/enableapi?apiid=ml.googleapis.com,compute_component,storage-component.googleapis.com)\n",
+        "3. [Enable the following APIs: Vertex AI APIs, Compute Engine APIs, and Cloud Storage.](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component,storage-component.googleapis.com)\n",
         "\n",
         "4. If you are running this notebook locally, you will need to install the [Cloud SDK]((https://cloud.google.com/sdk)).\n",
         "\n",
@@ -414,6 +412,8 @@
       },
       "outputs": [],
       "source": [
+        "import os\n",
+        "\n",
         "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
       ]
     },
@@ -471,7 +471,9 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+        "REGION = \"[your-region]\"  # @param {type:\"string\"}\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -612,9 +614,17 @@
         "    or SERVICE_ACCOUNT is None\n",
         "    or SERVICE_ACCOUNT == \"[your-service-account]\"\n",
         "):\n",
-        "    # Get your GCP project id from gcloud\n",
-        "    shell_output = !gcloud auth list 2>/dev/null\n",
-        "    SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
+        "    # Get your service account from gcloud\n",
+        "    if not IS_COLAB:\n",
+        "        shell_output = !gcloud auth list 2>/dev/null\n",
+        "        SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
+        "\n",
+        "    if IS_COLAB:\n",
+        "        shell_output = ! gcloud projects describe  $PROJECT_ID\n",
+        "        # print(\"shell_output=\", shell_output)\n",
+        "        project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n",
+        "        SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n",
+        "\n",
         "    print(\"Service Account:\", SERVICE_ACCOUNT)"
       ]
     },
@@ -667,6 +677,7 @@
         "from typing import NamedTuple\n",
         "\n",
         "import google.cloud.aiplatform as aip\n",
+        "from google.cloud import bigquery\n",
         "from kfp import dsl\n",
         "from kfp.v2 import compiler\n",
         "from kfp.v2.dsl import Artifact, Input, Metrics, Output, component"
@@ -1429,6 +1440,17 @@
         "- `validate_infrastructure`: Validate the deployed model serving infrastructure."
       ]
     },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "040e82bc1646"
+      },
+      "outputs": [],
+      "source": [
+        "DISPLAY_NAME = \"rapid-prototyping\""
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -1453,8 +1475,7 @@
         "    from google_cloud_pipeline_components.types import artifact_types\n",
         "    from google_cloud_pipeline_components.v1.bigquery import (\n",
         "        BigqueryCreateModelJobOp, BigqueryEvaluateModelJobOp,\n",
-        "        BigqueryExportModelJobOp, BigqueryPredictModelJobOp,\n",
-        "        BigqueryQueryJobOp)\n",
+        "        BigqueryExportModelJobOp)\n",
         "    from google_cloud_pipeline_components.v1.endpoint import (EndpointCreateOp,\n",
         "                                                              ModelDeployOp)\n",
         "    from google_cloud_pipeline_components.v1.model import ModelUploadOp\n",
@@ -1670,7 +1691,7 @@
         "PIPELINE_ROOT = f\"{BUCKET_URI}/pipeline_root\"\n",
         "image_prefix = REGION.split(\"-\")[0]\n",
         "BQML_SERVING_CONTAINER_IMAGE_URI = (\n",
-        "    f\"{image_prefix}-docker.pkg.dev/vertex-ai/prediction/tf2-cpu.2-6:latest\"\n",
+        "    f\"{image_prefix}-docker.pkg.dev/vertex-ai/prediction/tf2-cpu.2-8:latest\"\n",
         ")\n",
         "\n",
         "BQ_DATASET = \"rapid_prototype\"  # j90wipxexhrgq3cquanc5\"  # @param {type:\"string\"}\n",
@@ -1678,7 +1699,6 @@
         "BQ_LOCATION = BQ_LOCATION.upper()\n",
         "BQML_EXPORT_LOCATION = f\"{BUCKET_URI}/artifacts/bqml\"\n",
         "\n",
-        "DISPLAY_NAME = \"rapid-prototyping\"\n",
         "ENDPOINT_DISPLAY_NAME = f\"{DISPLAY_NAME}_endpoint\"\n",
         "\n",
         "compiler.Compiler().compile(\n",
@@ -1706,7 +1726,7 @@
         "    template_path=PIPELINE_JSON_PKG_PATH,\n",
         "    pipeline_root=PIPELINE_ROOT,\n",
         "    parameter_values=pipeline_params,\n",
-        "    enable_caching=True,\n",
+        "    enable_caching=False,\n",
         ")\n",
         "\n",
         "response = pipeline_job.submit()"
@@ -1758,96 +1778,72 @@
       },
       "outputs": [],
       "source": [
-        "delete = True  # set to True if you want to delete resources created in this tutorial.\n",
+        "delete_bucket = True\n",
         "\n",
+        "print(\"Will delete endpoint\")\n",
         "\n",
-        "delete_vertex_dataset = True and delete\n",
-        "delete_pipeline = True and delete\n",
-        "delete_model = True and delete\n",
-        "delete_endpoint = True and delete\n",
-        "delete_batchjob = True and delete\n",
-        "delete_bucket = True and delete\n",
-        "delete_bq_dataset = True and delete\n",
+        "endpoints = aip.Endpoint.list(\n",
+        "    filter=f\"display_name={DISPLAY_NAME}_endpoint\", order_by=\"create_time\"\n",
+        ")\n",
+        "endpoint = endpoints[0]\n",
+        "endpoint.undeploy_all()\n",
+        "aip.Endpoint.delete(endpoint.resource_name)\n",
+        "print(\"Deleted endpoint:\", endpoint)\n",
         "\n",
-        "try:\n",
-        "    if delete_endpoint and \"DISPLAY_NAME\" in globals():\n",
-        "        print(\"Will delete endpoint\")\n",
-        "        endpoints = aip.Endpoint.list(\n",
-        "            filter=f\"display_name={DISPLAY_NAME}_endpoint\", order_by=\"create_time\"\n",
-        "        )\n",
-        "        endpoint = endpoints[0]\n",
-        "        endpoint.undeploy_all()\n",
-        "        aip.Endpoint.delete(endpoint.resource_name)\n",
-        "        print(\"Deleted endpoint:\", endpoint)\n",
-        "except Exception as e:\n",
-        "    print(e)\n",
-        "\n",
-        "if delete_model and \"DISPLAY_NAME\" in globals():\n",
-        "    print(\"Will delete models\")\n",
-        "    suffix_list = [\"bqml\", \"automl\", \"best\"]\n",
-        "    for suffix in suffix_list:\n",
-        "        try:\n",
-        "            model_display_name = f\"{DISPLAY_NAME}_{suffix}\"\n",
-        "            print(\"Will delete model with name \" + model_display_name)\n",
-        "            models = aip.Model.list(\n",
-        "                filter=f\"display_name={model_display_name}\", order_by=\"create_time\"\n",
-        "            )\n",
-        "\n",
-        "            model = models[0]\n",
-        "            aip.Model.delete(model)\n",
-        "            print(\"Deleted model:\", model)\n",
-        "        except Exception as e:\n",
-        "            print(e)\n",
-        "\n",
-        "if delete_vertex_dataset and \"DISPLAY_NAME\" in globals():\n",
-        "    print(\"Will delete Vertex dataset\")\n",
+        "print(\"Will delete models\")\n",
+        "suffix_list = [\"bqml\", \"automl\", \"best\"]\n",
+        "for suffix in suffix_list:\n",
         "    try:\n",
-        "        datasets = aip.TabularDataset.list(\n",
-        "            filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
+        "        model_display_name = f\"{DISPLAY_NAME}_{suffix}\"\n",
+        "        print(\"Will delete model with name \" + model_display_name)\n",
+        "        models = aip.Model.list(\n",
+        "            filter=f\"display_name={model_display_name}\", order_by=\"create_time\"\n",
         "        )\n",
         "\n",
-        "        dataset = datasets[0]\n",
-        "        aip.TabularDataset.delete(dataset)\n",
-        "        print(\"Deleted Vertex dataset:\", dataset)\n",
+        "        model = models[0]\n",
+        "        aip.Model.delete(model)\n",
+        "        print(\"Deleted model:\", model)\n",
         "    except Exception as e:\n",
         "        print(e)\n",
         "\n",
         "\n",
-        "try:\n",
-        "    if delete_pipeline and \"DISPLAY_NAME\" in globals():\n",
-        "        pipelines = aip.PipelineJob.list(\n",
-        "            filter=f\"pipeline_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
-        "        )\n",
-        "        pipeline = pipelines[0]\n",
-        "        aip.PipelineJob.delete(pipeline)\n",
-        "        print(\"Deleted pipeline:\", pipeline)\n",
-        "except Exception as e:\n",
-        "    print(e)\n",
+        "print(\"Will delete Vertex dataset\")\n",
         "\n",
-        "if delete_bq_dataset and \"DISPLAY_NAME\" in globals():\n",
-        "    from google.cloud import bigquery\n",
+        "datasets = aip.TabularDataset.list(\n",
+        "    filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
+        ")\n",
         "\n",
-        "    try:\n",
-        "        # Construct a BigQuery client object.\n",
-        "\n",
-        "        bq_client = bigquery.Client(project=PROJECT_ID, location=BQ_LOCATION)\n",
-        "\n",
-        "        # TODO(developer): Set model_id to the ID of the model to fetch.\n",
-        "        dataset_id = f\"{PROJECT_ID}.{BQ_DATASET}\"\n",
-        "\n",
-        "        print(f\"Will delete BQ dataset '{dataset_id}' from location {BQ_LOCATION}.\")\n",
-        "        # Use the delete_contents parameter to delete a dataset and its contents.\n",
-        "        # Use the not_found_ok parameter to not receive an error if the dataset has already been deleted.\n",
-        "        bq_client.delete_dataset(\n",
-        "            dataset_id, delete_contents=True, not_found_ok=True\n",
-        "        )  # Make an API request.\n",
-        "\n",
-        "        print(f\"Deleted BQ dataset '{dataset_id}' from location {BQ_LOCATION}.\")\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
+        "dataset = datasets[0]\n",
+        "aip.TabularDataset.delete(dataset)\n",
+        "print(\"Deleted Vertex dataset:\", dataset)\n",
         "\n",
         "\n",
-        "if delete_bucket and \"BUCKET_URI\" in globals():\n",
+        "pipelines = aip.PipelineJob.list(\n",
+        "    filter=f\"pipeline_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
+        ")\n",
+        "pipeline = pipelines[0]\n",
+        "aip.PipelineJob.delete(pipeline)\n",
+        "print(\"Deleted pipeline:\", pipeline)\n",
+        "\n",
+        "\n",
+        "# Construct a BigQuery client object.\n",
+        "\n",
+        "bq_client = bigquery.Client(project=PROJECT_ID, location=BQ_LOCATION)\n",
+        "\n",
+        "# TODO(developer): Set dataset_id to the ID of the dataset to fetch.\n",
+        "dataset_id = f\"{PROJECT_ID}.{BQ_DATASET}\"\n",
+        "\n",
+        "print(f\"Will delete BQ dataset '{dataset_id}' from location {BQ_LOCATION}.\")\n",
+        "# Use the delete_contents parameter to delete a dataset and its contents.\n",
+        "# Use the not_found_ok parameter to not receive an error if the dataset has already been deleted.\n",
+        "bq_client.delete_dataset(\n",
+        "    dataset_id, delete_contents=True, not_found_ok=True\n",
+        ")  # Make an API request.\n",
+        "\n",
+        "print(f\"Deleted BQ dataset '{dataset_id}' from location {BQ_LOCATION}.\")\n",
+        "\n",
+        "\n",
+        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
         "    ! gsutil rm -r $BUCKET_URI"
       ]
     }

notebooks/community/ml_ops/stage3/mlops_formalization.ipynb

@@ -32,17 +32,23 @@
         "# E2E ML on GCP: MLOps stage 3 : formalization\n",
         "\n",
         "<table align=\"left\">\n",
+        "   <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/mlops_formalization.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/mlops_formalization.ipynb\">\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samplestree/main/notebooks/community/ml_ops/stage3/mlops_formalization.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/mlops_formalization.ipynb\">\n",
-        "      Open in Google Cloud Notebooks\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage3/mlops_formalization.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
         "    </a>\n",
-        "  </td>\n",
+        "  </td>           \n",
         "</table>\n",
         "<br/><br/><br/>"
       ]
@@ -182,6 +188,19 @@
       },
       "outputs": [],
       "source": [
+        "import os\n",
+        "\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
+        "\n",
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "USER_FLAG = \"\"\n",
+        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\"\n",
+        "\n",
         "ONCE_ONLY = False\n",
         "if ONCE_ONLY:\n",
         "    ! pip3 install -U tensorflow==2.5 $USER_FLAG\n",
@@ -231,6 +250,32 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "BF1j6f9HApxa"
+      },
+      "source": [
+        "## Before you begin\n",
+        "\n",
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI API and Dataflow API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,dataflow.googleapis.com).\n",
+        "\n",
+        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -334,6 +379,67 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gcp_authenticate"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
+        "\n",
+        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
+        "\n",
+        "**Otherwise**, follow these steps:\n",
+        "\n",
+        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
+        "\n",
+        "**Click Create service account**.\n",
+        "\n",
+        "In the **Service account name** field, enter a name, and click **Create**.\n",
+        "\n",
+        "In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex\" into the filter box, and select **Vertex Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "\n",
+        "Click Create. A JSON file that contains your key downloads to your local environment.\n",
+        "\n",
+        "Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "gcp_authenticate"
+      },
+      "outputs": [],
+      "source": [
+        "# If you are running this notebook in Colab, run this cell and follow the\n",
+        "# instructions to authenticate your GCP account. This provides access to your\n",
+        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
+        "# requests.\n",
+        "\n",
+        "import os\n",
+        "import sys\n",
+        "\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = \"google.colab\" in sys.modules\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
+        "    if \"google.colab\" in sys.modules:\n",
+        "        IS_COLAB = True\n",
+        "        from google.colab import auth as google_auth\n",
+        "\n",
+        "        google_auth.authenticate_user()\n",
+        "\n",
+        "    # If you are running this notebook locally, replace the string below with the\n",
+        "    # path to your service account key and run this cell to authenticate your GCP\n",
+        "    # account.\n",
+        "    elif not os.getenv(\"IS_TESTING\"):\n",
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {

notebooks/community/ml_ops/stage4/README.md

@@ -36,7 +36,7 @@ This stage may be done entirely by MLOps. We recommend:
 
 
 
-<img src='stage4.png'>
+<img src='stage4v3.png'>
 
 ## Notebooks
 
@@ -75,6 +75,25 @@ The steps performed include:
     - Query your pipeline run metadata.
 ```
 
+[Get started with Vertex ML Metadata and AutoML](get_started_with_vertex_ml_metadata_and_automl.ipynb)
+
+```
+The steps performed include:
+
+- Create a `Dataset` resource.
+- Create a corresponding `google.VertexDataset` artifact.
+- Train a model using `AutoML`.
+- Create a corresponding `google.VertexModel` artifact.
+- Create an `Endpoint` resource.
+- Create a corresponding `google.Endpoint` artifact.
+- Deploy the train model to the `Endpoint`.
+- Create an execution and context for the `AutoML` training job and deployment.
+- Add the corresponding artifacts and context to the execution.
+- Add artifact links (event) to the execution.
+- Display the execution graph.
+```
+
+
 Get started with custom model evaluation
 
 Get started with A/B Testing

notebooks/community/ml_ops/stage4/get_started_with_google_artifact_registry.ipynb

@@ -33,16 +33,22 @@
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage4/get_started_with_google_artifact_registry.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
         "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage4/get_started_with_google_artifact_registry.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage4/get_started_with_google_artifact_registry.ipynb\">\n",
-        "      Open in Google Cloud Notebooks\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage4/get_started_with_google_artifact_registry.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
         "    </a>\n",
-        "  </td>\n",
+        "  </td>                               \n",
         "</table>\n",
         "<br/><br/><br/>"
       ]
@@ -90,7 +96,7 @@
       "source": [
         "## Installations\n",
         "\n",
-        "Install *one time* the packages for executing the MLOps notebooks."
+        "Install the packages required for executing the notebook."
       ]
     },
     {
@@ -101,22 +107,20 @@
       },
       "outputs": [],
       "source": [
-        "ONCE_ONLY = False\n",
-        "if ONCE_ONLY:\n",
-        "    ! pip3 install -U tensorflow==2.5 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-transform==1.2 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-io==0.18 $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-bigquery $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-logging $USER_FLAG\n",
-        "    ! pip3 install --upgrade apache-beam[gcp] $USER_FLAG\n",
-        "    ! pip3 install --upgrade pyarrow $USER_FLAG\n",
-        "    ! pip3 install --upgrade cloudml-hypertune $USER_FLAG\n",
-        "    ! pip3 install --upgrade kfp $USER_FLAG\n",
-        "    ! pip3 install --upgrade torchvision $USER_FLAG\n",
-        "    ! pip3 install --upgrade rpy2 $USER_FLAG"
+        "import os\n",
+        "\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
+        "\n",
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "USER_FLAG = \"\"\n",
+        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\"\n",
+        "\n",
+        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG -q"
       ]
     },
     {
@@ -148,6 +152,36 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "before_you_begin"
+      },
+      "source": [
+        "## Before you begin\n",
+        "\n",
+        "### GPU runtime\n",
+        "\n",
+        "*Make sure you're running this notebook in a GPU runtime if you have that option. In Colab, select* **Runtime > Change Runtime Type > GPU**\n",
+        "\n",
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "2. [Make sure that billing is enabled for your project.](https://cloud.google.com/billing/docs/how-to/modify-project)\n",
+        "\n",
+        "3. [Enable the following APIs: Vertex AI APIs, Compute Engine APIs, and Cloud Storage.](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component,storage-component.googleapis.com)\n",
+        "\n",
+        "4. If you are running this notebook locally, you will need to install the [Cloud SDK]((https://cloud.google.com/sdk)).\n",
+        "\n",
+        "5. Enter your project ID in the cell below. Then run the  cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$`."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -224,7 +258,10 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -251,6 +288,67 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gcp_authenticate"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
+        "\n",
+        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
+        "\n",
+        "**Otherwise**, follow these steps:\n",
+        "\n",
+        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
+        "\n",
+        "**Click Create service account**.\n",
+        "\n",
+        "In the **Service account name** field, enter a name, and click **Create**.\n",
+        "\n",
+        "In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex\" into the filter box, and select **Vertex Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "\n",
+        "Click Create. A JSON file that contains your key downloads to your local environment.\n",
+        "\n",
+        "Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "gcp_authenticate"
+      },
+      "outputs": [],
+      "source": [
+        "# If you are running this notebook in Colab, run this cell and follow the\n",
+        "# instructions to authenticate your GCP account. This provides access to your\n",
+        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
+        "# requests.\n",
+        "\n",
+        "import os\n",
+        "import sys\n",
+        "\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = False\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
+        "    if \"google.colab\" in sys.modules:\n",
+        "        IS_COLAB = True\n",
+        "        from google.colab import auth as google_auth\n",
+        "\n",
+        "        google_auth.authenticate_user()\n",
+        "\n",
+        "    # If you are running this notebook locally, replace the string below with the\n",
+        "    # path to your service account key and run this cell to authenticate your GCP\n",
+        "    # account.\n",
+        "    elif not os.getenv(\"IS_TESTING\"):\n",
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {

notebooks/community/ml_ops/stage4/get_started_with_vertex_xai.ipynb

@@ -44,10 +44,11 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage4/get_started_with_vertex_xai.ipynb\">\n",
-        "      Open in Google Cloud Notebooks\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage4/get_started_with_vertex_xai.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
         "    </a>\n",
-        "  </td>\n",
+        "  </td>                \n",
         "</table>\n",
         "<br/><br/><br/>"
       ]
@@ -145,7 +146,7 @@
       "source": [
         "## Installations\n",
         "\n",
-        "Install *one time* the packages for executing the MLOps notebooks."
+        "Install the packages required for executing the notebook."
       ]
     },
     {
@@ -156,24 +157,24 @@
       },
       "outputs": [],
       "source": [
-        "ONCE_ONLY = False\n",
-        "if ONCE_ONLY:\n",
-        "    ! pip3 install -U tensorflow==2.5 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-transform==1.2 $USER_FLAG\n",
-        "    ! pip3 install -U tensorflow-io==0.18 $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-bigquery $USER_FLAG\n",
-        "    ! pip3 install --upgrade google-cloud-logging $USER_FLAG\n",
-        "    ! pip3 install --upgrade apache-beam[gcp] $USER_FLAG\n",
-        "    ! pip3 install --upgrade pyarrow $USER_FLAG\n",
-        "    ! pip3 install --upgrade cloudml-hypertune $USER_FLAG\n",
-        "    ! pip3 install --upgrade kfp $USER_FLAG\n",
-        "    ! pip3 install --upgrade torchvision $USER_FLAG\n",
-        "    ! pip3 install --upgrade rpy2 $USER_FLAG\n",
-        "    ! pip3 install --upgrade python-tabulate $USER_FLAG\n",
-        "    ! pip3 install -U opencv-python-headless==4.5.2.52 $USER_FLAG"
+        "import os\n",
+        "\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
+        "\n",
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "USER_FLAG = \"\"\n",
+        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\"\n",
+        "\n",
+        "! pip3 install -U tensorflow==2.5 $USER_FLAG -q\n",
+        "! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG -q\n",
+        "! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG -q\n",
+        "! pip3 install --upgrade python-tabulate $USER_FLAG -q\n",
+        "! pip3 install -U opencv-python-headless==4.5.2.52 $USER_FLAG -q"
       ]
     },
     {
@@ -205,6 +206,36 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "before_you_begin"
+      },
+      "source": [
+        "## Before you begin\n",
+        "\n",
+        "### GPU runtime\n",
+        "\n",
+        "*Make sure you're running this notebook in a GPU runtime if you have that option. In Colab, select* **Runtime > Change Runtime Type > GPU**\n",
+        "\n",
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "2. [Make sure that billing is enabled for your project.](https://cloud.google.com/billing/docs/how-to/modify-project)\n",
+        "\n",
+        "3. [Enable the following APIs: Vertex AI APIs, Compute Engine APIs, and Cloud Storage.](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component,storage-component.googleapis.com)\n",
+        "\n",
+        "4. If you are running this notebook locally, you will need to install the [Cloud SDK]((https://cloud.google.com/sdk)).\n",
+        "\n",
+        "5. Enter your project ID in the cell below. Then run the  cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$`."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -281,7 +312,10 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -308,6 +342,67 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gcp_authenticate"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
+        "\n",
+        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
+        "\n",
+        "**Otherwise**, follow these steps:\n",
+        "\n",
+        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
+        "\n",
+        "**Click Create service account**.\n",
+        "\n",
+        "In the **Service account name** field, enter a name, and click **Create**.\n",
+        "\n",
+        "In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex\" into the filter box, and select **Vertex Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "\n",
+        "Click Create. A JSON file that contains your key downloads to your local environment.\n",
+        "\n",
+        "Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "gcp_authenticate"
+      },
+      "outputs": [],
+      "source": [
+        "# If you are running this notebook in Colab, run this cell and follow the\n",
+        "# instructions to authenticate your GCP account. This provides access to your\n",
+        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
+        "# requests.\n",
+        "\n",
+        "import os\n",
+        "import sys\n",
+        "\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = False\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
+        "    if \"google.colab\" in sys.modules:\n",
+        "        IS_COLAB = True\n",
+        "        from google.colab import auth as google_auth\n",
+        "\n",
+        "        google_auth.authenticate_user()\n",
+        "\n",
+        "    # If you are running this notebook locally, replace the string below with the\n",
+        "    # path to your service account key and run this cell to authenticate your GCP\n",
+        "    # account.\n",
+        "    elif not os.getenv(\"IS_TESTING\"):\n",
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -331,7 +426,8 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
+        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
     {
@@ -342,8 +438,9 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
+        "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
       ]
     },
     {
@@ -363,7 +460,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_NAME"
+        "! gsutil mb -l $REGION $BUCKET_URI"
       ]
     },
     {
@@ -383,7 +480,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_NAME"
+        "! gsutil ls -al $BUCKET_URI"
       ]
     },
     {
@@ -428,7 +525,7 @@
       },
       "outputs": [],
       "source": [
-        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_NAME)"
+        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
       ]
     },
     {
@@ -506,7 +603,7 @@
         "if os.getenv(\"IS_TESTING_TF\"):\n",
         "    TF = os.getenv(\"IS_TESTING_TF\")\n",
         "else:\n",
-        "    TF = \"2-1\".replace(\".\", \"-\")\n",
+        "    TF = \"2-5\".replace(\".\", \"-\")\n",
         "\n",
         "if TF[0] == \"2\":\n",
         "    if TRAIN_GPU:\n",
@@ -660,7 +757,7 @@
         "dataset = aip.TabularDataset.create(\n",
         "    display_name=\"CIFAR10\" + \"_\" + TIMESTAMP,\n",
         "    gcs_source=gcs_source,\n",
-        "    labels={\"user_metadata\": BUCKET_NAME[5:]},\n",
+        "    labels={\"user_metadata\": BUCKET_NAME},\n",
         ")\n",
         "\n",
         "\n",
@@ -809,7 +906,7 @@
         "\n",
         "! cut -d, -f1-16 tmp.csv > batch.csv\n",
         "\n",
-        "gcs_input_uri = BUCKET_NAME + \"/test.csv\"\n",
+        "gcs_input_uri = BUCKET_URI + \"/test.csv\"\n",
         "\n",
         "! gsutil cp batch.csv $gcs_input_uri"
       ]
@@ -844,7 +941,7 @@
         "batch_predict_job = model.batch_predict(\n",
         "    job_display_name=\"cifar10_\" + TIMESTAMP,\n",
         "    gcs_source=gcs_input_uri,\n",
-        "    gcs_destination_prefix=BUCKET_NAME,\n",
+        "    gcs_destination_prefix=BUCKET_URI,\n",
         "    instances_format=\"csv\",\n",
         "    predictions_format=\"csv\",\n",
         "    generate_explanation=True,\n",
@@ -1510,7 +1607,7 @@
         "! rm -f custom.tar custom.tar.gz\n",
         "! tar cvf custom.tar custom\n",
         "! gzip custom.tar\n",
-        "! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_cifar10.tar.gz"
+        "! gsutil cp custom.tar.gz $BUCKET_URI/trainer_cifar10.tar.gz"
       ]
     },
     {
@@ -1578,7 +1675,7 @@
       },
       "outputs": [],
       "source": [
-        "MODEL_DIR = \"{}/{}\".format(BUCKET_NAME, TIMESTAMP)\n",
+        "MODEL_DIR = \"{}/{}\".format(BUCKET_URI, TIMESTAMP)\n",
         "\n",
         "EPOCHS = 20\n",
         "STEPS = 100\n",
@@ -2056,7 +2153,7 @@
       "source": [
         "import json\n",
         "\n",
-        "gcs_input_uri = BUCKET_NAME + \"/\" + \"test.jsonl\"\n",
+        "gcs_input_uri = BUCKET_URI + \"/\" + \"test.jsonl\"\n",
         "with tf.io.gfile.GFile(gcs_input_uri, \"w\") as f:\n",
         "    data = {serving_input: test_item_1.tolist()}\n",
         "    f.write(json.dumps(data) + \"\\n\")\n",
@@ -2097,7 +2194,7 @@
         "batch_predict_job = model.batch_predict(\n",
         "    job_display_name=\"cifar10_\" + TIMESTAMP,\n",
         "    gcs_source=gcs_input_uri,\n",
-        "    gcs_destination_prefix=BUCKET_NAME,\n",
+        "    gcs_destination_prefix=BUCKET_URI,\n",
         "    instances_format=\"jsonl\",\n",
         "    predictions_format=\"jsonl\",\n",
         "    machine_type=DEPLOY_COMPUTE,\n",
@@ -3018,7 +3115,7 @@
         "! rm -f custom.tar custom.tar.gz\n",
         "! tar cvf custom.tar custom\n",
         "! gzip custom.tar\n",
-        "! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_cifar10.tar.gz"
+        "! gsutil cp custom.tar.gz $BUCKET_URI/trainer_cifar10.tar.gz"
       ]
     },
     {
@@ -3086,7 +3183,7 @@
       },
       "outputs": [],
       "source": [
-        "MODEL_DIR = \"{}/{}\".format(BUCKET_NAME, TIMESTAMP)\n",
+        "MODEL_DIR = \"{}/{}\".format(BUCKET_URI, TIMESTAMP)\n",
         "\n",
         "EPOCHS = 20\n",
         "STEPS = 100\n",
@@ -3635,11 +3732,11 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil cp tmp1.jpg $BUCKET_NAME/tmp1.jpg\n",
-        "! gsutil cp tmp2.jpg $BUCKET_NAME/tmp2.jpg\n",
+        "! gsutil cp tmp1.jpg $BUCKET_URI/tmp1.jpg\n",
+        "! gsutil cp tmp2.jpg $BUCKET_URI/tmp2.jpg\n",
         "\n",
-        "test_item_1 = BUCKET_NAME + \"/tmp1.jpg\"\n",
-        "test_item_2 = BUCKET_NAME + \"/tmp2.jpg\""
+        "test_item_1 = BUCKET_URI + \"/tmp1.jpg\"\n",
+        "test_item_2 = BUCKET_URI + \"/tmp2.jpg\""
       ]
     },
     {
@@ -3677,7 +3774,7 @@
         "import base64\n",
         "import json\n",
         "\n",
-        "gcs_input_uri = BUCKET_NAME + \"/\" + \"test.jsonl\"\n",
+        "gcs_input_uri = BUCKET_URI + \"/\" + \"test.jsonl\"\n",
         "with tf.io.gfile.GFile(gcs_input_uri, \"w\") as f:\n",
         "    bytes = tf.io.read_file(test_item_1)\n",
         "    b64str = base64.b64encode(bytes.numpy()).decode(\"utf-8\")\n",
@@ -3722,7 +3819,7 @@
         "batch_predict_job = model.batch_predict(\n",
         "    job_display_name=\"cifar10_\" + TIMESTAMP,\n",
         "    gcs_source=gcs_input_uri,\n",
-        "    gcs_destination_prefix=BUCKET_NAME,\n",
+        "    gcs_destination_prefix=BUCKET_URI,\n",
         "    instances_format=\"jsonl\",\n",
         "    model_parameters=None,\n",
         "    machine_type=DEPLOY_COMPUTE,\n",
@@ -4361,7 +4458,7 @@
         "! rm -f custom.tar custom.tar.gz\n",
         "! tar cvf custom.tar custom\n",
         "! gzip custom.tar\n",
-        "! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_cifar10.tar.gz"
+        "! gsutil cp custom.tar.gz $BUCKET_URI/trainer_cifar10.tar.gz"
       ]
     },
     {
@@ -4401,7 +4498,7 @@
         "\n",
         "job = aip.CustomPythonPackageTrainingJob(\n",
         "    display_name=DISPLAY_NAME,\n",
-        "    python_package_gcs_uri=f\"{BUCKET_NAME}/trainer_cifar10.tar.gz\",\n",
+        "    python_package_gcs_uri=f\"{BUCKET_URI}/trainer_cifar10.tar.gz\",\n",
         "    python_module_name=\"trainer.task\",\n",
         "    container_uri=TRAIN_IMAGE,\n",
         "    model_serving_container_image_uri=DEPLOY_IMAGE,\n",
@@ -4436,7 +4533,7 @@
       },
       "outputs": [],
       "source": [
-        "MODEL_DIR = \"{}/{}\".format(BUCKET_NAME, TIMESTAMP)\n",
+        "MODEL_DIR = \"{}/{}\".format(BUCKET_URI, TIMESTAMP)\n",
         "DATASET_DIR = \"gs://cloud-samples-data/ai-platform/iris\"\n",
         "\n",
         "ROUNDS = 20\n",
@@ -5233,7 +5330,7 @@
         "! rm -f custom.tar custom.tar.gz\n",
         "! tar cvf custom.tar custom\n",
         "! gzip custom.tar\n",
-        "! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_cifar10.tar.gz"
+        "! gsutil cp custom.tar.gz $BUCKET_URI/trainer_cifar10.tar.gz"
       ]
     },
     {
@@ -5273,7 +5370,7 @@
         "\n",
         "job = aip.CustomPythonPackageTrainingJob(\n",
         "    display_name=DISPLAY_NAME,\n",
-        "    python_package_gcs_uri=f\"{BUCKET_NAME}/trainer_cifar10.tar.gz\",\n",
+        "    python_package_gcs_uri=f\"{BUCKET_URI}/trainer_cifar10.tar.gz\",\n",
         "    python_module_name=\"trainer.task\",\n",
         "    container_uri=TRAIN_IMAGE,\n",
         "    model_serving_container_image_uri=DEPLOY_IMAGE,\n",
@@ -5305,7 +5402,7 @@
       },
       "outputs": [],
       "source": [
-        "MODEL_DIR = \"{}/{}\".format(BUCKET_NAME, TIMESTAMP)\n",
+        "MODEL_DIR = \"{}/{}\".format(BUCKET_URI, TIMESTAMP)\n",
         "\n",
         "DIRECT = False\n",
         "if DIRECT:\n",
@@ -5836,17 +5933,7 @@
         "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
         "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
         "\n",
-        "Otherwise, you can delete the individual resources you created in this tutorial:\n",
-        "\n",
-        "- Dataset\n",
-        "- Pipeline\n",
-        "- Model\n",
-        "- Endpoint\n",
-        "- AutoML Training Job\n",
-        "- Batch Job\n",
-        "- Custom Job\n",
-        "- Hyperparameter Tuning Job\n",
-        "- Cloud Storage Bucket"
+        "Otherwise, you can delete the individual resources you created in this tutorial."
       ]
     },
     {
@@ -5857,61 +5944,10 @@
       },
       "outputs": [],
       "source": [
-        "delete_all = True\n",
+        "delete_bucket = False\n",
         "\n",
-        "if delete_all:\n",
-        "    # Delete the dataset using the Vertex dataset object\n",
-        "    try:\n",
-        "        if \"dataset\" in globals():\n",
-        "            dataset.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the model using the Vertex model object\n",
-        "    try:\n",
-        "        if \"model\" in globals():\n",
-        "            model.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the endpoint using the Vertex endpoint object\n",
-        "    try:\n",
-        "        if \"endpoint\" in globals():\n",
-        "            endpoint.undeploy_all()\n",
-        "            endpoint.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the AutoML or Pipeline training job\n",
-        "    try:\n",
-        "        if \"dag\" in globals():\n",
-        "            dag.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the custom training job\n",
-        "    try:\n",
-        "        if \"job\" in globals():\n",
-        "            job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the batch prediction job using the Vertex batch prediction object\n",
-        "    try:\n",
-        "        if \"batch_predict_job\" in globals():\n",
-        "            batch_predict_job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the hyperparameter tuning job using the Vertex hyperparameter tuning object\n",
-        "    try:\n",
-        "        if \"hpt_job\" in globals():\n",
-        "            hpt_job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    if \"BUCKET_NAME\" in globals():\n",
-        "        ! gsutil rm -r $BUCKET_NAME"
+        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "    ! gsutil rm -r $BUCKET_URI"
       ]
     }
   ],

notebooks/community/ml_ops/stage5/README.md

@@ -0,0 +1,59 @@
+# Stage 5: Deployment
+
+## Purpose
+
+Configure compute and networking requirements for containerized serving binaries for a production load.
+
+
+## Recommendations  
+
+The fifth stage in MLOps is deployment to production of the blessed model, which will replace the previous blessed model in production. This stage may be done entirely by MLOps. We recommend:
+
+- Deploy the blessed model from the Vertex Model Registry.
+- Use the Google Container Registry for the deployment container.
+- Attach, if any, serving function from the Vertex Model Registry to the deployed model.
+- Use Vertex Pipelines for the deployment.
+- For cloud models, deploy within the Google Cloud infrastructure.
+- Use Vertex Prediction traffic split for production rollout.
+- Use Vertex Prediction to set your criteria for scaling and load balancing.
+
+
+<img src='stage5v3.png'>
+
+## Notebooks
+
+### Get Started
+
+
+[Get started with Vertex AI Endpoints](get_started_with_vertex_endpoints.ipynb)
+
+```
+The steps include:
+
+- Creating an `Endpoint` resource.
+- List all `Endpoint` resources.
+- List `Endpoint` resources by query filter.
+- Configuring the serving binary of a `Model` resource for deployment to an `Endpoint` resource.
+- Deploying a single `Model` resource to an `Endpoint` resource.
+- Get deployment settings for a deployed `Model` resource.
+- Configuring auto-scaling.
+- Deploying multiple `Model` resources to an `Endpoint` resource and configuring a traffic split.
+- Dynamically change the traffic split for an `Endpoint` resource.
+- Undeploy a single `Model` resource to an `Endpoint` resource.
+- Undeploy all `Model` resources from an `Endpoint` resource.
+- Delete an `Endpoint` resource.
+- In pipeline: Create an `Endpoint` resource and deploy an existing `Model` resource to the `Endpoint` resource.
+- In pipeline: Deploy an existing `Model` resource to an existing `Endpoint` resource.
+```
+
+[Get started with Vertex AI Private Endpoints](get_started_with_vertex_private_endpoints.ipynb)
+
+```
+The steps performed include:
+
+- Creating a `Private Endpoint` resource.
+- Configure a VPC peering connection.
+- Configuring the serving binary of a `Model` resource for deployment to a `Private Endpoint` resource.
+- Deploying a `Model` resource to a `Private Endpoint` resource.
+- Send a prediction request to a `Private Endpoint`
+```

notebooks/community/ml_ops/stage6/README.md

@@ -23,7 +23,7 @@ This stage may be done entirely by MLOps. We recommend:
 - Features that dynamically change per example (e.g., bank balance) are stored in Vertex Feature Store.
 
 
-<img src='stage6.png'>
+<img src='stage6v2.png'>
 
 ## Notebooks
 
@@ -56,3 +56,149 @@ The steps performed include:
     - Upload and deploy the model serving container to Vertex AI Endpoint.
     - Make a prediction request.
 ```
+
+[Get started with Vertex AI Raw Prediction](get_started_with_raw_predict.ipynb)
+
+```
+The steps performed include:
+
+- Download a pretrained tabular classification model artifacts for a TensorFlow 1.x estimator.
+- Upload the TensorFlow estimator model as a `Vertex AI Model` resource.
+- Creating an `Endpoint` resource.
+- Deploying the `Model` resource to an `Endpoint` resource.
+- Make an online raw prediction to the `Model` resource instance deployed to the `Endpoint` resource.
+```
+
+
+[Get started with serving functions for TensorFlow model](get_started_with_tf_serving_function.ipynb)
+
+```
+The steps performed include:
+
+- Download a pretrained image classification model from TensorFlow Hub.
+- Create a serving function to receive compressed image data, and output decomopressed preprocessed data for the model input.
+- Upload the TensorFlow Hub model and serving function as a `Vertex AI Model` resource.
+- Creating an `Endpoint` resource.
+- Deploying the `Model` resource to an `Endpoint` resource.
+- Make an online prediction to the `Model` resource instance deployed to the `Endpoint` resource.
+```
+
+[Get started with TensorFlow Serving](get_started_with_tf_serving.ipynb)
+
+```
+The steps performed include:
+
+- Download a pretrained image classification model from TensorFlow Hub.
+- Create a serving function to receive compressed image data, and output decomopressed preprocessed data for the model input.
+- Upload the TensorFlow Hub model and serving function as a `Vertex AI Model` resource.
+- Creating an `Endpoint` resource.
+- Deploying the `Model` resource to an `Endpoint` resource with `TensorFlow Serving` serving binary.
+- Make an online prediction to the `Model` resource instance deployed to the `Endpoint` resource.
+```
+
+[Get started with FastAPI serving binary](get_started_with_fastapi.ipynb)
+
+```
+The steps performed include:
+
+- Download a pretrained image classification model from TensorFlow Hub.
+- Create a serving function to receive compressed image data, and output decomopressed preprocessed data for the model input.
+- Upload the TensorFlow Hub model and serving function as a `Vertex AI Model` resource.
+- Creating an `Endpoint` resource.
+- Deploying the `Model` resource to an `Endpoint` resource with `FastAPI` custom serving binary.
+- Make an online prediction to the `Model` resource instance deployed to the `Endpoint` resource.
+```
+
+[Get started with Vertex AI Prediction optimized TensorFlow run-time container](get_started_with_optimized_tfe_bert.ipynb)
+
+```
+The steps performed include:
+
+- Download a pretrained BERT model from TensorFlow Hub.
+- Fine-tune (transfer learning) the BERT model as a binary classifier.
+- Upload the TensorFlow Hub model as a `Vertex AI Model` resource, with standard TensorFlow serving container.
+- Upload the TensorFlow Hub model as a `Vertex AI Model` resource, with `Vertex AI Prediction optimized TensorFlow runtime` container
+- Create two `Endpoint` resources.
+- Deploying both `Model` resources to separate `Endpoint` resources.
+- Make the same online prediction requests to both `Model` resource instances deployed to the `Endpoint` resources.
+- Compare the prediction accuracy between the two deployed `Model` resources.
+- Configuring container settings for fine-tune control of optimizations.
+- Create a `Private Endpoint` resource.
+- Deploy the `Model` resoure with then `Vertex AI Prediction optimized TensorFlow runtime` to the `Private Endpoint` resource.
+- Make an online prediction request to the `Private Endpoint` resource.
+```
+
+[Get started with Nvidia Triton Server](get_started_with_nvidia_triton_server.ipynb)
+
+```
+The steps performed in this tutorial include:
+
+- Download the model artifacts from TensorFlow Hub.
+- Create Triton serving configuration file for the model.
+- Construct a custom container, with Triton serving image, for model deployment.
+- Locally test the custom container.
+- Upload the model as a Vertex AI Model resource.
+- Deploy the Vertex AI Model resource to a Vertex AI Endpoint resource.
+- Make a prediction request
+- Undeploy the Model resource and delete the Endpoint
+```
+
+[Get started with Vertex AI Matching Engine](get_started_with_matching_engine.ipynb)
+
+```
+The steps performed include:
+
+- Create ANN Index.
+- Create an IndexEndpoint with VPC Network
+- Deploy ANN Index
+- Perform online query
+- Deploy brute force Index.
+- Perform calibration between ANN and brute force index.
+```
+
+[Get started with Vertex AI Matching Engine using Two Towers builtin algorithm](get_started_with_matchine_engine_twotowers.ipynb)
+
+```
+The steps performed include:
+
+1. Train the `Two-Tower` algorithm to generate embeddings (encoder) for the dataset.
+2. Hyperparameter tune the trained `Two-Tower` encoder.
+3. Make example predictions (embeddings) from then trained encoder.
+4. Generate embeddings using the trained `Two-Tower` builtin algorithm.
+5. Store embeddings to format supported by `Matching Engine`.
+6. Create a `Matching Engine Index` for the embeddings.
+7. Deploy the `Matching Engine Index` to a `Index Endpoint`.
+8. Make a matching engine prediction request.
+```
+
+[Get started with Vertex AI Matching Engine using Swivel builtin algorithm](get_started_with_matchine_swivel.ipynb)
+
+```
+The steps performed include:
+
+1. Train the `Swivel` algorithm to generate embeddings (encoder) for the dataset.
+2. Hyperparameter tune the trained `Swivel` encoder.
+3. Make example predictions (embeddings) from then trained encoder.
+4. Generate embeddings using the trained `Swivel` builtin algorithm.
+5. Store embeddings to format supported by `Matching Engine`.
+6. Create a `Matching Engine Index` for the embeddings.
+7. Deploy the `Matching Engine Index` to a `Index Endpoint`.
+8. Make a matching engine prediction request.
+```
+
+[Get started with Explainable AI and custom model server](get_started_with_xai_and_custom_server.ipynb)
+
+```
+The steps performed include:
+
+- Locally train a Pytorch tabular classifier.
+- Locally test the trained model.
+- Build a HTTP server using FastAPI.
+- Create a custom serving container with the trained model and FastAPI server.
+- Locally test the custom serving container.
+- Push the custom serving container to the Artifact Registry.
+- Upload the custom serving container as a `Model` resource.
+- Deploy the `Model` resource to an `Endpoint` resource.
+- Make a prediction request to the deployed custom serving container.
+- Make an explanation request to the deployed custom serving container.
+```

notebooks/community/ml_ops/stage6/get_started_with_cpr.ipynb

@@ -32,16 +32,22 @@
         "# E2E ML on GCP: MLOps stage 6 : Get started with Custom Prediction Routine (CPR)\n",
         "<table align=\"left\">\n",
         "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage6/get_started_with_custom_predictions.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
         "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage6/get_started_with_custom_predictions.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage6/get_started_with_custom_predictions.ipynb\">\n",
-        "      Open in Vertex Workbench\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage6/get_started_with_custom_predictions.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
         "    </a>\n",
-        "  </td>\n",
+        "  </td>         \n",
         "</table>\n",
         "<br/><br/><br/>"
       ]
@@ -167,9 +173,22 @@
       },
       "outputs": [],
       "source": [
+        "import os\n",
+        "\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
+        "\n",
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "USER_FLAG = \"\"\n",
+        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\"\n",
+        "\n",
         "# Install the same dependencies used in the serving container in the notebook\n",
         "# environment.\n",
-        "%pip install -U --user -r src/requirements.txt"
+        "! pip3 install -U {USER_FLAG} -r src/requirements.txt"
       ]
     },
     {
@@ -202,6 +221,36 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "before_you_begin"
+      },
+      "source": [
+        "## Before you begin\n",
+        "\n",
+        "### GPU runtime\n",
+        "\n",
+        "*Make sure you're running this notebook in a GPU runtime if you have that option. In Colab, select* **Runtime > Change Runtime Type > GPU**\n",
+        "\n",
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "2. [Make sure that billing is enabled for your project.](https://cloud.google.com/billing/docs/how-to/modify-project)\n",
+        "\n",
+        "3. [Enable the following APIs: Vertex AI APIs, Compute Engine APIs, and Cloud Storage.](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component,storage-component.googleapis.com)\n",
+        "\n",
+        "4. If you are running this notebook locally, you will need to install the [Cloud SDK]((https://cloud.google.com/sdk)).\n",
+        "\n",
+        "5. Enter your project ID in the cell below. Then run the  cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$`."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -278,7 +327,10 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -305,6 +357,67 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gcp_authenticate"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
+        "\n",
+        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
+        "\n",
+        "**Otherwise**, follow these steps:\n",
+        "\n",
+        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
+        "\n",
+        "**Click Create service account**.\n",
+        "\n",
+        "In the **Service account name** field, enter a name, and click **Create**.\n",
+        "\n",
+        "In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex\" into the filter box, and select **Vertex Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "\n",
+        "Click Create. A JSON file that contains your key downloads to your local environment.\n",
+        "\n",
+        "Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "gcp_authenticate"
+      },
+      "outputs": [],
+      "source": [
+        "# If you are running this notebook in Colab, run this cell and follow the\n",
+        "# instructions to authenticate your GCP account. This provides access to your\n",
+        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
+        "# requests.\n",
+        "\n",
+        "import os\n",
+        "import sys\n",
+        "\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = False\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
+        "    if \"google.colab\" in sys.modules:\n",
+        "        IS_COLAB = True\n",
+        "        from google.colab import auth as google_auth\n",
+        "\n",
+        "        google_auth.authenticate_user()\n",
+        "\n",
+        "    # If you are running this notebook locally, replace the string below with the\n",
+        "    # path to your service account key and run this cell to authenticate your GCP\n",
+        "    # account.\n",
+        "    elif not os.getenv(\"IS_TESTING\"):\n",
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -2474,7 +2587,10 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil rm -rf {BUCKET_URI}\n",
+        "delete_bucket = False\n",
+        "\n",
+        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "    ! gsutil rm -rf {BUCKET_URI}\n",
         "\n",
         "! rm -rf src model instances.json instances.csv Dockerfile"
       ]