Merge branch 'main' into ml.googleapis_fix

.cloud-build/cleanup/cleanup-requirements.txt

@@ -1,2 +1 @@
-ratemate
 google-cloud-aiplatform

.cloud-build/cleanup/cleanup.py

@@ -1,13 +1,9 @@
 from typing import List
-from ratemate import RateLimit
-from resource_cleanup_manager import (
-    DatasetResourceCleanupManager,
-    ModelResourceCleanupManager,
-    EndpointResourceCleanupManager,
-    ResourceCleanupManager,
-)
 
-rate_limit = RateLimit(max_count=25, per=60, greedy=False)
+from resource_cleanup_manager import (DatasetResourceCleanupManager,
+                                      EndpointResourceCleanupManager,
+                                      ModelResourceCleanupManager,
+                                      ResourceCleanupManager)
 
 
 def run_cleanup_managers(managers: List[ResourceCleanupManager], is_dry_run: bool):
@@ -18,18 +14,17 @@ def run_cleanup_managers(managers: List[ResourceCleanupManager], is_dry_run: boo
         resources = manager.list()
         print(f"Found {len(resources)} {type_name}'s")
         for resource in resources:
-            try:
-                if not manager.is_deletable(resource):
-                    continue
+            if not manager.is_deletable(resource):
+                continue
 
-                if is_dry_run:
-                    resource_name = manager.resource_name(resource)
-                    print(f"Will delete '{type_name}': {resource_name}")
-                else:
-                    rate_limit.wait()  # wait before deleting
+            if is_dry_run:
+                resource_name = manager.resource_name(resource)
+                print(f"Will delete '{type_name}': {resource_name}")
+            else:
+                try:
                     manager.delete(resource)
-            except Exception as exception:
-                print(exception)
+                except Exception as exception:
+                    print(exception)
 
         print("")
 
@@ -43,7 +38,7 @@ if is_dry_run:
 managers = [
     DatasetResourceCleanupManager(),
     EndpointResourceCleanupManager(),
-    ModelResourceCleanupManager(),  # ModelResourceCleanupManager must follow EndpointResourceCleanupManager due to deployed models blocking model deletion.
+    ModelResourceCleanupManager(),
 ]
 
 run_cleanup_managers(managers=managers, is_dry_run=is_dry_run)

.cloud-build/cleanup/resource_cleanup_manager.py

@@ -1,5 +1,5 @@
 import abc
-from typing import Any, Type
+from typing import Any
 
 from google.cloud import aiplatform
 from google.cloud.aiplatform import base
@@ -41,7 +41,7 @@ class ResourceCleanupManager(abc.ABC):
         # Check that it wasn't created too recently, to prevent race conditions
         if time_difference <= RESOURCE_UPDATE_BUFFER_IN_SECONDS:
             print(
-                f"Skipping '{resource}' due to update_time being '{time_difference}', which is less than '{RESOURCE_UPDATE_BUFFER_IN_SECONDS}'."
+                f"Skipping '{resource}' due update_time being '{time_difference}', which is less than '{RESOURCE_UPDATE_BUFFER_IN_SECONDS}'."
             )
             return False
 
@@ -51,7 +51,7 @@ class ResourceCleanupManager(abc.ABC):
 class VertexAIResourceCleanupManager(ResourceCleanupManager):
     @property
     @abc.abstractmethod
-    def vertex_ai_resource(self) -> Type[base.VertexAiResourceNounWithFutureManager]:
+    def vertex_ai_resource(self) -> base.VertexAiResourceNounWithFutureManager:
         pass
 
     @property
@@ -61,9 +61,7 @@ class VertexAIResourceCleanupManager(ResourceCleanupManager):
     def list(self) -> Any:
         return self.vertex_ai_resource.list()
 
-    def resource_name(
-        self, resource: Type[base.VertexAiResourceNounWithFutureManager]
-    ) -> str:
+    def resource_name(self, resource: Any) -> str:
         return resource.display_name
 
     def delete(self, resource):
@@ -77,33 +75,12 @@ class VertexAIResourceCleanupManager(ResourceCleanupManager):
 
 class DatasetResourceCleanupManager(VertexAIResourceCleanupManager):
     vertex_ai_resource = aiplatform.datasets._Dataset
-    dataset_types = [
-        aiplatform.ImageDataset,
-        aiplatform.TabularDataset,
-        aiplatform.TextDataset,
-        aiplatform.TimeSeriesDataset,
-        aiplatform.VideoDataset,
-    ]
-
-    def list(self) -> Any:
-        return [
-            dataset
-            for dataset_type in self.dataset_types
-            for dataset in dataset_type.list()
-        ]
 
 
 class EndpointResourceCleanupManager(VertexAIResourceCleanupManager):
     vertex_ai_resource = aiplatform.Endpoint
 
     def delete(self, resource):
-        # TODO: Remove this once https://github.com/googleapis/python-aiplatform/issues/1441 is fixed
-        resource._sync_gca_resource()
-        for deployed_model_id in [
-            models.id for models in resource._gca_resource.deployed_models
-        ]:
-            resource._undeploy(deployed_model_id=deployed_model_id)
-
         resource.delete(force=True)
 
 

.cloud-build/execute_changed_notebooks_cli.py

@@ -62,18 +62,6 @@ parser.add_argument(
     help="The GCP region. This is used to inject a variable value into the notebook before running.",
     required=True,
 )
-parser.add_argument(
-    "--variable_service_account",
-    type=str,
-    help="A service account. This is used to inject a variable value into the notebook before running. This is not the account that will run the notebook.",
-    required=True,
-)
-parser.add_argument(
-    "--variable_vpc_network",
-    type=str,
-    help="The full VPC network name. See https://cloud.google.com/compute/docs/networks-and-firewalls#networks. Format is projects/{project}/global/networks/{network}, where {project} is a project number, as in '12345', and {network} is network name. See <https://cloud.google.com/compute/docs/reference/rest/v1/networks/insert> for details. This is used to inject a variable value into the notebook before running.",
-    required=False,
-)
 parser.add_argument(
     "--staging_bucket",
     type=str,
@@ -120,11 +108,9 @@ execute_changed_notebooks_helper.process_and_execute_notebooks(
     container_uri=args.container_uri,
     staging_bucket=args.staging_bucket,
     artifacts_bucket=args.artifacts_bucket,
-    should_parallelize=args.should_parallelize,
-    timeout=args.timeout,
     variable_project_id=args.variable_project_id,
     variable_region=args.variable_region,
-    variable_service_account=args.variable_service_account,
-    variable_vpc_network=args.variable_vpc_network,
     private_pool_id=args.private_pool_id,
+    should_parallelize=args.should_parallelize,
+    timeout=args.timeout,
 )

.cloud-build/execute_changed_notebooks_helper.py

@@ -17,18 +17,13 @@ import concurrent
 import dataclasses
 import datetime
 import functools
-import json
-import git
 import operator
 import os
 import pathlib
 import re
 import subprocess
-import utils
 from typing import List, Optional
-from utils import util
 
-import execute_notebook_helper
 import execute_notebook_remote
 import nbformat
 from google.cloud.devtools.cloudbuild_v1.types import BuildOperationMetadata
@@ -38,7 +33,6 @@ from utils import NotebookProcessors, util
 
 # A buffer so that workers finish before the orchestrating job
 WORKER_TIMEOUT_BUFFER_IN_SECONDS: int = 60 * 60
-PYTHON_VERSION = "3.9" # Set default python version
 
 
 def format_timedelta(delta: datetime.timedelta) -> str:
@@ -70,23 +64,13 @@ class NotebookExecutionResult:
     log_url: str
     output_uri: str
     build_id: str
-    logs_bucket: str
     error_message: Optional[str]
 
-    @property
-    def output_uri_web(self) -> Optional[str]:
-        if self.output_uri.startswith("gs://"):
-            return f"https://storage.googleapis.com/{self.output_uri[5:]}"
-        else:
-            return None
-
 
 def _process_notebook(
     notebook_path: str,
     variable_project_id: str,
     variable_region: str,
-    variable_service_account: str,
-    variable_vpc_network: Optional[str],
 ):
     # Read notebook
     with open(notebook_path) as f:
@@ -98,8 +82,6 @@ def _process_notebook(
         replacement_map={
             "PROJECT_ID": variable_project_id,
             "REGION": variable_region,
-            "SERVICE_ACCOUNT": variable_service_account,
-            "VPC_NETWORK": variable_vpc_network,
         },
     )
 
@@ -115,33 +97,6 @@ def _process_notebook(
         nbformat.write(nb, new_file)
 
 
-def _get_notebook_python_version(notebook_path: str) -> str:
-    """
-    Get the python version for running the notebook if it is specified in
-    the notebook.
-    """
-    python_version = PYTHON_VERSION
-
-    # Load the notebook
-    file = open(notebook_path)
-    src = file.read()
-    nb_json = json.loads(src)
-
-    #Iterate over the cells in the ipynb
-    for cell in nb_json['cells']:
-        if cell['cell_type'] == 'markdown':
-            markdown = str.join('', cell['source'])
-
-            # Look for the python version specification pattern
-            re_match = re.search('python version = (\d\.\d)', markdown, flags=re.IGNORECASE)
-            if re_match:
-                # get the version number
-                python_version = re_match.group(1)
-                break
-
-    return python_version
-
-
 def _create_tag(filepath: str) -> str:
     tag = os.path.basename(os.path.normpath(filepath))
     tag = re.sub("[^0-9a-zA-Z_.-]+", "-", tag)
@@ -161,10 +116,8 @@ def process_and_execute_notebook(
     artifacts_bucket: str,
     variable_project_id: str,
     variable_region: str,
-    variable_service_account: str,
-    variable_vpc_network: Optional[str],
     private_pool_id: Optional[str],
-    deadline: datetime.datetime,
+    deadline: datetime,
     notebook: str,
     should_get_tail_logs: bool = False,
 ) -> NotebookExecutionResult:
@@ -172,13 +125,6 @@ def process_and_execute_notebook(
 
     print(f"Running notebook: {notebook}")
 
-    # Handle empty strings
-    if not variable_vpc_network:
-        variable_vpc_network = None
-
-    if not private_pool_id:
-        private_pool_id = None
-
     # Create paths
     notebook_output_uri = "/".join([artifacts_bucket, pathlib.Path(notebook).name])
 
@@ -192,7 +138,6 @@ def process_and_execute_notebook(
         output_uri=notebook_output_uri,
         log_url="",
         build_id="",
-        logs_bucket="",
         error_message=None,
     )
 
@@ -200,17 +145,11 @@ def process_and_execute_notebook(
     time_start = datetime.datetime.now()
     operation = None
     try:
-        # Get the python version for running the notebook if specified
-        notebook_exec_python_version = _get_notebook_python_version(notebook_path=notebook)
-        print(f"Running notebook with python {notebook_exec_python_version}")
-
         # Pre-process notebook by substituting variable names
         _process_notebook(
             notebook_path=notebook,
             variable_project_id=variable_project_id,
             variable_region=variable_region,
-            variable_service_account=variable_service_account,
-            variable_vpc_network=variable_vpc_network,
         )
 
         # Upload the pre-processed code to a GCS bucket
@@ -230,13 +169,11 @@ def process_and_execute_notebook(
             private_pool_id=private_pool_id,
             private_pool_region=variable_region,
             timeout_in_seconds=timeout_in_seconds,
-            python_version=notebook_exec_python_version
         )
 
         operation_metadata = BuildOperationMetadata(mapping=operation.metadata)
         result.build_id = operation_metadata.build.id
         result.log_url = operation_metadata.build.log_url
-        result.logs_bucket = operation_metadata.build.logs_bucket
 
         # Block and wait for the result
         operation_result = operation.result()
@@ -294,40 +231,20 @@ def get_changed_notebooks(
 
     # Find notebooks
     notebooks = []
-
-    # Instantiate GitPython objects
-    repo = git.Repo(os.getcwd())
-    index = repo.index
-
     if base_branch:
-        # Get the point at which this branch branches off from main
-        branching_commits = repo.merge_base("HEAD", f"origin/{base_branch}")
-
-        if len(branching_commits) > 0:
-            branching_commit = branching_commits[0]
-            print(f"Looking for notebooks that changed from branch: {branching_commit}")
-
-            notebooks = [
-                diff.b_path
-                for diff in index.diff(branching_commit, paths=test_paths)
-                if diff.b_path is not None
-            ]
-        else:
-            notebooks = []
+        print(f"Looking for notebooks that changed from branch: {base_branch}")
+        notebooks = subprocess.check_output(
+            ["git", "diff", "--name-only", f"origin/{base_branch}..."] + test_paths
+        )
     else:
         print(f"Looking for all notebooks.")
-        notebooks_str = subprocess.check_output(["git", "ls-files"] + test_paths)
-        notebooks = notebooks_str.decode("utf-8").split("\n")
+        notebooks = subprocess.check_output(["git", "ls-files"] + test_paths)
 
+    notebooks = notebooks.decode("utf-8").split("\n")
     notebooks = [notebook for notebook in notebooks if notebook.endswith(".ipynb")]
     notebooks = [notebook for notebook in notebooks if len(notebook) > 0]
     notebooks = [notebook for notebook in notebooks if pathlib.Path(notebook).exists()]
 
-    if len(notebooks) > 0:
-        print(f"Found {len(notebooks)} notebooks:")
-        for notebook in notebooks:
-            print(f"\t{notebook}")
-
     return notebooks
 
 
@@ -336,13 +253,11 @@ def process_and_execute_notebooks(
     container_uri: str,
     staging_bucket: str,
     artifacts_bucket: str,
-    should_parallelize: bool,
-    timeout: int,
     variable_project_id: str,
     variable_region: str,
-    variable_service_account: str,
-    variable_vpc_network: Optional[str] = None,
-    private_pool_id: Optional[str] = None,
+    private_pool_id: Optional[str],
+    should_parallelize: bool,
+    timeout: int,
 ):
     """
     Run the notebooks that exist under the folders defined in the test_paths_file.
@@ -372,15 +287,14 @@ def process_and_execute_notebooks(
         timeout (str):
             Required. Timeout string according to https://cloud.google.com/build/docs/build-config-file-schema#timeout.
     """
+    notebook_execution_results: List[NotebookExecutionResult] = []
 
     # Calculate deadline
     deadline = datetime.datetime.now() + datetime.timedelta(
         seconds=max(timeout - WORKER_TIMEOUT_BUFFER_IN_SECONDS, 0)
     )
 
-    if len(notebooks) >= 1:
-        notebook_execution_results: List[NotebookExecutionResult] = []
-
+    if len(notebooks) > 0:
         print(f"Found {len(notebooks)} modified notebooks: {notebooks}")
 
         if should_parallelize and len(notebooks) > 1:
@@ -399,8 +313,6 @@ def process_and_execute_notebooks(
                             artifacts_bucket,
                             variable_project_id,
                             variable_region,
-                            variable_service_account,
-                            variable_vpc_network,
                             private_pool_id,
                             deadline,
                         ),
@@ -415,88 +327,49 @@ def process_and_execute_notebooks(
                     artifacts_bucket=artifacts_bucket,
                     variable_project_id=variable_project_id,
                     variable_region=variable_region,
-                    variable_service_account=variable_service_account,
-                    variable_vpc_network=variable_vpc_network,
                     private_pool_id=private_pool_id,
                     deadline=deadline,
                     notebook=notebook,
                 )
                 for notebook in notebooks
             ]
-
-        print("\n=== RESULTS ===\n")
-
-        results_sorted = sorted(
-            notebook_execution_results,
-            key=lambda result: result.is_pass,
-            reverse=True,
-        )
-
-        # Print results
-        print(
-            tabulate(
-                [
-                    [
-                        result.name,
-                        "PASSED" if result.is_pass else "FAILED",
-                        format_timedelta(result.duration),
-                        result.log_url,
-                        result.output_uri,
-                        result.output_uri_web,
-                        result.logs_bucket
-                    ]
-                    for result in results_sorted
-                ],
-                headers=[
-                    "build_tag",
-                    "status",
-                    "duration",
-                    "log_url",
-                    "output_uri",
-                    "output_uri_web",
-                    "logs_bucket"
-                ],
-            )
-        )
-
-        if len(notebooks) == 1:
-          print("="*100)
-          print("The notebook execution build log:\n")
-          print("="*100)
-
-          build_id = results_sorted[0].build_id
-          logs_bucket_name = (results_sorted[0].logs_bucket).removeprefix("gs://")
-          log_file_name = f"log-{build_id}.txt"
-
-          log_contents = util.download_blob_into_memory(
-            bucket_name=logs_bucket_name,
-            blob_name=log_file_name,
-            download_as_text=True
-            )
-
-          # Remove extra steps from the log
-          match = re.search("starting Step #4", log_contents, flags=re.IGNORECASE)
-
-          if match is not None:
-            match_index = match.span()[0]
-            print(log_contents[match_index:])
-          else:
-            print(log_contents)
-
-        print("\n=== END RESULTS===\n")
-
-        total_notebook_duration = functools.reduce(
-            operator.add,
-            [datetime.timedelta(seconds=0)]
-            + [result.duration for result in results_sorted],
-        )
-
-        print(
-            f"Cumulative notebook duration: {format_timedelta(total_notebook_duration)}"
-        )
-
-        # Raise error if any notebooks failed
-        if not all([result.is_pass for result in results_sorted]):
-            raise RuntimeError("Notebook failures detected. See logs for details")
     else:
         print("No notebooks modified in this pull request.")
+
+    print("\n=== RESULTS ===\n")
+
+    results_sorted = sorted(
+        notebook_execution_results,
+        key=lambda result: result.is_pass,
+        reverse=True,
+    )
+
+    # Print results
+    print(
+        tabulate(
+            [
+                [
+                    result.name,
+                    "PASSED" if result.is_pass else "FAILED",
+                    format_timedelta(result.duration),
+                    result.log_url,
+                ]
+                for result in results_sorted
+            ],
+            headers=["build_tag", "status", "duration", "log_url"],
+        )
+    )
+
+    print("\n=== END RESULTS===\n")
+
+    total_notebook_duration = functools.reduce(
+        operator.add,
+        [datetime.timedelta(seconds=0)]
+        + [result.duration for result in results_sorted],
+    )
+
+    print(f"Cumulative notebook duration: {format_timedelta(total_notebook_duration)}")
+
+    # Raise error if any notebooks failed
+    if not all([result.is_pass for result in results_sorted]):
+        raise RuntimeError("Notebook failures detected. See logs for details")

.cloud-build/execute_notebook_helper.py

@@ -26,9 +26,6 @@ from utils import util
 
 # This script is used to execute a notebook and write out the output notebook.
 
-# This is used to force papermill to use this kernel to run the notebook instead of any defined inside the notebook itself
-DEFAULT_KERNEL_NAME = "python3"
-
 
 def execute_notebook(
     notebook_source: str,
@@ -53,17 +50,6 @@ def execute_notebook(
 
     execution_exception = None
 
-    print("\n=== DOWNLOAD EXECUTED NOTEBOOK ===\n")
-    print(f"Please debug the executed notebook by downloading the executed notebook:")
-
-    print("Option 1. Using gsutil. Run the following command in your terminal.")
-    print(f'\tgsutil cp "{output_file_or_uri}" .')
-
-    print("Option 2. Using this link.")
-    print(f"\thttps://storage.googleapis.com/{output_file_or_uri[5:]}")
-
-    print("\n======\n")
-
     # Execute notebook
     try:
         # Execute notebook
@@ -72,7 +58,6 @@ def execute_notebook(
             output_path=notebook_source,
             progress_bar=should_log_output,
             request_save_on_cell_execute=should_log_output,
-            kernel_name=DEFAULT_KERNEL_NAME,
             log_output=should_log_output,
             stdout_file=sys.stdout if should_log_output else None,
             stderr_file=sys.stderr if should_log_output else None,
@@ -86,6 +71,10 @@ def execute_notebook(
             util.upload_file(notebook_source, remote_file_path=output_file_or_uri)
 
             print("\n=== EXECUTION FINISHED ===\n")
+            print(
+                f"Please debug the executed notebook by downloading: {output_file_or_uri}"
+            )
+            print("\n======\n")
         else:
             # Create directories if they don't exist
             if not os.path.exists(os.path.dirname(output_file_or_uri)):

.cloud-build/execute_notebook_remote.py

@@ -40,7 +40,6 @@ def execute_notebook_remote(
     private_pool_region: Optional[str],
     tag: Optional[str],
     timeout_in_seconds: Optional[int] = None,
-    python_version: Optional[str] = None
 ) -> operation.Operation:
     """Create and execute a single notebook on Google Cloud Build"""
     # Load build steps from YAML
@@ -51,12 +50,8 @@ def execute_notebook_remote(
         "_PYTHON_IMAGE": container_uri,
         "_NOTEBOOK_GCS_URI": notebook_uri,
         "_NOTEBOOK_OUTPUT_GCS_URI": notebook_output_uri,
-        "_PYTHON_VERSION" : f"python{python_version}"
     }
 
-    if python_version is not None:
-      substitutions["_PYTHON_VERSION"] = "python" + python_version
-
     build = cloudbuild_v1.Build()
 
     options: Optional[client_options.ClientOptions] = None

.cloud-build/notebook-execution-test-cloudbuild-single.yaml

@@ -4,35 +4,25 @@ steps:
     entrypoint: /bin/sh
     args:
     - -c
-    - 'gcloud config list --quiet'
+    - 'gcloud config list'
   # Check the Python version
   - name: ${_PYTHON_IMAGE}
     entrypoint: /bin/sh
     args:
     - -c
-    - ${_PYTHON_VERSION} .cloud-build/CheckPythonVersion.py -q
-  # Create a virtual environment
-  - name: ${_PYTHON_IMAGE}
-    entrypoint: /bin/sh
-    args:
-    - -c
-    - ${_PYTHON_VERSION} -m venv workspace/env
+    - 'python3 .cloud-build/CheckPythonVersion.py'
   # Install Python dependencies
   - name: ${_PYTHON_IMAGE}
     entrypoint: /bin/sh
     args:
     - -c
-    - . workspace/env/bin/activate &&
-      python -m pip -q install -U pip &&
-      python -m pip -q install -U -r .cloud-build/requirements.txt
+    - 'python3 -m pip install -U pip && python3 -m pip install -U --user -r .cloud-build/requirements.txt'
   # Install Python dependencies and run testing script
   - name: ${_PYTHON_IMAGE}
     entrypoint: /bin/sh
     args:
     - -c
-    - |
-      . workspace/env/bin/activate &&
-      python .cloud-build/execute_notebook_cli.py --notebook_source "${_NOTEBOOK_GCS_URI}" --output_file_or_uri "${_NOTEBOOK_OUTPUT_GCS_URI}"
+    - 'python3 -m pip install -U pip && python3 -m pip freeze && python3 .cloud-build/execute_notebook_cli.py --notebook_source "${_NOTEBOOK_GCS_URI}" --output_file_or_uri "${_NOTEBOOK_OUTPUT_GCS_URI}"'
     env:
     - 'IS_TESTING=1'
-timeout: 86400s
+timeout: 86400s

.cloud-build/notebook-execution-test-cloudbuild.yaml

@@ -4,42 +4,35 @@ steps:
     entrypoint: /bin/sh
     args:
     - -c
-    - gcloud config list --quiet
+    - 'gcloud config list'
   # Check the Python version
   - name: ${_PYTHON_IMAGE}
     entrypoint: /bin/sh
     args:
     - -c
-    - python3 .cloud-build/CheckPythonVersion.py -q
-  # Fetch full repo for diff purposes
-  - name: gcr.io/cloud-builders/git
-    args: [fetch, --unshallow, --quiet]
-  # Create a virtual environment
+    - 'python3 .cloud-build/CheckPythonVersion.py'
+  # Fetch base branch if required
   - name: ${_PYTHON_IMAGE}
     entrypoint: /bin/sh
     args:
     - -c
-    - python3 -m venv workspace/env
+    - 'if [ -n "${_BASE_BRANCH}" ]; then git fetch origin "${_BASE_BRANCH}":refs/remotes/origin/"${_BASE_BRANCH}"; else echo "Skipping fetch."; fi'
   # Install Python dependencies
   - name: ${_PYTHON_IMAGE}
     entrypoint: /bin/sh
     args:
     - -c
-    - . workspace/env/bin/activate &&
-      python3 -m pip -q install -U pip &&
-      python3 -m pip -q install -U -r .cloud-build/requirements.txt
+    - 'python3 -m pip install -U pip && python3 -m pip install -U --user -r .cloud-build/requirements.txt'
   # Install Python dependencies and run testing script
   # TODO: Only pass in private_pool_id if it is set
   - name: ${_PYTHON_IMAGE}
     entrypoint: /bin/sh
     args:
     - -c
-    - |
-      . workspace/env/bin/activate &&
-      python3 .cloud-build/execute_changed_notebooks_cli.py --test_paths_file "${_TEST_PATHS_FILE}" --base_branch "${_FORCED_BASE_BRANCH}" --container_uri ${_PYTHON_IMAGE} --staging_bucket ${_GCS_STAGING_BUCKET}  --artifacts_bucket ${_GCS_STAGING_BUCKET}/executed_notebooks/PR_${_PR_NUMBER}/BUILD_${BUILD_ID} --variable_project_id ${PROJECT_ID} --variable_region ${_GCP_REGION} --variable_service_account ${_GCP_SERVICE_ACCOUNT} --variable_vpc_network "${_GPC_VPC_NETWORK_NAME}" `if [ ! -z "${_PRIVATE_POOL_NAME}" ]; then echo "--private_pool_id ${_PRIVATE_POOL_NAME}"; fi`
+    - 'python3 -m pip install -U pip && python3 -m pip freeze && python3 .cloud-build/execute_changed_notebooks_cli.py --test_paths_file "${_TEST_PATHS_FILE}" --base_branch "${_FORCED_BASE_BRANCH}" --container_uri ${_PYTHON_IMAGE} --staging_bucket ${_GCS_STAGING_BUCKET}  --artifacts_bucket ${_GCS_STAGING_BUCKET}/executed_notebooks/PR_${_PR_NUMBER}/BUILD_${BUILD_ID} --variable_project_id ${PROJECT_ID} --variable_region ${_GCP_REGION} `if [ ! -z "${_PRIVATE_POOL_NAME}" ]; then echo "--private_pool_id ${_PRIVATE_POOL_NAME}"; fi`'
     env:
     - 'IS_TESTING=1'
 timeout: 86400s
 options:
   pool:
-    name: ${_PRIVATE_POOL_NAME}
+    name: ${_PRIVATE_POOL_NAME}

.cloud-build/requirements.txt

@@ -10,4 +10,3 @@ google-cloud-aiplatform
 google-cloud-storage
 google-cloud-build
 ratemate
-GitPython

.cloud-build/test_folders.txt

@@ -1,2 +1,3 @@
 notebooks/official
 notebooks/notebook_template.ipynb
+notebooks/community/ml_ops

.cloud-build/test_notebook_vm.txt

@@ -1,6 +1,5 @@
 notebooks/official/vizier/gapic-vizier-multi-objective-optimization.ipynb
 notebooks/official/pipelines/lightweight_functions_component_io_kfp.ipynb
+notebooks/official/matching_engine/intro-swivel.ipynb
 notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-locally-trained-models.ipynb
-notebooks/official/custom/custom-tabular-bq-managed-dataset.ipynb
-notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb
-.cloud-build/tests/python_version_test.ipynb
+notebooks/official/pipelines/metrics_viz_run_compare_kfp.ipynb

.cloud-build/test_single_notebook_vm.txt

@@ -1 +0,0 @@
-notebooks/official/custom/custom-tabular-bq-managed-dataset.ipynb

.cloud-build/tests/python_version_test.ipynb

@@ -1,61 +0,0 @@
-{
-  "cells": [
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "57a3d44ed8a8"
-      },
-      "source": [
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**_NOTE_**: This notebook has been tested in the following environment:\n",
-        "\n",
-        "* Python version = 3.7\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI API and Compute Engine API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component).\n",
-        "\n",
-        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "c6516f90311b"
-      },
-      "outputs": [],
-      "source": [
-        "# test if the right python version is being used\n",
-        "import sys\n",
-        "\n",
-        "actual_python_version = f\"{sys.version_info.major}.{sys.version_info.minor}\"\n",
-        "print(f\"Runtime python version: {actual_python_version}\")\n",
-        "\n",
-        "assert actual_python_version == \"3.7\", \"Wrong python version!\""
-      ]
-    }
-  ],
-  "metadata": {
-    "colab": {
-      "name": "python_version_test.ipynb",
-      "toc_visible": true
-    },
-    "kernelspec": {
-      "display_name": "Python 3",
-      "name": "python3"
-    }
-  },
-  "nbformat": 4,
-  "nbformat_minor": 0
-}

.cloud-build/utils/UpdateNotebookVariables.py

@@ -35,8 +35,8 @@ Variables in conditionals can also be replaced:
 
 def get_updated_value(content: str, variable_name: str, variable_value: str) -> str:
     return re.sub(
-        rf"({variable_name}.*? = .*?[\",\'])\[.+?\]([\",\'].*?)",
-        rf"\g<1>{variable_value}\g<2>",
+        rf"({variable_name}.*?=.*?[\",\'])\[.+?\]([\",\'].*?)",
+        rf"\1{variable_value}\2",
         content,
         flags=re.M,
     )
@@ -79,26 +79,3 @@ def test_region():
         variable_value="us-central1",
     )
     assert new_content == 'REGION = "us-central1"  # @param {type:"string"}'
-
-
-def test_region_equal_equals_ignore():
-    # Tests that == is ignored
-    new_content = get_updated_value(
-        content='REGION == "[your-region]"  # @param {type:"string"}',
-        variable_name="REGION",
-        variable_value="us-central1",
-    )
-    assert new_content == 'REGION == "[your-region]"  # @param {type:"string"}'
-
-
-def test_service_account():
-    # Tests that == is ignored
-    new_content = get_updated_value(
-        content='SERVICE_ACCOUNT = "[your-service-account]"  # @param {type:"string"}',
-        variable_name="SERVICE_ACCOUNT",
-        variable_value="12345-compute@developer.gserviceaccount.com",
-    )
-    assert (
-        new_content
-        == 'SERVICE_ACCOUNT = "12345-compute@developer.gserviceaccount.com"  # @param {type:"string"}'
-    )

.cloud-build/utils/util.py

@@ -3,7 +3,7 @@ import subprocess
 import tarfile
 import uuid
 from datetime import datetime
-from typing import Optional, Union
+from typing import Optional
 
 from google.auth import credentials as auth_credentials
 from google.cloud import storage
@@ -58,34 +58,3 @@ def archive_code_and_upload(staging_bucket: str):
     print(f"Uploaded source code archive to {source_archived_file_gcs}")
 
     return source_archived_file_gcs
-
-
-def download_blob_into_memory(
-  bucket_name: str,
-  blob_name: str,
-  download_as_text: Optional[bool]=False
-) -> Union[bytes, str]:
-    """
-    Downloads a blob into memory as byte or as text if
-    download_as_text is set to True.
-    """
-
-    storage_client = storage.Client()
-
-    bucket = storage_client.bucket(bucket_name)
-
-    # Construct a client side representation of a blob.
-    blob = bucket.blob(blob_name)
-
-    # Download the blob content
-    if download_as_text:
-      contents = blob.download_as_text()
-    else:
-      contents = blob.download_as_bytes()
-
-    print(
-        f"Downloaded storage object {blob_name} from bucket {bucket_name}."
-    )
-
-    return contents
-

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,11 +1,4 @@
-**REQUIRED:** Add a summary of your PR here, typically including why the change is needed and what was changed. Include any design alternatives for discussion purposes.
-
-<br>
---- YOUR PR SUMMARY GOES HERE ---
-<br><br><br>
-
-**REQUIRED:** Fill out the below checklists or remove if irrelevant
-1. If you are opening a PR for `Official Notebooks` under the [notebooks/official](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/notebooks/official) folder, follow this mandatory checklist:
+If you are opening a PR for `Official Notebooks` under the [notebooks/official](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/notebooks/official) folder, follow this mandatory checklist:
 - [ ] Use the [notebook template](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/notebook_template.ipynb) as a starting point.
 - [ ] Follow the style and grammar rules outlined in the above notebook template.
 - [ ] Verify the notebook runs successfully in Colab since the automated tests cannot guarantee this even when it passes.
@@ -14,15 +7,12 @@
 - [ ] This notebook has been added to the [CODEOWNERS](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/CODEOWNERS) file under the `Official Notebooks` section, pointing to the author or the author's team.
 - [ ] The Jupyter notebook cleans up any artifacts it has created (datasets, ML models, endpoints, etc) so as not to eat up unnecessary resources.
 
-<br>
 
-2. If you are opening a PR for `Community Notebooks` under the [notebooks/community](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/notebooks/community) folder:
+If you are opening a PR for `Community Notebooks` under the [notebooks/community](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/notebooks/community) folder:
 - [ ] This notebook has been added to the [CODEOWNERS](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/CODEOWNERS) file under the `Community Notebooks` section, pointing to the author or the author's team.
 - [ ] Passes all the required formatting and linting checks. You can locally test with these [instructions](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/CONTRIBUTING.md#code-quality-checks).
 
-<br>
-
-3. If you are opening a PR for `Community Content` under the [community-content](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/community-content) folder:
+If you are opening a PR for `Community Content` under the [community-content](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/community-content) folder:
 - [ ] Make sure your main `Content Directory Name` is descriptive, informative, and includes some of the key products and attributes of your content, so that it is differentiable from other content
 - [ ] The main content directory has been added to the [CODEOWNERS](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/community-content/CODEOWNERS) file under the `Community Content` section, pointing to the author or the author's team.
 - [ ] Passes all the required formatting and linting checks. You can locally test with these [instructions](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/CONTRIBUTING.md#code-quality-checks).

.github/workflows/ci.yaml

@@ -7,9 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Set up Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: '3.x'      
+      uses: actions/setup-python@v3
     - name: Fetch pull request branch
       uses: actions/checkout@v3
       with:

.github/workflows/linter/Dockerfile

@@ -1,20 +0,0 @@
-# To use this image, run this command with the desired notebook args from the top-level vertex-ai-samples directory:
-# 1. To lint all changed notebooks:
-# docker run -v ${PWD}:/setup/app gcr.io/python-docs-samples-tests/notebook_linter:latest
-# 2. To lint specific notebooks:
-# docker run -v ${PWD}:/setup/app gcr.io/python-docs-samples-tests/notebook_linter:latest notebooks/1.ipynb notebooks/2.ipynb
-
-FROM python:3.10
-
-WORKDIR setup
-
-COPY ./requirements.txt .
-COPY ./run_linter.sh .
-
-# Install dependencies.
-RUN pip install --upgrade pip
-RUN pip install -r requirements.txt
-
-WORKDIR app
-
-ENTRYPOINT ["/setup/run_linter.sh"]

.github/workflows/linter/requirements.txt

@@ -2,9 +2,8 @@ git+https://github.com/tensorflow/docs
 ipython
 jupyter
 nbconvert
-black==22.6.0
-pyupgrade==2.34.0
+black==22.3.0
+pyupgrade==2.31.1
 isort==5.10.1
 flake8==4.0.1
-nbqa==1.4.0
-
+nbqa==1.3.1

.github/workflows/linter/run_linter.sh

@@ -47,22 +47,12 @@ done
 
 echo "Test mode: $is_test"
 
-# Read in user-provided notebooks
-notebooks=()
-for arg in "$@"; do
-    if [[ $arg == *.ipynb ]]; then
-        notebooks+=("$arg")
-    fi
-done
-
 # Only check notebooks in test folders modified in this pull request.
 # Note: Use process substitution to persist the data in the array
-if [ ${#notebooks[@]} -eq 0 ]; then
-    echo "Checking for changed notebooked using git"
-    while read -r file || [ -n "$line" ]; do
-        notebooks+=("$file")
-    done < <(git diff --name-only main... | grep '\.ipynb$')
-fi
+notebooks=()
+while read -r file || [ -n "$line" ]; do
+    notebooks+=("$file")
+done < <(git diff --name-only main... | grep '\.ipynb$')
 
 problematic_notebooks=()
 if [ ${#notebooks[@]} -gt 0 ]; then
@@ -78,7 +68,7 @@ if [ ${#notebooks[@]} -gt 0 ]; then
 
             if [ "$is_test" = true ]; then
                 echo "Running nbfmt..."
-                python3 -m tensorflow_docs.tools.nbfmt --test "$notebook"
+                python3 -m tensorflow_docs.tools.nbfmt --remove_outputs --test "$notebook"
                 NBFMT_RTN=$?
                 # echo "Running black..."
                 # python3 -m nbqa black "$notebook" --check
@@ -103,7 +93,7 @@ if [ ${#notebooks[@]} -gt 0 ]; then
                 python3 -m nbqa isort "$notebook"
                 ISORT_RTN=$?
                 echo "Running nbfmt..."
-                python3 -m tensorflow_docs.tools.nbfmt "$notebook"
+                python3 -m tensorflow_docs.tools.nbfmt --remove_outputs "$notebook"
                 NBFMT_RTN=$?
                 echo "Running flake8..."
                 python3 -m nbqa flake8 "$notebook" --show-source --extend-ignore=W391,E501,F821,E402,F404,W503,E203,E722,W293,W291

CONTRIBUTING.md

@@ -48,8 +48,8 @@ then you will need to manually address them before submitting your PR.
 nbqa black "$notebook"
 nbqa pyupgrade "$notebook"
 nbqa isort "$notebook"
-nbqa flake8 "$notebook" --extend-ignore=W391,E501,F821,E402,F404,W503,E203,E722,W293,W291
 python3 -m tensorflow_docs.tools.nbfmt --remove_outputs "$notebook"
+nbqa flake8 "$notebook" --extend-ignore=W391,E501,F821,E402,F404,W503,E203,E722,W293,W291
 ```
 
 ## Code Reviews

community-content/CODEOWNERS

@@ -1,8 +1,6 @@
 *  @vertex-ai-samples-contributors  @GoogleCloudPlatform/cloudml-samples-owners
 /tf_agents_bandits_movie_recommendation_with_kfp_and_vertex_sdk	@yinghsienwu
-/pytorch_pre_built_images_deployment @googleapis/vertex-prediction-team
 /pytorch_text_classification_using_vertex_sdk_and_gcloud  @RajeshThallam
 /pytorch_text_classification_using_vertex_sdk_and_gcloud		@RajeshThallam @ultrons
 /sklearn_text_classification_from_script_using_vertex_sdk       @maxhardt
 /pluto_on_workbench @wkharold
-/cpr-examples @samthrasher

community-content/cpr-examples/timm_serving/.dockerignore

@@ -1,5 +0,0 @@
-testdata/*
-build.py
-test.py
-state_dict.pth
-config.json

community-content/cpr-examples/timm_serving/.gitignore

@@ -1,6 +0,0 @@
-cpr_model_server.py
-entrypoint.py
-state_dict.pth
-config.json
-**/__pycache__
-!testdata/**

community-content/cpr-examples/timm_serving/README.md

@@ -1,110 +0,0 @@
-# CPR Example: PyTorch Image Models (timm)
-
-## About CPR
-
-CPR ([custom prediction routines](https://github.com/googleapis/python-aiplatform/blob/main/google/cloud/aiplatform/prediction/README.md)) is a framework designed by Google Cloud developers to make it easier to combine machine learning models with custom preprocessing and postprocessing logic in a real-time serving application. 
-
-## Using this example
-
-This code is a self-contained example of a custom model server project built using CPR.
-
-As is, you can use it to serve the ViT-Small image classification model from Ross Wightman's [`timm`](https://github.com/rwightman/pytorch-image-models) library of image model implementations in PyTorch. Both CPU and GPU are supported.
-
-You can also consider using the code here as a template for your own CPR project if you want to use a different model from `timm`, a different PyTorch model, or an entirely different framework.
-
-### Requirements
-
-In order to use this example, you'll need Docker and Python 3 installed on your system.
-
-To get started, first create a virtual environment in an empty directory:
-```sh
-mkdir cpr-example
-python3 -m venv cpr-example
-cd cpr-example && source bin/activate
-``` 
-
-Then, clone the [vertex-ai-samples repo](https://github.com/GoogleCloudPlatform/vertex-ai-samples) in that directory:
-```sh
-git clone https://github.com/GoogleCloudPlatform/vertex-ai-samples.git
-cd vertex-ai-samples/community-content/cpr-examples/timm_serving
-```
-
-Finally, install the Python modules required to build and run the model server:
-```sh
-pip install -r requirements.txt
-```
-
-### Auth
-
-This example uses Google Cloud Storage for hosting model artifacts and Artifact Registry to store the container image. 
-You'll need to authorize yourself before you can interact with these.
-
-First, log in to GCP with application default credentials:
-```sh
-gcloud auth application-default login
-```
-
-Next, if you haven't done so already, set up the [gcloud credential helper](https://cloud.google.com/artifact-registry/docs/docker/authentication)
-for the Artifact Registry region where you intend to host the image.  
-```
-gcloud auth configure-docker <region>-docker.pkg.dev
-```
-
-
-### Predictor
-
-The `TimmPredictor` class in `timm_serving/predictor.py` implements most of the important logic for the server.
-
-- `load(artifacts_dir)`: The predictor's `load` method is called when the server starts up in order to set up the predictor, usually by loading model weights and any artifacts needed for preprocessing and postprocessing. In this example, we initialize the saved model from the `state_dict.pth` file located inside the `artifacts_dir` folder and create the preprocessing transform from the model config.
-
-- `preprocess`, `predict`, `postprocess`: These methods are applied in sequence to the deserialized JSON data from each request. 
-    - `preprocess` decodes images from base64 and apply cropping, scaling and normalizing transforms. 
-    - `predict` runs the ViT-Small model on the preprocessed images and returns class scores.
-    - `postprocess` finds the top five classes and packs the class names, probabilities, and indices in a serializable result.
-
-### Building the container
-
-To build the model server locally, run the build command:
-```sh
-python build.py build
-```
-
-You can edit configuration values such as the model server's base image, the name and tag assigned to the image, and the path where model weights are stored locally.
-
-When you run the build command, model weights are downloaded and the model server container is built.
-
-### Running local tests
-
-`test.py` contains a suite of unit tests for the predictor as well as end-to-end tests for the model server. 
-
-To run the tests:
-```sh
-python test.py
-```
-
-All of the test images are public domain.
-- [Cat](https://commons.wikimedia.org/wiki/File:Stray_cat_on_wall.jpg)
-- [Airplane](https://commons.wikimedia.org/wiki/File:Airplanes_jets.jpg)
-- The infamous [mandrill](https://commons.wikimedia.org/wiki/File:Wikipedia-sipi-image-db-mandrill-4.2.03.png)
-
-### Deploying to Vertex AI
-
-Before uploading or deploying the container, you'll need to modify `config.py` to set appropriate values for: 
-- `project_id`: Your GCP project id.
-- `region`: Region where the model will be uploaded and deployed.
-- `repository`: [Artifact Registry repository](https://cloud.google.com/artifact-registry/docs/repositories/create-repos) in your project where the container image will be uploaded. 
-- `artifacts_gcs_dir`: Folder in a [Google Cloud Storage bucket](https://cloud.google.com/storage/docs/creating-buckets) where the model weights will be uploaded.
-
-Once this is done, first upload the model:
-```sh
-python build.py upload
-```
-
-Then deploy it:
-```sh
-python build.py deploy
-```
-
-If you run the deploy command again, it will create a new endpoint. If you want to undeploy the model, you can do so using the Vertex AI dashboard on the Google Cloud console, or use `gcloud ai endpoints undeploy` from the command line.
-
-After deploying successfully, you can run `python build.py probe` to send a sample request to the deployed model. 

community-content/cpr-examples/timm_serving/build.py

@@ -1,117 +0,0 @@
-# Copyright 2022 Google LLC
-
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-
-#     https://www.apache.org/licenses/LICENSE-2.0
-
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Build the model server container."""
-import json
-import logging
-import os
-import pathlib
-from typing import Sequence
-
-from absl import app
-from absl import logging
-from config import CPRConfig
-from google.cloud import aiplatform
-from google.cloud.aiplatform import prediction as cpr
-import smart_open
-import timm
-from timm_serving import predictor
-import torch
-
-
-def build_container(config: CPRConfig, tag: str) -> cpr.LocalModel:
-    """Build the model server container.
-
-    Args:
-      tag: Output image tag.
-
-    Returns:
-      LocalModel exposing the built model server.
-    """
-    return cpr.LocalModel.build_cpr_model(
-        src_dir=os.path.join(os.getcwd()),
-        output_image_uri=tag,
-        base_image=config.base_image,
-        predictor=predictor.TimmPredictor,
-        requirements_path=os.path.join(os.getcwd(), "requirements.txt"),
-    )
-
-
-def save_model_artifact(destination: str) -> None:
-    """Save a copy of the model state dict."""
-    model = timm.create_model(predictor.TimmPredictor.TIMM_MODEL_NAME, pretrained=True)
-    dest_file = os.path.join(destination, predictor.TimmPredictor.WEIGHTS_FILE)
-    with smart_open.open(dest_file, "wb") as f:
-        torch.save(model, f)
-    logging.info("Saved model to %s", dest_file)
-    logging.info("%s parameters", sum(p.numel() for p in model.parameters()))
-
-
-def upload_model(config: CPRConfig) -> aiplatform.Model:
-    """Tag and upload the model server."""
-    ar_tag = (
-        f"{config.region}-docker.pkg.dev/{config.project_id}"
-        f"/{config.repository}/{config.image}"
-    )
-    local_model = build_container(config, tag=ar_tag)
-    aiplatform.init(project=config.project_id, location=config.region)
-    local_model.push_image()
-    aip_model = aiplatform.Model.upload(
-        local_model=local_model,
-        display_name=predictor.TimmPredictor.TIMM_MODEL_NAME,
-        artifact_uri=config.artifact_gcs_dir,
-    )
-    config.model_name = aip_model.resource_name
-    config.save()
-    return aip_model
-
-
-def deploy_model(config: CPRConfig) -> aiplatform.Endpoint:
-    """Deploy the model server to a Vertex Prediction endpoint."""
-    aiplatform.init(project=config.project_id, location=config.region)
-    aip_model = aiplatform.Model(model_name=config.model_name)
-    endpoint = aip_model.deploy(machine_type=config.machine_type)
-    config.endpoint_name = endpoint.resource_name
-    config.save()
-    return endpoint
-
-
-def probe_prediction(config: CPRConfig, request_path: str) -> None:
-    """Send a sample prediction request to the Vertex Prediction endpoint."""
-    aiplatform.init(project=config.project_id, location=config.region)
-    aip_endpoint = aiplatform.Endpoint(endpoint_name=config.endpoint_name)
-    with open(request_path) as f:
-        logging.info(aip_endpoint.predict(**json.load(f)))
-
-
-def main(argv: Sequence[str]):
-    config = CPRConfig()
-    if pathlib.Path(config.config_file).exists():
-        config.load()
-
-    actions = set(argv[1:])
-    if "build" in actions:
-        build_container(config, config.image)
-        save_model_artifact(config.artifact_local_dir)
-    if "upload" in actions:
-        save_model_artifact(config.artifact_gcs_dir)
-        upload_model(config)
-    if "deploy" in actions:
-        deploy_model(config)
-    if "probe" in actions:
-        probe_prediction(config, request_path="sample_request.json")
-
-
-if __name__ == "__main__":
-    app.run(main)

community-content/cpr-examples/timm_serving/config.py

@@ -1,76 +0,0 @@
-# Copyright 2022 Google LLC
-
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-
-#     https://www.apache.org/licenses/LICENSE-2.0
-
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import dataclasses
-import json
-
-
-@dataclasses.dataclass
-class CPRConfig(object):
-    """Configure the build process by editing the default values here.
-
-    config_file: File path used to save values in this config. (Some
-      values, such as the model name, are generated at build time and
-      depended on by future steps, so saving it allows this script to
-      deploy the model without re-uploading it, for example.)
-
-    base_image: Base Docker image on top of which the model server will
-      be built. By default, a Debian-based Python 3 image without GPU
-      support will be used.
-
-    image: Name and tag assigned to the built model server image.
-
-    artifact_local_dir: Local directory where a copy of the pretrained model weights
-      will be saved.
-
-    region: Google Cloud Region where the model will be uploaded during the
-      build process.
-
-    project_id: Google Cloud project ID.
-
-    repository: Name of the Artifact Registry repository where the container
-      will be uploaded.
-
-    artifact_gcs_dir: Location on GCS where a copy of the pretrained model
-      weights will be uploaded.
-
-    model_name: Full resource path of the uploaded model. This is a write-only
-      field, the value is generated by Vertex AI when the model is uploaded.
-
-    endpoint_name: Full resource path of the created endpoint. This is a
-      write-only field, the value is generated by Vertex AI when the model is
-      deployed to an endpoint.
-
-    machine_type: Machine type to use when deploying the model.
-    """
-
-    config_file: str = "config.json"
-    base_image: str = "python:3.10-bullseye"
-    image: str = "timm_predictor:latest"
-    artifact_local_dir: str = ""
-    region: str = "us-central1"
-    project_id: str = "<your project ID here>"
-    repository: str = "cpr-images"
-    artifact_gcs_dir: str = "gs://<your bucket ID here>/timm-vit224/"
-    model_name: str = ""
-    endpoint_name: str = ""
-    machine_type: str = "n1-standard-2"
-
-    def save(self):
-        with open(self.config_file, "w") as f:
-            json.dump(dataclasses.asdict(self), f, indent=2)
-
-    def load(self):
-        with open(self.config_file) as f:
-            self.__init__(**json.load(f))

community-content/cpr-examples/timm_serving/requirements.txt

@@ -1,8 +0,0 @@
-absl-py==1.1.0
-fastapi==0.75.2
-uvicorn==0.18.2
-timm==0.5.4
-smart_open==6.0.0
-
-google-cloud-storage>=1.26.0,<2.0.0dev
-google-cloud-aiplatform[prediction]>=1.16.0

community-content/cpr-examples/timm_serving/test.py

@@ -1,255 +0,0 @@
-"""Test the timm_serving predictor."""
-import base64
-import json
-import logging
-import os
-import pickle
-from typing import List, Dict
-
-from absl import flags
-from absl import logging
-from absl.testing import absltest
-from config import CPRConfig
-import fastapi
-from google.cloud import aiplatform
-from google.cloud.aiplatform import prediction as cpr
-import PIL
-from timm_serving import predictor
-import torch
-
-VIT_SMALL_PARAMS = 22878952
-
-
-def b64_encode_file(path: str) -> str:
-    """Encode a file's contents as base64.
-
-    Args:
-      path: Path to the file.
-
-    Returns:
-      Base64-encoded contents of the file.
-    """
-    with open(path, "rb") as f:
-        return str(base64.b64encode(f.read()), encoding="utf-8")
-
-
-def make_instance_dict(
-    image_paths: List[str], base64_encodings: List[str]
-) -> Dict[str, List[str]]:
-    """Generate a dictionary similar to a parsed prediction server request.
-
-    Args:
-      image_paths: Paths to image files to include.
-      base64_encodings: Pre-encoded base64 strings.
-
-    Returns:
-      Dictionary of instances in the format accepted by the preprocessor.
-    """
-    instances = [s for s in base64_encodings]
-    for path in image_paths:
-        instances.append(b64_encode_file(path))
-    return {"instances": instances}
-
-
-def count_parameters(model: torch.nn.Module):
-    """Count the parameters in a Pytorch model.
-
-    Args:
-      model: Pytorch model (nn.Module).
-
-    Returns:
-      Number of parameters in the model.
-
-    """
-    return sum(p.numel() for p in model.parameters())
-
-
-class PredictorUnitTests(absltest.TestCase):
-    """Unit tests for timm_serving.predictor."""
-
-    def setUp(self):
-        super().setUp()
-        self.config = CPRConfig()
-        try:
-            self.config.load()
-        except FileNotFoundError:
-            logging.info("No saved config file found, using default values.")
-        self.predictor = predictor.TimmPredictor()
-
-    def test_load_from_saved_state_dict_ok(self):
-        self.predictor.load(self.config.artifact_local_dir)
-        self.assertEqual(count_parameters(self.predictor._model), VIT_SMALL_PARAMS)
-
-    def test_load_bad_path(self):
-        with self.assertRaises(FileNotFoundError):
-            self.predictor.load("testdata/")
-        with self.assertRaisesRegex(ValueError, "not a directory"):
-            self.predictor.load("blah")
-
-    def test_load_bad_data(self):
-        with self.assertRaises(pickle.UnpicklingError):
-            self.predictor.load("testdata/bad_model_1")
-        with self.assertRaisesRegex(RuntimeError, "Invalid magic number"):
-            self.predictor.load("testdata/bad_model_2")
-
-    def test_preprocess_ok(self):
-        self.predictor.load(self.config.artifact_local_dir)
-        instance_dict = make_instance_dict(
-            base64_encodings=[],
-            image_paths=[
-                "testdata/airplane.jpg",
-                "testdata/mandrill.tiff",
-                "testdata/mandrill.tiff",
-                "testdata/cat_alpha.png",
-            ],
-        )
-        result = self.predictor.preprocess(instance_dict)
-        self.assertEqual(result.size(), torch.Size([4, 3, 224, 224]))
-        self.assertEqual(result.dtype, torch.float32)
-
-    def test_preprocess_no_instances(self):
-        self.predictor.load(self.config.artifact_local_dir)
-        with self.assertRaises(fastapi.HTTPException) as ctx:
-            self.predictor.preprocess({})
-        self.assertEqual(ctx.exception.status_code, 400)
-        self.assertRegex(ctx.exception.detail, 'must contain "instances"')
-
-    def test_preprocess_wrong_shape_instances(self):
-        self.predictor.load(self.config.artifact_local_dir)
-        instance_dict = {"instances": [[b64_encode_file("testdata/mandrill.tiff")]]}
-        with self.assertRaises(fastapi.HTTPException) as ctx:
-            self.predictor.preprocess(instance_dict)
-        self.assertEqual(ctx.exception.status_code, 400)
-        self.assertRegex(ctx.exception.detail, "not 'list'")
-
-    def test_preprocess_bad_base64(self):
-        self.predictor.load(self.config.artifact_local_dir)
-        instance_dict = make_instance_dict(base64_encodings=["!@#$"], image_paths=[])
-        with self.assertRaises(fastapi.HTTPException) as ctx:
-            self.predictor.preprocess(instance_dict)
-        self.assertEqual(ctx.exception.status_code, 400)
-        self.assertRegex(ctx.exception.detail, "[Bb]ase64")
-
-    def test_preprocess_not_image_data(self):
-        self.predictor.load(self.config.artifact_local_dir)
-        instance_dict = make_instance_dict(
-            base64_encodings=[], image_paths=["testdata/bad.jpg"]
-        )
-        with self.assertRaises(fastapi.HTTPException) as ctx:
-            self.predictor.preprocess(instance_dict)
-        self.assertEqual(ctx.exception.status_code, 400)
-        self.assertRegex(ctx.exception.detail, "image file")
-
-    def test_predict_ok(self):
-        self.predictor.load(self.config.artifact_local_dir)
-        inputs = torch.zeros(size=[2, 3, 224, 224], dtype=torch.float32)
-        if torch.cuda.device_count() > 0:
-            inputs = inputs.cuda()
-        result = self.predictor.predict(inputs)
-        self.assertEqual(result.size(), torch.Size([2, 1000]))
-        self.assertEqual(result.dtype, torch.float32)
-
-    def test_postprocess_ok(self):
-        class_probs = torch.zeros(size=[2, 1000])
-        class_probs[0, 0] = 1
-        class_probs[1, 123] = 1
-        result = self.predictor.postprocess(class_probs)
-        predictions = result["predictions"]
-        self.assertLen(predictions[0]["class_names"], 5)
-        self.assertLen(predictions[0]["indices"], 5)
-        self.assertLen(predictions[0]["probabilities"], 5)
-        self.assertLen(predictions[1]["class_names"], 5)
-        self.assertLen(predictions[1]["indices"], 5)
-        self.assertLen(predictions[1]["probabilities"], 5)
-        self.assertContainsSubsequence(predictions[0]["class_names"][0], "tench")
-        self.assertContainsSubsequence(
-            predictions[1]["class_names"][0], "spiny lobster"
-        )
-
-
-class ServerEndToEndTests(absltest.TestCase):
-    """End-to-end tests for the model server, using LocalEndpoint."""
-
-    def setUp(self):
-        super().setUp()
-        self.config = CPRConfig()
-        try:
-            self.config.load()
-        except FileNotFoundError:
-            logging.info("No saved config file found, using default values.")
-        self.local_model = cpr.LocalModel(
-            serving_container_spec=aiplatform.gapic.ModelContainerSpec(
-                image_uri=self.config.image
-            )
-        )
-
-        self.local_endpoint = self.local_model.deploy_to_local_endpoint(
-            artifact_uri=self.config.artifact_local_dir or os.getcwd()
-        )
-        self.local_endpoint.serve()
-
-    def tearDown(self):
-        self.local_endpoint.stop()
-        super().tearDown()
-
-    def test_e2e_healthcheck_ok(self):
-        health_check_response = self.local_endpoint.run_health_check()
-        self.assertEqual(health_check_response.status_code, 200)
-        self.assertEqual(health_check_response.content, b"{}")
-
-    def test_e2e_predict_ok(self):
-        predict_request = json.dumps(
-            make_instance_dict(
-                base64_encodings=[],
-                image_paths=[
-                    "testdata/mandrill.tiff",
-                ],
-            )
-        )
-        response = self.local_endpoint.predict(
-            request=predict_request, headers={"Content-Type": "application/json"}
-        )
-        logging.info(response.content)
-        self.assertEqual(response.status_code, 200)
-        predictions = response.json()["predictions"]
-        self.assertContainsSubsequence(predictions[0]["class_names"][0], "baboon")
-
-    def test_e2e_predict_bad_json_returns_400(self):
-        predict_request = "blah"
-        response = self.local_endpoint.predict(
-            request=predict_request, headers={"Content-Type": "application/json"}
-        )
-        logging.info(response.content)
-        self.assertEqual(response.status_code, 400)
-
-    def test_e2e_predict_no_instances_returns_400(self):
-        predict_request = json.dumps({})
-        response = self.local_endpoint.predict(
-            request=predict_request, headers={"Content-Type": "application/json"}
-        )
-        logging.info(response.content)
-        self.assertEqual(response.status_code, 400)
-
-    def test_e2e_predict_bad_base64_returns_400(self):
-        predict_request = json.dumps(
-            make_instance_dict(base64_encodings=["blah"], image_paths=[])
-        )
-        response = self.local_endpoint.predict(
-            request=predict_request, headers={"Content-Type": "application/json"}
-        )
-        logging.info(response.content)
-        self.assertEqual(response.status_code, 400)
-
-    def test_e2e_predict_bad_image_returns_400(self):
-        predict_request = json.dumps(
-            make_instance_dict(base64_encodings=[], image_paths=["testdata/bad.jpg"])
-        )
-        response = self.local_endpoint.predict(
-            request=predict_request, headers={"Content-Type": "application/json"}
-        )
-        logging.info(response.content)
-        self.assertEqual(response.status_code, 400)
-
-
-if __name__ == "__main__":
-    absltest.main()

community-content/cpr-examples/timm_serving/testdata/bad.jpg

@@ -1 +0,0 @@
-some non-image data

community-content/cpr-examples/timm_serving/testdata/bad.txt

@@ -1 +0,0 @@
-some non-image data

community-content/cpr-examples/timm_serving/testdata/bad_model_1/state_dict.pth

@@ -1 +0,0 @@
-blah

community-content/cpr-examples/timm_serving/timm_serving/predictor.py

@@ -1,178 +0,0 @@
-# Copyright 2022 Google LLC
-
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-
-#     https://www.apache.org/licenses/LICENSE-2.0
-
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Adapts a pretrained TIMM image classification model to the CPR framework.
-
-Documentation for the TIMM (Torch IMage Models) library is here:
-  https://rwightman.github.io/pytorch-image-models/
-
-Its source can also be found here:
-  https://github.com/rwightman/pytorch-image-models
-"""
-
-import base64
-import binascii
-import io
-import os
-from typing import Dict, List, Union
-
-from fastapi import HTTPException
-from google.cloud.aiplatform import prediction as cpr
-from pathlib import Path
-import PIL
-import smart_open
-import timm
-import torch
-import torch.nn.functional as F
-
-with open(Path(__file__).parent.absolute().joinpath("imagenet.txt")) as f:
-    IMAGENET_CLASSES = f.read().splitlines()
-
-
-class TimmPredictor(cpr.predictor.Predictor):
-    """Predictor class for image models based on TIMM."""
-
-    TIMM_MODEL_NAME = os.getenv("TIMM_MODEL_NAME", default="vit_small_patch32_224")
-    WEIGHTS_FILE = "state_dict.pth"
-    NUM_TOP_CLASSES_TO_RETURN = 5
-
-    def __init__(self):
-        self._cuda = torch.cuda.device_count() > 0
-
-    def load(self, artifacts_uri: str = ""):
-        """Initializes the model and preprocessing transforms.
-
-        Args:
-          artifacts_uri: Directory where state dict is stored. Can be a
-          GCS URI or local path.
-        """
-        if artifacts_uri:
-            artifact_path = os.path.join(artifacts_uri)
-            if not (os.path.isdir(artifact_path) or artifact_path.startswith("gs://")):
-                raise ValueError("Provided artifact_uri is not a directory.")
-        else:
-            artifact_path = os.getcwd()
-
-        artifact_path = os.path.join(artifact_path, self.WEIGHTS_FILE)
-        with smart_open.open(artifact_path, "rb") as f:
-            self._model = torch.load(f)
-
-        if self._cuda:
-            self._model.cuda()
-
-        config = timm.data.resolve_data_config(model=self.TIMM_MODEL_NAME, args=[])
-        self._transform = timm.data.create_transform(
-            is_training=False, use_prefetcher=False, **config
-        )
-
-    def preprocess(self, request_dict: Dict[str, List[str]]) -> torch.Tensor:
-        """Performs preprocessing.
-
-        By default, the server expects a request body consisting of a valid JSON
-        object. This will be parsed by the handler before it's evaluated by the
-        preprocess method.
-
-        Args:
-          request_dict: Parsed request body. We expect that the input consists of
-            a list of base64-encoded image files under the "instances" key. (Any
-            image format that PIL.image.open can handle is okay.)
-
-        Returns:
-          torch.Tensor containing the preprocessed images as a batch. If GPU is
-          available, the result tensor will be stored on GPU.
-        """
-
-        if "instances" not in request_dict:
-            raise HTTPException(
-                status_code=400,
-                detail='Request must contain "instances" as a top-level key.',
-            )
-
-        tensors = []
-
-        for (i, image) in enumerate(request_dict["instances"]):
-            # We use Base64 encoding to handle image data.
-            # This is probably the best we can do while still using JSON input.
-            # Overriding the input format requires building a custom Handler.
-            try:
-                image_bytes = base64.b64decode(image, validate=True)
-            except (binascii.Error, TypeError) as e:
-                raise HTTPException(
-                    status_code=400,
-                    detail=f"Base64 decoding of the input image at index {i} failed:"
-                    f" {str(e)}",
-                )
-
-            try:
-                pil_image = PIL.Image.open(io.BytesIO(image_bytes)).convert("RGB")
-            except PIL.UnidentifiedImageError:
-                raise HTTPException(
-                    status_code=400,
-                    detail=f"The input image at index {i} could not be identified as an"
-                    " image file.",
-                )
-
-            tensors.append(self._transform(pil_image))
-
-        with torch.inference_mode():
-            result = torch.stack(tensors)
-            if self._cuda:
-                result = result.cuda()
-        return result
-
-    def predict(self, instances: torch.Tensor) -> torch.Tensor:
-        """Performs prediction.
-
-        Args:
-          instances: torch.Tensor with type torch.float32 and shape
-            [?, 3, 224, 224], containing the pre-processed input images.
-
-        Returns:
-          Vector of scores with type torch.float32 and shape [?, 1000],
-            representing the model's estimate of the likelihood that the
-            input belongs to the Imagenet class with that index.
-        """
-        with torch.inference_mode():
-            class_scores = self._model(instances)
-        return class_scores
-
-    def postprocess(
-        self, class_scores: torch.Tensor
-    ) -> Dict[str, List[Dict[str, Union[str, int, float]]]]:
-        """Translate the model output into a classification result.
-
-        Args:
-          class_scores: torch.Tensor with type torch.float32 and shape
-            [?, 1000], containing the scores assigned to each class by
-            the model.
-
-        Returns:
-          Dictionary containing the list of classification results. Each
-            classification result contains the probabilities, class names, and
-            class indices of the classes with the top class scores as reported by
-            the model.
-        """
-        class_probs = F.softmax(class_scores, dim=1)
-        top_k = class_probs.topk(self.NUM_TOP_CLASSES_TO_RETURN)
-        top_k_values = top_k.values.numpy().tolist()
-        top_k_indices = top_k.indices.numpy().tolist()
-        predictions = [
-            dict(
-                probabilities=values,
-                indices=indices,
-                class_names=[IMAGENET_CLASSES[int(class_num)] for class_num in indices],
-            )
-            for (values, indices) in zip(top_k_values, top_k_indices)
-        ]
-        return {"predictions": predictions}

community-content/pytorch_image_classification_distributed_data_parallel_training_with_vertex_sdk/multi_node_ddp_gloo_vertex_training_with_custom_container.ipynb

@@ -0,0 +1,474 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "a6b56b1c7b76"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2021 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "c414a395a19b"
+      },
+      "source": [
+        "# PyTorch Image Classification Multi-Node Distributed Data Parallel Training on CPU using Vertex Training with Custom Container"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "b98238e32cf7"
+      },
+      "source": [
+        "<table align=\"left\">\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/community-content/pytorch_image_classification_distributed_data_parallel_training_with_vertex_sdk/multi_node_ddp_gloo_vertex_training_with_custom_container.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "03d216c7f7b1"
+      },
+      "source": [
+        "## Setup"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "c5ac73516218"
+      },
+      "outputs": [],
+      "source": [
+        "PROJECT_ID = \"YOUR PROJECT ID\"\n",
+        "BUCKET_NAME = \"gs://YOUR BUCKET NAME\"\n",
+        "REGION = \"YOUR REGION\"\n",
+        "SERVICE_ACCOUNT = \"YOUR SERVICE ACCOUNT\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "0b5ae674177e"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil ls -al $BUCKET_NAME"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "19a9b3bdd553"
+      },
+      "outputs": [],
+      "source": [
+        "content_name = \"pt-img-cls-multi-node-ddp-cust-cont\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "57bf6f8b4361"
+      },
+      "source": [
+        "## Local Training"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "e5d8a3443da0"
+      },
+      "outputs": [],
+      "source": [
+        "! ls trainer"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "07f79309472d"
+      },
+      "outputs": [],
+      "source": [
+        "! cat trainer/requirements.txt"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "e16cd8bb7483"
+      },
+      "outputs": [],
+      "source": [
+        "! pip install -r trainer/requirements.txt"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "0b8a210718c4"
+      },
+      "outputs": [],
+      "source": [
+        "! cat trainer/task.py"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "c0c6e7dfb3c6"
+      },
+      "outputs": [],
+      "source": [
+        "%run trainer/task.py --epochs 5 --no-cuda --local-mode"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "31dfdeede587"
+      },
+      "outputs": [],
+      "source": [
+        "! ls ./tmp"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "48d56ec621cc"
+      },
+      "outputs": [],
+      "source": [
+        "! rm -rf ./tmp"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8f3ea1210749"
+      },
+      "source": [
+        "## Vertex Training using Vertex SDK and Custom Container"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "93002a20a2a6"
+      },
+      "source": [
+        "### Build Custom Container"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "4130ce43fd08"
+      },
+      "outputs": [],
+      "source": [
+        "hostname = \"gcr.io\"\n",
+        "image_name = content_name\n",
+        "tag = \"latest\"\n",
+        "\n",
+        "custom_container_image_uri = f\"{hostname}/{PROJECT_ID}/{image_name}:{tag}\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2f1fc5b05240"
+      },
+      "outputs": [],
+      "source": [
+        "! cd trainer && docker build -t $custom_container_image_uri -f Dockerfile ."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b4f274f499ac"
+      },
+      "outputs": [],
+      "source": [
+        "! docker run --rm $custom_container_image_uri --epochs 5 --no-cuda --local-mode"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ee1a0a06d0b4"
+      },
+      "outputs": [],
+      "source": [
+        "! docker push $custom_container_image_uri"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cb763be12fc9"
+      },
+      "outputs": [],
+      "source": [
+        "! gcloud container images list --repository $hostname/$PROJECT_ID"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "10c8cc6b3334"
+      },
+      "source": [
+        "### Initialize Vertex SDK"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "1a12348169fa"
+      },
+      "outputs": [],
+      "source": [
+        "! pip install -r requirements.txt"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "42e981cefe41"
+      },
+      "outputs": [],
+      "source": [
+        "from google.cloud import aiplatform\n",
+        "\n",
+        "aiplatform.init(\n",
+        "    project=PROJECT_ID,\n",
+        "    staging_bucket=BUCKET_NAME,\n",
+        "    location=REGION,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "73c92c9298e9"
+      },
+      "source": [
+        "### Create a Vertex Tensorboard Instance"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "bde509558cd5"
+      },
+      "outputs": [],
+      "source": [
+        "content_name = content_name + \"-cpu\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6d7908c0083c"
+      },
+      "outputs": [],
+      "source": [
+        "tensorboard = aiplatform.Tensorboard.create(\n",
+        "    display_name=content_name,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "a1f0a4f54037"
+      },
+      "source": [
+        "#### Option: Use a Previously Created Vertex Tensorboard Instance\n",
+        "\n",
+        "```\n",
+        "tensorboard_name = \"Your Tensorboard Resource Name or Tensorboard ID\"\n",
+        "tensorboard = aiplatform.Tensorboard(tensorboard_name=tensorboard_name)\n",
+        "```"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "a4cac84e04ac"
+      },
+      "source": [
+        "### Run a Vertex SDK CustomContainerTrainingJob"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "f92e8fdd44ee"
+      },
+      "outputs": [],
+      "source": [
+        "display_name = content_name\n",
+        "gcs_output_uri_prefix = f\"{BUCKET_NAME}/{display_name}\"\n",
+        "\n",
+        "replica_count = 4\n",
+        "machine_type = \"n1-standard-4\"\n",
+        "\n",
+        "args = [\n",
+        "    \"--backend\",\n",
+        "    \"gloo\",\n",
+        "    \"--no-cuda\",\n",
+        "    \"--batch-size\",\n",
+        "    \"128\",\n",
+        "    \"--epochs\",\n",
+        "    \"25\",\n",
+        "]"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ae4c57df7e07"
+      },
+      "outputs": [],
+      "source": [
+        "custom_container_training_job = aiplatform.CustomContainerTrainingJob(\n",
+        "    display_name=display_name,\n",
+        "    container_uri=custom_container_image_uri,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "35cf3ecdf0df"
+      },
+      "outputs": [],
+      "source": [
+        "custom_container_training_job.run(\n",
+        "    args=args,\n",
+        "    base_output_dir=gcs_output_uri_prefix,\n",
+        "    replica_count=replica_count,\n",
+        "    machine_type=machine_type,\n",
+        "    tensorboard=tensorboard.resource_name,\n",
+        "    service_account=SERVICE_ACCOUNT,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "49d10dded73b"
+      },
+      "outputs": [],
+      "source": [
+        "print(f\"Custom Training Job Name: {custom_container_training_job.resource_name}\")\n",
+        "print(f\"GCS Output URI Prefix: {gcs_output_uri_prefix}\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "78398f52807b"
+      },
+      "source": [
+        "### Training Output Artifact"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "fc74422de1d1"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil ls $gcs_output_uri_prefix"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5e99a6a05b10"
+      },
+      "source": [
+        "## Clean Up Artifact"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b0c1b3f7466b"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil rm -rf $gcs_output_uri_prefix"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "multi_node_ddp_gloo_vertex_training_with_custom_container.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

community-content/pytorch_image_classification_distributed_data_parallel_training_with_vertex_sdk/multi_node_ddp_nccl_vertex_training_with_custom_container.ipynb

@@ -0,0 +1,347 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "a6b56b1c7b76"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2021 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "20a5ea0081d0"
+      },
+      "source": [
+        "# PyTorch Image Classification Multi-Node Distributed Data Parallel Training on GPU using Vertex Training with Custom Container"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8752d4a255fb"
+      },
+      "source": [
+        "<table align=\"left\">\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/community-content/pytorch_image_classification_distributed_data_parallel_training_with_vertex_sdk/multi_node_ddp_nccl_vertex_training_with_custom_container.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "03d216c7f7b1"
+      },
+      "source": [
+        "## Setup"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "c5ac73516218"
+      },
+      "outputs": [],
+      "source": [
+        "PROJECT_ID = \"YOUR PROJECT ID\"\n",
+        "BUCKET_NAME = \"gs://YOUR BUCKET NAME\"\n",
+        "REGION = \"YOUR REGION\"\n",
+        "SERVICE_ACCOUNT = \"YOUR SERVICE ACCOUNT\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "0b5ae674177e"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil ls -al $BUCKET_NAME"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "19a9b3bdd553"
+      },
+      "outputs": [],
+      "source": [
+        "content_name = \"pt-img-cls-multi-node-ddp-cust-cont\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5307fe28b633"
+      },
+      "source": [
+        "## Vertex Training using Vertex SDK and Custom Container"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "46cb58c7fbf9"
+      },
+      "source": [
+        "### Built Custom Container"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "97e66e9f9bab"
+      },
+      "outputs": [],
+      "source": [
+        "hostname = \"gcr.io\"\n",
+        "image_name = content_name\n",
+        "tag = \"latest\"\n",
+        "\n",
+        "custom_container_image_uri = f\"{hostname}/{PROJECT_ID}/{image_name}:{tag}\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ae9b29c4773f"
+      },
+      "source": [
+        "### Initialize Vertex SDK"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "dc1e84d5dec2"
+      },
+      "outputs": [],
+      "source": [
+        "! pip install -r requirements.txt"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6964be27b98e"
+      },
+      "outputs": [],
+      "source": [
+        "from google.cloud import aiplatform\n",
+        "\n",
+        "aiplatform.init(\n",
+        "    project=PROJECT_ID,\n",
+        "    staging_bucket=BUCKET_NAME,\n",
+        "    location=REGION,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "594a91f438f2"
+      },
+      "source": [
+        "### Create a Vertex Tensorboard Instance"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "93134273261e"
+      },
+      "outputs": [],
+      "source": [
+        "content_name = content_name + \"-gpu\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "c2bd82dbcd9b"
+      },
+      "outputs": [],
+      "source": [
+        "tensorboard = aiplatform.Tensorboard.create(\n",
+        "    display_name=content_name,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ebc593c6472e"
+      },
+      "source": [
+        "#### Option: Use a Previously Created Vertex Tensorboard Instance\n",
+        "\n",
+        "```\n",
+        "tensorboard_name = \"Your Tensorboard Resource Name or Tensorboard ID\"\n",
+        "tensorboard = aiplatform.Tensorboard(tensorboard_name=tensorboard_name)\n",
+        "```"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0769e8e34c2f"
+      },
+      "source": [
+        "### Run a Vertex SDK CustomContainerTrainingJob"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "023f33ece826"
+      },
+      "outputs": [],
+      "source": [
+        "display_name = content_name\n",
+        "gcs_output_uri_prefix = f\"{BUCKET_NAME}/{display_name}\"\n",
+        "\n",
+        "replica_count = 1\n",
+        "machine_type = \"n1-standard-4\"\n",
+        "accelerator_count = 4\n",
+        "accelerator_type = \"NVIDIA_TESLA_K80\"\n",
+        "\n",
+        "args = [\n",
+        "    \"--backend\",\n",
+        "    \"nccl\",\n",
+        "    \"--batch-size\",\n",
+        "    \"128\",\n",
+        "    \"--epochs\",\n",
+        "    \"25\",\n",
+        "]"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "d4b599e726ef"
+      },
+      "outputs": [],
+      "source": [
+        "custom_container_training_job = aiplatform.CustomContainerTrainingJob(\n",
+        "    display_name=display_name,\n",
+        "    container_uri=custom_container_image_uri,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "81321e3bdf7f"
+      },
+      "outputs": [],
+      "source": [
+        "custom_container_training_job.run(\n",
+        "    args=args,\n",
+        "    base_output_dir=gcs_output_uri_prefix,\n",
+        "    replica_count=replica_count,\n",
+        "    machine_type=machine_type,\n",
+        "    accelerator_count=accelerator_count,\n",
+        "    accelerator_type=accelerator_type,\n",
+        "    tensorboard=tensorboard.resource_name,\n",
+        "    service_account=SERVICE_ACCOUNT,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "5100712c2c4c"
+      },
+      "outputs": [],
+      "source": [
+        "print(f\"Custom Training Job Name: {custom_container_training_job.resource_name}\")\n",
+        "print(f\"GCS Output URI Prefix: {gcs_output_uri_prefix}\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "f9b77676e5a6"
+      },
+      "source": [
+        "### Training Output Artifact"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "0e171ce95ace"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil ls $gcs_output_uri_prefix"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "cf1b74a12b87"
+      },
+      "source": [
+        "## Clean Up Artifact"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "a0b15089c341"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil rm -rf $gcs_output_uri_prefix"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "multi_node_ddp_nccl_vertex_training_with_custom_container.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

community-content/pytorch_pre_built_images_deployment/README.md

@@ -1,30 +0,0 @@
-# PyTorch Deployment on Google Cloud: Text Classification
-
-**This is an Experimental release**, covered by the Pre-GA Offerings Terms of your Google Cloud Platform [Terms of Service](https://cloud.google.com/terms).
-
-Experiments are focused on validating a prototype and are not guaranteed to be released. They are not intended for production use or covered by any SLA, support obligation, or deprecation policy and might be subject to backward-incompatible changes.
-
-**Kindly drop us a note before you run any scale tests.**
-
-**Do not hesitate to contact vertexai-prediction-preview-feedback@google.com if you have any questions or run into any issues.**
-
-The projects need to be added to the allowlist in order to deploy PyTorch models using Vertex AI Prediction pre-built PyTorch images. If you are interested in the feature, please send an email to vertexai-prediction-preview-feedback@google.com to provide your project numbers OR project ids.
-
-## Overview
-
-In the PyTorch on Google Cloud series of blog posts, we aim to share how to deploy PyTorch models at scale on [Vertex AI](https://cloud.google.com/vertex-ai).
-
-This tutorial on text classification shows how to deploy a PyTorch based text classification model on [Vertex AI](https://cloud.google.com/vertex-ai/docs/start/client-libraries#python) using Vertex SDK and [`gcloud ai`](https://cloud.google.com/sdk/gcloud/reference/beta/ai).
-
-## Notebooks
-
-| <h4>Notebook</h4> | <h4>Description</h4>                       |
-| :-------- | :------- |
-| [pytorch-text-classification-vertex-ai-deploy.ipynb](./pytorch-text-classification-vertex-ai-deploy.ipynb) | Notebook to show deploying a PyTorch model on Vertex AI |
-
-## Folders
-
-
-| <h4>Folder Name</h4> | <h4>Description</h4>                       |
-| :-------- | :------- |
-| [`predictor`](./predictor) | Folder with custom prediction handler to deploy a PyTorch model to Vertex Prediction. In the [notebook](./pytorch-text-classification-vertex-ai-deploy.ipynb), this folder is used for deploying a PyTorch model on Vertex AI using Vertex Prediction pre-built PyTorch images |

community-content/pytorch_pre_built_images_deployment/predictor/custom_handler.py

@@ -1,91 +0,0 @@
-
-import os
-import json
-import logging
-
-import torch
-from transformers import AutoModelForSequenceClassification, AutoTokenizer
-from ts.torch_handler.base_handler import BaseHandler
-
-logger = logging.getLogger(__name__)
-
-
-class TransformersClassifierHandler(BaseHandler):
-    """
-    The handler takes an input string and returns the classification text 
-    based on the serialized transformers checkpoint.
-    """
-    def __init__(self):
-        super(TransformersClassifierHandler, self).__init__()
-        self.initialized = False
-
-    def initialize(self, ctx):
-        """ Loads the model.pt file and initialized the model object.
-        Instantiates Tokenizer for preprocessor to use
-        Loads labels to name mapping file for post-processing inference response
-        """
-        self.manifest = ctx.manifest
-
-        properties = ctx.system_properties
-        model_dir = properties.get("model_dir")
-        self.device = torch.device("cuda:" + str(properties.get("gpu_id")) if torch.cuda.is_available() else "cpu")
-
-        # Read model serialize/pt file
-        serialized_file = self.manifest["model"]["serializedFile"]
-        model_pt_path = os.path.join(model_dir, serialized_file)
-        if not os.path.isfile(model_pt_path):
-            raise RuntimeError("Missing the model.pt or pytorch_model.bin file")
-        
-        # Load model
-        self.model = AutoModelForSequenceClassification.from_pretrained(model_dir)
-        self.model.to(self.device)
-        self.model.eval()
-        logger.debug('Transformer model from path {0} loaded successfully'.format(model_dir))
-        
-        # Ensure to use the same tokenizer used during training
-        self.tokenizer = AutoTokenizer.from_pretrained('bert-base-cased')
-
-        # Read the mapping file, index to object name
-        mapping_file_path = os.path.join(model_dir, "index_to_name.json")
-
-        if os.path.isfile(mapping_file_path):
-            with open(mapping_file_path) as f:
-                self.mapping = json.load(f)
-        else:
-            logger.warning('Missing the index_to_name.json file. Inference output will default.')
-            self.mapping = {"0": "Negative",  "1": "Positive"}
-
-        self.initialized = True
-
-    def preprocess(self, data):
-        """ Preprocessing input request by tokenizing
-            Extend with your own preprocessing steps as needed
-        """
-        text = data[0].get("data")
-        if text is None:
-            text = data[0].get("body")
-        sentences = text.decode('utf-8')
-        logger.info("Received text: '%s'", sentences)
-
-        # Tokenize the texts
-        tokenizer_args = ((sentences,))
-        inputs = self.tokenizer(*tokenizer_args,
-                                padding='max_length',
-                                max_length=128,
-                                truncation=True,
-                                return_tensors = "pt")
-        return inputs
-
-    def inference(self, inputs):
-        """ Predict the class of a text using a trained transformer model.
-        """
-        prediction = self.model(inputs['input_ids'].to(self.device))[0].argmax().item()
-
-        if self.mapping:
-            prediction = self.mapping[str(prediction)]
-
-        logger.info("Model predicted: '%s'", prediction)
-        return [prediction]
-
-    def postprocess(self, inference_output):
-        return inference_output

community-content/pytorch_pre_built_images_deployment/predictor/index_to_name.json

@@ -1,5 +0,0 @@
-
-{
-    "0": "Negative", 
-    "1": "Positive"
-}

community-content/pytorch_text_classification_using_vertex_sdk_and_gcloud/pytorch-text-classification-vertex-ai-train-tune-deploy.ipynb

@@ -658,8 +658,8 @@
       },
       "outputs": [],
       "source": [
-        "dataset = load_dataset(\"imdb\")\n",
-        "dataset"
+        "datasets = load_dataset(\"imdb\")\n",
+        "datasets"
       ]
     },
     {
@@ -668,7 +668,7 @@
         "id": "RzfPtOMoIrIu"
       },
       "source": [
-        "The `dataset` object itself is [`DatasetDict`](https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasetdict), which contains one key for the training, validation and test set."
+        "The `datasets` object itself is [`DatasetDict`](https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasetdict), which contains one key for the training, validation and test set."
       ]
     },
     {
@@ -681,12 +681,12 @@
       "source": [
         "print(\n",
         "    \"Total # of rows in training dataset {} and size {:5.2f} MB\".format(\n",
-        "        dataset[\"train\"].shape[0], dataset[\"train\"].size_in_bytes / (1024 * 1024)\n",
+        "        datasets[\"train\"].shape[0], datasets[\"train\"].size_in_bytes / (1024 * 1024)\n",
         "    )\n",
         ")\n",
         "print(\n",
         "    \"Total # of rows in test dataset {} and size {:5.2f} MB\".format(\n",
-        "        dataset[\"test\"].shape[0], dataset[\"test\"].size_in_bytes / (1024 * 1024)\n",
+        "        datasets[\"test\"].shape[0], datasets[\"test\"].size_in_bytes / (1024 * 1024)\n",
         "    )\n",
         ")"
       ]
@@ -708,7 +708,7 @@
       },
       "outputs": [],
       "source": [
-        "dataset[\"train\"][0]"
+        "datasets[\"train\"][0]"
       ]
     },
     {
@@ -728,7 +728,7 @@
       },
       "outputs": [],
       "source": [
-        "label_list = dataset[\"train\"].unique(\"label\")\n",
+        "label_list = datasets[\"train\"].unique(\"label\")\n",
         "label_list"
       ]
     },
@@ -779,7 +779,7 @@
       },
       "outputs": [],
       "source": [
-        "show_random_elements(dataset[\"train\"])"
+        "show_random_elements(datasets[\"train\"])"
       ]
     },
     {
@@ -883,7 +883,7 @@
       },
       "outputs": [],
       "source": [
-        "example = dataset[\"train\"][4]\n",
+        "example = datasets[\"train\"][4]\n",
         "print(example)"
       ]
     },
@@ -920,7 +920,7 @@
       "source": [
         "# Dataset loading repeated here to make this cell idempotent\n",
         "# Since we are over-writing datasets variable\n",
-        "dataset = load_dataset(\"imdb\")\n",
+        "datasets = load_dataset(\"imdb\")\n",
         "\n",
         "# Mapping labels to ids\n",
         "# NOTE: We can extract this automatically but the `Unique` method of the datasets\n",
@@ -948,7 +948,7 @@
         "\n",
         "\n",
         "# apply preprocessing function to input examples\n",
-        "dataset = dataset.map(preprocess_function, batched=True, load_from_cache_file=True)"
+        "datasets = datasets.map(preprocess_function, batched=True, load_from_cache_file=True)"
       ]
     },
     {
@@ -1091,8 +1091,8 @@
         "trainer = Trainer(\n",
         "    model,\n",
         "    args,\n",
-        "    train_dataset=dataset[\"train\"],\n",
-        "    eval_dataset=dataset[\"test\"],\n",
+        "    train_dataset=datasets[\"train\"],\n",
+        "    eval_dataset=datasets[\"test\"],\n",
         "    data_collator=default_data_collator,\n",
         "    tokenizer=tokenizer,\n",
         "    compute_metrics=compute_metrics,\n",

community-content/tf_agents_bandits_movie_recommendation_with_kfp_and_vertex_sdk/mlops_pipeline_tf_agents_bandits_movie_recommendation/src/logger/requirements.txt

@@ -1,4 +1,4 @@
 google-cloud-bigquery==2.20.0
-tensorflow==2.7.2
+tensorflow==2.5.3
 pillow==9.0.1
 tf-agents==0.8.0

community-content/tf_agents_bandits_movie_recommendation_with_kfp_and_vertex_sdk/mlops_pipeline_tf_agents_bandits_movie_recommendation/src/prediction_container/requirements.txt

@@ -1,4 +1,4 @@
 google-cloud-pubsub==2.5.0
 pillow==9.0.1
 tf-agents==0.8.0
-tensorflow==2.7.2
+tensorflow==2.5.3

community-content/tf_agents_bandits_movie_recommendation_with_kfp_and_vertex_sdk/mlops_pipeline_tf_agents_bandits_movie_recommendation/src/simulator/requirements.txt

@@ -1,5 +1,5 @@
 dataclasses==0.6
 google-cloud-aiplatform==1.8.1
-tensorflow==2.7.2
+tensorflow==2.5.3
 pillow==9.0.1
 tf-agents==0.8.0

community-content/tf_keras_image_classification_distributed_multi_worker_with_vertex_sdk/trainer/requirements.txt

@@ -1 +1 @@
-tensorflow==2.7.2
+tensorflow==2.5.3

community-content/tf_keras_text_classification_distributed_single_worker_gpus_with_gcloud_local_run_and_vertex_sdk/trainer/requirements.txt

@@ -1 +1 @@
-tensorflow==2.7.2
+tensorflow==2.5.3

notebooks/README.md

@@ -1,5 +1,5 @@
-The [official](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/notebooks/official) folder contains notebooks organized by Google Cloud product. These are tested weekly and maintained by Google.
+The [official](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/notebooks/official) folder contains notebooks organized by Google Cloud product.
 
-The [community](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/notebooks/community) folder contains notebooks that may be created by Google or external contributors. They are not necessary maintained.
+The [community](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/notebooks/community) folder contains notebooks that aren't officially supported by Google.
 
 Contributions to the repo should use the [notebook template](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/notebook_template.ipynb) as a starting point.

notebooks/community/CODEOWNERS

@@ -5,32 +5,19 @@
 
 /sdk/sdk_* @andrewferlitsch
 /gapic @andrewferlitsch
-/gapic/custom/showcase_custom_image_classification_online_explain_example_based_api.ipynb @inardini
 /ml_ops @andrewferlitsch
-/model_monitoring/* @andrewferlitsch
+/model_monitoring/* @mco-gh
 /structured_data/rapid_prototyping_* @rafael-carvalho
 
 /managed_notebooks/
-/bigquery_ml/ @polong
 /sdk/SDK_FBProphet_Forecasting_Online.ipynb @brianchunkang
 /pipelines/google_cloud_pipeline_components_TPU_model_train_upload_deploy.ipynb @brianchunkang
-/explainable_ai/SDK_Custom_Container_XAI.ipynb @brianchunkang
 /matching_engine/sdk_matching_engine_for_indexing.ipynb @ivanmkc
 /matching_engine/matching_engine_for_indexing.ipynb @yinghsienwu
-/matching_engine/stream_update_for_matching_engine.ipynb @peterping666
 /sdk/pytorch_lightning_custom_container_training.ipynb @brianchunkang
 /tensorboard @yfang1
 /feature_store @nayaknishant @morgandu
-/prediction @googleapis/vertex-prediction-team
 /vertex_endpoints/tf_hub_obj_detection/deploy_tfhub_object_detection_on_vertex_endpoints.ipynb @entrpn
 /vertex_endpoints/nvidia-triton/nvidia-triton-custom-container-prediction.ipynb @RajeshThallam
 /vertex_endpoints/optimized_tensorflow_runtime @vlasenkoalexey
-/notebooks/community/ml_ops/stage2/get_started_with_visionapi_and_automl.ipynb @mansari
-/notebooks/community/neo4j/graph_paysim.ipynb @benofben @laeg
-/notebooks/community/ml_ops/stage1/get_started_with_visionapi_and_vertex_datasets.ipynb @mansari
-/notebooks/community/pipelines/google_cloud_pipeline_components_bqml_pipeline_demand_forecasting.ipynb @inardini
-/notebooks/community/ml_ops/stage2/get_started_vertex_hpt_r_kernel.ipynb @fhirschmann
-/notebooks/community/ml_ops/stage2/get_started_vertex_training_r_using_r_kernel.ipynb @fhirschmann
-/notebooks/community/vertex-ai-samples/notebooks/community/model_registry/vertex_ai_model_registry_bqml_custom_model_versioning.ipynb @inardini
-/notebooks/community/vertex-ai-samples/notebooks/community/model_registry/vertex_ai_model_registry_automl_model_versioning.ipynb @inardini
-/notebooks/community/vizier/conversions_vertex_vizier_and_open_source_vizier.ipynb @halio-g
+/notebooks/community/ml_ops/stage2/get_started_with_visionapi_and_automl.ipynb @mansari

notebooks/community/feature_store/mobile_gaming/mobile_gaming_feature_store.ipynb

@@ -32,18 +32,18 @@
         "<table align=\"left\">\n",
         "\n",
         "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/vertex-ai-samples/blob/main/notebooks/community/feature_store/mobile_gaming/mobile_gaming_feature_store.ipynb\">\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/notebook_template.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/feature_store/mobile_gaming/mobile_gaming_feature_store.ipynb\">\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/notebook_template.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/feature_store/mobile_gaming/mobile_gaming_feature_store.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/notebook_template.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
@@ -54,52 +54,52 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "tvgnzT1CKxrO"
+        "id": "7FZeBEwdXS4d"
       },
       "source": [
         "## Overview\n",
-        " \n",
-        "Imagine you are a member of the Data Science team working on the same Mobile Gaming application reported in the [Churn prediction for game developers using Google Analytics 4 (GA4) and BigQuery ML](https://cloud.google.com/blog/topics/developers-practitioners/churn-prediction-game-developers-using-google-analytics-4-ga4-and-bigquery-ml) blog post.\n",
-        " \n",
-        "Business wants to use that information in real-time to take immediate intervention actions in-game to prevent churn. In particular, for each player, they want to provide gaming incentives like new items or bonus packs depending on the customer demographic,  behavioral information and the resulting propensity of return.\n",
-        " \n",
-        "Last year, Google Cloud announced Vertex AI, a managed machine learning (ML) platform that allows data science teams to accelerate the deployment and maintenance of ML models. One of the platform building blocks is Vertex AI Feature store which provides a managed service for low latency scalable feature serving. Also it is a centralized feature repository with easy APIs to search & discover features and feature monitoring capabilities to track drift and other quality issues.\n",
-        " \n",
-        "In this notebook, we will show how the role of Vertex AI Feature Store in a ready to production scenario when the user's activities within the first 24 hours of last engagement and the gaming platform would consume in order to improve UX. Below you can find the high level picture of the system\n",
-        " \n",
+        "\n",
+        "Imagine you are a member of the Data Science team working on the same Mobile Gaming application reported in the [Churn prediction for game developers using Google Analytics 4 (GA4) and BigQuery ML](https://cloud.google.com/blog/topics/developers-practitioners/churn-prediction-game-developers-using-google-analytics-4-ga4-and-bigquery-ml) blog post. \n",
+        "\n",
+        "Business wants to use that information in real-time to take immediate intervention actions in-game to prevent churn. In particular, for each player, they want to provide gaming incentives like new items or bonus packs depending on the customer demographic,  behavioral information and the resulting propensity of return. \n",
+        "\n",
+        "Last year, Google Cloud announced Vertex AI, a managed machine learning (ML) platform that allows data science teams to accelerate the deployment and maintenance of ML models. One of the platform building blocks is Vertex AI Feature store which provides a managed service for low latency scalable feature serving. Also it is a centralized feature repository with easy APIs to search & discover features and feature monitoring capabilities to track drift and other quality issues. \n",
+        "\n",
+        "In this notebook, we will show how the role of Vertex AI Feature Store in a ready to production scenario when the user's activities within the first 24 hours of last engagment and the gaming platform would consume in order to improver UX. Below you can find the high level picture of the system\n",
+        "\n",
         "<img src=\"./assets/mobile_gaming_architecture_1.png\">\n",
-        " \n",
-        " \n",
+        "\n",
+        "\n",
         "### Dataset\n",
-        " \n",
+        "\n",
         "The dataset is the public sample export data from an actual mobile game app called \"Flood It!\" (Android, iOS)\n",
-        " \n",
+        "\n",
         "### Objective\n",
-        " \n",
+        "\n",
         "In the following notebook, you will learn how Vertex AI Feature store\n",
-        " \n",
-        "1.  Provide a centralized feature repository with easy APIs to search & discover features and fetch them for training/serving.\n",
-        " \n",
-        "2.  Simplify deployments of models for Online Prediction, via low latency scalable feature serving.\n",
-        " \n",
-        "3.  Mitigate training serving skew and data leakage by performing point in time lookups to fetch historical data for training.\n",
-        " \n",
+        "\n",
+        "1.   Provide a centralized feature repository with easy APIs to search & discover features and fetch them for training/serving. \n",
+        "\n",
+        "2.   Simplify deployments of models for Online Prediction, via low latency scalable feature serving.\n",
+        "\n",
+        "3.   Mitigate training serving skew and data leakage by performing point in time lookups to fetch historical data for training.\n",
+        "\n",
         "**Notice that we assume that already know how to set up a Vertex AI Feature store. In case you are not, please check out [this detailed notebook](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/feature_store/gapic-feature-store.ipynb).**\n",
-        " \n",
-        " \n",
-        "### Costs\n",
-        " \n",
+        "\n",
+        "\n",
+        "### Costs \n",
+        "\n",
         "This tutorial uses billable components of Google Cloud:\n",
-        " \n",
+        "\n",
         "* Vertex AI\n",
         "* BigQuery\n",
         "* Cloud Storage\n",
-        " \n",
+        "\n",
         "Learn about [Vertex AI\n",
         "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
         "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
         "Calculator](https://cloud.google.com/products/calculator/)\n",
-        "to generate a cost estimate based on your projected usage.\n"
+        "to generate a cost estimate based on your projected usage."
       ]
     },
     {
@@ -110,7 +110,7 @@
       "source": [
         "### Set up your local development environment\n",
         "\n",
-        "**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets\n",
+        "**If you are using Colab or Google Cloud Notebooks**, your environment already meets\n",
         "all the requirements to run this notebook. You can skip this step."
       ]
     },
@@ -159,7 +159,7 @@
       "source": [
         "### Install additional packages\n",
         "\n",
-        "Install additional package dependencies not installed in your notebook environment, such as XGBoost. Use the latest major GA version of each package."
+        "Install additional package dependencies not installed in your notebook environment, such as {XGBoost, AdaNet, or TensorFlow Hub TODO: Replace with relevant packages for the tutorial}. Use the latest major GA version of each package."
       ]
     },
     {
@@ -172,15 +172,12 @@
       "source": [
         "import os\n",
         "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
+        "# The Google Cloud Notebook product has specific requirements\n",
+        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
         "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "# Google Cloud Notebook requires dependencies to be installed with '--user'\n",
         "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "if IS_GOOGLE_CLOUD_NOTEBOOK:\n",
         "    USER_FLAG = \"--user\""
       ]
     },
@@ -188,11 +185,11 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "_vr6BYED_5my"
+        "id": "SzEo6DeE2GOP"
       },
       "outputs": [],
       "source": [
-        "! pip3 install {USER_FLAG} --upgrade pip -q\n",
+        "! pip3 install {USER_FLAG} --upgrade pip\n",
         "! pip3 install {USER_FLAG} --upgrade google-cloud-aiplatform==1.11.0 -q --no-warn-conflicts\n",
         "! pip3 install {USER_FLAG} git+https://github.com/googleapis/python-aiplatform.git@main # For features monitoring\n",
         "! pip3 install {USER_FLAG} --upgrade google-cloud-bigquery==2.24.0 -q --no-warn-conflicts\n",
@@ -252,7 +249,7 @@
         "\n",
         "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
         "\n",
-        "1. [Enable the APIs](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,notebooks.googleapis.com, ). \n",
+        "1. [Enable the Vertex AI API and Compute Engine API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component). \n",
         "\n",
         "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
         "\n",
@@ -310,76 +307,18 @@
       "outputs": [],
       "source": [
         "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
-        "    PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
+        "    PROJECT_ID = \"\"  # @param {type:\"string\"}"
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "set_gcloud_project_id"
+        "id": "dEjRdjxBuDsi"
       },
       "outputs": [],
       "source": [
-        "! gcloud config set project $PROJECT_ID"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "23988890fef6"
-      },
-      "source": [
-        "#### Get your project number (Optional)\n",
-        "\n",
-        "Now that the project ID is set, you get your corresponding project number."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "2d6950574e1d"
-      },
-      "outputs": [],
-      "source": [
-        "shell_output = ! gcloud projects list --filter=\"PROJECT_ID:'{PROJECT_ID}'\" --format='value(PROJECT_NUMBER)'\n",
-        "PROJECT_NUMBER = shell_output[0]\n",
-        "print(\"Project Number:\", PROJECT_NUMBER)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "region"
-      },
-      "source": [
-        "#### Region\n",
-        "\n",
-        "You can also change the `REGION` variable, which is used for operations\n",
-        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
-        "\n",
-        "- Americas: `us-central1`\n",
-        "- Europe: `europe-west4`\n",
-        "- Asia Pacific: `asia-east1`\n",
-        "\n",
-        "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
-        "\n",
-        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "jIcZV7-C2RrX"
-      },
-      "outputs": [],
-      "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "!gcloud config set project $PROJECT_ID #change it"
       ]
     },
     {
@@ -414,7 +353,7 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
+        "**If you are using Google Cloud Notebooks**, your environment is already\n",
         "authenticated. Skip this step."
       ]
     },
@@ -437,13 +376,9 @@
         "3. In the **Service account name** field, enter a name, and\n",
         "   click **Create**.\n",
         "\n",
-        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list and add the following roles:\n",
-        "   - BigQuery Admin\n",
-        "   - Storage Admin\n",
-        "   - Storage Object Admin\n",
-        "   - Vertex AI Administrator\n",
-        "   - Vertex AI Feature Store Admin\n",
-        "\n",
+        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
+        "into the filter box, and select\n",
+        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
         "\n",
         "5. Click *Create*. A JSON file that contains your key downloads to your\n",
         "local environment.\n",
@@ -460,19 +395,19 @@
       },
       "outputs": [],
       "source": [
+        "import os\n",
+        "import sys\n",
+        "\n",
         "# If you are running this notebook in Colab, run this cell and follow the\n",
         "# instructions to authenticate your GCP account. This provides access to your\n",
         "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
         "# requests.\n",
         "\n",
-        "import os\n",
-        "import sys\n",
+        "# The Google Cloud Notebook product has specific requirements\n",
+        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
         "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = \"google.colab\" in sys.modules\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
+        "# If on Google Cloud Notebooks, then don't execute this code\n",
+        "if not IS_GOOGLE_CLOUD_NOTEBOOK:\n",
         "    if \"google.colab\" in sys.modules:\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
@@ -512,8 +447,8 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+        "BUCKET_URI = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "REGION = \"[your-region]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -524,9 +459,11 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"-aip-\" + TIMESTAMP\n",
-        "    BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"-aip-\" + TIMESTAMP\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -549,6 +486,26 @@
         "! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "994afa65eaa2"
+      },
+      "source": [
+        "Run the following cell to grant access to your Cloud Storage resources from Vertex AI Feature store"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "psP1rPU9TRnX"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil uniformbucketlevelaccess set on $BUCKET_URI"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -569,78 +526,6 @@
         "! gsutil ls -al $BUCKET_URI"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "set_service_account"
-      },
-      "source": [
-        "#### Service Account (Optional)\n",
-        "\n",
-        "If you do not want to use your project's Compute Engine service account, set `SERVICE_ACCOUNT` to another service account ID."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "MQVV9haf2Rra"
-      },
-      "outputs": [],
-      "source": [
-        "SERVICE_ACCOUNT = \"[your-service-account]\"  # @param {type:\"string\"}"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "autoset_service_account"
-      },
-      "outputs": [],
-      "source": [
-        "if (\n",
-        "    SERVICE_ACCOUNT == \"\"\n",
-        "    or SERVICE_ACCOUNT is None\n",
-        "    or SERVICE_ACCOUNT == \"[your-service-account]\"\n",
-        "):\n",
-        "    # Get your service account from gcloud\n",
-        "    if not IS_COLAB:\n",
-        "        shell_output = !gcloud auth list 2>/dev/null\n",
-        "        SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
-        "\n",
-        "    else:  # IS_COLAB:\n",
-        "        shell_output = ! gcloud projects describe  $PROJECT_ID\n",
-        "        project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n",
-        "        SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n",
-        "\n",
-        "    print(\"Service Account:\", SERVICE_ACCOUNT)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "set_service_account:pipelines"
-      },
-      "source": [
-        "#### Set service account access\n",
-        "\n",
-        "Run the following commands to grant your service account access. You only need to run this step once per service account."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "U4UpQThc2Rrb"
-      },
-      "outputs": [],
-      "source": [
-        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
-        "\n",
-        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -652,18 +537,6 @@
         "You create the BigQuery dataset to store the data along the demo."
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "8615339fa4ca"
-      },
-      "outputs": [],
-      "source": [
-        "BQ_DATASET = \"Mobile_Gaming\"  # @param {type:\"string\"}\n",
-        "LOCATION = \"US\""
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -672,6 +545,9 @@
       },
       "outputs": [],
       "source": [
+        "BQ_DATASET = \"Mobile_Gaming\"  # @param {type:\"string\"}\n",
+        "LOCATION = \"US\"\n",
+        "\n",
         "!bq mk --location=$LOCATION --dataset $PROJECT_ID:$BQ_DATASET"
       ]
     },
@@ -724,9 +600,12 @@
       "outputs": [],
       "source": [
         "# Data Engineering and Feature Engineering\n",
-        "TODAY = \"2022-06-16\"\n",
+        "TODAY = \"2018-10-03\"\n",
+        "TOMORROW = \"2018-10-04\"\n",
         "LABEL_TABLE = f\"label_table_{TODAY}\".replace(\"-\", \"\")\n",
-        "FEATURES_TABLE = f\"wide_features_table_{TODAY}\"  # @param {type:\"string\"}\n",
+        "FEATURES_TABLE = \"wide_features_table\"  # @param {type:\"string\"}\n",
+        "FEATURES_TABLE_TODAY = f\"wide_features_table_{TODAY}\".replace(\"-\", \"\")\n",
+        "FEATURES_TABLE_TOMORROW = f\"wide_features_table_{TOMORROW}\".replace(\"-\", \"\")\n",
         "FEATURESTORE_ID = \"mobile_gaming\"  # @param {type:\"string\"}\n",
         "ENTITY_TYPE_ID = \"user\"\n",
         "\n",
@@ -1068,37 +947,13 @@
         "You will cover those steps in details below."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "init_aip:mbsdk,all"
-      },
-      "source": [
-        "### Initialize Vertex AI SDK for Python\n",
-        "\n",
-        "Initialize the Vertex AI SDK for Python for your project and corresponding bucket."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "poLJ0fV52Rrc"
-      },
-      "outputs": [],
-      "source": [
-        "vertex_ai.init(project=PROJECT_ID, location=REGION, staging_bucket=BUCKET_URI)"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "4ffd54e97270"
       },
       "source": [
-        "### Initialize BigQuery SDK for Python\n",
-        "\n",
-        "Initialize the BigQuery AI SDK for Python for your project and corresponding bucket."
+        "## Initiate clients"
       ]
     },
     {
@@ -1109,53 +964,55 @@
       },
       "outputs": [],
       "source": [
-        "bq_client = bigquery.Client(project=PROJECT_ID, location=LOCATION)"
+        "bq_client = bigquery.Client(project=PROJECT_ID, location=LOCATION)\n",
+        "vertex_ai.init(project=PROJECT_ID, location=REGION, staging_bucket=BUCKET_URI)"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "WnUQO2IHC9pZ"
+        "id": "zmMWIpCwsET9"
       },
       "source": [
         "## Identify users and build your features\n",
-        " \n",
-        "This section we will have static features we want to fetch from Vertex AI Feature Store. In particular, we will cover the following steps:\n",
-        " \n",
+        "\n",
+        "This section we will static features we want to fetch from Vertex AI Feature Store. In particular, we will cover the following steps:\n",
+        "\n",
         "1. Identify users, process demographic features and process behavioral features within the last 24 hours using **BigQuery**\n",
-        " \n",
+        "\n",
         "2. Set up the feature store\n",
-        " \n",
+        "\n",
         "3. Register features using **Vertex AI Feature Store** and the SDK.\n",
-        " \n",
-        "Below you have a picture that shows the process.\n",
-        " \n",
+        "\n",
+        "Below you have a picture that shows the process. \n",
+        "\n",
         "<img src=\"./assets/feature_store_ingestion_2.png\">\n",
-        " \n",
-        " \n",
-        "The original dataset contains raw event data we cannot ingest in the feature store as they are. We need to pre-process the raw data in order to get user features.\n",
-        " \n",
-        "**Notice we simulate those transformations in different points of time (today and tomorrow).**\n"
+        "\n",
+        "\n",
+        "\n",
+        "The original dataset contains raw event data we cannot ingest in the feature store as they are. We need to pre-process the raw data in order to get user features. \n",
+        "\n",
+        "**Notice we simulate those transformations in different point of time (today and tomorrow).**\n"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "e9zIrwhpDF2q"
+        "id": "8avYy5QOv02s"
       },
       "source": [
         "### Label, Demographic and Behavioral Transformations\n",
-        " \n",
-        "This section is based on the [Churn prediction for game developers using Google Analytics 4 (GA4) and BigQuery ML](https://cloud.google.com/blog/topics/developers-practitioners/churn-prediction-game-developers-using-google-analytics-4-ga4-and-bigquery-ml?utm_source=linkedin&utm_medium=unpaidsoc&utm_campaign=FY21-Q2-Google-Cloud-Tech-Blog&utm_content=google-analytics-4&utm_term=-) blog article by Minhaz Kazi and Polong Lin.\n",
-        " \n",
-        "You will adapt it to turn a batch churn prediction (using features within the first 24h user of first engagement) into a real-time churn prediction (using features within the first 6h user of last engagement).\n"
+        "\n",
+        "This section is based on the [Churn prediction for game developers using Google Analytics 4 (GA4) and BigQuery ML](https://cloud.google.com/blog/topics/developers-practitioners/churn-prediction-game-developers-using-google-analytics-4-ga4-and-bigquery-ml?utm_source=linkedin&utm_medium=unpaidsoc&utm_campaign=FY21-Q2-Google-Cloud-Tech-Blog&utm_content=google-analytics-4&utm_term=-) blog article by Minhaz Kazi and Polong Lin. \n",
+        "\n",
+        "You will adapt it in order to turn a batch churn prediction (using features within the first 24h user of first engagment) in a real-time churn prediction (using features within the first 24h user of last engagment)."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "RQX5m8UiC_px"
+        "id": "YO28RAITh6L-"
       },
       "outputs": [],
       "source": [
@@ -1182,27 +1039,27 @@
         "  SELECT\n",
         "    event_timestamp,\n",
         "    user_pseudo_id,\n",
-        "    SUM(IF(event_name = 'user_engagement', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 21600000000 PRECEDING\n",
+        "    SUM(IF(event_name = 'user_engagement', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 86400000000 PRECEDING\n",
         "      AND CURRENT ROW ) AS cnt_user_engagement,\n",
-        "    SUM(IF(event_name = 'level_start_quickplay', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 21600000000 PRECEDING\n",
+        "    SUM(IF(event_name = 'level_start_quickplay', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 86400000000 PRECEDING\n",
         "      AND CURRENT ROW ) AS cnt_level_start_quickplay,\n",
-        "    SUM(IF(event_name = 'level_end_quickplay', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 21600000000 PRECEDING\n",
+        "    SUM(IF(event_name = 'level_end_quickplay', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 86400000000 PRECEDING\n",
         "      AND CURRENT ROW ) AS cnt_level_end_quickplay,\n",
-        "    SUM(IF(event_name = 'level_complete_quickplay', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 21600000000 PRECEDING\n",
+        "    SUM(IF(event_name = 'level_complete_quickplay', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 86400000000 PRECEDING\n",
         "      AND CURRENT ROW ) AS cnt_level_complete_quickplay,\n",
-        "    SUM(IF(event_name = 'level_reset_quickplay', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 21600000000 PRECEDING\n",
+        "    SUM(IF(event_name = 'level_reset_quickplay', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 86400000000 PRECEDING\n",
         "      AND CURRENT ROW ) AS cnt_level_reset_quickplay,\n",
-        "    SUM(IF(event_name = 'post_score', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 21600000000 PRECEDING\n",
+        "    SUM(IF(event_name = 'post_score', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 86400000000 PRECEDING\n",
         "      AND CURRENT ROW ) AS cnt_post_score,\n",
-        "    SUM(IF(event_name = 'spend_virtual_currency', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 21600000000 PRECEDING\n",
+        "    SUM(IF(event_name = 'spend_virtual_currency', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 86400000000 PRECEDING\n",
         "      AND CURRENT ROW ) AS cnt_spend_virtual_currency,\n",
-        "    SUM(IF(event_name = 'ad_reward', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 21600000000 PRECEDING\n",
+        "    SUM(IF(event_name = 'ad_reward', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 86400000000 PRECEDING\n",
         "      AND CURRENT ROW ) AS cnt_ad_reward,\n",
-        "    SUM(IF(event_name = 'challenge_a_friend', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 21600000000 PRECEDING\n",
+        "    SUM(IF(event_name = 'challenge_a_friend', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 86400000000 PRECEDING\n",
         "      AND CURRENT ROW ) AS cnt_challenge_a_friend,\n",
-        "    SUM(IF(event_name = 'completed_5_levels', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 21600000000 PRECEDING\n",
+        "    SUM(IF(event_name = 'completed_5_levels', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 86400000000 PRECEDING\n",
         "      AND CURRENT ROW ) AS cnt_completed_5_levels,\n",
-        "    SUM(IF(event_name = 'use_extra_steps', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 21600000000 PRECEDING\n",
+        "    SUM(IF(event_name = 'use_extra_steps', 1, 0)) OVER (PARTITION BY user_pseudo_id ORDER BY event_timestamp ASC RANGE BETWEEN 86400000000 PRECEDING\n",
         "      AND CURRENT ROW ) AS cnt_use_extra_steps,\n",
         "  FROM (\n",
         "    SELECT\n",
@@ -1214,7 +1071,7 @@
         "\n",
         "SELECT\n",
         "    -- PARSE_TIMESTAMP('%Y-%m-%d %H:%M:%S', CONCAT('{TODAY}', ' ', STRING(TIME_TRUNC(CURRENT_TIME(), SECOND))), 'UTC') as timestamp,\n",
-        "    TIMESTAMP_ADD(PARSE_TIMESTAMP('%Y-%m-%d %H:%M:%S', FORMAT_TIMESTAMP('%Y-%m-%d %H:%M:%S', TIMESTAMP_MICROS(beh.event_timestamp))), INTERVAL 1351 DAY) AS timestamp,\n",
+        "    PARSE_TIMESTAMP('%Y-%m-%d %H:%M:%S', FORMAT_TIMESTAMP('%Y-%m-%d %H:%M:%S', TIMESTAMP_MICROS(beh.event_timestamp))) AS timestamp,\n",
         "    dem.*,\n",
         "    CAST(IFNULL(beh.cnt_user_engagement, 0) AS FLOAT64)  AS cnt_user_engagement,\n",
         "    CAST(IFNULL(beh.cnt_level_start_quickplay, 0) AS FLOAT64) AS cnt_level_start_quickplay,\n",
@@ -1240,7 +1097,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "oGYxLCSnD068"
+        "id": "Z6CjIOmDsET-"
       },
       "outputs": [],
       "source": [
@@ -1250,31 +1107,31 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "xQLIlsTCD_nk"
+        "id": "Lx__2-assET-"
       },
       "source": [
         "## Create a Vertex AI Feature store and ingest your features\n",
-        " \n",
-        "Now you have a wide table of features. It is time to ingest them into the feature store.\n",
-        " \n",
+        "\n",
+        "Now you have the wide table of features. It is time to ingest them into the feature store. \n",
+        "\n",
         "Before to moving on, you may have a question: **Why do I need a feature store**\n",
         "in this scenario at that point?\n",
-        " \n",
-        "One of the reasons would be to make those features accessible across teams by calculating once and reuse them many times. And in order to make it possible you need also be able to monitor those features over time to guarantee freshness and in case have a new feature engineering run to refresh them.\n",
-        " \n",
-        "If it is not your case, I will give even more reasons about why you should consider a feature store in the following sections. Just keep following me for now.\n",
-        " \n",
-        "One of the most important things is related to its data model. As you can see in the picture below, Vertex AI Feature Store organizes resources hierarchically in the following order: `Featurestore -> EntityType -> Feature`. You must create these resources before you can ingest data into Vertex AI Feature Store.\n",
-        " \n",
+        "\n",
+        "One of the reason would be to make those features accessable across team by calculating once and reuse them many times. And in order to make it possible you need also be able to monitor those features over time to guarantee freshness and in case have a new feature engineerign run to refresh them. \n",
+        "\n",
+        "If it is not your case, I will give even more reasons about why you should consider feature store in the following sections. Just keep following me for now.\n",
+        "\n",
+        "One of the most important thing is related to its data model. As you can see in the picture below, Vertex AI Feature Store organizes resources hierarchically in the following order: `Featurestore -> EntityType -> Feature`. You must create these resources before you can ingest data into Vertex AI Feature Store.\n",
+        "\n",
         "<img src=\"./assets/feature_store_data_model_3.png\">\n",
-        " \n",
-        "In our case we are going to create **mobile_gaming** featurestore resource containing **user** entity type and all its associated **features** such as country or the number of times a user challenged a friend (cnt_challenge_a_friend).\n"
+        "\n",
+        "In our case we are going to create **mobile_gaming** featurestore resource containing **user** entity type and all its associated **features** such as country or the number of times a user challenged a friend (cnt_challenge_a_friend)."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "VR7BJEozED_Q"
+        "id": "8dNlxda2sET_"
       },
       "source": [
         "### Create featurestore, ```mobile_gaming```\n",
@@ -1286,7 +1143,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "vUFqtYU-EDTR"
+        "id": "t2en8I7TSe4b"
       },
       "outputs": [],
       "source": [
@@ -1307,7 +1164,7 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "mUlCwfdpEHJG"
+        "id": "rN-vlvPUsET_"
       },
       "source": [
         "### Create the ```User``` entity type and its features\n",
@@ -1319,7 +1176,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "PnCU1wBND3W7"
+        "id": "CbZ2RQ5XbuRq"
       },
       "outputs": [],
       "source": [
@@ -1337,7 +1194,7 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "bT9LXzu1EOvW"
+        "id": "B2PIAprPmnhB"
       },
       "source": [
         "### Set Feature Monitoring\n",
@@ -1351,7 +1208,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "8WBlYUkOERaI"
+        "id": "N6im2c3ymiwC"
       },
       "outputs": [],
       "source": [
@@ -1374,7 +1231,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "92X4-7PFETj5"
+        "id": "gp9xaLQXn0CS"
       },
       "outputs": [],
       "source": [
@@ -1397,18 +1254,18 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "hxAuZjt3EWFo"
+        "id": "ustwKOMle8Qp"
       },
       "source": [
         "### Create features\n",
         "\n",
-        "In order to ingest features, you need to provide feature configuration and create them as featurestore resources."
+        "In order to ingest features, you need to provide feature configuration and create them as featurestore resources.\n"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "hRXO2I5VEYwt"
+        "id": "ijeZCTKIfCRL"
       },
       "source": [
         "#### Create Feature configuration\n",
@@ -1421,7 +1278,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "K26NEYZIEbvE"
+        "id": "vX_uYmjUgd9x"
       },
       "outputs": [],
       "source": [
@@ -1502,7 +1359,7 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "FjzMd1XbEfdo"
+        "id": "ErkruXPJkPuy"
       },
       "source": [
         "#### Create features using `batch_create_features` method\n",
@@ -1514,7 +1371,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "nqlgCDI9pbCD"
+        "id": "ZsCAO_IfsEUC"
       },
       "outputs": [],
       "source": [
@@ -1531,19 +1388,19 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "7zpFV7wAppkC"
+        "id": "9WisJk18qqgs"
       },
       "source": [
         "### Search features\n",
         "\n",
-        "Vertex AI Feature store supports searching capabilities. Below you have a simple example that shows how to filter a feature based on its name. "
+        "Vertex AI Feature store supports serching capabilities. Below you have a simple example that show how to filter a feature based on its name. "
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "BJXYLLOfppCL"
+        "id": "JzqyarMZqvZS"
       },
       "outputs": [],
       "source": [
@@ -1555,7 +1412,7 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "is9C_6-QpxG3"
+        "id": "ugtBfW5gsEUD"
       },
       "source": [
         "## Ingest features \n",
@@ -1590,7 +1447,7 @@
         "        entity_id_field=ENTITY_ID_FIELD,\n",
         "        disable_online_serving=False,\n",
         "        worker_count=10,\n",
-        "        sync=False,\n",
+        "        sync=True,\n",
         "    )\n",
         "except RuntimeError as error:\n",
         "    print(error)"
@@ -1599,7 +1456,7 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "8lCMpDPGp-oQ"
+        "id": "3Yv8MenWXrRX"
       },
       "source": [
         "# Train and deploy a real-time churn ML model using Vertex AI Training and Endpoints\n",
@@ -1610,34 +1467,34 @@
         "\n",
         "<img src=\"./assets/train_model_4.png\">\n",
         "\n",
-        "Let's dive into each step of this process."
+        "Let's dive into each step of this process.\n"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "VMrvnuyjqGfY"
+        "id": "saZZ3zWKX1YK"
       },
       "source": [
-        "## Fetch training data with point-in-time query using BigQuery and Vertex AI Feature store \n",
-        " \n",
-        "As we mentioned above, in real time churn prediction, it is so important defining the label you want to predict with your model.\n",
-        " \n",
-        "Let's assume that you decide to predict the churn probability over the next hour. So now you have your label. Next step is to define your training sample. But let's think about that for a second.\n",
-        " \n",
-        "In that churn real time system, you have a high volume of transactions you could use to calculate those features which keep floating and are collected constantly over time. It implies that you always get fresh data to reconstruct features. And depending on when you decide to calculate one feature or another you can end up with a set of features that are not aligned in time.\n",
-        " \n",
+        "## Fetch training data with point-in-time query using BigQuery and Vertex AI Feature store  \n",
+        "\n",
+        "As we mentioned above, in real time churn prediction, it is so important defining the label you want to predict with your model. \n",
+        "\n",
+        "Let's assume that you decide to predict the churn probability over the last 24 hr. So now you have your label. Next step is to define your training sample. But let's think about that for a second. \n",
+        "\n",
+        "In that churn real time system, you have a high volume of transactions you could use to calculate those features which keep floating and are collected constantly over time. It implies that you always get fresh data to reconstruct features. And depending on when you decide to calculate one feature or another you can end up with a set of features that are not aligned in time. \n",
+        "\n",
         "When you have labels available, it would be incredibly difficult to say which set of features contains the most up to date historical information associated with the label you want to predict. And, when you are not able to guarantee that, the performance of your model would be badly affected because you serve no representative features of the data and the label from the field when it goes live. So you need a way to get the most updated features you calculated over time before the label becomes available in order to avoid this informational skew.\n",
-        " \n",
-        "**With the Vertex AI Feature store, you can fetch feature values corresponding to a particular timestamp thanks to point-in-time lookup capability.** In our case, it would be the timestamp associated with the label you want to predict with your model. In this way, you will avoid data leakage and you will get the most updated features to train your model.\n",
-        " \n",
-        "Let's see how to do that.\n"
+        "\n",
+        "**With the Vertex AI Feature store, you can fetch feature values corresponding to a particular timestamp thanks to point-in-time lookup capability.** In our case, it would be the timestamp associated to the label you want to predict with your model. In this way, you will avoid data leakage and you will get the most updated features to train your model. \n",
+        "\n",
+        "Let's see how to do that. \n"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "RE_Pvmu-qdDt"
+        "id": "YHNbIHqFcQiM"
       },
       "source": [
         "### Define query for reading instances at a specific point in time\n",
@@ -1649,7 +1506,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "bUDVw7l-qF2x"
+        "id": "DGUm0bYqhVV4"
       },
       "outputs": [],
       "source": [
@@ -1661,13 +1518,13 @@
         "  # get training threshold ----------------------------------------------------------------------------------\n",
         "  get_training_threshold AS (\n",
         "  SELECT\n",
-        "    (MAX(event_timestamp) - 10800000000) AS training_thrs\n",
+        "    (MAX(event_timestamp) - 86400000000) AS training_thrs\n",
         "  FROM\n",
         "    `firebase-public-project.analytics_153293282.events_*`\n",
         "  WHERE\n",
         "    event_name=\"user_engagement\"\n",
         "    AND\n",
-        "    TIMESTAMP_ADD(PARSE_TIMESTAMP('%Y-%m-%d %H:%M:%S', FORMAT_TIMESTAMP('%Y-%m-%d %H:%M:%S', TIMESTAMP_MICROS(event_timestamp))), INTERVAL 1351 DAY) < '{TODAY}'),\n",
+        "    PARSE_TIMESTAMP('%Y-%m-%d %H:%M:%S', FORMAT_TIMESTAMP('%Y-%m-%d %H:%M:%S', TIMESTAMP_MICROS(event_timestamp))) < '{TODAY}'),\n",
         "\n",
         "  # query to create label -----------------------------------------------------------------------------------\n",
         "  get_label AS (\n",
@@ -1692,7 +1549,7 @@
         "    WHERE\n",
         "      event_name=\"user_engagement\"\n",
         "    AND\n",
-        "    TIMESTAMP_ADD(PARSE_TIMESTAMP('%Y-%m-%d %H:%M:%S', FORMAT_TIMESTAMP('%Y-%m-%d %H:%M:%S', TIMESTAMP_MICROS(event_timestamp))), INTERVAL 1351 DAY) < '{TODAY}'\n",
+        "    PARSE_TIMESTAMP('%Y-%m-%d %H:%M:%S', FORMAT_TIMESTAMP('%Y-%m-%d %H:%M:%S', TIMESTAMP_MICROS(event_timestamp))) < '{TODAY}'\n",
         "    GROUP BY\n",
         "      user_pseudo_id )\n",
         "  GROUP BY\n",
@@ -1812,7 +1669,7 @@
       "source": [
         "!mkdir -m 777 -p trainer data/ingest data/raw model config\n",
         "!gsutil -m cp -r $GCS_DESTINATION_OUTPUT_URI/*.csv data/ingest\n",
-        "!head -n 2000 data/ingest/*.csv > data/raw/sample.csv"
+        "!head -n 1000 data/ingest/*.csv > data/raw/sample.csv"
       ]
     },
     {
@@ -2214,7 +2071,7 @@
       },
       "outputs": [],
       "source": [
-        "TRAIN_JOB_RESOURCE_NAME = \"[your-train-job-resource-name]\"  # @param {type:\"string\"}"
+        "TRAIN_JOB_RESOURCE_NAME = \"\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -2309,32 +2166,32 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "1TNzL_EGrVUm"
+        "id": "7c9330928aa1"
       },
       "source": [
         "# Serve ML features at scale with low latency\n",
-        " \n",
-        "At that time, you are ready **to deploy our simple model which would requires fetching preprocessed attributes as input features in real time**.\n",
-        " \n",
+        "\n",
+        "At that time, you are ready **to deploy our simple model which would requires fetching preprocessed attributes as input features in real time**. \n",
+        "\n",
         "Below you can see how it works\n",
-        " \n",
-        "<center><img src=\"./assets/online_serving_5.png\" width=\"800\"/></center>\n",
-        " \n",
-        "But think about those features for a second.\n",
-        " \n",
-        "Your behavioral features used to train your model, they cannot be computed when you are going to serve the model online.\n",
-        " \n",
-        "How could you compute the number of times a user challenged a friend within the last 24 hours on the fly?\n",
-        " \n",
-        "You need to be computed this feature on the server side and serve it with low latency. And because Bigquery is not optimized for those read operations, we need a different service that allows singleton lookup where the result is a single row with many columns.\n",
-        " \n",
-        "Also, even if it was not the case, when you deploy a model that requires preprocessing your data, you need to be sure to reproduce the same preprocessing steps you had when you trained it. If you are not able to do that a skew between training and serving data would happen and it will badly affect your model performance (and in the worst scenario break your serving system).\n",
-        " \n",
-        "You need a way to mitigate that in a way you don't need to implement those preprocessing steps online but just serve the same aggregated features you already have for training to generate online prediction.\n",
-        " \n",
-        "These are other valuable reasons to introduce Vertex AI Feature Store. With it, you have a service which helps you to serve features at scale with low latency as they were available at training time mitigating in that way possible training-serving skew.\n",
-        " \n",
-        "Now that you know **why you need a feature store**, let's conclude this journey by deploying your model using a feature store to retrieve features online, pass them to the endpoint and generate predictions.\n"
+        "\n",
+        "<img src=\"./assets/online_serving_5.png\" width=\"600\">\n",
+        "\n",
+        "But think about those features for a second. \n",
+        "\n",
+        "Your behavioral features used to trained your model, they cannot be computed when you are going to serve the model online. \n",
+        "\n",
+        "How could you compute the number of time a user challenged a friend withing the last 24 hours on the fly?\n",
+        "\n",
+        "You simply can't do that. You need to be computed this feature on the server side and serve it with low latency. And becuase Bigquery is not optimized for those read operations, we need a different service that allows singleton lookup where the result is a single row with many columns.\n",
+        "\n",
+        "Also, even if it was not the case, when you deploy a model that requires preprocessing your data, you need to be sure to reproduce the same preprocessing steps you had when you trained it. If you are not able to do that a skew between training and serving data would happen and it will affect badly your model performance (and in the worst scenario break your serving system). \n",
+        "\n",
+        "You need a way to mitigate that in a way you don't need to implement those preprocessing steps online but just serve the same aggregated features you already have for training to generate online prediction. \n",
+        "\n",
+        "These are other valuable reasons to introduce Vertex AI Feature Store. With it, you have a service which helps you to serve feature at scale with low latency as they were available at training time mitigating in that way possible training-serving skew.\n",
+        "\n",
+        "Now that you know **why you need a feature store**, let's closing this journey by deploying your model and use feature store to retrieve features online, pass them to endpoint and generate predictions.\n"
       ]
     },
     {
@@ -2364,13 +2221,13 @@
       },
       "outputs": [],
       "source": [
-        "simulate_prediction(endpoint=endpoint, n_requests=10, latency=1)"
+        "simulate_prediction(endpoint=endpoint, n_requests=1000, latency=1)"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "8d3S1d1urZOy"
+        "id": "TpV-iwP9qw9c"
       },
       "source": [
         "## Cleaning up\n",
@@ -2410,12 +2267,11 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "FMXT2akXrZOy"
+        "id": "sx_vKniMq9ZX"
       },
       "outputs": [],
       "source": [
         "# Delete bucket\n",
-        "delete_bucket = False\n",
         "if (delete_bucket or os.getenv(\"IS_TESTING\")) and \"BUCKET_URI\" in globals():\n",
         "    ! gsutil -m rm -r $BUCKET_URI"
       ]

notebooks/community/feature_store/sdk-feature-store-pandas.ipynb

@@ -29,28 +29,18 @@
         "id": "JAPoU8Sm5E6e"
       },
       "source": [
-        "# Using Vertex AI Feature Store with pandas DataFrame\n",
-        "\n",
         "<table align=\"left\">\n",
-        "    <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/feature_store/sdk-feature-store-pandas.ipynb\">\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/feature_store/sdk-feature-store-pandas.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/feature_store/sdk-feature-store-pandas.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
-        "    \n",
-        "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/feature_store/sdk-feature-store-pandas.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> \n",
-        "        Run in Colab\n",
-        "    </a>\n",
-        "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/feature_store/sdk-feature-store-pandas.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
-        "    </a>\n",
-        "  </td>\n",
         "</table>"
       ]
     },
@@ -62,29 +52,7 @@
       "source": [
         "## Overview\n",
         "\n",
-        "This notebook introduces Pandas support for Feature Store using Vertex AI SDK. For pre-requisites and introduction on Vertex AI SDK and Feature Store native support, please go through this [Colab notebook](https://colab.sandbox.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/feature_store/sdk-feature-store.ipynb). "
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "DxF5JWRVT5PP"
-      },
-      "source": [
-        "### Objective\n",
-        "\n",
-        "In this notebook, you learn how to use `Vertex AI Feature Store` with pandas DataFrame.\n",
-        "\n",
-        "The steps performed include:\n",
-        "\n",
-        "- Ingest Feature values from Pandas DataFrame into Feature Store's Entity types.\n",
-        "- Read Entity Feature values from Online Feature Store into Pandas DataFrame.\n",
-        "- Batch serve Feature values from your Feature Store into Pandas DataFrame.\n",
-        "\n",
-        "You also learn how Vertex AI Feature Store can be useful in the below scenarios:\n",
-        "\n",
-        "- Online serving with updated feature values.\n",
-        "- Point-in-time correctness to fetch feature values for training."
+        "This Colab introduces Pandas support of Vertex AI SDK Feature Store. For pre-requisite and introduction for Vertex AI SDK Feature Store native support, please see this [Colab](https://colab.sandbox.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/feature_store/sdk-feature-store.ipynb). "
       ]
     },
     {
@@ -95,7 +63,27 @@
       "source": [
         "### Dataset\n",
         "\n",
-        "This tutorial uses a movie recommendation dataset as an example throughout all the notebooks including this one. The original task is to train a model to predict if a user is going to watch a movie and serve the model online."
+        "This Colab uses a movie recommendation dataset as an example throughout all the sessions. The task is to train a model to predict if a user is going to watch a movie and serve this model online."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "DxF5JWRVT5PP"
+      },
+      "source": [
+        "### Objective\n",
+        "\n",
+        "In this notebook, you will learn how to:\n",
+        "\n",
+        "    * Ingest Feature Values from Pandas DataFrame into featurestore's entity types.\n",
+        "    * Read Entity Feature Values from Online Feature Store into Pandas DataFrame.\n",
+        "    * Batch Serve Feature Values from your featurestore to Pandas DataFrame.\n",
+        "\n",
+        "We will also discuss how Vertex AI Feature Store can be useful in the below scenarios:\n",
+        "\n",
+        "    * online serving with updated feature values\n",
+        "    * point-in-time correctness to fetch feature values for training"
       ]
     },
     {
@@ -109,9 +97,11 @@
         "This tutorial uses billable components of Google Cloud:\n",
         "\n",
         "* Vertex AI\n",
+        "* Cloud BigQuery\n",
         "\n",
         "Learn about [Vertex AI\n",
-        "pricing](https://cloud.google.com/vertex-ai/pricing) and use the [Pricing\n",
+        "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
+        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
         "Calculator](https://cloud.google.com/products/calculator/)\n",
         "to generate a cost estimate based on your projected usage."
       ]
@@ -133,7 +123,7 @@
       "source": [
         "### Install additional packages\n",
         "\n",
-        "To run this notebook, you need to install the following packages for Python."
+        "For this Colab, you need the Vertex SDK for Python."
       ]
     },
     {
@@ -152,14 +142,35 @@
         "# Google Cloud Notebook requires dependencies to be installed with '--user'\n",
         "USER_FLAG = \"\"\n",
         "if IS_GOOGLE_CLOUD_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "    \n",
-        "! pip install -U {USER_FLAG} --upgrade google-cloud-aiplatform \\\n",
-        "                                        google-cloud-bigquery \\\n",
-        "                                        google-cloud-bigquery-storage \\\n",
-        "                                        avro \\\n",
-        "                                        pyarrow \\\n",
-        "                                        pandas -q"
+        "    USER_FLAG = \"--user\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Kd0kgDqVZyRe"
+      },
+      "outputs": [],
+      "source": [
+        "! pip uninstall {USER_FLAG} -y google-cloud-aiplatform\n",
+        "! pip uninstall {USER_FLAG} -y google-cloud-bigquery\n",
+        "! pip uninstall {USER_FLAG} -y google-cloud-bigquery-storage\n",
+        "! pip uninstall {USER_FLAG} -y google-cloud-aiplatform"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "wUswAmpiN2l-"
+      },
+      "outputs": [],
+      "source": [
+        "! pip install {USER_FLAG} --upgrade google-cloud-aiplatform\n",
+        "! pip install {USER_FLAG} --upgrade google-cloud-bigquery\n",
+        "! pip install {USER_FLAG} --upgrade google-cloud-bigquery-storage\n",
+        "! pip install {USER_FLAG} avro"
       ]
     },
     {
@@ -170,7 +181,7 @@
       "source": [
         "### Restart the kernel\n",
         "\n",
-        "After you install the packages, you need to restart the notebook kernel so that it can find the packages."
+        "After you install the SDK, you need to restart the notebook kernel so it can find the packages. You can restart kernel from *Kernel -> Restart Kernel*, or running the following:"
       ]
     },
     {
@@ -227,17 +238,6 @@
         "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "dcdfccf50581"
-      },
-      "outputs": [],
-      "source": [
-        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -246,56 +246,37 @@
       },
       "outputs": [],
       "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
-        "    # Get your GCP project id from gcloud\n",
-        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "import os\n",
+        "\n",
+        "PROJECT_ID = \"\"\n",
+        "\n",
+        "# Get your Google Cloud project ID from gcloud\n",
+        "if not os.getenv(\"IS_TESTING\"):\n",
+        "    shell_output = !gcloud config list --format 'value(core.project)' 2>/dev/null\n",
         "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID:\", PROJECT_ID)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "09021c90b34c"
-      },
-      "outputs": [],
-      "source": [
-        "! gcloud config set project $PROJECT_ID"
+        "    print(\"Project ID: \", PROJECT_ID)"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "f41eda68c379"
+        "id": "qJYoRfYng0XZ"
       },
       "source": [
-        "#### Region\n",
-        "\n",
-        "You can also change the `REGION` variable, which is used for operations\n",
-        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
-        "\n",
-        "- Americas: `us-central1`\n",
-        "- Europe: `europe-west4`\n",
-        "- Asia Pacific: `asia-east1`\n",
-        "\n",
-        "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
-        "\n",
-        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)"
+        "Otherwise, set your project ID here."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "5c615e53149f"
+        "id": "riG_qUokg0XZ"
       },
       "outputs": [],
       "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
+        "    PROJECT_ID = \"\"  # @param {type:\"string\"}\n",
+        "print(\"Project ID: \", PROJECT_ID)"
       ]
     },
     {
@@ -395,6 +376,8 @@
         "import pandas as pd\n",
         "from google.cloud import aiplatform\n",
         "\n",
+        "REGION = \"\"  # @param {type:\"string\"}\n",
+        "\n",
         "aiplatform.init(project=PROJECT_ID, location=REGION)"
       ]
     },
@@ -413,11 +396,11 @@
         "id": "buQBIv3ZL3A0"
       },
       "source": [
-        "### Create Feature Store\n",
+        "### Create Featurestore\n",
         "\n",
-        "The method to create a Feature Store returns a\n",
+        "The method to create a Featurestore returns a\n",
         "[long-running operation](https://google.aip.dev/151) (LRO). An LRO starts an asynchronous job. LROs are returned for other API\n",
-        "methods too, such as updating or deleting a featurestore. Running the code cell creates a featurestore and prints the process logs."
+        "methods too, such as updating or deleting a featurestore. Running the code cell will create a featurestore and print the process log."
       ]
     },
     {
@@ -442,7 +425,7 @@
       "source": [
         "### Create Entity Types\n",
         "\n",
-        "Entity types can be created within the Featurestore class. Below, you create the `Users` entity type and `Movies` entity type. Process logs are printed in the output for each cell."
+        "Entity types can be created within the Featurestore class. Below, create the Users entity type and Movies entity type. A process log will be printed out."
       ]
     },
     {
@@ -480,7 +463,7 @@
       },
       "source": [
         "### Create Features\n",
-        "Features can be created within each entity type. Add defining features to the `Users` entity type and `Movies` entity type by using the following methods."
+        "Features can be created within each entity type. Add defining features to the Users entity type and Movies entity type by using the following methods."
       ]
     },
     {
@@ -564,7 +547,7 @@
         "id": "BlqJ-QdTcs6W"
       },
       "source": [
-        "#### Get data from source files"
+        "#### Entity Type Source Files"
       ]
     },
     {
@@ -660,7 +643,7 @@
         "id": "bgb0WGwX5OW6"
       },
       "source": [
-        "#### Ingest Feature Values into _Users_ Entity Type"
+        "#### Ingest Feature Values into Users Entity Type"
       ]
     },
     {
@@ -685,7 +668,7 @@
         "id": "PCAdQ3cF5OW6"
       },
       "source": [
-        "#### Ingest Feature Values into _Movies_ Entity Type"
+        "#### Ingest Feature Values into Movies Entity Type"
       ]
     },
     {
@@ -751,9 +734,9 @@
         "id": "AK2Glzkq5OW7"
       },
       "source": [
-        "## Batch Serve Feature Values from Vertex AI Feature Store\n",
+        "## Batch Serve Featurestore's Feature Values from Vertex AI Feature Store\n",
         "\n",
-        "Batch Serving is used to fetch a large batch of feature values for high-throughput, and is typically used for training a model or batch prediction. In this section, you learn how to prepare training examples by using the Feature Store's batch serve function."
+        "Batch Serving is used to fetch a large batch of feature values for high-throughput, and is typically used for training a model or batch prediction. In this section, you will learn how to prepare for training examples by using the Featurestore's batch serve function."
       ]
     },
     {
@@ -762,7 +745,7 @@
         "id": "hxsotHUe5OW7"
       },
       "source": [
-        "#### Read instances from source file"
+        "#### Read Instances Source File"
       ]
     },
     {
@@ -773,8 +756,7 @@
       },
       "outputs": [],
       "source": [
-        "GCS_READ_INSTANCES_CSV_URI = \"gs://cloud-samples-data-us-central1/vertex-ai/feature-store/datasets/movie_prediction.csv\"\n",
-        "READ_INSTANCES_CSV_FN = \"data.csv\""
+        "GCS_READ_INSTANCES_CSV_URI = \"gs://cloud-samples-data-us-central1/vertex-ai/feature-store/datasets/movie_prediction.csv\""
       ]
     },
     {
@@ -794,7 +776,7 @@
         "id": "T5DW1MFt5OW7"
       },
       "source": [
-        "#### Load CSV file into a Pandas DataFrame"
+        "#### Load Csv File into a Pandas DataFrame"
       ]
     },
     {
@@ -805,7 +787,7 @@
       },
       "outputs": [],
       "source": [
-        "read_instances_df = pd.read_csv(READ_INSTANCES_CSV_FN)\n",
+        "read_instances_df = pd.read_csv(read_instances_csv_fn)\n",
         "print(read_instances_df)"
       ]
     },
@@ -837,7 +819,7 @@
         "id": "ao1dC5Pc5OW8"
       },
       "source": [
-        "#### Batch Serve Feature Values from Movie Predictions Feature Store"
+        "#### Batch Serve Feature Values from Movie Predictions Featurestore"
       ]
     },
     {
@@ -873,8 +855,7 @@
         "id": "XN84znoI5OW8"
       },
       "source": [
-        "#### Feature Values from last ingestion\n",
-        "Recall read from the Entity Type shows Feature Values from the last ingestion."
+        "#### Recall Read from the Entity Type Shows Feature Values from the Last Ingestion"
       ]
     },
     {
@@ -894,7 +875,7 @@
         "id": "feTUJjqG5OW9"
       },
       "source": [
-        "#### Ingest updated Feature Values"
+        "#### Ingest Updated Feature Values"
       ]
     },
     {
@@ -934,8 +915,7 @@
         "id": "s47WCIvL5OW9"
       },
       "source": [
-        "#### Latest Feature Values\n",
-        "Read from the Entity Type shows updated Feature values from the latest ingestion."
+        "#### Read from the Entity Type Shows Updated Feature Values from the Latest Ingestion"
       ]
     },
     {
@@ -968,8 +948,7 @@
         "id": "R1YGRNsW5OW9"
       },
       "source": [
-        "#### Missing data\n",
-        "Recall Batch Serve from the last ingestion has some missing data in it."
+        "#### Recall Batch Serve From the Last Ingestion Has Missing Data"
       ]
     },
     {
@@ -989,7 +968,7 @@
         "id": "abQRF6mx5OW-"
       },
       "source": [
-        "#### Backfill/Correct point-in-time data"
+        "#### Backfill/Correct Point-in-Time Data"
       ]
     },
     {
@@ -1030,7 +1009,7 @@
         "id": "WXb4JUhu5OW-"
       },
       "source": [
-        "#### Ingest backfilled/corrected point-in-time data from dataframe"
+        "#### Ingest Backfill/Correct Point-in-Time Data"
       ]
     },
     {
@@ -1071,8 +1050,7 @@
         "id": "1e62Ku6W5OW_"
       },
       "source": [
-        "#### Latest ingestion with imputed missing data\n",
-        "Batch Serve from the latest ingestion with backfill/correction has reduced missing data."
+        "#### Batch Serve From the Latest Ingestion with Backfill/Correction Has Reduced Missing Data"
       ]
     },
     {

notebooks/community/managed_notebooks/ads_targetting/training-multi-class-classification-model-for-ads-targeting-usecase.ipynb

@@ -1,55 +1,29 @@
 {
   "cells": [
     {
-      "cell_type": "code",
-      "execution_count": null,
+      "cell_type": "markdown",
       "metadata": {
         "id": "503077811e70"
       },
-      "outputs": [],
-      "source": [
-        "# Copyright 2022 Google LLC\n",
-        "#\n",
-        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
-        "# you may not use this file except in compliance with the License.\n",
-        "# You may obtain a copy of the License at\n",
-        "#\n",
-        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
-        "#\n",
-        "# Unless required by applicable law or agreed to in writing, software\n",
-        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
-        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
-        "# See the License for the specific language governing permissions and\n",
-        "# limitations under the License."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "e885ac09bc73"
-      },
       "source": [
         "# Train a multi-class classification model for ads-targeting\n",
-        "<table align=\"left\">\n",
         "\n",
-        "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/ads_targetting/training-multi-class-classification-model-for-ads-targeting-usecase.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
-        "    </a>\n",
-        "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/ads_targetting/training-multi-class-classification-model-for-ads-targeting-usecase.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
-        "      View on GitHub\n",
-        "    </a>\n",
-        "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/ads_targetting/training-multi-class-classification-model-for-ads-targeting-usecase.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
-        "    </a>\n",
-        "  </td>                                                                                               \n",
-        "</table>"
+        "## Table of contents\n",
+        "\n",
+        "* [Overview](#section-1)\n",
+        "* [Dataset](#section-2)\n",
+        "* [Objective](#section-3)\n",
+        "* [Costs](#section-4)\n",
+        "* [Tutorial](#section-5)\n",
+        "\t- [Fetch the data from BigQuery](#section-5)\n",
+        "    - [Preprocess the data](#section-6)\n",
+        "    - [Train a TensorFlow model](#section-7)\n",
+        "    - [Run the model on test data](#section-8)\n",
+        "    - [Automating the execution of the notebook using executor](#section-9)\n",
+        "    - [Scheduled runs on executor](#section-10)\n",
+        "    - [Parameterizing the variables](#section-11)\n",
+        "* [Save the model to a Cloud Storage path](#section-12)\n",
+        "* [Clean up](#section-13)\n"
       ]
     },
     {
@@ -59,19 +33,23 @@
       },
       "source": [
         "## Overview\n",
+        "<a name=\"section-1\"></a>\n",
         "\n",
         "This tutorial demonstrates how to build a machine learning model for an ads-targeting use case. Ads-targeting is an advertisement technique where chosen or tailor-made ads are shown to the customers based on their past behavior and preferences. Targeted ads are meant to reach specific customers based on demographics, psychographics, behavior, and other second-order activities that are learned usually through data collected from the customers.\n",
         "\n",
-        "*Note: If you are using [Vertex AI Workbench managed notebooks](https://cloud.google.com/vertex-ai/docs/workbench/managed/create-instance) instance use the `TensorFlow 2 (Local)` kernel. Some components of this notebook may not work in other notebook environments.*\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "1bea2b6e9b25"
-      },
-      "source": [
+        "*Note: This notebook file was designed to run in a [Vertex AI Workbench managed notebooks](https://cloud.google.com/vertex-ai/docs/workbench/managed/create-instance) instance using the `TensorFlow 2 (Local)` kernel. Some components of this notebook may not work in other notebook environments.*\n",
+        "\n",
+        "## Dataset\n",
+        "<a name=\"section-2\"></a>\n",
+        "\n",
+        "This tutorial uses the `looker-private-demo.ecomm` dataset in BigQuery. The dataset consists of information about various advertisement campaigns including the demographics of users who have clicked and made some purchases after seeing the ads. For this tutorial, the top three campaigns from the USA are selected from this dataset and user information for those who have made purchases shall be used to train a model with the campaigns as the classes. The idea is to see if the advertisement and the user data can be used to identify which campaign is best-suited for the user.\n",
+        "\n",
+        "The dataset can be accessed by pinning the `looker-private-demo` project in BigQuery. Instead of going to the BigQuery user interface, this process can be performed from the JupyterLab user interface on a Vertex AI Workbench managed notebooks instance. Vertex AI Workbench managed notebooks instances support browsing through the datasets and tables from BigQuery through its BigQuery integration. \n",
+        "\n",
+        "<img src=\"images/Bigquery_UI_new.PNG\"></img>\n",
+        "\n",
         "## Objective\n",
+        "<a name=\"section-3\"></a>\n",
         "\n",
         "This tutorial demonstrates how to collect data from BigQuery, preprocess it, and train a multi-class classification model on an E-commerce dataset. The steps performed include the following:\n",
         "\n",
@@ -81,31 +59,10 @@
         "- Evaluate the loss for the trained model\n",
         "- Automate the notebook execution using the executor feature\n",
         "- Save the model to a Cloud Storage path\n",
-        "- Clean up the created resources"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "34d623e6dfa3"
-      },
-      "source": [
-        "## Dataset\n",
+        "- Clean up the created resources\n",
         "\n",
-        "This tutorial uses the `looker-private-demo.ecomm` dataset in BigQuery. The dataset consists of information about various advertisement campaigns including the demographics of users who have clicked and made some purchases after seeing the ads. For this tutorial, the top three campaigns from the USA are selected from this dataset and user information for those who have made purchases shall be used to train a model with the campaigns as the classes. The idea is to see if the advertisement and the user data can be used to identify which campaign is best-suited for the user.\n",
-        "\n",
-        "The dataset can be accessed by pinning the `looker-private-demo` project in BigQuery. If you are using Vertex AI Workbench managed notebooks instance, instead of going to the BigQuery user interface, this process can be performed from the JupyterLab user interface. Vertex AI Workbench managed notebooks instances support browsing through the datasets and tables from BigQuery through its BigQuery integration. \n",
-        "\n",
-        "<img src=\"images/Bigquery_UI_new.PNG\"></img>"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "ee02650bb7fd"
-      },
-      "source": [
         "### Costs \n",
+        "<a name=\"section-4\"></a>\n",
         "\n",
         "This tutorial uses billable components of Google Cloud:\n",
         "\n",
@@ -121,121 +78,6 @@
         "to generate a cost estimate based on your projected usage."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "y320EIk-kXT7"
-      },
-      "source": [
-        "### Set up your local development environment\n",
-        "\n",
-        "**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets\n",
-        "all the requirements to run this notebook. You can skip this step.\n",
-        "\n",
-        "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
-        "You need the following:\n",
-        "\n",
-        "* The Google Cloud SDK\n",
-        "* Git\n",
-        "* Python 3\n",
-        "* virtualenv\n",
-        "* Jupyter notebook running in a virtual environment with Python 3\n",
-        "\n",
-        "The Google Cloud guide to [Setting up a Python development\n",
-        "environment](https://cloud.google.com/python/setup) and the [Jupyter\n",
-        "installation guide](https://jupyter.org/install) provide detailed instructions\n",
-        "for meeting these requirements. The following steps provide a condensed set of\n",
-        "instructions:\n",
-        "\n",
-        "1. [Install and initialize the Cloud SDK.](https://cloud.google.com/sdk/docs/)\n",
-        "\n",
-        "1. [Install Python 3.](https://cloud.google.com/python/setup#installing_python)\n",
-        "\n",
-        "1. [Install\n",
-        "   virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv)\n",
-        "   and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
-        "\n",
-        "1. To install Jupyter, run `pip3 install jupyter` on the\n",
-        "command-line in a terminal shell.\n",
-        "\n",
-        "1. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
-        "\n",
-        "1. Open this notebook in the Jupyter Notebook Dashboard."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "1DouUvNOkXT8"
-      },
-      "source": [
-        "### Install additional packages\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "Ayt1jhFXkXT9"
-      },
-      "outputs": [],
-      "source": [
-        "import os\n",
-        "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\""
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "95826791kXT_"
-      },
-      "outputs": [],
-      "source": [
-        "! pip3 install {USER_FLAG} --upgrade pandas-gbq 'google-cloud-bigquery[bqstorage,pandas]' tensorflow sklearn protobuf==3.20.1 -q \\\n",
-        "                                     "
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "aNeMRbpukXUA"
-      },
-      "source": [
-        "### Restart the kernel\n",
-        "\n",
-        "After you install the additional packages, you need to restart the notebook kernel so it can find the packages."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "dJ_yvi_9kXUB"
-      },
-      "outputs": [],
-      "source": [
-        "# Automatically restart kernel after installs\n",
-        "import os\n",
-        "\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    # Automatically restart kernel after installs\n",
-        "    import IPython\n",
-        "\n",
-        "    app = IPython.Application.instance()\n",
-        "    app.kernel.do_shutdown(True)"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -255,67 +97,34 @@
       },
       "outputs": [],
       "source": [
-        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "5bf9979b96ff"
-      },
-      "outputs": [],
-      "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
-        "    # Get your GCP project id from gcloud\n",
-        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "PROJECT_ID = \"\"\n",
+        "\n",
+        "# Get your Google Cloud project ID from gcloud\n",
+        "if not os.getenv(\"IS_TESTING\"):\n",
+        "    shell_output=!gcloud config list --format 'value(core.project)' 2>/dev/null\n",
         "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID:\", PROJECT_ID)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "07-xo93jlC6l"
-      },
-      "outputs": [],
-      "source": [
-        "! gcloud config set project $PROJECT_ID"
+        "    print(\"Project ID: \", PROJECT_ID)"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "03d8d65b914d"
+        "id": "d0058f55f8cf"
       },
       "source": [
-        "#### Region\n",
-        "\n",
-        "You can also change the `REGION` variable, which is used for operations\n",
-        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
-        "\n",
-        "- Americas: `us-central1`\n",
-        "- Europe: `europe-west4`\n",
-        "- Asia Pacific: `asia-east1`\n",
-        "\n",
-        "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
-        "\n",
-        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
+        "Otherwise, set your project ID here."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "3281bedf6d3c"
+        "id": "19579640c063"
       },
       "outputs": [],
       "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
+        "    PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -342,74 +151,6 @@
         "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "OoPGk5KOkXUG"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
-        "authenticated. Skip this step.\n",
-        "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions\n",
-        "when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "1. In the Cloud Console, go to the [**Create service account key**\n",
-        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
-        "\n",
-        "2. Click **Create service account**.\n",
-        "\n",
-        "3. In the **Service account name** field, enter a name, and\n",
-        "   click **Create**.\n",
-        "\n",
-        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
-        "into the filter box, and select\n",
-        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
-        "local environment.\n",
-        "\n",
-        "6. Enter the path to your service account key as the\n",
-        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "Teyy6LGqkXUG"
-      },
-      "outputs": [],
-      "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = \"google.colab\" in sys.modules\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -420,8 +161,20 @@
         "\n",
         "**The following steps are required, regardless of your notebook environment.**\n",
         "\n",
+        "When you submit a training job using the Cloud SDK, you upload a Python package\n",
+        "containing your training code to a Cloud Storage bucket. Vertex AI runs\n",
+        "the code from this package. In this tutorial, Vertex AI also saves the\n",
+        "trained model that results from your job in the same bucket. Using this model artifact, you can then\n",
+        "create Vertex AI model and endpoint resources in order to serve\n",
+        "online predictions.\n",
+        "\n",
         "Set the name of your Cloud Storage bucket below. It must be unique across all\n",
-        "Cloud Storage buckets.\n"
+        "Cloud Storage buckets.\n",
+        "\n",
+        "You may also change the `REGION` variable, which is used for operations\n",
+        "throughout the rest of this notebook. Make sure to [choose a region where Vertex AI services are\n",
+        "available](https://cloud.google.com/vertex-ai/docs/general/locations#available_regions). You may\n",
+        "not use a Multi-Regional Storage bucket for training with Vertex AI."
       ]
     },
     {
@@ -432,8 +185,8 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "REGION = \"[your-region]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -444,9 +197,8 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
-        "    BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
       ]
     },
     {
@@ -466,7 +218,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_URI"
+        "! gsutil mb -l $REGION $BUCKET_NAME"
       ]
     },
     {
@@ -486,36 +238,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_URI"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "bmnMD2MjkXUJ"
-      },
-      "source": [
-        "### Import libraries and define constants"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "oqtZRqDEkXUJ"
-      },
-      "outputs": [],
-      "source": [
-        "import warnings\n",
-        "\n",
-        "import pandas as pd\n",
-        "from sklearn.model_selection import train_test_split\n",
-        "from sklearn.preprocessing import StandardScaler\n",
-        "from tensorflow.keras import Sequential\n",
-        "from tensorflow.keras.layers import Dense\n",
-        "from tensorflow.keras.utils import to_categorical\n",
-        "\n",
-        "warnings.filterwarnings(\"ignore\")"
+        "! gsutil ls -al $BUCKET_NAME"
       ]
     },
     {
@@ -526,16 +249,8 @@
       "source": [
         "## Tutorial\n",
         "\n",
-        "### Fetch the data from BigQuery \n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "5c07be8840ae"
-      },
-      "source": [
-        "If you are using ***Vertex AI Workbench managed notebooks instance***, below cell which starts with \"#@bigquery\" will be a SQL Query. If you are using Vertex AI Workbench user managed notebooks instance or Colab it will be a markdown cell."
+        "### Fetch the data from BigQuery \n",
+        "<a name=\"section-5\"></a>"
       ]
     },
     {
@@ -616,7 +331,7 @@
         "id": "923fdd823683"
       },
       "source": [
-        "If you are using Vertex AI Workbench managed notebooks instance, once the results from BigQuery are displayed in the above cell, click the **Query and load as DataFrame** button and execute the generated code stub to fetch the data into the current notebook as a dataframe.\n",
+        "Once the results from BigQuery are displayed in the above cell, click the **Query and load as DataFrame** button and execute the generated code stub to fetch the data into the current notebook as a dataframe.\n",
         "\n",
         "*Note: By default the data is loaded into a `df` variable, though this can be changed before executing the cell if required.*"
       ]
@@ -633,7 +348,7 @@
         "# Comment out otherwise for speed-up.\n",
         "from google.cloud.bigquery import Client\n",
         "\n",
-        "client = Client(project=PROJECT_ID)\n",
+        "client = Client()\n",
         "\n",
         "query = \"\"\"WITH traindata AS (\n",
         "SELECT b.* except(ad_event_id, user_id), c.* except(id), d.* except(keyword_id, ad_id), a.amount, a.device_type, e.name\n",
@@ -664,6 +379,44 @@
       },
       "source": [
         "### Preprocess the data\n",
+        "<a name=\"section-6\"></a>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e8503e799eec"
+      },
+      "source": [
+        "Import the required libraries."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "5b11973ccf76"
+      },
+      "outputs": [],
+      "source": [
+        "import warnings\n",
+        "\n",
+        "import pandas as pd\n",
+        "from sklearn.model_selection import train_test_split\n",
+        "from sklearn.preprocessing import StandardScaler\n",
+        "from tensorflow.keras import Sequential\n",
+        "from tensorflow.keras.layers import Dense\n",
+        "from tensorflow.keras.utils import to_categorical\n",
+        "\n",
+        "warnings.filterwarnings(\"ignore\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e48d156d8bb6"
+      },
+      "source": [
         "Select the necessary columns from the E-commerce data and divide them based on their type (numerical/categorical)."
       ]
     },
@@ -688,22 +441,13 @@
         "num_cols = [\"age\", \"cpc_bid_amount\", \"quality_score\", \"amount\"]"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "9bd71de0d37e"
-      },
-      "source": [
-        "#### Select top three campaigns"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "ace612851261"
       },
       "source": [
-        "From the current dataset, only the top three campaigns will be chosen to target the users. All the relevant information about the advertisement and the user who purchased an item after seeing the advertisement is available in the dataframe already. "
+        "From the current dataset, only the top three camapigns will be chosen to target the users. All the relevant information about the advertisement and the user who purchased an item after seeing the advertisement is available in the dataframe already. "
       ]
     },
     {
@@ -737,22 +481,13 @@
         "df[\"name\"] = df[\"name\"].map({\"Tops & Tees\": 0, \"Active\": 1, \"Accessories\": 2})"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "c2d5338b1b95"
-      },
-      "source": [
-        "#### One-hot encode the categorical variables"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "8902f763d1ca"
       },
       "source": [
-        "After one-hot encoding, the first level-column is dropped to avoid the [dummy-variable trap](https://en.wikipedia.org/wiki/Dummy_variable_(statistics)) scenario. This process is called *dummy-encoding*."
+        "One-hot encode the categorical variables. After one-hot encoding, the first level-column is dropped to avoid the [dummy-variable trap](https://en.wikipedia.org/wiki/Dummy_variable_(statistics)) scenario. This process is called *dummy-encoding*."
       ]
     },
     {
@@ -786,7 +521,7 @@
         "id": "3abf027eda2d"
       },
       "source": [
-        "#### Split the data into train and test."
+        "Split the data into train and test."
       ]
     },
     {
@@ -811,7 +546,7 @@
         "id": "d1a32b9d9640"
       },
       "source": [
-        "#### Scale the data."
+        "Scale the data."
       ]
     },
     {
@@ -834,7 +569,16 @@
       },
       "source": [
         "### Train a TensorFlow model\n",
-        "#### Convert the target column to a categorical encoded colum (one-hot encoded)."
+        "<a name=\"section-7\"></a>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3e7656556a48"
+      },
+      "source": [
+        "Convert the target column to a categorical encoded colum (one-hot encoded)."
       ]
     },
     {
@@ -855,7 +599,7 @@
         "id": "3dd0014a7e1d"
       },
       "source": [
-        "#### Define hyperparameters for model training. \n",
+        "Define hyperparameters for model training. \n",
         "\n",
         "*Note: Comment or remove the parameters from the following cell if they are provided already as an input parameter through the executor feature.*"
       ]
@@ -880,7 +624,7 @@
         "id": "406b731f576b"
       },
       "source": [
-        "#### Define the architecture and compile the model."
+        "Define the architecture and compile the model."
       ]
     },
     {
@@ -920,7 +664,7 @@
         "id": "4ab12c34f258"
       },
       "source": [
-        "#### Fit the model."
+        "Fit the model."
       ]
     },
     {
@@ -940,7 +684,8 @@
         "id": "51a2d0b52df3"
       },
       "source": [
-        "### Run the model on test data\n"
+        "### Run the model on test data\n",
+        "<a name=\"section-8\"></a>"
       ]
     },
     {
@@ -949,7 +694,7 @@
         "id": "f08445f2cd02"
       },
       "source": [
-        "#### Evaluate the model on test data."
+        "Evaluate the model on test data."
       ]
     },
     {
@@ -964,24 +709,16 @@
         "print(f\"Test results - Loss: {test_results}\")"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "81ef0e081340"
-      },
-      "source": [
-        "**Please note that executor feature is available only in Vertex AI Workbench managed notebooks**"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "9769168778e8"
       },
       "source": [
-        "### Automating the execution of the notebook using executor in Vertex AI Workbench managed notebooks instance\n",
+        "### Automating the execution of the notebook using executor\n",
+        "<a name=\"section-9\"></a>\n",
         "\n",
-        "If you are using Vertex AI Workbench managed notebooks instance, the executor can help you run a notebook file from start to end, with your choice of the environment, machine type, input parameters, and other characteristics. After setting up an execution, the notebook is executed as a job in Vertex AI custom training. Your jobs can be monitored from the <b>Notebook Executor</b> pane in the menu on the left.\n",
+        "The executor can help you run a notebook file from start to end, with your choice of the environment, machine type, input parameters, and other characteristics. After setting up an execution, the notebook is executed as a job in Vertex AI custom training. Your jobs can be monitored from the <b>Notebook Executor</b> pane in the menu on the left.\n",
         "\n",
         "<img src=\"images/executor.png\"></img>\n",
         "\n",
@@ -994,9 +731,10 @@
         "id": "cf486c351581"
       },
       "source": [
-        "### Scheduled runs on executor in Vertex AI Workbench managed notebooks instance\n",
+        "### Scheduled runs on executor\n",
+        "<a name=\"section-10\"></a>\n",
         "\n",
-        "Vertex AI Workbench managed noteboook runs can also be scheduled recurringly with the executor. To do so, select <b>Schedule-based recurring executions</b> as the run type instead of <b>One-time execution</b>. The frequency of the job and the time when it executes is provided when you create the execution.\n",
+        "Notebook runs can also be scheduled recurringly with the executor. To do so, select <b>Schedule-based recurring executions</b> as the run type instead of <b>One-time execution</b>. The frequency of the job and the time when it executes is provided when you create the execution.\n",
         "\n",
         "<img src=\"images/executor_scheduled_runs2.png\"></img>"
       ]
@@ -1008,8 +746,9 @@
       },
       "source": [
         "### Parameterizing the variables\n",
+        "<a name=\"section-11\"></a>\n",
         "\n",
-        "If you are using Vertex AI Workbench managed notebooks instance, executor lets you run a notebook with different sets of input parameters. If required, constants in the notebook can be treated as arguments to a function, and when you submit the execution, you can provide those constants as input parameters.\n",
+        "Executor lets you run a notebook with different sets of input parameters. If required, constants in the notebook can be treated as arguments to a function, and when you submit the execution, you can provide those constants as input parameters.\n",
         "\n",
         "<img src=\"images/executor_input_parameters.png\"></img>\n",
         "\n",
@@ -1023,6 +762,7 @@
       },
       "source": [
         "### Save the model to a Cloud Storage path\n",
+        "<a name=\"section-12\"></a>\n",
         "\n",
         "TensorFlow's `model.save()` method supports Cloud Storage paths as well as the local file paths while writing the model object to a file. It needs to be ensured that the service account being used to run this notebook has `write` permissions to the specified Cloud Storage path."
       ]
@@ -1035,7 +775,7 @@
       },
       "outputs": [],
       "source": [
-        "GCS_PATH = BUCKET_URI + \"/path-to-save/\"\n",
+        "GCS_PATH = \"gs://\" + BUCKET_NAME + \"/[path-to-save]/\"\n",
         "model.save(GCS_PATH)"
       ]
     },
@@ -1046,6 +786,7 @@
       },
       "source": [
         "## Clean up\n",
+        "<a name=\"section-13\"></a>\n",
         "\n",
         "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
         "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
@@ -1061,11 +802,7 @@
       },
       "outputs": [],
       "source": [
-        "# Delete the Cloud Storage bucket\n",
-        "\n",
-        "delete_bucket = False\n",
-        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
-        "    ! gsutil -m rm -r $BUCKET_URI"
+        "! gsutil -m rm -r [cloud-storage-folder-path-to-delete]"
       ]
     }
   ],

notebooks/community/managed_notebooks/predictive_maintainance/predictive_maintenance_usecase.ipynb

@@ -1,62 +1,17 @@
 {
   "cells": [
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "18ebbd838e32"
-      },
-      "outputs": [],
-      "source": [
-        "# Copyright 2022 Google LLC\n",
-        "#\n",
-        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
-        "# you may not use this file except in compliance with the License.\n",
-        "# You may obtain a copy of the License at\n",
-        "#\n",
-        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
-        "#\n",
-        "# Unless required by applicable law or agreed to in writing, software\n",
-        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
-        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
-        "# See the License for the specific language governing permissions and\n",
-        "# limitations under the License."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "aef73cfa8725"
       },
       "source": [
-        "# Predictive Maintenance using Vertex AI\n",
-        "\n",
-        "<table align=\"left\">\n",
-        "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/predictive_maintainance/predictive_maintenance_usecase.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
-        "      View on GitHub\n",
-        "    </a>\n",
-        "  </td>\n",
-        "    <td>\n",
-        "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/predictive_maintainance/predictive_maintenance_usecase.ipynb\">\n",
-        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\\\" alt=\"Colab logo\"> Run in Colab\n",
-        "        </a>\n",
-        "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/predictive_maintainance/predictive_maintenance_usecase.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
-        "    </a>\n",
-        "  </td>\n",
-        "</table>\n",
-        "<br/><br/><br/>\n",
-        "\n",
+        "# Predictive Maintenance \n",
         "\n",
         "## Table of contents\n",
         "* [Overview](#section-1)\n",
-        "* [Objective](#section-2)\n",
-        "* [Dataset](#section-3)\n",
+        "* [Dataset](#section-2)\n",
+        "* [Objective](#section-3)\n",
         "* [Costs](#section-4)\n",
         "* [Data analysis](#section-5)\n",
         "* [Fit a regression model](#section-6)\n",
@@ -67,32 +22,24 @@
         "  * [Create an endpoint](#section-11)\n",
         "  * [Deploy the model to the created endpoint](#section-12)\n",
         "  * [Test calling the endpoint](#section-13)\n",
-        "* [Clean up](#section-14)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "e10c5167a061"
-      },
-      "source": [
+        "* [Clean up](#section-14)\n",
+        "\n",
+        "\n",
         "## Overview\n",
         "<a name=\"section-1\"></a>\n",
         "\n",
-        "In this notebook, you go through a predictive maintenance usecase on industrial data using machine learning techniques, deploy the machine learning model on Vertex AI, and automate the workflow using the executor feature of Vertex AI Workbench.\n",
+        "This notebook demonstrates how to perform predictive maintenance on industrial data using machine learning techniques, deploy the machine learning model on Vertex AI, and automate the workflow using the executor feature of Vertex AI Workbench.\n",
         "\n",
-        "*Note: This notebook file is developed to run in a [Vertex AI Workbench managed notebooks](https://console.cloud.google.com/vertex-ai/workbench/list/managed) instance using the XGBoost (Local) kernel. Some components of this notebook may not work in other notebook environments.*"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "fead9e83ebd7"
-      },
-      "source": [
-        "### Objective\n",
+        "*Note: This notebook file was developed to run in a [Vertex AI Workbench managed notebooks](https://console.cloud.google.com/vertex-ai/workbench/list/managed) instance using the XGBoost (Local) kernel. Some components of this notebook may not work in other notebook environments.*\n",
+        "\n",
+        "## Dataset\n",
         "<a name=\"section-2\"></a>\n",
         "\n",
+        "The dataset used in this notebook is a part of the [NASA Turbofan Engine Degradation Simulation dataset](https://ti.arc.nasa.gov/tech/dash/groups/pcoe/prognostic-data-repository/), which consists of simulated time-series data for four sets of fleet engines under different combinations of operational conditions and fault modes. In this notebook, only one of the engine's simulated data (FD001) has been used to analyze and train a model that can predict the engine's remaining useful life.\n",
+        "\n",
+        "## Objectives\n",
+        "<a name=\"section-3\"></a>\n",
+        "\n",
         "The objectives of this notebook include:\n",
         "\n",
         "- Loading the required dataset from a Cloud Storage bucket.\n",
@@ -102,28 +49,9 @@
         "- Evaluating the model.\n",
         "- Running the notebook end-to-end as a training job using Executor.\n",
         "- Deploying the model on Vertex AI.\n",
-        "- Clean up."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "a71f4d96bf80"
-      },
-      "source": [
-        "### Dataset\n",
-        "<a name=\"section-3\"></a>\n",
+        "- Clean up.\n",
         "\n",
-        "The dataset used in this notebook is a part of the [NASA Turbofan Engine Degradation Simulation dataset](https://ti.arc.nasa.gov/tech/dash/groups/pcoe/prognostic-data-repository/), which consists of simulated time-series data for four sets of fleet engines under different combinations of operational conditions and fault modes. A version of this dataset which is saved to a public Cloud Storage bucket is used in this notebook. In this notebook, one of the engine's simulated data (FD001) is used to analyze and train a model that can predict the engine's remaining useful life."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "36c53c95b4b9"
-      },
-      "source": [
-        "### Costs\n",
+        "## Costs\n",
         "<a name=\"section-4\"></a>\n",
         "\n",
         "This tutorial uses the following billable components of Google Cloud:\n",
@@ -141,126 +69,24 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "629f52f6efe1"
+        "id": "5b15a97278df"
       },
       "source": [
+        "## Before you begin\n",
+        "\n",
         "### Kernel selection\n",
-        "Select <b>XGBoost</b> kernel while running this notebook on Vertex AI Workbench's managed instances. Otherwise, ensure that the following libraries are installed in the environment where this notebook is being run.\n",
+        "Select <b>XGBoost</b> kernel while running this notebook on Vertex AI Workbench managed notebooks instances or ensure that the following libraries are installed in the environment where this notebook is being run.\n",
         "- XGBoost\n",
         "- Pandas\n",
         "- Seaborn\n",
         "- Sklearn\n",
         "\n",
-        "Along with the above libraries, th`e following google-cloud libraries are also used in this notebook.\n",
+        "Along with the above libraries, the following google-cloud libraries are also used in this notebook.\n",
         "\n",
         "- google.cloud.aiplatform\n",
-        "- google.cloud.storage"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "16bee0754628"
-      },
-      "source": [
-        "## Installation\n",
+        "- google.cloud.storage\n",
         "\n",
-        "Install the following packages to run this notebook outside Vertex AI Workbench's managed instances."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "69520a67e54c"
-      },
-      "outputs": [],
-      "source": [
-        "import os\n",
-        "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "    \n",
-        "! pip3 install {USER_FLAG} --upgrade google-cloud-aiplatform \\\n",
-        "                                    google-cloud-storage \\\n",
-        "                                    xgboost \\\n",
-        "                                    seaborn \\\n",
-        "                                    sklearn \\\n",
-        "                                    fsspec \\\n",
-        "                                    gcsfs \\\n",
-        "                                    pandas -q"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "eda79cca981d"
-      },
-      "source": [
-        "### Restart the kernel\n",
-        "\n",
-        "Once you've installed the additional packages, you need to restart the notebook kernel so it can find the packages."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "e200999cabe5"
-      },
-      "outputs": [],
-      "source": [
-        "import os\n",
-        "\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    # Automatically restart kernel after installs\n",
-        "    import IPython\n",
-        "\n",
-        "    app = IPython.Application.instance()\n",
-        "    app.kernel.do_shutdown(True)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "5b15a97278df"
-      },
-      "source": [
-        "## Before you begin \n",
-        "\n",
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com).\n",
-        "\n",
-        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "5aee4379e8e5"
-      },
-      "source": [
-        "#### Set your project ID\n",
+        "### Set your project ID\n",
         "\n",
         "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
       ]
@@ -273,67 +99,36 @@
       },
       "outputs": [],
       "source": [
-        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "5bf9979b96ff"
-      },
-      "outputs": [],
-      "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
-        "    # Get your GCP project id from gcloud\n",
-        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "import os\n",
+        "\n",
+        "PROJECT_ID = \"\"\n",
+        "\n",
+        "# Get your Google Cloud project ID from gcloud\n",
+        "if not os.getenv(\"IS_TESTING\"):\n",
+        "    shell_output = !gcloud config list --format 'value(core.project)' 2>/dev/null\n",
         "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID:\", PROJECT_ID)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "09021c90b34c"
-      },
-      "outputs": [],
-      "source": [
-        "! gcloud config set project $PROJECT_ID"
+        "    print(\"Project ID: \", PROJECT_ID)"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "9658ecf524b1"
+        "id": "750bf2883c2d"
       },
       "source": [
-        "#### Region\n",
-        "\n",
-        "You can also change the `REGION` variable, which is used for operations\n",
-        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. It is recommended that you choose the region closest to you.\n",
-        "\n",
-        "- Americas: `us-central1`\n",
-        "- Europe: `europe-west4`\n",
-        "- Asia Pacific: `asia-east1`\n",
-        "\n",
-        "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
-        "\n",
-        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
+        "Otherwise, set your project ID here."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "5c615e53149f"
+        "id": "3c6db1ca88b9"
       },
       "outputs": [],
       "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
+        "    PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -342,9 +137,9 @@
         "id": "f66f96816fd0"
       },
       "source": [
-        "#### UUID\n",
+        "#### Timestamp\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append it onto the name of resources you create in this tutorial."
       ]
     },
     {
@@ -355,84 +150,9 @@
       },
       "outputs": [],
       "source": [
-        "import random\n",
-        "import string\n",
+        "from datetime import datetime\n",
         "\n",
-        "\n",
-        "# Generate a uuid of a specifed length(default=8)\n",
-        "def generate_uuid(length: int = 8) -> str:\n",
-        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
-        "\n",
-        "\n",
-        "UUID = generate_uuid()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "df899ce9999c"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
-        "authenticated.\n",
-        "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions\n",
-        "when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "1. In the Cloud Console, go to the [**Create service account key**\n",
-        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
-        "\n",
-        "2. Click **Create service account**.\n",
-        "\n",
-        "3. In the **Service account name** field, enter a name, and\n",
-        "   click **Create**.\n",
-        "\n",
-        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
-        "into the filter box, and select\n",
-        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
-        "local environment.\n",
-        "\n",
-        "6. Enter the path to your service account key as the\n",
-        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "201e8e760d22"
-      },
-      "outputs": [],
-      "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = \"google.colab\" in sys.modules\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
     {
@@ -441,18 +161,11 @@
         "id": "ea53caa30628"
       },
       "source": [
-        "### Create a Cloud Storage bucket\n",
+        "## Select or Create a Cloud Storage Bucket for storing the model\n",
         "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
+        "When you create a model resource on Vertex AI using the Cloud SDK, you need to give a Cloud Storage bucket URI of the model where the model is stored. Using the model saved, you can then create Vertex AI model and endpoint resources in order to serve online predictions.\n",
         "\n",
-        "\n",
-        "When you create a model in Vertex AI using the Cloud SDK, you give a Cloud Storage path where the trained model is saved. \n",
-        "In this tutorial, Vertex AI saves the trained model to a Cloud Storage bucket. Using this model artifact, you can then\n",
-        "create Vertex AI model and endpoint resources in order to serve\n",
-        "online predictions.\n",
-        "\n",
-        "Set the name of your Cloud Storage bucket below. It must be unique across all\n",
-        "Cloud Storage buckets."
+        "Set the name of your Cloud Storage bucket below. It must be unique across all Cloud Storage buckets. You may also change the `REGION` variable, which is used for operations throughout the rest of this notebook. Make sure to choose a region where Vertex AI services are available."
       ]
     },
     {
@@ -463,8 +176,9 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+        "BUCKET_NAME = \"[your-bucket-name]\"\n",
+        "BUCKET_URI = f\"gs://{BUCKET_NAME}\"\n",
+        "REGION = \"us-central1\""
       ]
     },
     {
@@ -475,9 +189,13 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
-        "    BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+        "# Set a default bucketname in case bucket name is not given\n",
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None:\n",
+        "    from datetime import datetime\n",
+        "\n",
+        "    TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
+        "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
       ]
     },
     {
@@ -497,7 +215,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_URI"
+        "! gsutil mb -l $REGION $BUCKET_NAME"
       ]
     },
     {
@@ -517,7 +235,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_URI"
+        "! gsutil ls -al $BUCKET_NAME"
       ]
     },
     {
@@ -526,7 +244,7 @@
         "id": "4c0f6aac282a"
       },
       "source": [
-        "### Import the required libraries"
+        "## Import the required libraries"
       ]
     },
     {
@@ -569,7 +287,7 @@
       "outputs": [],
       "source": [
         "# load the data from the source\n",
-        "INPUT_PATH = \"gs://cloud-samples-data/ai-platform-unified/datasets/tabular/predictive_maintenance.csv\"  # data source\n",
+        "INPUT_PATH = \"gs://vertex_ai_managed_services_demo/mfg_predictive_maintenance/train_FD001.txt\"  # data source\n",
         "raw_data = pd.read_csv(INPUT_PATH, sep=\" \", header=None)\n",
         "# check the data\n",
         "print(raw_data.shape)\n",
@@ -774,7 +492,7 @@
         "id": "8197cdef2cff"
       },
       "source": [
-        "As the current objective is to predict the remaining useful life (RUL) of each unit (ID), the target variable needs to be identified. Since you're dealing with a timeseries data that represents the lifetime of a unit, remaining useful life of a unit can be calculated by subtracting the current cycle from the maximum cycle of that unit.\n",
+        "As the current objective is to predict the remaining useful life (RUL) of each unit (ID), the target variable needs to be identified. Since we're dealing with a timeseries data that represents the lifetime of a unit, remaining useful life of a unit can be calculated by subtracting the current cycle from the maximum cycle of that unit.\n",
         "\n",
         "\t\t\t\t\tRUL = Max. Cycle - Current Cycle    \n",
         "## RUL calculation and Feature selection"
@@ -1092,7 +810,6 @@
         "## Running a notebook end-to-end using executor\n",
         "<a name=\"section-9\"></a>\n",
         "\n",
-        "**Note:** This section can only be considered when running this notebook on Managed instances from Vertex AI Workbench.\n",
         "### Automating the notebook execution\n",
         "All the steps followed until now can be run as a training job without using any additional code using the Vertex AI Workbench executor. The executor can help you run a notebook file from start to end, with your choice of the environment, machine type, input parameters, and other characteristics. After setting up an execution, the notebook is executed as a job in Vertex AI custom training. Your jobs can be monitored from the Executor pane in the left sidebar.\n",
         "\n",
@@ -1100,13 +817,13 @@
         "\n",
         "The executor also lets you choose the environment and machine type while automating the runs similar to Vertex AI training jobs without switching to the training jobs UI. Apart from the custom container that replicates the existing kernel by default, pre-built environments like TensorFlow Enterprise, PyTorch, and others can also be selected to run the notebook. The required compute power can be specified by choosing from the list of machine types available, including GPUs.\n",
         "\n",
-        "### Scheduled runs on executor\n",
+        "## Scheduled runs on executor\n",
         "\n",
         "Notebook runs can also be scheduled recurringly with the executor. To do so, select Schedule-based recurring executions as the run type instead of One-time execution. The frequency of the job and the time when it executes is provided when you create the execution.\n",
         "\n",
         "<img src=\"https://storage.googleapis.com/gweb-cloudblog-publish/images/7_Vertex_AI_Workbench.max-1100x1100.jpg\">\n",
         "\n",
-        "### Parameterizing the variables\n",
+        "## Parameterizing the variables\n",
         "\n",
         "The executor lets you run a notebook with different sets of input parameters. If you’ve added parameter tags to any of your notebook cells, you can pass in your parameter values to the executor. More about how to use this feature can be found on this [blog](https://cloud.google.com/blog/products/ai-machine-learning/schedule-and-execute-notebooks-with-vertex-ai-workbench).\n",
         "\n",
@@ -1138,37 +855,6 @@
         "ARTIFACT_GCS_PATH = f\"gs://{BUCKET_NAME}/{BLOB_PATH}\""
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "1aa75b3d4616"
-      },
-      "source": [
-        "Give a display name to the Vertex AI model resource."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "02ca350dba6c"
-      },
-      "outputs": [],
-      "source": [
-        "# Set the model-dsiplay-name\n",
-        "MODEL_DISPLAY_NAME = \"[your-model-display-name]\"  # @param {type:\"string\"}\n",
-        "\n",
-        "# Otherwise, use the default name\n",
-        "if (\n",
-        "    MODEL_DISPLAY_NAME == \"[your-model-display-name]\"\n",
-        "    or MODEL_DISPLAY_NAME is None\n",
-        "    or MODEL_DISPLAY_NAME == \"\"\n",
-        "):\n",
-        "    MODEL_DISPLAY_NAME = \"pred_maint_model_\" + UUID\n",
-        "\n",
-        "print(MODEL_DISPLAY_NAME)"
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -1205,28 +891,6 @@
         "Next, create an endpoint resource for deploying the model."
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "e1e0cd571992"
-      },
-      "outputs": [],
-      "source": [
-        "# Set the endpoint-dsiplay-name\n",
-        "ENDPOINT_DISPLAY_NAME = \"[your-endpoint-display-name]\"  # @param {type:\"string\"}\n",
-        "\n",
-        "# Otherwise, use the default name\n",
-        "if (\n",
-        "    ENDPOINT_DISPLAY_NAME == \"[your-endpoint-display-name]\"\n",
-        "    or ENDPOINT_DISPLAY_NAME is None\n",
-        "    or ENDPOINT_DISPLAY_NAME == \"\"\n",
-        "):\n",
-        "    ENDPOINT_DISPLAY_NAME = \"pred_maint_endpoint_\" + UUID\n",
-        "\n",
-        "print(ENDPOINT_DISPLAY_NAME)"
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -1235,7 +899,6 @@
       },
       "outputs": [],
       "source": [
-        "# Create the Endpoint resource\n",
         "endpoint = aiplatform.Endpoint.create(display_name=ENDPOINT_DISPLAY_NAME)\n",
         "\n",
         "print(endpoint.display_name)\n",
@@ -1252,11 +915,18 @@
         "<a name=\"section-12\"></a>\n",
         "\n",
         "\n",
-        "Configure the following parameters and deploy the model to the created endpoint.\n",
-        "\n",
-        "- `endpoint`: The `Endpoint` object created using Vertex AI SDK.\n",
-        "- `deployed_model_display_name`: A display-name for the deployment.\n",
-        "- `machine_type`: Type of the machine required for the deployment environment. See [here](https://cloud.google.com/vertex-ai/docs/predictions/configure-compute) for references."
+        "Configure the deployment name, machine type, and other parameters for the deployment and deploy the model to the created endpoint."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ca41cac871d6"
+      },
+      "outputs": [],
+      "source": [
+        "MACHINE_TYPE = \"n1-standard-2\""
       ]
     },
     {
@@ -1270,8 +940,8 @@
         "# deploy the model to the endpoint\n",
         "model.deploy(\n",
         "    endpoint=endpoint,\n",
-        "    deployed_model_display_name=MODEL_DISPLAY_NAME + \"_deployment\",\n",
-        "    machine_type=\"n1-standard-2\",\n",
+        "    deployed_model_display_name=DEPLOYED_MODEL_NAME,\n",
+        "    machine_type=MACHINE_TYPE,\n",
         ")\n",
         "\n",
         "model.wait()\n",
@@ -1314,15 +984,7 @@
         "## Clean up\n",
         "<a name=\"section-14\"></a>\n",
         "\n",
-        "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
-        "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
-        "\n",
-        "Otherwise, you can delete the individual resources you created in this tutorial:\n",
-        "* Vertex AI Model\n",
-        "* Vertex AI Endpoint\n",
-        "* Cloud Storage bucket\n",
-        "\n",
-        "Set `delete_bucket` to **True** to delete the Cloud Storage bucket."
+        "Undeploy the model from the endpoint."
       ]
     },
     {
@@ -1333,19 +995,68 @@
       },
       "outputs": [],
       "source": [
-        "# Undeploy all the models from the endpoint\n",
-        "endpoint.undeploy_all()\n",
-        "\n",
-        "# Delete the endpoint resource\n",
-        "endpoint.delete()\n",
-        "\n",
-        "# Delete the model resource\n",
-        "model.delete()\n",
-        "\n",
-        "# Delete the Cloud Storage bucket\n",
-        "delete_bucket = False\n",
-        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
-        "    ! gsutil -m rm -r $BUCKET_URI"
+        "DEPLOYED_MODEL_ID = \"\"\n",
+        "endpoint.undeploy(deployed_model_id=DEPLOYED_MODEL_ID)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "96e427b77791"
+      },
+      "source": [
+        "Delete the endpoint."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ace028ac23ea"
+      },
+      "outputs": [],
+      "source": [
+        "endpoint.delete()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "4b77998d0512"
+      },
+      "source": [
+        "Delete the model."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "e034150a4c94"
+      },
+      "outputs": [],
+      "source": [
+        "model.delete()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "23cb2deb122d"
+      },
+      "source": [
+        "Remove the contents of the Cloud Storage bucket."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "98aaac27d85d"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil -m rm -r $BUCKET_URI"
       ]
     }
   ],

notebooks/community/managed_notebooks/pricing_optimization/pricing-optimization.ipynb

@@ -0,0 +1,823 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d1cc1c1fa076"
+      },
+      "source": [
+        "# Pricing Optimization \n",
+        "## Table of contents\n",
+        "* [Overview](#section-1)\n",
+        "* [Dataset](#section-2)\n",
+        "* [Objective](#section-3)\n",
+        "* [Costs](#section-4)\n",
+        "* [Create a BigQuery dataset](#section-5)\n",
+        "* [Load the dataset from Cloud Storage](#section-6)\n",
+        "* [Data analysis](#section-7)\n",
+        "* [Preprocess the data for training](#section-8)\n",
+        "* [Train the model using BigQuery ML](#section-9)\n",
+        "* [Generate forecasts from the model](#section-10)\n",
+        "* [Interpret the results to choose the best price](#section-11)\n",
+        "* [Clean up](#section-12)\n",
+        "\n",
+        "## Overview\n",
+        "<a name=\"section-1\"></a>\n",
+        "\n",
+        "This notebook demonstrates analysis of pricing optimization on [CDM Pricing Data](https://github.com/trifacta/trifacta-google-cloud/tree/main/design-pattern-pricing-optimization) and automating the workflow using Vertex AI Workbench managed notebooks.\n",
+        "\n",
+        "*Note: This notebook file was developed to run in a [Vertex AI Workbench managed notebooks](https://console.cloud.google.com/vertex-ai/workbench/list/managed) instance using the Python (Local) kernel. Some components of this notebook may not work in other notebook environments.*\n",
+        "\n",
+        "## Dataset\n",
+        "<a name=\"section-2\"></a>\n",
+        "\n",
+        "The dataset used in this notebook is a part of the [CDM Pricing dataset](https://github.com/trifacta/trifacta-google-cloud/blob/main/design-pattern-pricing-optimization/CDM_Pricing_large_table.csv), which consists of product sales information on specified dates.\n",
+        "\n",
+        "## Objective\n",
+        "<a name=\"section-3\"></a>\n",
+        "\n",
+        "The objective of this notebook is to build a pricing optimization model using Vertex AI. The following steps have been followed:  \n",
+        "\n",
+        "- Load the required dataset from a Cloud Storage bucket.\n",
+        "- Analyze the fields present in the dataset.\n",
+        "- Process the data to build a model.\n",
+        "- Build a BigQuery ML forecast model on the processed data.\n",
+        "- Get forecasted values from the BigQuery ML model.\n",
+        "- Interpret the forecasts to identify the best prices.\n",
+        "- Clean up.\n",
+        "\n",
+        "## Costs\n",
+        "<a name=\"section-4\"></a>\n",
+        "\n",
+        "This tutorial uses the following billable components of Google Cloud:\n",
+        "\n",
+        "- Vertex AI\n",
+        "- BigQuery\n",
+        "- Cloud Storage\n",
+        "\n",
+        "\n",
+        "Learn about [Vertex AI\n",
+        "pricing](https://cloud.google.com/vertex-ai/pricing), [BigQuery pricing](https://cloud.google.com/bigquery/pricing) and [Cloud Storage\n",
+        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
+        "Calculator](https://cloud.google.com/products/calculator/)\n",
+        "to generate a cost estimate based on your projected usage.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5ed1f5e85640"
+      },
+      "source": [
+        "## Before you begin\n",
+        "\n",
+        "### Set your project ID\n",
+        "\n",
+        "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "c3f30148b66d"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "PROJECT_ID = \"\"\n",
+        "\n",
+        "# Get your Google Cloud project ID from gcloud\n",
+        "if not os.getenv(\"IS_TESTING\"):\n",
+        "    shell_output = !gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "    PROJECT_ID = shell_output[0]\n",
+        "    print(\"Project ID: \", PROJECT_ID)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "750bf2883c2d"
+      },
+      "source": [
+        "Otherwise, set your project ID here."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3c6db1ca88b9"
+      },
+      "outputs": [],
+      "source": [
+        "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
+        "    PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2a1c270c7d34"
+      },
+      "source": [
+        "### Import the required libraries and define constants\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "acc6fac1fa55"
+      },
+      "outputs": [],
+      "source": [
+        "import matplotlib.pyplot as plt\n",
+        "import pandas as pd\n",
+        "import seaborn as sns\n",
+        "from google.cloud import bigquery\n",
+        "from google.cloud.bigquery import Client"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "a06006dff8f9"
+      },
+      "outputs": [],
+      "source": [
+        "DATASET = \"[your-bigquery-dataset-id]\"  # set the BigQuery dataset-id\n",
+        "TRAINING_DATA_TABLE = \"[your-bigquery-table-id-to-store-the-training-data]\"  # set the BigQuery table-id to store the training data"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "016c3d47cc69"
+      },
+      "source": [
+        "## Create a BigQuery dataset\n",
+        "<a name=\"section-5\"></a>\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "12ccd8d7956e"
+      },
+      "source": [
+        "#@bigquery\n",
+        "-- create a dataset in BigQuery\n",
+        "\n",
+        "CREATE SCHEMA pricing_optimization\n",
+        "OPTIONS(\n",
+        "  location=\"us\"\n",
+        "  )"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "c106b978a79b"
+      },
+      "source": [
+        "## Load the dataset from Cloud Storage\n",
+        "<a name=\"section-6\"></a>\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "8aeae9da9796"
+      },
+      "outputs": [],
+      "source": [
+        "DATA_LOCATION = \"gs://cloud-samples-data/ai-platform-unified/datasets/tabular/cdm_pricing_large_table.csv\"\n",
+        "df = pd.read_csv(DATA_LOCATION)\n",
+        "print(df.shape)\n",
+        "df.head()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "7b98d5f09842"
+      },
+      "source": [
+        "You will build a forecast model on this data and thus determine the best price for a product. For this type of model, you will not be using many fields: only the sales and price related ones. For the current execrcise, focus on the following fields:\n",
+        "\n",
+        "- `Product_ID`\n",
+        "- `Customer_Hierarchy`\n",
+        "- `Fiscal_Date`\n",
+        "- `List_Price_Converged`\n",
+        "- `Invoiced_quantity_in_Pieces`\n",
+        "- `Net_Sales`\n",
+        "\n",
+        "## Data Analysis\n",
+        "<a name=\"section-7\"></a>\n",
+        "\n",
+        "First, explore the data and distributions.\n",
+        "\n",
+        "Select the required columns from the dataframe."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "af4b41c5eb1f"
+      },
+      "outputs": [],
+      "source": [
+        "id_col = \"Product_ID\"\n",
+        "date_col = \"Fiscal_Date\"\n",
+        "categ_cols = [\"Customer_Hierarchy\"]\n",
+        "num_cols = [\"List_Price_Converged\", \"Invoiced_quantity_in_Pieces\", \"Net_Sales\"]\n",
+        "\n",
+        "df = df[[id_col, date_col] + categ_cols + num_cols].copy()\n",
+        "df.head()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3d780043ee5b"
+      },
+      "source": [
+        "Check the column types and null values in the dataframe."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "f54c445a1288"
+      },
+      "outputs": [],
+      "source": [
+        "df.info()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "cd817b414c4d"
+      },
+      "source": [
+        "This data description reveals that there are no null values in the data. Also, the field `Fiscal_Date` which is a date field is loaded as an object type. \n",
+        "\n",
+        "Change the type of the date field to datetime."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b160fac085c8"
+      },
+      "outputs": [],
+      "source": [
+        "df[\"Fiscal_Date\"] = pd.to_datetime(df[\"Fiscal_Date\"], infer_datetime_format=True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "fb4778578064"
+      },
+      "source": [
+        "Plot the distributions for the categorical fields."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "dd0467cd57c3"
+      },
+      "outputs": [],
+      "source": [
+        "for i in categ_cols:\n",
+        "    df[i].value_counts(normalize=True).plot(kind=\"bar\")\n",
+        "    plt.title(i)\n",
+        "    plt.show()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "145deed255e0"
+      },
+      "source": [
+        "Plot the distributions for the numerical fields."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "f934137c6d82"
+      },
+      "outputs": [],
+      "source": [
+        "for i in num_cols:\n",
+        "    _, ax = plt.subplots(1, 2, figsize=(10, 4))\n",
+        "    df[i].plot(kind=\"box\", ax=ax[0])\n",
+        "    df[i].plot(kind=\"hist\", ax=ax[1])\n",
+        "    ax[0].set_title(i + \"-Boxplot\")\n",
+        "    ax[1].set_title(i + \"-Histogram\")\n",
+        "    plt.show()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "f9b9c2e58380"
+      },
+      "source": [
+        "Check the maximum date and minimum date in Fiscal_Date column."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2a10aa689f9d"
+      },
+      "outputs": [],
+      "source": [
+        "print(df[\"Fiscal_Date\"].max())\n",
+        "print(df[\"Fiscal_Date\"].min())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "4834f63e2e59"
+      },
+      "source": [
+        "Check the product distribution across each category."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "4664877f5304"
+      },
+      "outputs": [],
+      "source": [
+        "grp_cols = [\"Customer_Hierarchy\", \"Product_ID\"]\n",
+        "grp_df = df[grp_cols].groupby(by=grp_cols).count().reset_index()\n",
+        "grp_df.groupby(\"Customer_Hierarchy\").nunique()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "01ed02b9c8fd"
+      },
+      "source": [
+        "Check the percentage changes in the orders based on the percentage changes in the price."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "0b2c428cb135"
+      },
+      "outputs": [],
+      "source": [
+        "# aggregate the data\n",
+        "df_aggr = (\n",
+        "    df.groupby([\"Product_ID\", \"List_Price_Converged\"])\n",
+        "    .agg({\"Fiscal_Date\": min, \"Invoiced_quantity_in_Pieces\": sum, \"Net_Sales\": sum})\n",
+        "    .reset_index()\n",
+        ")\n",
+        "# rename the aggregated columns\n",
+        "df_aggr.rename(\n",
+        "    columns={\n",
+        "        \"Fiscal_Date\": \"First_price_date\",\n",
+        "        \"Invoiced_quantity_in_Pieces\": \"Total_ordered_pieces\",\n",
+        "        \"Net_Sales\": \"Total_net_sales\",\n",
+        "    },\n",
+        "    inplace=True,\n",
+        ")\n",
+        "\n",
+        "# sort values chronologically\n",
+        "df_aggr.sort_values(by=[\"Product_ID\", \"First_price_date\"], inplace=True)\n",
+        "df_aggr.reset_index(drop=True, inplace=True)\n",
+        "\n",
+        "# add columns for previous values\n",
+        "df_aggr[\"Previous_List\"] = df_aggr.groupby([\"Product_ID\"])[\n",
+        "    \"List_Price_Converged\"\n",
+        "].shift()\n",
+        "df_aggr[\"Previous_Total_ordered_pieces\"] = df_aggr.groupby([\"Product_ID\"])[\n",
+        "    \"Total_ordered_pieces\"\n",
+        "].shift()\n",
+        "\n",
+        "# average price change across sku's\n",
+        "df_aggr[\"price_change_perc\"] = (\n",
+        "    (df_aggr[\"List_Price_Converged\"] - df_aggr[\"Previous_List\"])\n",
+        "    / df_aggr[\"Previous_List\"].fillna(0)\n",
+        "    * 100\n",
+        ")\n",
+        "df_aggr[\"order_change_perc\"] = (\n",
+        "    (df_aggr[\"Total_ordered_pieces\"] - df_aggr[\"Previous_Total_ordered_pieces\"])\n",
+        "    / df_aggr[\"Previous_Total_ordered_pieces\"].fillna(0)\n",
+        "    * 100\n",
+        ")\n",
+        "\n",
+        "# plot a scatterplot to visualize the changes\n",
+        "sns.scatterplot(\n",
+        "    x=\"price_change_perc\",\n",
+        "    y=\"order_change_perc\",\n",
+        "    data=df_aggr,\n",
+        "    hue=\"Product_ID\",\n",
+        "    legend=False,\n",
+        ")\n",
+        "plt.title(\"Percentage of change in price vs order\")\n",
+        "plt.show()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8259e916fe25"
+      },
+      "source": [
+        "For most of the products, the percentage change in orders are high where the percentage changes in the prices are low. This suggests that too much change in the prices can affect the number of orders. \n",
+        "\n",
+        "**Note**: There seem to be some outliers in the data as percentage changes greater than 800 are found. In the current exercise, do not take any manual measures to deal with outliers as you will create a BigQuery ML timeseries model that already deals with outliers.\n",
+        "\n",
+        "## Preprocess the data for training\n",
+        "<a name=\"section-8\"></a>\n",
+        "\n",
+        "Check which `Product_ID`'s  have the maximum orders."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "f5cbc7709c6a"
+      },
+      "outputs": [],
+      "source": [
+        "df_orders = df.groupby([\"Product_ID\", \"Customer_Hierarchy\"], as_index=False)[\n",
+        "    \"Invoiced_quantity_in_Pieces\"\n",
+        "].sum()\n",
+        "df_orders.loc[\n",
+        "    df_orders.groupby(\"Customer_Hierarchy\")[\"Invoiced_quantity_in_Pieces\"].idxmax()\n",
+        "]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "fd6d227e513e"
+      },
+      "source": [
+        "From the above result, you can infer the following:\n",
+        "\n",
+        "- Under the **Food** category, **SKU 62** has the maximum orders.\n",
+        "- Under the **Manufacturing** category, **SKU 17** has the maximum orders.\n",
+        "- Under the **Paper** category, **SKU 107** has the maximum orders.\n",
+        "- Under the **Publishing** category, **SKU 8** has the maximum orders.\n",
+        "- Under the **Utilities** category, **SKU 140** has the maximum orders.\n",
+        "\n",
+        "Given that there are too many ids and only a few records for most of them, consider only the above `Product_ID`s for which there are a maximum number of orders. \n",
+        "\n",
+        "**Note**: The `Invoiced_quantity_in_Pieces` field seems to be a *float* type rather than an *int* type as it should be. This could be because the data itself might be averaged in the first place."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2dbc0d64d157"
+      },
+      "source": [
+        "Check the various prices available for these `Product_ID`s."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "acc1dbd2d838"
+      },
+      "outputs": [],
+      "source": [
+        "df_type_food = df[(df[\"Product_ID\"] == \"SKU 62\") & (df[\"Customer_Hierarchy\"] == \"Food\")]\n",
+        "print(\"Food :\")\n",
+        "print(df_type_food[\"List_Price_Converged\"].value_counts())\n",
+        "df_type_manuf = df[\n",
+        "    (df[\"Product_ID\"] == \"SKU 17\") & (df[\"Customer_Hierarchy\"] == \"Manufacturing\")\n",
+        "]\n",
+        "print(\"Manufacturing :\")\n",
+        "print(df_type_manuf[\"List_Price_Converged\"].value_counts())\n",
+        "df_type_paper = df[\n",
+        "    (df[\"Product_ID\"] == \"SKU 107\") & (df[\"Customer_Hierarchy\"] == \"Paper\")\n",
+        "]\n",
+        "print(\"Paper :\")\n",
+        "print(df_type_paper[\"List_Price_Converged\"].value_counts())\n",
+        "df_type_pub = df[\n",
+        "    (df[\"Product_ID\"] == \"SKU 8\") & (df[\"Customer_Hierarchy\"] == \"Publishing\")\n",
+        "]\n",
+        "print(\"Publishing :\")\n",
+        "print(df_type_pub[\"List_Price_Converged\"].value_counts())\n",
+        "df_type_util = df[\n",
+        "    (df[\"Product_ID\"] == \"SKU 140\") & (df[\"Customer_Hierarchy\"] == \"Utilities\")\n",
+        "]\n",
+        "print(\"Utilities :\")\n",
+        "print(df_type_util[\"List_Price_Converged\"].value_counts())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "f023af578c0f"
+      },
+      "source": [
+        "In the publishing category, `Product_ID` `SKU 8` and `SKU 17` are less than or equal to two different prices in the entire data and so you will exclude them and consider the rest for building the forecast model. The idea here is to train a forecast model on the timeseries data for products with different prices.\n",
+        "\n",
+        "Join the data for all the `Product_ID`s into one dataframe and remove duplicate records."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "a44771cc4c20"
+      },
+      "outputs": [],
+      "source": [
+        "df_final = pd.concat([df_type_food, df_type_paper, df_type_util])\n",
+        "df_final = (\n",
+        "    df_final[\n",
+        "        [\n",
+        "            \"Product_ID\",\n",
+        "            \"Fiscal_Date\",\n",
+        "            \"Customer_Hierarchy\",\n",
+        "            \"List_Price_Converged\",\n",
+        "            \"Invoiced_quantity_in_Pieces\",\n",
+        "        ]\n",
+        "    ]\n",
+        "    .drop_duplicates()\n",
+        "    .reset_index(drop=True)\n",
+        ")\n",
+        "df_final.head()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "add5063df368"
+      },
+      "source": [
+        "Save the data to a BigQuery table."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "fd82ba56571f"
+      },
+      "outputs": [],
+      "source": [
+        "bq_client = bigquery.Client(project=PROJECT_ID)\n",
+        "\n",
+        "job_config = bigquery.LoadJobConfig(\n",
+        "    # Specify a (partial) schema. All columns are always written to the\n",
+        "    # table. The schema is used to assist in data type definitions.\n",
+        "    schema=[\n",
+        "        bigquery.SchemaField(\"Product_ID\", bigquery.enums.SqlTypeNames.STRING),\n",
+        "        bigquery.SchemaField(\"Fiscal_Date\", bigquery.enums.SqlTypeNames.DATE),\n",
+        "        bigquery.SchemaField(\"List_Price_Converged\", bigquery.enums.SqlTypeNames.FLOAT),\n",
+        "        bigquery.SchemaField(\n",
+        "            \"Invoiced_quantity_in_Pieces\", bigquery.enums.SqlTypeNames.FLOAT\n",
+        "        ),\n",
+        "    ],\n",
+        "    # Optionally, set the write disposition. BigQuery appends loaded rows\n",
+        "    # to an existing table by default, but with WRITE_TRUNCATE write\n",
+        "    # disposition it replaces the table with the loaded data.\n",
+        "    write_disposition=\"WRITE_TRUNCATE\",\n",
+        ")\n",
+        "\n",
+        "# save the dataframe to a table in the created dataset\n",
+        "job = bq_client.load_table_from_dataframe(\n",
+        "    df_final,\n",
+        "    \"{}.{}.{}\".format(PROJECT_ID, DATASET, TRAINING_DATA_TABLE),\n",
+        "    job_config=job_config,\n",
+        ")  # Make an API request.\n",
+        "job.result()  # Wait for the job to complete."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "fca77641b03b"
+      },
+      "source": [
+        "# Train the model using BigQuery ML\n",
+        "<a name=\"section-9\"></a>\n",
+        "\n",
+        "Train an [Arima-Plus](https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-create-time-series) model on the data using BigQuery ML."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "cded27507891"
+      },
+      "source": [
+        "#@bigquery\n",
+        "create or replace model pricing_optimization.bqml_arima\n",
+        "options\n",
+        " (model_type = 'ARIMA_PLUS',\n",
+        "  time_series_timestamp_col = 'Fiscal_Date',\n",
+        "  time_series_data_col = 'Invoiced_quantity_in_Pieces',\n",
+        "  time_series_id_col = 'ID'\n",
+        " ) as\n",
+        "select\n",
+        " Fiscal_Date,\n",
+        " Concat(Product_ID,\"_\" ,Cast(List_Price_Converged as string)) as ID,\n",
+        " Invoiced_quantity_in_Pieces\n",
+        "from\n",
+        " pricing_optimization.TRAINING_DATA\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "332fd11ff32b"
+      },
+      "source": [
+        "## Generate forecasts from the model\n",
+        "<a name=\"section-10\"></a>\n",
+        "\n",
+        "Predict the sales for the next 30 days for each id and save to a dataframe."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ef926cdbf28e"
+      },
+      "outputs": [],
+      "source": [
+        "client = Client()\n",
+        "\n",
+        "query = '''\n",
+        "DECLARE HORIZON STRING DEFAULT \"30\"; #number of values to forecast\n",
+        "DECLARE CONFIDENCE_LEVEL STRING DEFAULT \"0.90\"; ## required confidence level\n",
+        "\n",
+        "EXECUTE IMMEDIATE format(\"\"\"\n",
+        "    SELECT\n",
+        "      *\n",
+        "    FROM \n",
+        "      ML.FORECAST(MODEL pricing_optimization.bqml_arima, \n",
+        "                  STRUCT(%s AS horizon, \n",
+        "                         %s AS confidence_level)\n",
+        "                 )\n",
+        "    \"\"\",HORIZON,CONFIDENCE_LEVEL)'''\n",
+        "job = client.query(query)\n",
+        "dfforecast = job.to_dataframe()\n",
+        "dfforecast.head()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "608c7de72dae"
+      },
+      "source": [
+        "## Interpret the results to choose the best price\n",
+        "<a name=\"section-11\"></a>\n",
+        "\n",
+        "Calculate average forecast values for the forecast duration."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "e1e193680400"
+      },
+      "outputs": [],
+      "source": [
+        "dfforecast_avg = (\n",
+        "    dfforecast[[\"ID\", \"forecast_value\"]].groupby(\"ID\", as_index=False).mean()\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5ce395d652a3"
+      },
+      "source": [
+        "Extract the ID and Price fields from the ID field."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "452c56fa58ed"
+      },
+      "outputs": [],
+      "source": [
+        "dfforecast_avg[\"Product_ID\"] = dfforecast_avg[\"ID\"].apply(lambda x: x.split(\"_\")[0])\n",
+        "dfforecast_avg[\"Price\"] = dfforecast_avg[\"ID\"].apply(lambda x: x.split(\"_\")[1])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3cee67f4028f"
+      },
+      "source": [
+        "Plot the average forecasted sales vs. the price of the product."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "fb351c8f383d"
+      },
+      "outputs": [],
+      "source": [
+        "for i in dfforecast_avg[\"Product_ID\"].unique():\n",
+        "    dfforecast_avg[dfforecast_avg[\"Product_ID\"] == i].set_index(\"Price\").sort_values(\n",
+        "        \"forecast_value\"\n",
+        "    ).plot(kind=\"bar\")\n",
+        "    plt.title(\"Price vs. Average Sales for \" + i)\n",
+        "    plt.show()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "67ff3acc74a5"
+      },
+      "source": [
+        "Based on the plots for price vs. the average forecasted orders, it can be said that to use the maximum orders, each of the considered `Product_ID`s can follow the below prices:\n",
+        "\n",
+        "- SKU 107's price range can be from 4.44 - 4.73 units\n",
+        "- SKU 140's price can be 1.95 units\n",
+        "- SKU 62's price can be 4.23 units\n",
+        "\n",
+        "\n",
+        "## Clean Up\n",
+        "<a name=\"section-12\"></a>\n",
+        "\n",
+        "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
+        "\n",
+        "Otherwise, you can delete the individual resources you created in this tutorial. The following code deletes the entire dataset."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "d78908b8134d"
+      },
+      "outputs": [],
+      "source": [
+        "# Construct a BigQuery client object.\n",
+        "client = bigquery.Client()\n",
+        "\n",
+        "# TODO(developer): Set model_id to the ID of the model to fetch.\n",
+        "dataset_id = \"{PROJECT}.{DATASET}\".format(PROJECT=PROJECT_ID, DATASET=DATASET)\n",
+        "\n",
+        "# Use the delete_contents parameter to delete a dataset and its contents.\n",
+        "# Use the not_found_ok parameter to not receive an error if the dataset has already been deleted.\n",
+        "client.delete_dataset(\n",
+        "    dataset_id, delete_contents=True, not_found_ok=True\n",
+        ")  # Make an API request.\n",
+        "\n",
+        "print(\"Deleted dataset '{}'.\".format(dataset_id))"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "pricing-optimization.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/community/managed_notebooks/subscriber_churn_prediction/telecom-subscriber-churn-prediction.ipynb

@@ -1,56 +1,33 @@
 {
   "cells": [
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "18ebbd838e32"
-      },
-      "outputs": [],
-      "source": [
-        "# Copyright 2022 Google LLC\n",
-        "#\n",
-        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
-        "# you may not use this file except in compliance with the License.\n",
-        "# You may obtain a copy of the License at\n",
-        "#\n",
-        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
-        "#\n",
-        "# Unless required by applicable law or agreed to in writing, software\n",
-        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
-        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
-        "# See the License for the specific language governing permissions and\n",
-        "# limitations under the License."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "64f7165bd1ac"
       },
       "source": [
-        "# Telecom subscriber churn prediction on Vertex AI\n",
+        "# Telecom subscriber churn prediction on Vertex AI"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "1ecf72573623"
+      },
+      "source": [
+        "## Table of contents\n",
         "\n",
-        "<table align=\"left\">\n",
-        "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/subscriber_churn_prediction/telecom-subscriber-churn-prediction.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
-        "      View on GitHub\n",
-        "    </a>\n",
-        "  </td>\n",
-        "    <td>\n",
-        "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/subscriber_churn_prediction/telecom-subscriber-churn-prediction.ipynb\">\n",
-        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\\\" alt=\"Colab logo\"> Run in Colab\n",
-        "        </a>\n",
-        "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/subscriber_churn_prediction/telecom-subscriber-churn-prediction.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
-        "    </a>\n",
-        "  </td>\n",
-        "</table>\n",
-        "<br/><br/><br/>"
+        "* [Overview](#section-1)\n",
+        "* [Dataset](#section-2)\n",
+        "* [Objective](#section-3)\n",
+        "* [Costs](#section-4)\n",
+        "* [Perform EDA](#section-5)\n",
+        "* [Train a logistic regression model using scikit-learn](#section-6)\n",
+        "* [Evaluate the trained model](#section-7)\n",
+        "* [Save the model to a Cloud Storage path](#section-8)\n",
+        "* [Create a model with Explainable AI support in Vertex AI](#section-9)\n",
+        "* [Get explanations from the model](#section-10)\n",
+        "* [Clean up](#section-11)\n"
       ]
     },
     {
@@ -60,28 +37,35 @@
       },
       "source": [
         "## Overview\n",
+        "<a name=\"section-1\"></a>\n",
         "\n",
-        "This example demonstrates building a subscriber churn prediction model on a [telecom customer churn dataset](https://www.kaggle.com/c/customer-churn-prediction-2020/overview). The generated churn model is further deployed to Vertex AI Endpoints and explanations are generated using the Explainable AI feature of Vertex AI. "
+        "This example demonstrates building a subscriber churn prediction model on a [telecom customer churn dataset](https://www.kaggle.com/c/customer-churn-prediction-2020/overview). The generated churn model is further deployed to Vertex AI Endpoints and explanations are generated using the Explainable AI feature of Vertex AI. \n",
+        "\n",
+        "*Note: This notebook file was designed to run in a [Vertex AI Workbench managed notebooks](https://cloud.google.com/vertex-ai/docs/workbench/managed/create-instance) instance using the `Python (Local)` kernel. Some components of this notebook may not work in other notebook environments.*"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "8c7726146192"
+        "id": "4bae972f3229"
       },
       "source": [
-        "### Objective\n",
+        "## Dataset\n",
+        "<a name=\"section-2\"></a>\n",
         "\n",
-        "This tutorial shows you how to do exploratory data analysis, preprocess data, train, deploy and get predictions from a churn prediction model on a tabular churn dataset. The objectives of this tutorial are as follows:\n",
+        "The dataset used in this tutorial is publicly available at Kaggle. See [Customer Churn Prediction 2020](https://www.kaggle.com/c/customer-churn-prediction-2020/data). "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "75e3d160163c"
+      },
+      "source": [
+        "## Objective\n",
+        "<a name=\"section-3\"></a>\n",
         "\n",
-        "This tutorial uses the following Google Cloud ML services and resources:\n",
-        "\n",
-        "- `Vertex AI Model` resource\n",
-        "- `Vertex AI Endpoint` resource\n",
-        "- `Vertex Explainable AI`\n",
-        "- Google Cloud Storage\n",
-        "\n",
-        "The steps performed include:\n",
+        "This tutorial shows you how to do exploratory data analysis, preprocess data, and train a churn prediction model on a tabular churn dataset. The steps include the following:\n",
         "\n",
         "- Load data from a Cloud Storage path\n",
         "- Perform exploratory data analysis (EDA)\n",
@@ -95,24 +79,14 @@
         "- Undeploy the model resource"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "4bae972f3229"
-      },
-      "source": [
-        "### Dataset\n",
-        "\n",
-        "The dataset used in this tutorial is Telecom-Customer Churn dataset publicly available on Kaggle. See [Customer Churn Prediction 2020](https://www.kaggle.com/c/customer-churn-prediction-2020/data). This dataset is used to build and deploy a churn prediction model using Vertex AI in this notebook."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "556bf343c423"
       },
       "source": [
-        "### Costs \n",
+        "## Costs \n",
+        "<a name=\"section-4\"></a>\n",
         "\n",
         "\n",
         "This tutorial uses billable components of Google Cloud:\n",
@@ -127,57 +101,13 @@
         "to generate a cost estimate based on your projected usage."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "f0b0e0803638"
-      },
-      "source": [
-        "### Set up your local development environment\n",
-        "\n",
-        "**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets\n",
-        "all the requirements to run this notebook. You can skip this step.\n",
-        "\n",
-        "**_NOTE_**: This notebook has been tested in the following environment:\n",
-        "\n",
-        "* Python version = 3.9\n",
-        "\n",
-        "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
-        "You need the following:\n",
-        "\n",
-        "* The Google Cloud SDK\n",
-        "\n",
-        "The Google Cloud guide to [Setting up a Python development\n",
-        "environment](https://cloud.google.com/python/setup) and the [Jupyter\n",
-        "installation guide](https://jupyter.org/install) provide detailed instructions\n",
-        "for meeting these requirements. The following steps provide a condensed set of\n",
-        "instructions:\n",
-        "\n",
-        "1. [Install and initialize the Cloud SDK.](https://cloud.google.com/sdk/docs/)\n",
-        "\n",
-        "1. [Install Python 3.](https://cloud.google.com/python/setup#installing_python)\n",
-        "\n",
-        "1. [Install\n",
-        "   virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv)\n",
-        "   and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
-        "\n",
-        "1. To install Jupyter, run `pip3 install jupyter` on the\n",
-        "command-line in a terminal shell.\n",
-        "\n",
-        "1. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
-        "\n",
-        "1. Open this notebook in the Jupyter Notebook Dashboard."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "44b8ae8e2d19"
       },
       "source": [
-        "## Installation\n",
-        "\n",
-        "Install the following packages required to execute this notebook. "
+        "## Installation"
       ]
     },
     {
@@ -190,55 +120,95 @@
       "source": [
         "import os\n",
         "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
+        "# The Google Cloud Notebook product has specific requirements\n",
+        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
         "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
         "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "    \n",
-        "! pip3 install {USER_FLAG} --upgrade google-cloud-aiplatform \\\n",
-        "                                    google-cloud-storage \\\n",
-        "                                    category_encoders \\\n",
-        "                                    seaborn \\\n",
-        "                                    scikit-learn \\\n",
-        "                                    pandas \\\n",
-        "                                    fsspec \\\n",
-        "                                    gcsfs -q "
+        "# Google Cloud Notebook requires dependencies to be installed with '--user'\n",
+        "if IS_GOOGLE_CLOUD_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\""
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "b24902cde81b"
+        "id": "606337930991"
       },
       "source": [
-        "### Restart the kernel\n",
+        "Install the latest version of the Vertex AI client library.\n",
         "\n",
-        "After you install the additional packages, you need to restart the notebook kernel so it can find the packages."
+        "Run the following command in your virtual environment to install the Vertex SDK for Python:"
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "c61d171395d7"
+        "id": "9f52f949a77b"
       },
       "outputs": [],
       "source": [
-        "# Automatically restart kernel after installs\n",
-        "import os\n",
-        "\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    # Automatically restart kernel after installs\n",
-        "    import IPython\n",
-        "\n",
-        "    app = IPython.Application.instance()\n",
-        "    app.kernel.do_shutdown(True)"
+        "! pip install {USER_FLAG} --upgrade google-cloud-aiplatform"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e67139e68463"
+      },
+      "source": [
+        "Install the Cloud Storage library:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2ad918f94f5d"
+      },
+      "outputs": [],
+      "source": [
+        "! pip install {USER_FLAG} --upgrade google-cloud-storage"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "eb0c1e24a8f0"
+      },
+      "source": [
+        "Install the `category_encoders` library:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "deb95a7f2104"
+      },
+      "outputs": [],
+      "source": [
+        "! pip install --upgrade category_encoders"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "184560c1b742"
+      },
+      "source": [
+        "Install the `seaborn` library for the EDA step. If a Vertex AI Workbench managed notebooks instance is being used, this step is optional as the library is already available in the `Python (Local)` kernel."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "0d99cdcdc470"
+      },
+      "outputs": [],
+      "source": [
+        "! pip install --upgrade seaborn"
       ]
     },
     {
@@ -247,7 +217,7 @@
         "id": "b012ef94ce80"
       },
       "source": [
-        "## Before you begin\n",
+        "## Before you begin \n",
         "\n",
         "### Set up your Google Cloud project\n",
         "\n",
@@ -257,9 +227,9 @@
         "\n",
         "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
         "\n",
-        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com). \n",
+        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com).\n",
         "\n",
-        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
         "\n",
         "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
         "Cloud SDK uses the right project for all the commands in this notebook.\n",
@@ -286,67 +256,34 @@
       },
       "outputs": [],
       "source": [
-        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "6855b42885bf"
-      },
-      "outputs": [],
-      "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
-        "    # Get your GCP project id from gcloud\n",
-        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "PROJECT_ID = \"\"\n",
+        "\n",
+        "# Get your Google Cloud project ID from gcloud\n",
+        "if not os.getenv(\"IS_TESTING\"):\n",
+        "    shell_output=!gcloud config list --format 'value(core.project)' 2>/dev/null\n",
         "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID:\", PROJECT_ID)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "f2e3c0f2cbfb"
-      },
-      "outputs": [],
-      "source": [
-        "! gcloud config set project $PROJECT_ID"
+        "    print(\"Project ID: \", PROJECT_ID)"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "60d535f443ac"
+        "id": "6855b42885bf"
       },
       "source": [
-        "#### Region\n",
-        "\n",
-        "You can also change the `REGION` variable, which is used for operations\n",
-        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. It is recommended that you choose the region closest to you.\n",
-        "\n",
-        "- Americas: `us-central1`\n",
-        "- Europe: `europe-west4`\n",
-        "- Asia Pacific: `asia-east1`\n",
-        "\n",
-        "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
-        "\n",
-        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
+        "Otherwise, set your project ID here."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "3aaadaaf9b30"
+        "id": "59255d2246fd"
       },
       "outputs": [],
       "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
+        "    PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -355,9 +292,9 @@
         "id": "e663bd062c6f"
       },
       "source": [
-        "#### UUID\n",
+        "#### Timestamp\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append it onto the name of resources you create in this tutorial."
       ]
     },
     {
@@ -368,84 +305,9 @@
       },
       "outputs": [],
       "source": [
-        "import random\n",
-        "import string\n",
+        "from datetime import datetime\n",
         "\n",
-        "\n",
-        "# Generate a uuid of a specifed length(default=8)\n",
-        "def generate_uuid(length: int = 8) -> str:\n",
-        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
-        "\n",
-        "\n",
-        "UUID = generate_uuid()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "3ffa6b6c7cdb"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
-        "authenticated. \n",
-        "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions\n",
-        "when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "1. In the Cloud Console, go to the [**Create service account key**\n",
-        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
-        "\n",
-        "2. Click **Create service account**.\n",
-        "\n",
-        "3. In the **Service account name** field, enter a name, and\n",
-        "   click **Create**.\n",
-        "\n",
-        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
-        "into the filter box, and select\n",
-        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
-        "local environment.\n",
-        "\n",
-        "6. Enter the path to your service account key as the\n",
-        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "2b72272258fc"
-      },
-      "outputs": [],
-      "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = \"google.colab\" in sys.modules\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS '[your-service-account-key-path]'"
+        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
     {
@@ -465,7 +327,12 @@
         "online predictions.\n",
         "\n",
         "Set the name of your Cloud Storage bucket below. It must be unique across all\n",
-        "Cloud Storage buckets."
+        "Cloud Storage buckets.\n",
+        "\n",
+        "You may also change the `REGION` variable, which is used for operations\n",
+        "throughout the rest of this notebook. Make sure to [choose a region where Vertex AI services are\n",
+        "available](https://cloud.google.com/vertex-ai/docs/general/locations#available_regions). You may\n",
+        "not use a Multi-Regional Storage bucket for training with Vertex AI."
       ]
     },
     {
@@ -476,8 +343,8 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "REGION = \"[your-region]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -488,9 +355,8 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
-        "    BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
       ]
     },
     {
@@ -510,7 +376,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
+        "! gsutil mb -l $REGION $BUCKET_NAME"
       ]
     },
     {
@@ -530,7 +396,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_URI"
+        "! gsutil ls -al $BUCKET_NAME"
       ]
     },
     {
@@ -563,15 +429,13 @@
         "import pandas as pd\n",
         "\n",
         "%matplotlib inline\n",
-        "import pickle\n",
         "# configure to don't display the warnings\n",
         "import warnings\n",
         "\n",
         "import category_encoders as ce\n",
+        "import joblib\n",
         "import seaborn as sns\n",
         "from google.cloud import aiplatform, storage\n",
-        "from google.cloud.aiplatform_v1.types import SampledShapleyAttribution\n",
-        "from google.cloud.aiplatform_v1.types.explanation import ExplanationParameters\n",
         "from sklearn.linear_model import LogisticRegression\n",
         "from sklearn.metrics import confusion_matrix, plot_roc_curve\n",
         "from sklearn.model_selection import train_test_split\n",
@@ -586,13 +450,7 @@
         "id": "e37354341588"
       },
       "source": [
-        "### Load data from Cloud Storage using Pandas\n",
-        "\n",
-        "The Telecom-Customer Churn dataset from [Kaggle](https://www.kaggle.com/c/customer-churn-prediction-2020/overview) is made available on a public Cloud Storage bucket at: \n",
-        "\n",
-        "```gs://cloud-samples-data/vertex-ai/managed_notebooks/telecom_churn_prediction/train.csv```\n",
-        "\n",
-        "Use Pandas to read data directly from the URI."
+        "### Load data from Cloud Storage path using Pandas"
       ]
     },
     {
@@ -616,7 +474,8 @@
         "id": "ca1a5b9da481"
       },
       "source": [
-        "## Perform EDA\n"
+        "## Perform EDA\n",
+        "<a name=\"section-5\"></a>"
       ]
     },
     {
@@ -1007,7 +866,8 @@
         "id": "229cfba1fd32"
       },
       "source": [
-        "## Train a logistic regression model using scikit-learn\n"
+        "## Train a logistic regression model using scikit-learn\n",
+        "<a name=\"section-6\"></a>"
       ]
     },
     {
@@ -1037,7 +897,8 @@
         "id": "18e23a047402"
       },
       "source": [
-        "## Evaluate the trained model\n"
+        "## Evaluate the trained model\n",
+        "<a name=\"section-7\"></a>"
       ]
     },
     {
@@ -1226,7 +1087,8 @@
         "id": "a7fc7b467b6f"
       },
       "source": [
-        "## Save the model to a Cloud Storage path\n"
+        "## Save the model to a Cloud Storage path\n",
+        "<a name=\"section-8\"></a>"
       ]
     },
     {
@@ -1235,7 +1097,7 @@
         "id": "ae43e214775f"
       },
       "source": [
-        "Save the trained model to a local file `model.pkl`."
+        "Save the trained model to a local file `model.joblib`."
       ]
     },
     {
@@ -1246,17 +1108,14 @@
       },
       "outputs": [],
       "source": [
-        "FILE_NAME = \"model.pkl\"\n",
-        "with open(FILE_NAME, \"wb\") as file:\n",
-        "    pickle.dump(model, file)\n",
+        "FILE_NAME = \"model.joblib\"\n",
+        "joblib.dump(model, FILE_NAME)\n",
         "\n",
         "# Upload the saved model file to Cloud Storage\n",
         "BLOB_PATH = (\n",
         "    \"[your-blob-path]\"  # leave blank if no folders inside the bucket are needed.\n",
         ")\n",
         "\n",
-        "if BLOB_PATH == (\"[your-blob-path]\"):\n",
-        "    BLOB_PATH = \"\"\n",
         "\n",
         "BLOB_NAME = BLOB_PATH + FILE_NAME\n",
         "\n",
@@ -1272,10 +1131,9 @@
       },
       "source": [
         "## Create a model with Explainable AI support in Vertex AI\n",
+        "<a name=\"section-9\"></a>\n",
         "\n",
-        "Before creating a model, configure the explanations for the model. For further details, see [Configuring explanations in Vertex AI](https://cloud.google.com/vertex-ai/docs/explainable-ai/configuring-explanations#scikit-learn-and-xgboost-pre-built-containers).\n",
-        "\n",
-        "Set a display name for the model resource."
+        "Before creating a model, configure the explanations for the model. For further details, see [Configuring explanations in Vertex AI](https://cloud.google.com/vertex-ai/docs/explainable-ai/configuring-explanations#scikit-learn-and-xgboost-pre-built-containers)."
       ]
     },
     {
@@ -1286,13 +1144,10 @@
       },
       "outputs": [],
       "source": [
-        "# Set the model display name\n",
-        "MODEL_DISPLAY_NAME = \"[your-model-display-name]\"  # @param {type:\"string\"}\n",
-        "\n",
-        "if MODEL_DISPLAY_NAME == \"[your-model-display-name]\":\n",
-        "    MODEL_DISPLAY_NAME = \"subscriber_churn_model\"\n",
-        "\n",
+        "MODEL_DISPLAY_NAME = \"[your-model-display-name]\"\n",
         "ARTIFACT_GCS_PATH = f\"gs://{BUCKET_NAME}/{BLOB_PATH}\"\n",
+        "PROJECT = \"[your-project-id]\"\n",
+        "LOCATION = REGION\n",
         "\n",
         "# Feature-name(Inp_feature) and Output-name(Model_output) can be arbitrary\n",
         "exp_metadata = {\"inputs\": {\"Inp_feature\": {}}, \"outputs\": {\"Model_output\": {}}}"
@@ -1308,17 +1163,15 @@
       "source": [
         "# Create a Vertex AI model resource with support for explanations\n",
         "\n",
-        "\n",
-        "aiplatform.init(project=PROJECT_ID, location=REGION)\n",
+        "aiplatform.init(project=PROJECT, location=LOCATION)\n",
+        "explanation_parameters = {\"sampledShapleyAttribution\": {\"pathCount\": 25}}\n",
         "\n",
         "model = aiplatform.Model.upload(\n",
         "    display_name=MODEL_DISPLAY_NAME,\n",
         "    artifact_uri=ARTIFACT_GCS_PATH,\n",
-        "    serving_container_image_uri=\"us-docker.pkg.dev/vertex-ai/prediction/sklearn-cpu.1-0:latest\",\n",
+        "    serving_container_image_uri=\"us-docker.pkg.dev/vertex-ai/prediction/sklearn-cpu.0-24:latest\",\n",
         "    explanation_metadata=exp_metadata,\n",
-        "    explanation_parameters=ExplanationParameters(\n",
-        "        sampled_shapley_attribution=SampledShapleyAttribution(path_count=25)\n",
-        "    ),\n",
+        "    explanation_parameters=explanation_parameters,\n",
         ")\n",
         "\n",
         "model.wait()\n",
@@ -1339,7 +1192,7 @@
         "gcloud beta ai models upload \\\n",
         "  --region=$REGION \\\n",
         "  --display-name=$MODEL_DISPLAY_NAME \\\n",
-        "  --container-image-uri=\"us-docker.pkg.dev/vertex-ai/prediction/sklearn-cpu.1-0:latest\" \\\n",
+        "  --container-image-uri=\"us-docker.pkg.dev/vertex-ai/prediction/sklearn-cpu.0-24:latest\" \\\n",
         "  --artifact-uri=$ARTIFACT_GCS_PATH \\\n",
         "  --explanation-method=sampled-shapley \\\n",
         "  --explanation-path-count=25 \\\n",
@@ -1364,9 +1217,7 @@
       },
       "outputs": [],
       "source": [
-        "ENDPOINT_DISPLAY_NAME = \"[your-endpoint-display-name]\"  # @param {type:\"string\"}\n",
-        "if ENDPOINT_DISPLAY_NAME == \"[your-endpoint-display-name]\":\n",
-        "    ENDPOINT_DISPLAY_NAME = \"subsc_churn_endpoint\""
+        "ENDPOINT_DISPLAY_NAME = \"[your-endpoint-display-name]\""
       ]
     },
     {
@@ -1378,13 +1229,33 @@
       "outputs": [],
       "source": [
         "endpoint = aiplatform.Endpoint.create(\n",
-        "    display_name=ENDPOINT_DISPLAY_NAME, project=PROJECT_ID, location=REGION\n",
+        "    display_name=ENDPOINT_DISPLAY_NAME, project=PROJECT, location=LOCATION\n",
         ")\n",
         "\n",
         "print(endpoint.display_name)\n",
         "print(endpoint.resource_name)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ae4c69ef8a8c"
+      },
+      "source": [
+        "Save the endpoint ID after the endpoint is created."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6aa73d9a88d3"
+      },
+      "outputs": [],
+      "source": [
+        "ENDPOINT_ID = \"[your-endpoint-id]\""
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -1404,11 +1275,8 @@
       },
       "outputs": [],
       "source": [
-        "DEPLOYED_MODEL_NAME = \"[deployment-model-name]\"  # @param {type:\"string\"}\n",
-        "MACHINE_TYPE = \"n1-standard-4\"\n",
-        "\n",
-        "if DEPLOYED_MODEL_NAME == \"[deployment-model-name]\":\n",
-        "    DEPLOYED_MODEL_NAME = \"subsc_churn_deployment\""
+        "DEPLOYED_MODEL_NAME = \"[deployment-model-name]\"\n",
+        "MACHINE_TYPE = \"n1-standard-4\""
       ]
     },
     {
@@ -1438,7 +1306,7 @@
         "id": "359c43e630cb"
       },
       "source": [
-        "To ensure the model is deployed, the ID of the deployed model can be checked using the `endpoint.list_models()` method."
+        "Save the ID of the deployed model. The ID of the deployed model can also checked using the `endpoint.list_models()` method."
       ]
     },
     {
@@ -1449,7 +1317,7 @@
       },
       "outputs": [],
       "source": [
-        "endpoint.list_models()"
+        "DEPLOYED_MODEL_ID = \"[your-deployed-model-id]\""
       ]
     },
     {
@@ -1458,7 +1326,8 @@
         "id": "21a50d4e9946"
       },
       "source": [
-        "## Get explanations from the deployed model\n"
+        "## Get explanations from the deployed model\n",
+        "<a name=\"section-10\"></a>"
       ]
     },
     {
@@ -1467,7 +1336,7 @@
         "id": "7b50c31e0552"
       },
       "source": [
-        "Get explanations for a test instance from the hosted model."
+        "Get explanations for some test instances from the hosted model."
       ]
     },
     {
@@ -1478,8 +1347,8 @@
       },
       "outputs": [],
       "source": [
-        "# format a test instance as the request's payload\n",
-        "test_json = [X_test.iloc[0].tolist()]"
+        "# format the top 2 test instances as the request's payload\n",
+        "test_json = {\"instances\": [X_test.iloc[0].tolist(), X_test.iloc[1].tolist()]}"
       ]
     },
     {
@@ -1516,13 +1385,15 @@
         "    return\n",
         "\n",
         "\n",
-        "def explain_tabular_sample(project: str, location: str, endpoint, instances: list):\n",
+        "def explain_tabular_sample(\n",
+        "    project: str, location: str, endpoint_id: str, instances: list\n",
+        "):\n",
         "    \"\"\"\n",
         "    Function to make an explanation request for the specified payload and generate feature attribution plots\n",
         "    \"\"\"\n",
         "    aiplatform.init(project=project, location=location)\n",
         "\n",
-        "    # endpoint = aiplatform.Endpoint(endpoint_id)\n",
+        "    endpoint = aiplatform.Endpoint(endpoint_id)\n",
         "\n",
         "    response = endpoint.explain(instances=instances)\n",
         "    print(\"#\" * 10 + \"Explanations\" + \"#\" * 10)\n",
@@ -1551,8 +1422,8 @@
         "    return response\n",
         "\n",
         "\n",
-        "# Get explanations for the test instance\n",
-        "prediction = explain_tabular_sample(PROJECT_ID, REGION, endpoint, test_json)"
+        "test_json = [X_test.iloc[0].tolist(), X_test.iloc[1].tolist()]\n",
+        "prediction = explain_tabular_sample(PROJECT, LOCATION, ENDPOINT_ID, test_json)"
       ]
     },
     {
@@ -1562,16 +1433,12 @@
       },
       "source": [
         "## Clean up\n",
+        "<a name=\"section-11\"></a>\n",
         "\n",
         "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
         "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
         "\n",
-        "Otherwise, you can delete the individual resources you created in this tutorial:\n",
-        "* Vertex AI Model\n",
-        "* Vertex AI Endpoint\n",
-        "* Cloud Storage bucket\n",
-        "\n",
-        "Set `delete_bucket` to *True* to delete the Cloud Storage bucket."
+        "Otherwise, you can delete the individual resources you created in this tutorial:"
       ]
     },
     {
@@ -1582,19 +1449,44 @@
       },
       "outputs": [],
       "source": [
-        "# Undeploy model\n",
-        "endpoint.undeploy_all()\n",
-        "\n",
-        "# Delete the endpoint\n",
-        "endpoint.delete()\n",
-        "\n",
-        "# Delete the model\n",
-        "model.delete()\n",
-        "\n",
-        "# Delete the Cloud Storage bucket\n",
-        "delete_bucket = True\n",
-        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
-        "    ! gsutil -m rm -r $BUCKET_URI"
+        "# undeploy the model\n",
+        "endpoint.undeploy(deployed_model_id=DEPLOYED_MODEL_ID)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "67e88dd0b2da"
+      },
+      "outputs": [],
+      "source": [
+        "# delete the endpoint\n",
+        "endpoint.delete()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "023779d4ac48"
+      },
+      "outputs": [],
+      "source": [
+        "# delete the model\n",
+        "model.delete()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "dd3b12107312"
+      },
+      "outputs": [],
+      "source": [
+        "# remove the contents of the Cloud Storage bucket\n",
+        "! gsutil -m rm -r $BUCKET_NAME"
       ]
     }
   ],

notebooks/community/ml_ops/README.md

@@ -14,5 +14,5 @@ The purpose of this set of notebooks and markdown files is to demonstrate Google
 4. [Evaluation](stage4)
 5. [Deployment](stage5)
 6. [Serving](stage6)
-7. Monitoring(stage7)
+7. Monitoring
 8. Continuous Training

notebooks/community/ml_ops/setup.md

@@ -1,43 +0,0 @@
-## Before you begin
-
-### Set up your Google Cloud project
-
-**The following steps are required, regardless of your notebook environment.**
-
-1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.
-
-1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).
-
-1. [Enable the Vertex AI, BigQuery, Compute Engine and Cloud Storage APIs](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,bigquery,compute_component,storage_component).
-
-1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).
-
-1. Enter your project ID in the cell below. Then run the cell to make sure the
-Cloud SDK uses the right project for all the commands in this notebook.
-
-**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands.
-
-### Set up your local development environment
-
-**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets all the requirements to run this notebook. You can skip this step.
-
-**Otherwise**, make sure your environment meets this notebook's requirements. You need the following:
-
-- The Cloud Storage SDK
-- Python 3
-- virtualenv
-- Jupyter notebook running in a virtual environment with Python 3
-
-The Cloud Storage guide to [Setting up a Python development environment](https://cloud.google.com/python/setup) and the [Jupyter installation guide](https://jupyter.org/install) provide detailed instructions for meeting these requirements. The following steps provide a condensed set of instructions:
-
-1. [Install and initialize the SDK](https://cloud.google.com/sdk/docs/).
-
-2. [Install Python 3](https://cloud.google.com/python/setup#installing_python).
-
-3. [Install virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv) and create a virtual environment that uses Python 3.  Activate the virtual environment.
-
-4. To install Jupyter, run `pip3 install jupyter` on the command-line in a terminal shell.
-
-5. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.
-
-6. Open this notebook in the Jupyter Notebook Dashboard.

notebooks/community/ml_ops/setup.py

@@ -1,112 +0,0 @@
-import os
-import sys
-import argparse
-import subprocess
-import random
-import string
-
-parser = argparse.ArgumentParser()
-parser.add_argument('--bucket', dest='bucket_required', action='store_true', 
-                    default=False, help='Bucket required')
-parser.add_argument('--email', dest='email_required', action='store_true', 
-                    default=False, help='Email required')
-parser.add_argument('--sa', dest='sa_required', action='store_true', 
-                    default=False, help='Service account required')
-parser.add_argument('--packages', dest='extra_packages',
-                    default='', type=str, help='additional required packages')
-args = parser.parse_args()
-
-extra_pkgs = args.extra_packages
-
-
-# Installation
-
-
-# The Vertex AI Workbench Notebook product has specific requirements
-IS_WORKBENCH_NOTEBOOK = os.getenv("DL_ANACONDA_HOME")
-IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(
-    "/opt/deeplearning/metadata/env_version"
-)
-IS_COLAB = "google.colab" in sys.modules
-
-# Vertex AI Notebook requires dependencies to be installed with '--user'
-USER_FLAG = ""
-if IS_WORKBENCH_NOTEBOOK:
-    USER_FLAG = "--user"
-
-# not used
-'''
-print("Installing packages")
-os.system(f"pip3 install --upgrade --quiet {USER_FLAG} google-cloud-aiplatform {args.extra_packages}")
-print("Done installation")
-'''
-
-# Authenticate
-if IS_COLAB:
-    from google.colab import auth as google_auth
-
-    google_auth.authenticate_user()
-
-
-# project ID
-if IS_WORKBENCH_NOTEBOOK:
-    shell_output = subprocess.check_output("gcloud config list --format 'value(core.project)' 2>/dev/null", shell=True)
-    PROJECT_ID = shell_output[0:-1].decode('utf-8')
-    print("PROJECT ID: ", PROJECT_ID)
-else:
-    PROJECT_ID = input("Enter PROJECT_ID: ")
-    os.system(f"gcloud config set project {PROJECT_ID}")
-
-# email
-if args.email_required:
-    shell_output = subprocess.check_output("gcloud config list --format 'value(core.account)' 2>/dev/null", shell=True)
-    EMAIL_ADDR = shell_output[0:-1].decode('utf-8')
-    if EMAIL_ADDR == '':
-            EMAIL_ADDR = input("Enter Email Address: ")
-    print("EMAIL_ADDR: ", EMAIL_ADDR)
-
-# region
-shell_output = subprocess.check_output("gcloud config list --format 'value(ai.region)'", shell=True)
-REGION = shell_output[0:-1].decode('utf-8')
-if REGION == '':
-    REGION = input("Enter REGION: ")
-print("REGION: ", REGION)
-
-# multi-region
-MULTI_REGION = REGION.split('-')[0]
-
-    
-# UUID
-# Generate a uuid of a specifed length(default=8)
-def generate_uuid(length: int = 8) -> str:
-    return "".join(random.choices(string.ascii_lowercase + string.digits, k=length))
-
-
-UUID = generate_uuid()
-print("UUID", UUID)
-    
-# Bucket
-if args.bucket_required:
-    BUCKET_NAME = PROJECT_ID + "aip-" + UUID
-    BUCKET_URI = f"gs://{BUCKET_NAME}"
-    os.system(f"gsutil mb -l {REGION} {BUCKET_URI}")
-    print("BUCKET_URI", BUCKET_URI)
-    
-    
-# Project Number
-if args.sa_required:
-    if IS_WORKBENCH_NOTEBOOK:
-        shell_output = subprocess.check_output("gcloud auth list 2>/dev/null", shell=True)
-        SERVICE_ACCOUNT = shell_output[:-1].decode('utf-8').split('\n')[2].strip()
-        PROJECT_NUMBER = SERVICE_ACCOUNT.split('-')[0]
-    else:
-        shell_output = subprocess.check_output(f"gcloud projects describe {PROJECT_ID}", shell=True)
-        try:
-            PROJECT_NUMBER = shell_output[:-1].decode('utf-8').split('\n')[7].split(':')[-1].strip().replace("'", "")
-            SERVICE_ACCOUNT = f"{PROJECT_NUMBER}-compute@developer.gserviceaccount.com"
-        except:
-            PROJECT_NUMBER = input("Enter project number: ")
-            SERVICE_ACCOUNT = f"{PROJECT_NUMBER}-compute@developer.gserviceaccount.com"
-    
-    print("SERVICE_ACCOUNT", SERVICE_ACCOUNT)
-    print("PROJECT_NUMBER", PROJECT_NUMBER)

notebooks/community/ml_ops/stage1/README.md

@@ -28,25 +28,25 @@ The first stage in MLOps is the collection and preparation for the purpose of de
 
 ### Get Started
 
-[Get started with Dataflow](community/ml_ops/stage1/get_started_dataflow.ipynb)
-
-In this tutorial, you learn how to use `Dataflow` for training with `Vertex AI`.
+[Get Started with BQ datasets](get_started_bq_datasets.ipynb)
 
+```
 The steps performed include:
 
-- Offline preprocessing of data:
-    - Serially - w/o dataflow
-    - Parallel - with dataflow
-- Upstream preprocessing of data:
-    - tabular data
-    - image data
+- Create a Vertex AI `Dataset` resource from `BigQuery` table -- compatible for `AutoML` training.
+- Extract a copy of the dataset from `BigQuery` to a CSV file in Cloud Storage -- compatible for `AutoML` or custom training.
+- Select rows from a `BigQuery` dataset into a `pandas` dataframe -- compatible for custom training.
+- Select rows from a `BigQuery` dataset into a `tf.data.Dataset` -- compatible for custom training `TensorFlow` models.
+- Select rows from extracted CSV files into a `tf.data.Dataset` -- compatible for custom training `TensorFlow` models.
+- Create a `BigQuery` dataset from CSV files.
+- Extract data from `BigQuery` table into a `DMatrix` -- compatible for custom training `XGBoost` models.
+```
 
-[Get started with Vertex AI datasets](community/ml_ops/stage1/get_started_vertex_datasets.ipynb)
-
-In this tutorial, you learn how to use `Vertex AI Dataset` for training with `Vertex AI`.
+[Get Started with Vertex datasets](get_started_vertex_datasets.ipynb)
 
+```
 The steps performed include:
-
+  
 - Create a Vertex AI `Dataset` resource for:
     - image data
     - text data
@@ -61,25 +61,24 @@ The steps performed include:
 - Detect anomalies in new data using TensorFlow Data Validation.
 - Generate a TFRecord feature specification using TensorFlow Transform from the data schema.
 - Export a dataset and convert to TFRecords.
+```
 
-[Get started with BigQuery datasets](community/ml_ops/stage1/get_started_bq_datasets.ipynb)
-
-In this tutorial, you learn how to use `BigQuery` as a dataset for training with `Vertex AI`.
+[Get Started with Dataflow](get_started_dataflow.ipynb)
 
+```
 The steps performed include:
 
-- Create a Vertex AI `Dataset` resource from `BigQuery` table -- compatible for `AutoML` training.
-- Extract a copy of the dataset from `BigQuery` to a CSV file in Cloud Storage -- compatible for `AutoML` or custom training.
-- Select rows from a `BigQuery` dataset into a `pandas` dataframe -- compatible for custom training.
-- Select rows from a `BigQuery` dataset into a `tf.data.Dataset` -- compatible for custom training `TensorFlow` models.
-- Select rows from extracted CSV files into a `tf.data.Dataset` -- compatible for custom training `TensorFlow` models.
-- Create a `BigQuery` dataset from CSV files.
-- Extract data from `BigQuery` table into a `DMatrix` -- compatible for custom training `XGBoost` models.
+- Offline preprocessing of data:
+    - Serially - w/o dataflow
+    - Parallel - with dataflow
+- Upstream preprocessing of data:
+    - tabular data
+    - image data
+```
 
-[Get started with Vertex AI Data Labeling](community/ml_ops/stage1/get_started_with_data_labeling.ipynb)
-
-In this tutorial, you learn how to use the `Vertex AI Data Labeling` service.
+[Get Started with Data Labeling](get_started_data_labeling.ipynb)
 
+```
 The steps performed include:
 
 - Create a Specialist Pool for data labelers.
@@ -87,28 +86,15 @@ The steps performed include:
 - Submit the data labeling job.
 - List data labeling jobs.
 - Cancel a data labeling job.
-
-
-
-[Create an unlabelled Vertex AI AutoML text entity extraction dataset from PDFs using Vision API](community/ml_ops/stage1/get_started_with_visionapi_and_vertex_datasets.ipynb)
-
-In this tutorial, you learn to use `Vision API` to extract text from PDF files stored on a Cloud Storage bucket. You then process the results and create an unlabelled `Vertex AI Dataset`, compatible with `AutoML`, for text entity extraction.
-
-The steps performed include:
-
-1. Using `Vision API` to perform Optical Character Recognition (OCR) to extract text from PDF files.
-2. Processing the results and saving them to text files.
-3. Generating a `Vertex AI Dataset` import file.
-4. Creating a new unlabelled text entity extraction `Vertex AI Dataset` resource in `Vertex AI`.
-
+```
 
 ### E2E Stage Example
 
 [Stage 1: Data Management](mlops_data_management.ipynb)
-
-
+  
 ```
 The steps performed include:
+
 - Explore and visualize the data.
 - Create a Vertex AI `Dataset` resource from `BigQuery` table -- for AutoML training.
 - Extract a copy of the dataset to a CSV file in Cloud Storage.
@@ -118,3 +104,4 @@ The steps performed include:
 - Generate a TFRecord feature specification using TensorFlow Data Validation from the data schema.
 - Preprocess a portion of the BigQuery data using `Dataflow` -- for custom training.
 ```
+

notebooks/community/ml_ops/stage1/get_started_bq_datasets.ipynb

@@ -64,6 +64,17 @@
         "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 1 : data management: get started with BigQuery datasets."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dataset:gsod,lrg"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the GSOD dataset from [BigQuery public datasets](https://cloud.google.com/bigquery/public-data). In this version of the dataset you consider the fields year, month and day to predict the value of mean daily temperature (mean_temp)."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -120,7 +131,7 @@
         "    - XGBoost model training:\n",
         "      - Use BigQuery ML built-in XGBoost training.\n",
         "      - Alternatively, create a DMatrix generator from CSV files extracted from BigQuery table.\n",
-        "    - PyTorch model training:\n",
+        "    - Pytorch model training:\n",
         "        - Extract the BigQuery to a pandas dataframe.\n",
         "        - Preprocess the data in the dataframe.\n",
         "        - Create a DataLoader generator from the pandas dataframe.\n",
@@ -129,26 +140,8 @@
         "- Alternatively:\n",
         "    - Extract the BigQuery table to CSV files.\n",
         "    - Preprocess the CSV files.\n",
-        "    - Create a tf.data.Dataset generator from the CSV files."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "dataset:gsod,lrg"
-      },
-      "source": [
-        "### Dataset\n",
-        "\n",
-        "The dataset used for this tutorial is the GSOD dataset from [BigQuery public datasets](https://cloud.google.com/bigquery/public-data). In this version of the dataset you consider the fields year, month and day to predict the value of mean daily temperature (mean_temp)."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "9e483012a752"
-      },
-      "source": [
+        "    - Create a tf.data.Dataset generator from the CSV files.\n",
+        "    \n",
         "### Costs\n",
         "This tutorial uses billable components of Google Cloud:\n",
         "\n",
@@ -181,7 +174,7 @@
         "import os\n",
         "\n",
         "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
         "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
         "    \"/opt/deeplearning/metadata/env_version\"\n",
         ")\n",
@@ -191,8 +184,13 @@
         "if IS_WORKBENCH_NOTEBOOK:\n",
         "    USER_FLAG = \"--user\"\n",
         "\n",
-        "extra_pkgs = \"tensorflow tensorflow-io==0.18 pyarrow xgboost google-cloud-bigquery\"\n",
-        "! pip3 install --upgrade --quiet {USER_FLAG} google-cloud-aiplatform $extra_pkgs"
+        "# Install the packages\n",
+        "! pip3 install --upgrade pyarrow $USER_FLAG -q\n",
+        "! pip3 install --upgrade google-cloud-bigquery $USER_FLAG -q\n",
+        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG -q\n",
+        "! pip3 install -U xgboost $USER_FLAG -q\n",
+        "! pip3 install -U tensorflow $USER_FLAG -q\n",
+        "! pip3 install -U tensorflow-io==0.18 $USER_FLAG -q"
       ]
     },
     {
@@ -214,9 +212,9 @@
       },
       "outputs": [],
       "source": [
-        "import sys\n",
+        "import os\n",
         "\n",
-        "if \"google.colab\" in sys.modules:\n",
+        "if not os.getenv(\"IS_TESTING\"):\n",
         "    # Automatically restart kernel after installs\n",
         "    import IPython\n",
         "\n",
@@ -227,42 +225,274 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "fc8fb52b5cca"
+        "id": "84cd83853240"
       },
       "source": [
-        "### Common setup\n",
+        "## Before you begin\n",
         "\n",
-        "Now, execute the common setup for the notebook tutorials."
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI, BigQuery, Compute Engine and Cloud Storage APIs](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,bigquery,compute_component,storage_component).\n",
+        "\n",
+        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "project_id"
+      },
+      "source": [
+        "#### Set your project ID\n",
+        "\n",
+        "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "001a0fcd5d78"
+        "id": "set_project_id"
       },
       "outputs": [],
       "source": [
-        "# Common code setup for notebook tutorials\n",
-        "\n",
-        "! wget https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/setup.py -O setup.py\n",
-        "\n",
-        "%run setup.py --bucket"
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "d809f07a8935"
+        "id": "autoset_project_id"
       },
       "outputs": [],
       "source": [
-        "# Other Common setup instructions for notebook tutorials\n",
+        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
+        "    # Get your GCP project id from gcloud\n",
+        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "    PROJECT_ID = shell_output[0]\n",
+        "    print(\"Project ID:\", PROJECT_ID)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "set_gcloud_project_id"
+      },
+      "outputs": [],
+      "source": [
+        "! gcloud config set project $PROJECT_ID"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "region"
+      },
+      "source": [
+        "#### Region\n",
         "\n",
-        "! wget https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/setup.md -O setup.md\n",
+        "You can also change the `REGION` variable, which is used for operations\n",
+        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
         "\n",
-        "%load setup.md"
+        "- Americas: `us-central1`\n",
+        "- Europe: `europe-west4`\n",
+        "- Asia Pacific: `asia-east1`\n",
+        "\n",
+        "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
+        "\n",
+        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "region"
+      },
+      "outputs": [],
+      "source": [
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "timestamp"
+      },
+      "source": [
+        "#### Timestamp\n",
+        "\n",
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "timestamp"
+      },
+      "outputs": [],
+      "source": [
+        "from datetime import datetime\n",
+        "\n",
+        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "77c385f0db59"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
+        "\n",
+        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
+        "\n",
+        "**Otherwise**, follow these steps:\n",
+        "\n",
+        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
+        "\n",
+        "1. **Click Create service account**.\n",
+        "\n",
+        "2. In the **Service account name** field, enter a name, and click **Create**.\n",
+        "\n",
+        "3. In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex AI\" into the filter box, and select **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "\n",
+        "4. Click Create. A JSON file that contains your key downloads to your local environment.\n",
+        "\n",
+        "5. Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "535223fa4b84"
+      },
+      "outputs": [],
+      "source": [
+        "i  # If you are running this notebook in Colab, run this cell and follow the\n",
+        "# instructions to authenticate your GCP account. This provides access to your\n",
+        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
+        "# requests.\n",
+        "\n",
+        "import os\n",
+        "import sys\n",
+        "\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = False\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
+        "    if \"google.colab\" in sys.modules:\n",
+        "        IS_COLAB = True\n",
+        "        from google.colab import auth as google_auth\n",
+        "\n",
+        "        google_auth.authenticate_user()\n",
+        "\n",
+        "    # If you are running this notebook locally, replace the string below with the\n",
+        "    # path to your service account key and run this cell to authenticate your GCP\n",
+        "    # account.\n",
+        "    elif not os.getenv(\"IS_TESTING\"):\n",
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "bucket:custom"
+      },
+      "source": [
+        "### Create a Cloud Storage bucket\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "When you create a dataset resource using the Vertex SDK, you can provide a Cloud Storage bucket that contains the data. Vertex AI creates the dataset resource from the data. In this tutorial, Vertex AI also creates a dataset resource from your data in the Cloud Storage bucket.\n",
+        "\n",
+        "Set the name of your Cloud Storage bucket below. Bucket names must be globally unique across all Google Cloud projects, including those outside of your organization."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "bucket"
+      },
+      "outputs": [],
+      "source": [
+        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "autoset_bucket"
+      },
+      "outputs": [],
+      "source": [
+        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
+        "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "create_bucket"
+      },
+      "source": [
+        "**Only if your bucket doesn't already exist**: Run the following cell to create your Cloud Storage bucket."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "create_bucket"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil mb -l $REGION $BUCKET_URI"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "validate_bucket"
+      },
+      "source": [
+        "Finally, validate access to your Cloud Storage bucket by examining its contents:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "validate_bucket"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil ls -al $BUCKET_URI"
       ]
     },
     {
@@ -383,7 +613,7 @@
       "outputs": [],
       "source": [
         "dataset = aiplatform.TabularDataset.create(\n",
-        "    display_name=\"NOAA historical weather data\" + \"_\" + UUID,\n",
+        "    display_name=\"NOAA historical weather data\" + \"_\" + TIMESTAMP,\n",
         "    bq_source=[IMPORT_FILE],\n",
         "    labels={\"user_metadata\": BUCKET_URI[5:]},\n",
         ")\n",
@@ -458,7 +688,7 @@
         "gcs_source = IMPORT_FILES\n",
         "\n",
         "dataset = aiplatform.TabularDataset.create(\n",
-        "    display_name=\"NOAA historical weather data\" + \"_\" + UUID,\n",
+        "    display_name=\"NOAA historical weather data\" + \"_\" + TIMESTAMP,\n",
         "    gcs_source=gcs_source,\n",
         "    labels={\"user_metadata\": BUCKET_URI[5:]},\n",
         ")\n",
@@ -500,10 +730,10 @@
         "    or BQ_MY_DATASET is None\n",
         "    or BQ_MY_DATASET == \"[your-dataset-name]\"\n",
         "):\n",
-        "    BQ_MY_DATASET = \"mlops_dataset_\" + UUID\n",
+        "    BQ_MY_DATASET = \"mlops_dataset_\" + TIMESTAMP\n",
         "\n",
         "if BQ_MY_TABLE == \"\" or BQ_MY_TABLE is None or BQ_MY_TABLE == \"[your-view-name]\":\n",
-        "    BQ_MY_TABLE = \"mlops_view_\" + UUID"
+        "    BQ_MY_TABLE = \"mlops_view_\" + TIMESTAMP"
       ]
     },
     {
@@ -902,7 +1132,7 @@
       "source": [
         "dataframe[\"station_number\"] = pd.to_numeric(dataframe[\"station_number\"])\n",
         "labels = dataframe[\"mean_temp\"]\n",
-        "data = dataframe.drop([\"mean_temp\"], axis=1)\n",
+        "data = dataframe.drop(4)\n",
         "\n",
         "dtrain = xgb.DMatrix(data, label=labels)"
       ]

notebooks/community/ml_ops/stage1/get_started_dataflow.ipynb

@@ -44,7 +44,7 @@
         "        </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage1/get_started_dataflow.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_dataflow.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
@@ -65,6 +65,17 @@
         "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 1 : data management: get started with Dataflow."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dataset:gsod,lrg"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the GSOD dataset from [BigQuery public datasets](https://cloud.google.com/bigquery/public-data). The version of the dataset you use only the fields year, month and day to predict the value of mean daily temperature (mean_temp)."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -126,34 +137,6 @@
         "Alternately for AutoML tabular model training, you can reconfigure the otherwise default preprocessing."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "dataset:gsod,lrg"
-      },
-      "source": [
-        "### Dataset\n",
-        "\n",
-        "The dataset used for this tutorial is the GSOD dataset from [BigQuery public datasets](https://cloud.google.com/bigquery/public-data). The version of the dataset you use only the fields year, month and day to predict the value of mean daily temperature (mean_temp)."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "9e483012a752"
-      },
-      "source": [
-        "### Costs\n",
-        "This tutorial uses billable components of Google Cloud:\n",
-        "\n",
-        "- Vertex AI\n",
-        "- Cloud Storage\n",
-        "- BigQuery\n",
-        "- Dataflow\n",
-        "\n",
-        "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing), [Cloud Storage pricing](https://cloud.google.com/storage/pricing), [BigQuery pricing](https://cloud.google.com/bigquery/pricing), and [Dataflow pricing](https://cloud.google.com/dataflow/pricing) and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -176,7 +159,7 @@
         "import os\n",
         "\n",
         "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
         "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
         "    \"/opt/deeplearning/metadata/env_version\"\n",
         ")\n",
@@ -186,9 +169,13 @@
         "if IS_WORKBENCH_NOTEBOOK:\n",
         "    USER_FLAG = \"--user\"\n",
         "\n",
-        "extra_pkgs = \"tensorflow==2.5 tensorflow-data-validation==1.2 tensorflow-transform==1.2 \\\n",
-        "              tensorflow-io==0.18 pyarrow pandas apache-beam[gcp] google-cloud-bigquery\"\n",
-        "! pip3 install --upgrade --quiet {USER_FLAG} google-cloud-aiplatform $extra_pkgs"
+        "! pip3 install -U tensorflow==2.5 $USER_FLAG -q\n",
+        "! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG -q\n",
+        "! pip3 install -U tensorflow-transform==1.2 $USER_FLAG -q\n",
+        "! pip3 install -U tensorflow-io==0.18 $USER_FLAG -q\n",
+        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG -q\n",
+        "! pip3 install --upgrade google-cloud-bigquery $USER_FLAG -q\n",
+        "! pip3 install --upgrade apache-beam[gcp] $USER_FLAG -q"
       ]
     },
     {
@@ -210,9 +197,9 @@
       },
       "outputs": [],
       "source": [
-        "import sys\n",
+        "import os\n",
         "\n",
-        "if \"google.colab\" in sys.modules:\n",
+        "if not os.getenv(\"IS_TESTING\"):\n",
         "    # Automatically restart kernel after installs\n",
         "    import IPython\n",
         "\n",
@@ -223,42 +210,279 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "fc8fb52b5cca"
+        "id": "84cd83853240"
       },
       "source": [
-        "### Common setup\n",
+        "## Before you begin\n",
         "\n",
-        "Now, execute the common setup for the notebook tutorials."
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI, BigQuery, Compute Engine and Cloud Storage APIs](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,bigquery,compute_component,storage_component).\n",
+        "\n",
+        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "project_id"
+      },
+      "source": [
+        "#### Set your project ID\n",
+        "\n",
+        "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "001a0fcd5d78"
+        "id": "set_project_id"
       },
       "outputs": [],
       "source": [
-        "# Common code setup for notebook tutorials\n",
-        "\n",
-        "! wget https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/setup.py -O setup.py\n",
-        "\n",
-        "%run setup.py --bucket"
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "d809f07a8935"
+        "id": "autoset_project_id"
       },
       "outputs": [],
       "source": [
-        "# Other Common setup instructions for notebook tutorials\n",
+        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
+        "    # Get your GCP project id from gcloud\n",
+        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "    PROJECT_ID = shell_output[0]\n",
+        "    print(\"Project ID:\", PROJECT_ID)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "set_gcloud_project_id"
+      },
+      "outputs": [],
+      "source": [
+        "! gcloud config set project $PROJECT_ID"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "region"
+      },
+      "source": [
+        "#### Region\n",
         "\n",
-        "! wget https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/setup.md -O setup.md\n",
+        "You can also change the `REGION` variable, which is used for operations\n",
+        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
         "\n",
-        "%load setup.md "
+        "- Americas: `us-central1`\n",
+        "- Europe: `europe-west4`\n",
+        "- Asia Pacific: `asia-east1`\n",
+        "\n",
+        "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
+        "\n",
+        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "region"
+      },
+      "outputs": [],
+      "source": [
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "timestamp"
+      },
+      "source": [
+        "#### Timestamp\n",
+        "\n",
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "timestamp"
+      },
+      "outputs": [],
+      "source": [
+        "from datetime import datetime\n",
+        "\n",
+        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "77c385f0db59"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
+        "\n",
+        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
+        "\n",
+        "**Otherwise**, follow these steps:\n",
+        "\n",
+        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
+        "\n",
+        "1. **Click Create service account**.\n",
+        "\n",
+        "2. In the **Service account name** field, enter a name, and click **Create**.\n",
+        "\n",
+        "3. In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex AI\" into the filter box, and select **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "\n",
+        "4. Click Create. A JSON file that contains your key downloads to your local environment.\n",
+        "\n",
+        "5. Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "535223fa4b84"
+      },
+      "outputs": [],
+      "source": [
+        "# If you are running this notebook in Colab, run this cell and follow the\n",
+        "# instructions to authenticate your GCP account. This provides access to your\n",
+        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
+        "# requests.\n",
+        "\n",
+        "import os\n",
+        "import sys\n",
+        "\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = False\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
+        "    if \"google.colab\" in sys.modules:\n",
+        "        IS_COLAB = True\n",
+        "        from google.colab import auth as google_auth\n",
+        "\n",
+        "        google_auth.authenticate_user()\n",
+        "\n",
+        "    # If you are running this notebook locally, replace the string below with the\n",
+        "    # path to your service account key and run this cell to authenticate your GCP\n",
+        "    # account.\n",
+        "    elif not os.getenv(\"IS_TESTING\"):\n",
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "bucket:custom"
+      },
+      "source": [
+        "### Create a Cloud Storage bucket\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "When you submit a custom training job using the Vertex SDK, you upload a Python package\n",
+        "containing your training code to a Cloud Storage bucket. Vertex AI runs\n",
+        "the code from this package. In this tutorial, Vertex AI also saves the\n",
+        "trained model that results from your job in the same bucket. You can then\n",
+        "create an `Endpoint` resource based on this output in order to serve\n",
+        "online predictions.\n",
+        "\n",
+        "Set the name of your Cloud Storage bucket below. Bucket names must be globally unique across all Google Cloud projects, including those outside of your organization."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "bucket"
+      },
+      "outputs": [],
+      "source": [
+        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "autoset_bucket"
+      },
+      "outputs": [],
+      "source": [
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
+        "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "create_bucket"
+      },
+      "source": [
+        "**Only if your bucket doesn't already exist**: Run the following cell to create your Cloud Storage bucket."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "create_bucket"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil mb -l $REGION $BUCKET_URI"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "validate_bucket"
+      },
+      "source": [
+        "Finally, validate access to your Cloud Storage bucket by examining its contents:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "validate_bucket"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil ls -al $BUCKET_URI"
       ]
     },
     {
@@ -1078,7 +1302,7 @@
       },
       "outputs": [],
       "source": [
-        "delete_storage = False\n",
+        "delete_storage = True\n",
         "\n",
         "if delete_storage or os.getenv(\"IS_TESTING\"):\n",
         "    if \"BUCKET_URI\" in globals():\n",

notebooks/community/ml_ops/stage1/get_started_vertex_datasets.ipynb

@@ -29,16 +29,16 @@
         "id": "title:generic,gcp"
       },
       "source": [
-        "# E2E ML on GCP: MLOps stage 1 : data management: get started with Vertex AI datasets\n",
+        "# E2E ML on GCP: MLOps stage 1 : data management: get started with Vertex datasets\n",
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_vertex_datasets.ipynb\">\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_vertex_datasets.ipynb\">\n",
         "     <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_vertex_datasets.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_vertex_datasets.ipynb\">\n",
         "<img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
@@ -136,26 +136,9 @@
         "            - Create a tf.data.Dataset generator from the CSV index file.\n",
         "        - If text strings are in text files:\n",
         "            - Using the JSON index file, convert the text files and labels to TFRecords.\n",
-        "            - Create a tf.data.Dataset from the TFRecords."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "533dd6fe83c8"
-      },
-      "source": [
-        "### Datasets\n",
+        "            - Create a tf.data.Dataset from the TFRecords.\n",
         "\n",
-        "This tutorial uses a variety of public datasets to demonstrate using a `Vertex AI` managed dataset."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "9e483012a752"
-      },
-      "source": [
+        "    \n",
         "### Costs\n",
         "This tutorial uses billable components of Google Cloud:\n",
         "\n",
@@ -188,7 +171,7 @@
         "import os\n",
         "\n",
         "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
         "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
         "    \"/opt/deeplearning/metadata/env_version\"\n",
         ")\n",
@@ -205,7 +188,7 @@
         "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG -q\n",
         "! pip3 install --upgrade google-cloud-bigquery $USER_FLAG -q\n",
         "! pip3 install -U tensorflow-io==0.18 $USER_FLAG -q\n",
-        "! pip3 install --upgrade db-dtypes $USER_FLAG -q! pip3 install --upgrade future $USER_FLAG -q"
+        "! pip3 install --upgrade future $USER_FLAG -q"
       ]
     },
     {
@@ -243,8 +226,6 @@
         "id": "cb082379ed5b"
       },
       "source": [
-        "## Before you begin\n",
-        "\n",
         "### Set up your Google Cloud project\n",
         "\n",
         "**The following steps are required, regardless of your notebook environment.**\n",
@@ -282,22 +263,36 @@
       },
       "outputs": [],
       "source": [
-        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
+        "import os\n",
+        "\n",
+        "PROJECT_ID = \"\"\n",
+        "\n",
+        "if not os.getenv(\"IS_TESTING\"):\n",
+        "    # Get your Google Cloud project ID from gcloud\n",
+        "    shell_output = !gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "    PROJECT_ID = shell_output[0]\n",
+        "    print(\"Project ID: \", PROJECT_ID)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "37c0a68ff20d"
+      },
+      "source": [
+        "Otherwise, set your project ID here."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "nWlzLu5ELxWd"
+        "id": "autoset_project_id"
       },
       "outputs": [],
       "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
-        "    # Get your GCP project id from gcloud\n",
-        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
-        "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID:\", PROJECT_ID)"
+        "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
+        "    PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -413,11 +408,12 @@
         "import sys\n",
         "\n",
         "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = \"google.colab\" in sys.modules\n",
+        "IS_COLAB = False\n",
         "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
         "    \"DL_ANACONDA_HOME\"\n",
         "):\n",
         "    if \"google.colab\" in sys.modules:\n",
+        "        IS_COLAB = True\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
         "        google_auth.authenticate_user()\n",
@@ -562,7 +558,7 @@
       },
       "outputs": [],
       "source": [
-        "aip.init(project=PROJECT_ID, location=REGION, staging_bucket=BUCKET_URI)"
+        "aip.init(project=PROJECT_ID, location=REGION)"
       ]
     },
     {
@@ -616,13 +612,26 @@
         "Learn more about [All dataset documentation](https://cloud.google.com/vertex-ai/docs/datasets/datasets)"
       ]
     },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "import_file:flowers,csv,icn"
+      },
+      "outputs": [],
+      "source": [
+        "IMPORT_FILE = (\n",
+        "    \"gs://cloud-samples-data/vision/automl_classification/flowers/all_data_v2.csv\"\n",
+        ")"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "create_dataset:image,icn"
       },
       "source": [
-        "### Create an Image Dataset\n",
+        "### Create the Dataset\n",
         "\n",
         "Next, create the `Dataset` resource using the `create` method for the `ImageDataset` class, which takes the following parameters:\n",
         "\n",
@@ -637,19 +646,6 @@
         "Learn more about [ImageDataset](https://cloud.google.com/vertex-ai/docs/datasets/prepare-image)."
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "import_file:flowers,csv,icn"
-      },
-      "outputs": [],
-      "source": [
-        "IMPORT_FILE = (\n",
-        "    \"gs://cloud-samples-data/vision/automl_classification/flowers/all_data_v2.csv\"\n",
-        ")"
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -667,13 +663,24 @@
         "print(dataset.resource_name)"
       ]
     },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "import_file:hmdb,csv,vcn"
+      },
+      "outputs": [],
+      "source": [
+        "IMPORT_FILE = \"gs://automl-video-demo-data/hmdb_split1_5classes_train_inf.csv\""
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "create_dataset:video,vcn"
       },
       "source": [
-        "### Create a Video Dataset\n",
+        "### Create the Dataset\n",
         "\n",
         "Next, create the `Dataset` resource using the `create` method for the `VideoDataset` class, which takes the following parameters:\n",
         "\n",
@@ -687,17 +694,6 @@
         "Learn more about [VideoDataset](https://cloud.google.com/vertex-ai/docs/datasets/prepare-video)."
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "import_file:hmdb,csv,vcn"
-      },
-      "outputs": [],
-      "source": [
-        "IMPORT_FILE = \"gs://automl-video-demo-data/hmdb_split1_5classes_train_inf.csv\""
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -715,13 +711,24 @@
         "print(dataset.resource_name)"
       ]
     },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "import_file:happydb,csv,tcn"
+      },
+      "outputs": [],
+      "source": [
+        "IMPORT_FILE = \"gs://cloud-ml-data/NL-classification/happiness.csv\""
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "create_dataset:text,tcn"
       },
       "source": [
-        "### Create a Text Dataset\n",
+        "### Create the Dataset\n",
         "\n",
         "Next, create the `Dataset` resource using the `create` method for the `TextDataset` class, which takes the following parameters:\n",
         "\n",
@@ -736,17 +743,6 @@
         "Learn more about [TextDataset](https://cloud.google.com/vertex-ai/docs/datasets/prepare-text)."
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "import_file:happydb,csv,tcn"
-      },
-      "outputs": [],
-      "source": [
-        "IMPORT_FILE = \"gs://cloud-ml-data/NL-classification/happiness.csv\""
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -764,24 +760,6 @@
         "print(dataset.resource_name)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "create_dataset:tabular,bq,lrg,v2"
-      },
-      "source": [
-        "### Create a Tabular Dataset\n",
-        "\n",
-        "#### CSV input data\n",
-        "\n",
-        "Next, create the `Dataset` resource using the `create` method for the `TabularDataset` class for CSV input data, which takes the following parameters:\n",
-        "\n",
-        "- `display_name`: The human readable name for the `Dataset` resource.\n",
-        "- `gcs_source`: A list of one or more dataset index files to import the data items into the `Dataset` resource.\n",
-        "\n",
-        "Learn more about [TabularDataset from CSV files](https://cloud.google.com/vertex-ai/docs/datasets/create-dataset-api#aiplatform_create_dataset_tabular_gcs_sample-python)"
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -790,50 +768,27 @@
       },
       "outputs": [],
       "source": [
-        "IMPORT_FILE = \"gs://cloud-samples-data/tables/iris_1000.csv\""
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "create_dataset:tabular,bq,lrg,v2"
-      },
-      "outputs": [],
-      "source": [
-        "dataset = aip.TabularDataset.create(\n",
-        "    display_name=\"example\" + \"_\" + TIMESTAMP, gcs_source=[IMPORT_FILE]\n",
-        ")\n",
-        "\n",
-        "print(dataset.resource_name)"
+        "IMPORT_FILE = \"bq://bigquery-public-data.samples.gsod\"\n",
+        "BQ_TABLE = \"bigquery-public-data.samples.gsod\""
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "854dd1e0195c"
+        "id": "create_dataset:tabular,bq,lrg,v2"
       },
       "source": [
-        "#### BigQuery input data\n",
+        "### Create the Dataset\n",
         "\n",
-        "Next, create the `Dataset` resource using the `create` method for the `TabularDataset` class for BigQuery table input, which takes the following parameters:\n",
+        "#### CSV input data\n",
+        "\n",
+        "Next, create the `Dataset` resource using the `create` method for the `TabularDataset` class, which takes the following parameters:\n",
         "\n",
         "- `display_name`: The human readable name for the `Dataset` resource.\n",
-        "- `bq_source`: A list of one or more BigQuery tables to import the data items into the `Dataset` resource.\n",
+        "- `gcs_source`: A list of one or more dataset index files to import the data items into the `Dataset` resource.\n",
+        "- `labels`: User defined metadata. In this example, you store the location of the Cloud Storage bucket containing the user defined data.\n",
         "\n",
-        "Learn more about [TabularDataset from BigQuery table](https://cloud.google.com/vertex-ai/docs/datasets/create-dataset-api#aiplatform_create_dataset_tabular_bigquery_sample-pythonn)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "86343c146300"
-      },
-      "outputs": [],
-      "source": [
-        "IMPORT_FILE = \"bq://bigquery-public-data.samples.gsod\"\n",
-        "BQ_TABLE = \"bigquery-public-data.samples.gsod\""
+        "Learn more about [TabularDataset from CSV files](https://cloud.google.com/vertex-ai/docs/datasets/create-dataset-api#aiplatform_create_dataset_tabular_gcs_sample-python)"
       ]
     },
     {
@@ -851,82 +806,6 @@
         "print(dataset.resource_name)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "82e9fe20ce71"
-      },
-      "source": [
-        "#### Dataframe input data\n",
-        "\n",
-        "Next, create the `Dataset` resource using the `create_from_dataframe` method for the `TabularDataset` class for pandas dataframe input, which takes the following parameters:\n",
-        "\n",
-        "- `display_name`: The human readable name for the `Dataset` resource.\n",
-        "- `df_source`: The pandas dataframe to import the data items into the `Dataset` resource.\n",
-        "- `staging_path`: The BigQuery table to store the imported data."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "3805f945ffdd"
-      },
-      "outputs": [],
-      "source": [
-        "# Download the table.\n",
-        "table = bigquery.TableReference.from_string(BQ_TABLE)\n",
-        "\n",
-        "rows = bqclient.list_rows(\n",
-        "    table,\n",
-        "    max_results=10000,\n",
-        "    selected_fields=[\n",
-        "        bigquery.SchemaField(\"station_number\", \"STRING\"),\n",
-        "        bigquery.SchemaField(\"year\", \"INTEGER\"),\n",
-        "        bigquery.SchemaField(\"month\", \"INTEGER\"),\n",
-        "        bigquery.SchemaField(\"day\", \"INTEGER\"),\n",
-        "        bigquery.SchemaField(\"mean_temp\", \"FLOAT\"),\n",
-        "    ],\n",
-        ")\n",
-        "\n",
-        "dataframe = rows.to_dataframe()\n",
-        "print(dataframe.head())"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "create_dataset:tabular,bq,lrg,v2"
-      },
-      "outputs": [],
-      "source": [
-        "dataset = aip.TabularDataset.create_from_dataframe(\n",
-        "    display_name=\"example\" + \"_\" + TIMESTAMP,\n",
-        "    df_source=dataframe,\n",
-        "    staging_path=f\"bq://{PROJECT_ID}.samples.gsod\",\n",
-        ")\n",
-        "\n",
-        "print(dataset.resource_name)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "create_dataset:tabular,forecast,v2"
-      },
-      "source": [
-        "### Create a Time Series Dataset\n",
-        "\n",
-        "Next, create the `Dataset` resource using the `create` method for the `TimeSeriesDataset` class, which takes the following parameters:\n",
-        "\n",
-        "- `display_name`: The human readable name for the `Dataset` resource.\n",
-        "- `gcs_source`: A list of one or more dataset index files to import the data items into the `Dataset` resource.\n",
-        "- `bq_source`: Alternatively, import data items from a BigQuery table into the `Dataset` resource.\n",
-        "\n",
-        "Learn more about [TimeSeriesDataset](https://cloud.google.com/vertex-ai/docs/datasets/prepare-tabular)."
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -938,6 +817,23 @@
         "IMPORT_FILE = \"gs://cloud-samples-data/ai-platform/covid/bigquery-public-covid-nyt-us-counties-train.csv\""
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "create_dataset:tabular,forecast,v2"
+      },
+      "source": [
+        "### Create the Dataset\n",
+        "\n",
+        "Next, create the `Dataset` resource using the `create` method for the `TimeSeriesDataset` class, which takes the following parameters:\n",
+        "\n",
+        "- `display_name`: The human readable name for the `Dataset` resource.\n",
+        "- `gcs_source`: A list of one or more dataset index files to import the data items into the `Dataset` resource.\n",
+        "- `bq_source`: Alternatively, import data items from a BigQuery table into the `Dataset` resource.\n",
+        "\n",
+        "Learn more about [TimeSeriesDataset](https://cloud.google.com/vertex-ai/docs/datasets/prepare-tabular)."
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -1383,7 +1279,7 @@
         "import os\n",
         "import sys\n",
         "\n",
-        "# If on Workbench AI Notebook, then don't execute this code\n",
+        "# If on Google Cloud Notebook, then don't execute this code\n",
         "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
         "    if \"google.colab\" in sys.modules:\n",
         "        ! pip3 install fsspec\n",
@@ -1724,15 +1620,11 @@
       },
       "outputs": [],
       "source": [
-        "delete_bucket = False\n",
-        "\n",
         "# Delete the dataset using the Vertex dataset object\n",
-        "datasets = aip.TabularDataset.list(filter=f'display_name=\"example_{TIMESTAMP}\"')\n",
-        "for dataset in datasets:\n",
-        "    dataset.delete()\n",
+        "dataset.delete()\n",
         "\n",
         "# Delete the bucket\n",
-        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "if os.getenv(\"IS_TESTING\"):\n",
         "    ! gsutil rm -r $BUCKET_URI"
       ]
     }

notebooks/community/ml_ops/stage1/get_started_with_data_labeling.ipynb

@@ -65,6 +65,17 @@
         "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 1 : data management: get started with Vertex AI Data Labeling service."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dataset:flowers,icn"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the [Flowers dataset](https://www.tensorflow.org/datasets/catalog/tf_flowers) from [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/overview). The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket. The trained model predicts the type of flower an image is from a class of five flowers: daisy, dandelion, rose, sunflower, or tulip."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -91,17 +102,6 @@
         "Learn more about [Request a Vertex AI Data Labeling job](https://cloud.google.com/vertex-ai/docs/datasets/data-labeling-job)."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "dataset:flowers,icn"
-      },
-      "source": [
-        "### Dataset\n",
-        "\n",
-        "The dataset used for this tutorial is the [Flowers dataset](https://www.tensorflow.org/datasets/catalog/tf_flowers) from [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/overview). The version of the dataset you use in this tutorial is stored in a public Cloud Storage bucket. The trained model predicts the type of flower an image is from a class of five flowers: daisy, dandelion, rose, sunflower, or tulip."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -146,7 +146,7 @@
         "import os\n",
         "\n",
         "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
         "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
         "    \"/opt/deeplearning/metadata/env_version\"\n",
         ")\n",
@@ -167,7 +167,7 @@
         "id": "restart"
       },
       "source": [
-        "### Restart the kernel\n",
+        "### Restart the Kernel\n",
         "\n",
         "Once you've installed the Vertex AI SDK and Google *cloud-storage*, you need to restart the notebook kernel so it can find the packages.\n"
       ]
@@ -212,7 +212,7 @@
         "\n",
         "3. [Enable the Vertex AI APIs and Compute Engine APIs.](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component)\n",
         "\n",
-        "4. [Google Cloud SDK](https://cloud.google.com/sdk) is already installed in Vertex AI Workbench Notebooks.\n",
+        "4. [Google Cloud SDK](https://cloud.google.com/sdk) is already installed in Google Cloud Notebooks.\n",
         "\n",
         "5. Enter your project ID in the cell below. Then run the  cell to make sure the\n",
         "Cloud SDK uses the right project for all the commands in this notebook.\n",
@@ -220,17 +220,6 @@
         "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands.\n"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "WReHDGG5g0XY"
-      },
-      "source": [
-        "#### Set your project ID\n",
-        "\n",
-        "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -374,8 +363,15 @@
         "### Authenticate your Google Cloud account\n",
         "\n",
         "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
-        "authenticated. \n",
-        "\n",
+        "authenticated. Skip this step."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "32e1cd21a5d5"
+      },
+      "source": [
         "**If you are using Colab**, run the cell below and follow the instructions\n",
         "when prompted to authenticate your account via oAuth.\n",
         "\n",

notebooks/community/ml_ops/stage1/mlops_data_management.ipynb

@@ -39,15 +39,18 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
+        "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/mlops_data_management.ipynb\">\n",
+        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\\\" alt=\"Colab logo\"> Run in Colab\n",
+        "        </a>\n",
+        "  </td>\n",
+        "  <td>\n",
         "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage1/mlops_data_management.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
         "  </td>\n",
         "</table>\n",
-        "<br/><br/><br/>\n",
-        "\n",
-        "*Note: This notebook is not supported for execution in Colab*"
+        "<br/><br/><br/>"
       ]
     },
     {
@@ -62,6 +65,17 @@
         "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 1 : data management."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dataset:bq,chicago,lbn"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the [Chicago Taxi](https://www.kaggle.com/chicago/chicago-taxi-trips-bq). The version of the dataset you will use in this tutorial is stored in a public BigQuery table. The trained model predicts whether someone would leave a tip for a taxi fare."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -106,34 +120,6 @@
         " - Preprocess the data with `Dataflow`"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "dataset:bq,chicago,lbn"
-      },
-      "source": [
-        "### Dataset\n",
-        "\n",
-        "The dataset used for this tutorial is the [Chicago Taxi](https://www.kaggle.com/chicago/chicago-taxi-trips-bq). The version of the dataset used in this tutorial is stored in a public BigQuery table. The trained model predicts whether someone leaves a tip for a taxi fare."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "9e483012a752"
-      },
-      "source": [
-        "### Costs\n",
-        "This tutorial uses billable components of Google Cloud:\n",
-        "\n",
-        "- Vertex AI\n",
-        "- Cloud Storage\n",
-        "- BigQuery\n",
-        "- Dataflow\n",
-        "\n",
-        "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing), [Cloud Storage pricing](https://cloud.google.com/storage/pricing), [BigQuery pricing](https://cloud.google.com/bigquery/pricing), and [Dataflow pricing](https://cloud.google.com/dataflow/pricing) and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -156,7 +142,7 @@
         "import os\n",
         "\n",
         "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
         "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
         "    \"/opt/deeplearning/metadata/env_version\"\n",
         ")\n",
@@ -166,20 +152,20 @@
         "if IS_WORKBENCH_NOTEBOOK:\n",
         "    USER_FLAG = \"--user\"\n",
         "\n",
-        "ONCE_ONLY = False\n",
+        "ONCE_ONLY = True\n",
         "if ONCE_ONLY:\n",
-        "    ! pip3 install -U {USER_FLAG} -q tensorflow==2.5 \\\n",
-        "                                     tensorflow-data-validation==1.2 \\\n",
-        "                                     tensorflow-transform==1.2 \\\n",
-        "                                     tensorflow-io==0.18 \n",
-        "    \n",
-        "    ! pip3 install --upgrade {USER_FLAG} -q google-cloud-aiplatform[tensorboard] \\\n",
-        "                                            google-cloud-pipeline-components \\\n",
-        "                                            google-cloud-bigquery \\\n",
-        "                                            google-cloud-logging \\\n",
-        "                                            apache-beam[gcp] \\\n",
-        "                                            pyarrow \\\n",
-        "                                            cloudml-hypertune\n"
+        "    ! pip3 install -U tensorflow==2.5 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-transform==1.2 $USER_FLAG\n",
+        "    ! pip3 install -U tensorflow-io==0.18 $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-bigquery $USER_FLAG\n",
+        "    ! pip3 install --upgrade google-cloud-logging $USER_FLAG\n",
+        "    ! pip3 install --upgrade apache-beam[gcp] $USER_FLAG\n",
+        "    ! pip3 install --upgrade pyarrow $USER_FLAG\n",
+        "    ! pip3 install --upgrade cloudml-hypertune $USER_FLAG\n",
+        "    ! pip3 install --upgrade kfp $USER_FLAG"
       ]
     },
     {
@@ -351,7 +337,7 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. \n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
         "\n",
         "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
         "\n",
@@ -387,11 +373,12 @@
         "import sys\n",
         "\n",
         "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = \"google.colab\" in sys.modules\n",
+        "IS_COLAB = False\n",
         "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
         "    \"DL_ANACONDA_HOME\"\n",
         "):\n",
         "    if \"google.colab\" in sys.modules:\n",
+        "        IS_COLAB = True\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
         "        google_auth.authenticate_user()\n",
@@ -413,7 +400,7 @@
         "\n",
         "**The following steps are required, regardless of your notebook environment.**\n",
         "\n",
-        "When you submit a custom training job using the Vertex AI SDK, you upload a Python package\n",
+        "When you submit a custom training job using the Vertex SDK, you upload a Python package\n",
         "containing your training code to a Cloud Storage bucket. Vertex AI runs\n",
         "the code from this package. In this tutorial, Vertex AI also saves the\n",
         "trained model that results from your job in the same bucket. You can then\n",
@@ -662,7 +649,7 @@
       },
       "outputs": [],
       "source": [
-        "bqclient = bigquery.Client(project=PROJECT_ID)"
+        "bqclient = bigquery.Client()"
       ]
     },
     {
@@ -785,11 +772,6 @@
         "LIMIT = 300000\n",
         "YEAR = 2020\n",
         "\n",
-        "# First, create the dataset entry\n",
-        "dataset = bigquery.Dataset(f\"{PROJECT_ID}.{BQ_DATASET}\")\n",
-        "dataset.location = \"US\"\n",
-        "dataset = bqclient.create_dataset(dataset, timeout=30)\n",
-        "\n",
         "query = f\"\"\"\n",
         "CREATE OR REPLACE TABLE `{BQ_TABLE_COPY}`\n",
         "AS (\n",
@@ -1230,7 +1212,7 @@
         "import setuptools\n",
         "\n",
         "REQUIRED_PACKAGES = [\n",
-        "    \"google-cloud-aiplatform\",\n",
+        "    \"google-cloud-aiplatform==1.4.2\",\n",
         "    \"tensorflow-transform==1.2.0\",\n",
         "    \"tensorflow-data-validation==1.2.0\",\n",
         "]\n",

notebooks/community/ml_ops/stage2/README.md

@@ -35,10 +35,77 @@ The second stage in MLOps is experimenting in developing one or more baseline mo
 
 ### Get Started
 
-[Get started with Vertex AI Training for R](community/ml_ops/stage2/get_started_vertex_training_r.ipynb)
+[Get Started with Vertex Experiments and Vertex ML Metadata](get_started_vertex_experiments.ipynb)
 
-In this tutorial, you learn how to use `Vertex AI Training` for training a R custom model.
+```
+The steps performed include:
 
+- Use Python logging to log training configuration/results locally.
+- Use Google Cloud Logging to log training configuration/results in cloud storage.
+- Create a Vertex AI `Experiment` resource.
+- Instantiate an experiment run.
+- Log parameters for the run.
+- Log metrics for the run.
+- Display the logged experiment run.
+```
+
+[Get Started with Vertex TensorBoard](get_started_vertex_tensorboard.ipynb)
+
+```
+The steps performed include:
+
+- Create a TensorBoard callback when training a model.
+- Using Tensorboard with locally trained model.
+- Using Vertex AI TensorBoard with Vertex AI Training.
+```
+
+[Get Started with Custom Training Packages (Tensorflow)](get_started_vertex_training.ipynb)
+
+```
+The steps performed include:
+
+- Training using a single Python script.
+- Training using a Python package.
+- Training using a custom training image.
+- Laying out a training package.
+```
+
+[Get Started with Custom Training Packages (Scikit-Learn)](get_started_vertex_training_sklearn.ipynb)
+
+```
+The steps performed include:
+
+- Training using a Python package.
+- Report accuracy when hyperparameter tuning.
+- Save the model artifacts to Cloud Storage using GCSFuse.
+- Create a `Vertex AI Model` resource.
+```
+
+[Get Started with Custom Training Packages (XGBoost)](get_started_vertex_training_xgboost.ipynb)
+
+```
+The steps performed include:
+
+- Training using a Python package.
+- Report accuracy when hyperparameter tuning.
+- Save the model artifacts to Cloud Storage using GCSFuse.
+- Create a `Vertex AI Model` resource.
+```
+
+[Get Started with Custom Training Packages (Pytorch)](get_started_vertex_training_pytorch.ipynb)
+
+```
+The steps performed include:
+
+- Single node training using a Python package.
+- Report accuracy when hyperparameter tuning.
+- Save the model artifacts to Cloud Storage using GCSFuse.
+- Create a `Vertex AI Model` resource.
+```
+
+[Get Started with Custom Training Packages (R)](get_started_vertex_training_r.ipynb)
+
+```
 The steps performed include:
 
 - Locally train an R model in a notebook using %%R magic commands
@@ -50,57 +117,95 @@ The steps performed include:
 - Create a R-to-Python training package.
 - Create a training image for training the model.
 - Train a R model using `Vertex AI Trainingh` service with the R-to-Python training package.
+```
 
-[Get started with Logging](community/ml_ops/stage2/get_started_with_logging.ipynb)
-
-In this tutorial, you learn how to use Python and Cloud logging awhen training with `Vertex AI`.
-
-The steps performed include:
-
-- Use Python logging to log training configuration/results locally.
-- Use Google Cloud Logging to log training configuration/results in cloud storage.
-
-[Get started with Vertex AI Hyperparameter Tuning for XGBoost] (community/ml_ops/stage2/get_started_vertex_hpt_xgboost.ipynb)
-
-In this tutorial, you learn how to use `Vertex AI Hyperparameter Tuning` for training a XGBoost custom model.
+[Get Started with Custom Training Packages (LightGBM)](get_started_vertex_training_lightgbm.ipynb)
 
+```
 The steps performed include:
 
 - Training using a Python package.
-- Report accuracy when hyperparameter tuning.
 - Save the model artifacts to Cloud Storage using GCSFuse.
+- Construct a FastAPI prediction server.
+- Construct a Dockerfile deployment image.
+- Test the deployment image locally.
 - Create a `Vertex AI Model` resource.
+```
 
-[Get started with Vertex AI Training for XGBoost](community/ml_ops/stage2/get_started_vertex_training_xgboost.ipynb)
-
-In this tutorial, you learn how to use `Vertex AI Training` for training a XGBoost custom model.
+[Get Started with Distributed Training](get_started_vertex_distributed_training.ipynb)
 
+```
 The steps performed include:
 
-- Training using a Python package.
-- Report accuracy when hyperparameter tuning.
-- Save the model artifacts to Cloud Storage using GCSFuse.
-- Create a `Vertex AI Model` resource.
+- `MirroredStrategy`: Train on a single VM with multiple GPUs.
+- `MultiWorkerMirroredStrategy`: Train on multiple VMs with automatic setup of replicas.
+- `MultiWorkerMirroredStrategy`: Train on multiple VMs with fine grain control of replicas.
+- `ReductionServer`: Train on multiple VMS and sync updates across VMS with `Vertex AI Reduction Server`.
+- `TPUTraining`: Train with multiple Cloud TPUs.
+```
 
-[Get started with TabNet builtin algorithm for training tabular models](community/ml_ops/stage2/get_started_with_tabnet.ipynb)
-
-In this notebook, you learn how to run `Vertex AI TabNet` built algorithm for training custom tabular models.
+[Get Started with Vizier Hyperparameter Tuning](get_started_vertex_vizier.ipynb)
 
+```
 The steps performed include:
 
-- Get the training data.
-- Configure training parameters for the `Vertex AI TabNet` container.
-- Train the model using `Vertex AI Training` using CSV data.
-- Upload the model as a `Vertex AI Model` resource.
-- Deploy the `Vertex AI Model` resource to a `Vertex AI Endpoint` resource.
-- Make a prediction with the deployed model.
-- Hyperparameter tuning the `Vertex AI TabNet` model.
-- Train the model using `Vertex AI Training` using BigQuery table.
+- Hyperparameter tuning with Random algorithm.
+- Hyperparameter tuning with Vizier (Bayesian) algorithm.
+```
 
-[Get started with prebuilt TFHub models](community/ml_ops/stage2/get_started_with_tfhub_models.ipynb)
+[Get Started with AutoML Training](get_started_automl_training.ipynb)
 
-In this tutorial, you learn how to use `Vertex AI Training` with prebuilt models from TensorFlow Hub.
+```
+The steps performed include:
 
+- Train an image model.
+- Export the image model as an edge model.
+- Train a tabular model.
+- Export the tabular model as a cloud model.
+- Train a text model.
+```
+
+[Get Started with BQML Training](get_started_bqml_training.ipynb)
+
+```
+The steps performed include:
+
+- Create a local BigQuery table in your project
+- Train a BQML model
+- Evaluate the BQML model
+- Export the BQML model as a cloud model
+- Upload the exported model as a `Vertex AI Model` resource
+- Hyperparameter tune a BQML model with `Vertex AI Vizier`
+- Automatically register a BQML model to `Vertex AI Model Registry`
+```
+
+[Get Started with Vertex Feature Store](get_started_vertex_feature_store.ipynb)
+
+```
+The steps performed include:
+
+- Creating a Vertex AI `Featurestore` resource.
+    - Creating `EntityType` resources for the `Featurestore` resource.
+    - Creating `Feature` resources for each `EntityType` resource.
+- Import feature values (entity data items) into `Featurestore` resource from Cloud Storage.
+- Import feature values (entity data items) into `Featurestore` resource from pandas DataFrame.
+- Perform online serving from a `Featurestore` resource.
+- Perform batch serving from a `Featurestore` resource.
+```
+
+[Get Started with Google CMEK Training](get_started_with_cmek_training.ipynb)
+
+```
+The steps performed include:
+
+- Creating a customer managed encryption key.
+- Creating an image dataset with CMEK encryption.
+- Train an AutoML model with CMEK encryption.
+```
+
+[Get Started with TensorFlow Hub models](get_started_with_tfhub_models.ipynb)
+
+```
 The steps performed include:
 
 - Download a TensorFlow Hub prebuilt model.
@@ -111,76 +216,37 @@ The steps performed include:
     - Get model architecture from TensorFlow Hub
     - Train then model
     - Save model artifacts and upload as Vertex AI Model resource.
+```
 
-[Get started with BigQuery ML Training](community/ml_ops/stage2/get_started_bqml_training.ipynb)
-
-In this tutorial, you learn how to use `BigQueryML` (BQML) for training with `Vertex AI`.
-
+[Get Started with Vertex AI TabNet builtin algorithm](get_started_with_tabnet.ipynb)
+```
 The steps performed include:
 
-- Create a local BigQuery table in your project
-- Train a BQML model
-- Evaluate the BQML model
-- Export the BQML model as a cloud model
-- Upload the exported model as a `Vertex AI Model` resource
-- Hyperparameter tune a BQML model with `Vertex AI Vizier`
-- Automatically register a BQML model to `Vertex AI Model Registry`
-
-[Get started with Vertex AI Vizier](community/ml_ops/stage2/get_started_vertex_vizier.ipynb)
-
-In this tutorial, you learn how to use `Vertex AI Vizier` for when training with `Vertex AI`.
-
+- Get the training data.
+- Configure training parameters for the Vertex AI TabNet container.
+- Train the model using Vertex AI Training using CSV data.
+- Upload the model as a Vertex AI Model resource.
+- Deploy the Vertex AI Model resource to a Vertex AI Endpoint resource.
+- Make a prediction with the deployed model.
+- Hyperparameter tuning the Vertex AI TabNet model.
+- Train the model using Vertex AI Training using BigQuery table.
+```
+[Get Started with Vertex AI TabNet builtin algorithm](get_started_with_tabnet.ipynb)
+```
 The steps performed include:
 
-- Hyperparameter tuning with Random algorithm.
-- Hyperparameter tuning with Vizier (Bayesian) algorithm.
-- Suggesting trials and updating results for Vizier study
-
-[Get started with distributed training using DASK](community/ml_ops/stage2/get_started_with_distributed_training_xgboost.ipynb)
-
-In this tutorial, you learn how to use `Vertex AI Training` for distributed training of XGBoost model using the OSS package DASK. Additionally, you learn to construct and deploy a custom serving container using a Flask web server.
-
-The steps performed include:
-
-- Construct an XGBoost training script using DASK for distributed training.
-- Construct a custom training container.
-- Configure a distributed custom training job.
-- Execute the custom training job.
-- Construct a custom serving container using Flask.
-- Upload the trained XGBoost model as a `Vertex AI Model` resource.
-- Create a `Vertex AI Endpoint` resource.
-- Deploy the `Vertex AI Model` resource to `Vertex AI Endpoint` resource.
-- Make a prediction.
-
-[Get started with Vertex AI TensorBoard](community/ml_ops/stage2/get_started_vertex_tensorboard.ipynb)
-
-In this tutorial, you learn how to use `Vertex AI TensorBoard` when training with `Vertex AI`.
-
-The steps performed include:
-
-- Create a TensorBoard callback when training a model.
-- Using TensorBoard with locally trained model.
-- Using Vertex AI TensorBoard with Vertex AI Training.
-
-[Get started with Vertex AI Training for R using R Kernel](community/ml_ops/stage2/get_started_vertex_training_r_using_r_kernel.ipynb)
-
-In this tutorial, you learn how to use `Vertex AI`, using an R kernel, for training and deploying an R custom model.
-
-The steps performed include:
-
-- Create a custom R training script
-- Create a custom R serving script
-- Create a custom R deployment (serving) container.
-- Train the model using `Vertex AI` custom training.
-- Create an `Endpoint` resouce.
-- Deploy the `Model` resource (trained R model) to the `Endpoint` resource.
-- Make an online prediction.
-
-
-[Get started Vision API test preprocessing and AutoML text model generation](community/ml_ops/stage2/get_started_with_visionapi_and_automl.ipynb)
-
-In this tutorial, you create an `AutoML` text entity extraction model pre-existing extracted data by generating a custom import file. You deploy this mode for online prediction from a Python script using the `BigQuery`, `Vision AI`, Cloud Storage and `Vertex AI SDK` for Python. 
+- Get the training data.
+- Configure training parameters for the Vertex AI TabNet container.
+- Train the model using Vertex AI Training using CSV data.
+- Upload the model as a Vertex AI Model resource.
+- Deploy the Vertex AI Model resource to a Vertex AI Endpoint resource.
+- Make a prediction with the deployed model.
+- Hyperparameter tuning the Vertex AI TabNet model.
+- Train the model using Vertex AI Training using BigQuery table.
+```
 
+[Get Started with Vision API and AutoML](get_started_with_visionapi_and_automl.ipynb)
+```
 The steps performed include:
 
 - Preprocess training files using `Vision AI` APIs to extract the text from PDF files.
@@ -191,126 +257,8 @@ The steps performed include:
 - Deploy the `Vertex AI Model` resource to a serving `Endpoint` resource.
 - Make a prediction.
 - Undeploy the `Model`.
+```
 
-[Get started with Vertex AI Experiments](community/ml_ops/stage2/get_started_vertex_experiments.ipynb)
-
-In this tutorial, you learn how to use `Vertex AI Experiments` when training with `Vertex AI`.
-
-The steps performed include:
-
-- Local (notebook) Training
-    - Create an experiment
-    - Create a first run in the experiment
-    - Log parameters and metrics
-    - Create artifact lineage
-    - Visualize the experiment results
-    - Execute a second run
-    - Compare the two runs in the experiment
-- Cloud (`Vertex AI`) Training
-    - Within the training script:
-        - Create an experiment
-        - Log parameters and metrics
-        - Create artifact lineage
-    - Create a `Vertex AI Training` custom job
-    - Execute the custom job
-    - Visualize the experiment results
-
-[AutoML Image Classfication Training with Customer Managed Encryption Keys (CMEK)](community/ml_ops/stage2/get_started_with_cmek_training.ipynb)
-
-In this tutorial, you learn how to use a customer managed encryption key (CMEK) for `Vertex AI AutoML` training.
-
-The steps performed include:
-
-- Creating a customer managed encryption key.
-- Creating an image dataset with CMEK encryption.
-- Train an AutoML model with CMEK encryption.
-
-[Get started with Vertex AI Feature Store](community/ml_ops/stage2/get_started_vertex_feature_store.ipynb)
-
-In this tutorial, you learn how to use `Vertex AI Feature Store` when training and predicting with `Vertex AI`.
-
-The steps performed include:
-
-- Creating a Vertex AI `Featurestore` resource.
-    - Creating `EntityType` resources for the `Featurestore` resource.
-    - Creating `Feature` resources for each `EntityType` resource.
-- Import feature values (entity data items) into `Featurestore` resource.
-    - From a Cloud Storage location.
-    - From a pandas DataFrame.
-- Perform online serving from a `Featurestore` resource.
-- Perform batch serving from a `Featurestore` resource.
-
-[Get started with AutoML Training](community/ml_ops/stage2/get_started_automl_training.ipynb)
-
-In this tutorial, you learn how to use `AutoML` for training with `Vertex AI`.
-
-The steps performed include:
-
-- Train an image model
-- Export the image model as an edge model
-- Train a tabular model
-- Export the tabular model as a cloud model
-- Train a text model
-- Train a video model
-
-[Get started with Vertex AI Training for LightGBM](community/ml_ops/stage2/get_started_vertex_training_lightgbm.ipynb)
-
-In this tutorial, you learn how to use `Vertex AI Training` for training a LightGBM custom model.
-
-The steps performed include:
-
-- Training using a Python package.
-- Save the model artifacts to Cloud Storage using GCSFuse.
-- Construct a FastAPI prediction server.
-- Construct a Dockerfile deployment image.
-- Test the deployment image locally.
-- Create a `Vertex AI Model` resource.
-
-[Get started with Vertex AI Training for Scikit-Learn](community/ml_ops/stage2/get_started_vertex_training_sklearn.ipynb)
-
-In this tutorial, you learn how to use `Vertex AI Training` for training a Scikit-Learn custom model.
-
-The steps performed include:
-
-- Training using a Python package.
-- Report accuracy when hyperparameter tuning.
-- Save the model artifacts to Cloud Storage using GCSFuse.
-- Create a `Vertex AI Model` resource.
-
-[Get started with Vertex AI Training](community/ml_ops/stage2/get_started_vertex_training.ipynb)
-
-In this tutorial, you learn how to use `Vertex AI Training` for custom models when training with `Vertex AI`.
-
-The steps performed include:
-
-- Training using a single Python script.
-- Training using a Python package.
-- Training using a custom training image.
-- Laying out a training package.
-
-
-[Get started with Vertex AI Training for Pytorch](community/ml_ops/stage2/get_started_vertex_training_pytorch.ipynb)
-
-In this tutorial, you learn how to use `Vertex AI Training` for training a Pytorch custom model.
-
-The steps performed include:
-
-- Single node training using a Python package.
-- Report accuracy when hyperparameter tuning.
-- Save the model artifacts to Cloud Storage using GCSFuse.
-- Create a `Vertex AI Model` resource.
-
-[Get started with Vertex AI Distributed Training](community/ml_ops/stage2/get_started_vertex_distributed_training.ipynb)
-
-In this tutorial, you learn how to use `Vertex AI Distributed Training` for when training with `Vertex AI`.
-
-The steps performed include:
-
-- `MirroredStrategy`: Train on a single VM with multiple GPUs.
-- `MultiWorkerMirroredStrategy`: Train on multiple VMs with automatic setup of replicas.
-- `MultiWorkerMirroredStrategy`: Train on multiple VMs with fine grain control of replicas.
-- `ReductionServer`: Train on multiple VMS and sync updates across VMS with `Vertex AI Reduction Server`.
-- `TPUTraining`: Train with multiple Cloud TPUs.
 
 ### E2E Stage Example
 
@@ -318,6 +266,7 @@ The steps performed include:
 
 ```
 The steps performed include:
+
 - Review the `Dataset` resource created during stage 1.
 - Train an AutoML tabular binary classifier model in the background.
 - Build the experimental model architecture.

notebooks/community/ml_ops/stage2/get_started_automl_training.ipynb

@@ -40,7 +40,7 @@
         "  </td>\n",
         "  <td>\n",
         "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_automl_training.ipynb\">\n",
-        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\\\" alt=\"Colab logo\"> Run in Colab\n",
         "        </a>\n",
         "  </td>\n",
         "  <td>\n",
@@ -65,6 +65,52 @@
         "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 2 : experimentation: get started with AutoML Training."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dataset:flowers,icn"
+      },
+      "source": [
+        "### Datasets\n",
+        "\n",
+        "#### Image\n",
+        "\n",
+        "The image dataset used for this tutorial is the [Flowers dataset](https://www.tensorflow.org/datasets/catalog/tf_flowers) from [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/overview). The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket. The trained model predicts the type of flower in a given image from a class of five flowers: daisy, dandelion, rose, sunflower, or tulip."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dataset:gsod,lrg"
+      },
+      "source": [
+        "#### Tabular\n",
+        "\n",
+        "The tabular dataset used for this tutorial is the GSOD dataset from [BigQuery public datasets](https://cloud.google.com/bigquery/public-data). The version of the dataset you use only the fields year, month and day to predict the value of mean daily temperature (mean_temp)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dataset:happydb,tcn"
+      },
+      "source": [
+        "#### Text\n",
+        "\n",
+        "The text dataset used for this tutorial is the [Happy Moments dataset](https://www.kaggle.com/ritresearch/happydb) from [Kaggle Datasets](https://www.kaggle.com/ritresearch/happydb). The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "98eb93ec6faa"
+      },
+      "source": [
+        "#### Video\n",
+        "\n",
+        "The video dataset used for this tutorial is the golf swing recognition portion of the [Human Motion dataset](https://todo) from [MIT](http://cbcl.mit.edu/publications/ps/Kuehne_etal_iccv11.pdf). The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket. The trained model will predict the start frame where a golf swing begins."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -78,7 +124,6 @@
         "This tutorial uses the following Google Cloud ML services:\n",
         "\n",
         "- `AutoML Training`\n",
-        "- `Vertex AI Datasets`\n",
         "\n",
         "The steps performed include:\n",
         "\n",
@@ -105,31 +150,6 @@
         "* **You want to establish a baseline metric before experimenting with a custom model**"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "dataset:flowers,icn"
-      },
-      "source": [
-        "### Datasets\n",
-        "\n",
-        "#### Image\n",
-        "\n",
-        "The image dataset used for this tutorial is the [Flowers dataset](https://www.tensorflow.org/datasets/catalog/tf_flowers) from [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/overview). The version of the dataset in this tutorial is stored in a public Cloud Storage bucket. The trained model predicts the type of flower in a given image from a class of five flowers: daisy, dandelion, rose, sunflower, or tulip.\n",
-        "\n",
-        "#### Tabular\n",
-        "\n",
-        "The tabular dataset used for this tutorial is the GSOD dataset from [BigQuery public datasets](https://cloud.google.com/bigquery/public-data). The version of the dataset you use only the fields year, month and day to predict the value of mean daily temperature (mean_temp).\n",
-        "\n",
-        "#### Text\n",
-        "\n",
-        "The text dataset used for this tutorial is the [Happy Moments dataset](https://www.kaggle.com/ritresearch/happydb) from [Kaggle Datasets](https://www.kaggle.com/ritresearch/happydb). The version of the dataset you use in this tutorial is stored in a public Cloud Storage bucket.\n",
-        "\n",
-        "#### Video\n",
-        "\n",
-        "The video dataset used for this tutorial is the golf swing recognition portion of the [Human Motion dataset](https://todo) from [MIT](http://cbcl.mit.edu/publications/ps/Kuehne_etal_iccv11.pdf). The version of the dataset you use in this tutorial is stored in a public Cloud Storage bucket. The trained model predicts the start frame where a golf swing begins."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -167,7 +187,7 @@
         "import os\n",
         "\n",
         "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
         "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
         "    \"/opt/deeplearning/metadata/env_version\"\n",
         ")\n",
@@ -218,8 +238,6 @@
         "id": "project_id"
       },
       "source": [
-        "## Before you begin\n",
-        "\n",
         "### Set up your Google Cloud project\n",
         "\n",
         "**The following steps are required, regardless of your notebook environment.**\n",
@@ -235,15 +253,8 @@
         "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
         "Cloud SDK uses the right project for all the commands in this notebook.\n",
         "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "5aee4379e8e5"
-      },
-      "source": [
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands.\n",
+        "\n",
         "#### Set your project ID\n",
         "\n",
         "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
@@ -499,7 +510,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 1,
+      "execution_count": null,
       "metadata": {
         "id": "import_aip:mbsdk"
       },
@@ -569,142 +580,6 @@
         "Learn more about [AutoML Model Types](https://cloud.google.com/vertex-ai/docs/start/automl-model-types)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "data_preparation:image,u_dataset"
-      },
-      "source": [
-        "### Data preparation\n",
-        "\n",
-        "The Vertex `Dataset` resource for images has some requirements for your data:\n",
-        "\n",
-        "- Images must be stored in a Cloud Storage bucket.\n",
-        "- Each image file must be in an image format (PNG, JPEG, BMP, ...).\n",
-        "- There must be an index file stored in your Cloud Storage bucket that contains the path and label for each image.\n",
-        "- The index file must be either CSV or JSONL.\n",
-        "\n",
-        "Learn more about [Preparing image data](https://cloud.google.com/vertex-ai/docs/datasets/prepare-image)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "data_import_format:icn,u_dataset,csv"
-      },
-      "source": [
-        "#### CSV\n",
-        "\n",
-        "For image classification, the CSV index file has the requirements:\n",
-        "\n",
-        "- No heading.\n",
-        "- First column is the Cloud Storage path to the image.\n",
-        "- Second column is the label.\n",
-        "- Any remaining columns are additional labels for multi-label image classification.\n",
-        "\n",
-        "For image object detection, the CSV index file has the requirements:\n",
-        "\n",
-        "- No heading.\n",
-        "- First column is the Cloud Storage path to the image.\n",
-        "- Second column is the label.\n",
-        "- Third/Fourth columns are the upper left corner of bounding box. Coordinates are normalized, between 0 and 1.\n",
-        "- Fifth/Sixth/Seventh columns are not used and should be 0.\n",
-        "- Eighth/Ninth columns are the lower right corner of the bounding box.\n",
-        "\n",
-        "##### ML_USE\n",
-        "\n",
-        "Each row may additionally specify which split to assign the data item to when the dataset is split for training; otherwise, the dataset will be randomly split: 80/10/10.\n",
-        "\n",
-        "The `ml_use` assignment is specified by prepending a column for specifying the assignment -- as the first column. The value may be one of: training, test, or validation."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "data_import_format:isg,u_dataset,jsonl"
-      },
-      "source": [
-        "#### JSONL\n",
-        "\n",
-        "For image classification, the JSONL index file has the requirements:\n",
-        "\n",
-        "- Each data item is a separate JSON object, on a separate line.\n",
-        "- The key/value pair `image_gcs_uri` is the Cloud Storage path to the image.\n",
-        "- The key/value pair `display_name` is the label for the image.\n",
-        "\n",
-        "    { 'image_gcs_uri': image, \n",
-        "      'classification_annotations': \n",
-        "          { 'display_name': label\n",
-        "          }\n",
-        "    }\n",
-        "    \n",
-        "For multi-label, the labels are specified as a list of `display_name` key/value pairs:\n",
-        "\n",
-        "    { 'image_gcs_uri': image, \n",
-        "      'classification_annotations': [\n",
-        "          { 'display_name': label1\n",
-        "          },\n",
-        "          { 'display_name': labelN\n",
-        "          },\n",
-        "       ]\n",
-        "    }\n",
-        "    \n",
-        "For object detection, the JSONL index file has the requirements:\n",
-        "\n",
-        "- Each data item is a separate JSON object, on a separate line.\n",
-        "- The key/value pair `image_gcs_uri` is the Cloud Storage path to the image.\n",
-        "- The key/value pair `bounding_box_annotations` is a list of:\n",
-        "    - `display_name`: The label of the object\n",
-        "    - `x_min`, `y_min`, `x_max`, `y_max`: The coordinates for the bounding box\n",
-        "\n",
-        "{\n",
-        "  \"image_gcs_uri\": image,\n",
-        "  \"bounding_box_annotations\": [\n",
-        "    {\n",
-        "      \"display name\": label,\n",
-        "      \"x_min\": \"X_MIN\",\n",
-        "      \"y_min\": \"Y_MIN\",\n",
-        "      \"x_max\": \"X_MAX\",\n",
-        "      \"y_max\": \"Y_MAX\"\n",
-        "      }\n",
-        "    },\n",
-        "    {\n",
-        "      \"displayName\": \"OBJECT2_LABEL\",\n",
-        "      \"x_min\": \"X_MIN\",\n",
-        "      \"y_min\": \"Y_MIN\",\n",
-        "      \"x_max\": \"X_MAX\",\n",
-        "      \"y_max\": \"Y_MAX\"\n",
-        "    }\n",
-        "  ]\n",
-        "}\n",
-        "\n",
-        "\n",
-        "For image segmentation, the JSONL index file has the requirements:\n",
-        "\n",
-        "- Each data item is a separate JSON object, on a separate line.\n",
-        "- The key/value pair `image_gcs_uri` is the Cloud Storage path to the image.\n",
-        "- The key/value pair `category_mask_uri`  is the Cloud Storage path to the mask image in PNG format.\n",
-        "- The key/value pair `'annotation_spec_colors'` is a list mapping mask colors to a label.\n",
-        "  - The key/value pair pair `display_name` is the label for the pixel color mask.\n",
-        "  - The key/value pair pair `color` are the RGB normalized pixel values (between 0 and 1) of the mask for the corresponding label.\n",
-        "\n",
-        "    { 'image_gcs_uri': image, \n",
-        "      'segmentation_annotations': { 'category_mask_uri': mask_image, 'annotation_spec_colors' : [ \n",
-        "          { 'display_name': label, 'color': {\"red\": value, \"blue\", value, \"green\": value} }, ...\n",
-        "      ] \n",
-        "    }\n",
-        "    \n",
-        "##### ML_USE\n",
-        "\n",
-        "Each JSONL object may additionally specify which split to assign the data item to when the dataset is split for training; otherwise, the dataset will be randomly split: 80/10/10.\n",
-        "\n",
-        "\"data_item_resource_labels\": {\n",
-        "      \"aiplatform.googleapis.com/ml_use\": \"training|test|validation\"\n",
-        "    }\n",
-        "\n",
-        "*Note*: The dictionary key fields may alternatively be in camelCase. For example, 'image_gcs_uri' can also be 'imageGcsUri'."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -1205,42 +1080,6 @@
         "Learn more about [AutoML Model Types](https://cloud.google.com/vertex-ai/docs/start/automl-model-types)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "data_preparation:tabular,u_dataset"
-      },
-      "source": [
-        "### Data preparation\n",
-        "\n",
-        "The Vertex AI `Dataset` resource for tabular has a couple of requirements for your tabular data.\n",
-        "\n",
-        "- Must be in a CSV file or a BigQuery table.\n",
-        "\n",
-        "Learn more about [Preparing tabular data](https://cloud.google.com/vertex-ai/docs/datasets/prepare-tabular)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "data_import_format:lbn,u_dataset,csv"
-      },
-      "source": [
-        "#### CSV\n",
-        "\n",
-        "For tabular models, the CSV file has a few requirements:\n",
-        "\n",
-        "- The first row must be the heading -- note how this is different from Image, Text and Video where the requirement is no heading.\n",
-        "- All but one column are features.\n",
-        "- One column is the label, which you will specify when you subsequently create the training pipeline.\n",
-        "\n",
-        "##### ML_USE\n",
-        "\n",
-        "Each row may additionally specify which split to assign the data item to when the dataset is split for training; otherwise, the dataset will be randomly split: 80/10/10.\n",
-        "\n",
-        "The `ml_use` assignment is specified by prepending a column for specifying the assignment -- as the first column. The value may be one of: training, test, or validation."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -1600,155 +1439,6 @@
         "Learn more about [AutoML Model Types](https://cloud.google.com/vertex-ai/docs/start/automl-model-types)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "data_preparation:text,u_dataset"
-      },
-      "source": [
-        "### Data preparation\n",
-        "\n",
-        "The Vertex AI `Dataset` resource for text has a couple of requirements for your text data.\n",
-        "\n",
-        "- Text examples must be stored in a CSV or JSONL file.\n",
-        "\n",
-        "Learn more about [Preparing text data](https://cloud.google.com/vertex-ai/docs/datasets/prepare-text)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "data_import_format:tcn,u_dataset,csv"
-      },
-      "source": [
-        "#### CSV\n",
-        "\n",
-        "For text classification, the CSV file has a few requirements:\n",
-        "\n",
-        "- No heading.\n",
-        "- First column is the text example or Cloud Storage path to text file (.txt suffix).\n",
-        "- Second column the label.\n",
-        "- Any remaining columns are additional labels for multi-label text classification.\n",
-        "\n",
-        "For text sentiment analysis, the CSV file has a few requirements:\n",
-        "\n",
-        "- No heading.\n",
-        "- First column is the text example or Cloud Storage path to text file (.txt suffix).\n",
-        "- Second column is the sentiment value.\n",
-        "- Third column is the maximum possible sentiment value.\n",
-        "\n",
-        "##### ML_USE\n",
-        "\n",
-        "Each row may additionally specify which split to assign the data item to when the dataset is split for training; otherwise, the dataset will be randomly split: 80/10/10.\n",
-        "\n",
-        "The `ml_use` assignment is specified by prepending a column for specifying the assignment -- as the first column. The value may be one of: training, test, or validation."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "766c838de8a0"
-      },
-      "source": [
-        "#### JSONL \n",
-        "\n",
-        "For text classification, the JSONL file has a few requirements:\n",
-        "\n",
-        "- Each data item is a separate JSON object, on a separate line.\n",
-        "- The key/value pair `text_gcs_uri` is the Cloud Storage path to the text file.\n",
-        "- The key/value pair `text_content` is the alternate way of specifying the text as inlined.\n",
-        "- The key/value pair `display_name` is the label for the text.\n",
-        "\n",
-        "{\n",
-        "  \"classification_annotation\": {\n",
-        "    \"display_name\": label\n",
-        "  },\n",
-        "  \"text_content\": text\n",
-        "}\n",
-        "{\n",
-        "  \"classification_annotation\": {\n",
-        "    \"display_name\": label\n",
-        "  },\n",
-        "  \"text_gcs_uri\": \"gcs_uri_to_file\"\n",
-        "}\n",
-        "\n",
-        "    \n",
-        "For multi-label, the labels are specified as a list of `display_name` key/value pairs:\n",
-        "\n",
-        "      'classification_annotations': [\n",
-        "          { 'display_name': label1\n",
-        "          },\n",
-        "          { 'display_name': labelN\n",
-        "          },\n",
-        "       ]\n",
-        "\n",
-        "For text sentiment analysis, the JSONL file has a few requirements:\n",
-        "\n",
-        "- Each data item is a separate JSON object, on a separate line.\n",
-        "- The key/value pair `text_gcs_uri` is the Cloud Storage path to the text file.\n",
-        "- The key/value pair `text_content` is the alternate way of specifying the text as inlined.\n",
-        "- The key/value pair `sentiment` is the sentiment value as an integer value greater than 0.\n",
-        "- The key/value pair `sentiment_max`is the maximum possible value for the sentiment.\n",
-        "\n",
-        "{\n",
-        "  \"sentiment_annotation\": {\n",
-        "    \"sentiment\": number,\n",
-        "    \"sentiment_max\": number\n",
-        "  },\n",
-        "  \"text_content\": text,\n",
-        "}\n",
-        "{\n",
-        "  \"sentiment_annotation\": {\n",
-        "    \"sentiment\": number,\n",
-        "    \"sentiment_max\": number\n",
-        "  },\n",
-        "  \"text_gcs_uri\": \"gcs_uri_to_file\"\n",
-        "}\n",
-        "\n",
-        "\n",
-        "For text entity extraction, the JSONL file has a few requirements:\n",
-        "\n",
-        "- Each data item is a separate JSON object, on a separate line.\n",
-        "- The key/value pair `text_gcs_uri` is the Cloud Storage path to the text file.\n",
-        "- The key/value pair `text_content` is the alternate way of specifying the text as inlined.\n",
-        "- The key/value pair `start_offset` is the character offset of the start of the text.\n",
-        "- The key/value pair `end_offset` is the character offset of the end of the text.\n",
-        "- The key/value pair `display_name` is the label for the text.\n",
-        "\n",
-        "{\n",
-        "    \"text_segment_annotations\": [\n",
-        "      {\n",
-        "        \"start_offset\":number,\n",
-        "        \"end_offset\":number,\n",
-        "        \"display_name\": label\n",
-        "      },\n",
-        "      ...\n",
-        "    ],\n",
-        "    \"textContent\": \"inline_text\"\n",
-        "}\n",
-        "{\n",
-        "    \"textSegmentAnnotations\": [\n",
-        "      {\n",
-        "        \"start_offset\": number,\n",
-        "        \"end_offset\": number,\n",
-        "        \"displayName\": label\n",
-        "      },\n",
-        "      ...\n",
-        "    ],\n",
-        "    \"text_gcs_uri\": \"gcs_uri_to_file\"\n",
-        "}\n",
-        "\n",
-        "##### ML_USE\n",
-        "\n",
-        "Each JSONL object may additionally specify which split to assign the data item to when the dataset is split for training; otherwise, the dataset will be randomly split: 80/10/10.\n",
-        "\n",
-        "\"data_item_resource_labels\": {\n",
-        "      \"aiplatform.googleapis.com/ml_use\": \"training|test|validation\"\n",
-        "    }\n",
-        "\n",
-        "*Note*: The dictionary key fields may alternatively be in camelCase. For example, 'text_gcs_uri' can also be 'textGcsUri'."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -2071,144 +1761,6 @@
         "Learn more about [AutoML Model Types](https://cloud.google.com/vertex-ai/docs/start/automl-model-types)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "data_preparation:text,u_dataset"
-      },
-      "source": [
-        "### Data preparation\n",
-        "\n",
-        "The Vertex AI `Dataset` resource for text has a couple of requirements for your text data.\n",
-        "\n",
-        "- Text examples must be stored in a CSV or JSONL file.\n",
-        "\n",
-        "Learn more about [Preparing video data](https://cloud.google.com/vertex-ai/docs/datasets/prepare-video)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "427212b48840"
-      },
-      "source": [
-        "#### CSV\n",
-        "\n",
-        "For video classification, the CSV file has a few requirements:\n",
-        "\n",
-        "- No heading.\n",
-        "- First column is the Cloud Storage path to video file.\n",
-        "- Second column the label.\n",
-        "- Third column is the start time (seconds) in the video to classify.\n",
-        "- Fourth column is the end time (seconds) in the video to classify.\n",
-        "\n",
-        "For multi-label classification, each label is a separate row entry.\n",
-        "\n",
-        "For video object tracking, the CSV file has a few requirements:\n",
-        "\n",
-        "- No heading.\n",
-        "- First column is the Cloud Storage path to video file.\n",
-        "- Second column the label.\n",
-        "- Third column is unused (blank).\n",
-        "- Fourth column is the start time (seconds) in the video to start tracking the object.\n",
-        "- The fifth through eighth columns are the vertices of the object to track.\n",
-        "    - x_min\n",
-        "    - y_min\n",
-        "    - x_max\n",
-        "    - y_max\n",
-        "    \n",
-        "For action recognition, the CSV file has a few requirements:\n",
-        "\n",
-        "- No heading.\n",
-        "- Each row can be one of the following four formats:\n",
-        "\n",
-        "VIDEO_URI, TIME_SEGMENT_START, TIME_SEGMENT_END, LABEL, ANNOTATION_FRAME_TIMESTAMP\n",
-        "\n",
-        "VIDEO_URI, , , LABEL, ANNOTATION_FRAME_TIMESTAMP\n",
-        "\n",
-        "VIDEO_URI, TIME_SEGMENT_START, TIME_SEGMENT_END, LABEL, ANNOTATION_SEGMENT_START, ANNOTATION_SEGMENT_END\n",
-        "\n",
-        "VIDEO_URI, , , LABEL, ANNOTATION_SEGMENT_START, ANNOTATION_SEGMENT_END\n",
-        "\n",
-        "\n",
-        "##### ML_USE\n",
-        "\n",
-        "Each row may additionally specify which split to assign the data item to when the dataset is split for training; otherwise, the dataset will be randomly split: 80/10/10.\n",
-        "\n",
-        "The `ml_use` assignment is specified by prepending a column for specifying the assignment -- as the first column. The value may be one of: training, or test."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "461301339727"
-      },
-      "source": [
-        "#### JSONL\n",
-        "\n",
-        "For video classification, the CSV file has a few requirements:\n",
-        "\n",
-        "- Each data item is a separate JSON object, on a separate line.\n",
-        "- The key/value pair `video_gcs_uri` is the Cloud Storage path to the text file.\n",
-        "- The key/value pair `display_name` is the label for the text.\n",
-        "- The key/value pair `start_time` is the start time (seconds) for classifying.\n",
-        "- The key/value pair `end_time` is the end time (seconds) for classifying.\n",
-        "\n",
-        "\n",
-        "    {\n",
-        "        \"video_gcs_uri\": video,\n",
-        "        \"time_segment_annotations\": [{\n",
-        "            \"display_name\": label,\n",
-        "            \"start_time\": \"start_time_of_segment\",\n",
-        "            \"end_time\": \"end_time_of_segment\"\n",
-        "        }]\n",
-        "    }\n",
-        "\n",
-        "For video object tracking, the CSV file has a few requirements:\n",
-        "\n",
-        "- Each data item is a separate JSON object, on a separate line.\n",
-        "- The key/value pair `video_gcs_uri` is the Cloud Storage path to the text file.\n",
-        "\n",
-        "    {\n",
-        "        \"video_gcs_uri\": video,\n",
-        "        \"temporal_bounding_box_annotations\": [{\n",
-        "            \"display_name\": label,\n",
-        "            \"x_min\": \"leftmost_coordinate_of_the_bounding box\",\n",
-        "            \"x_max\": \"rightmost_coordinate_of_the_bounding box\",\n",
-        "            \"y_min\": \"topmost_coordinate_of_the_bounding box\",\n",
-        "            \"y_max\": \"bottommost_coordinate_of_the_bounding box\",\n",
-        "            \"time_offset\": \"timeframe_object-detected\"\n",
-        "        }]\n",
-        "    }\n",
-        "\n",
-        "For video action recognition, the CSV file has a few requirements:\n",
-        "\n",
-        "- Each data item is a separate JSON object, on a separate line.\n",
-        "- The key/value pair `video_gcs_uri` is the Cloud Storage path to the text file.\n",
-        "\n",
-        "    {\n",
-        "      \"video_gcs_uri': video,\n",
-        "      \"time_segments\": [{\n",
-        "        \"start_time\": \"start_time_of_fully_annotated_segment\",\n",
-        "        \"end_time\": \"end_time_of_segment\"}],\n",
-        "      \"time_segment_annotations\": [{\n",
-        "        \"display_name\": label,\n",
-        "        \"start_time\": \"start_time_of_segment\",\n",
-        "        \"end_time\": \"end_time_of_segment\"\n",
-        "      }]\n",
-        "    }\n",
-        "\n",
-        "##### ML_USE\n",
-        "\n",
-        "Each JSONL object may additionally specify which split to assign the data item to when the dataset is split for training; otherwise, the dataset will be randomly split: 80/20.\n",
-        "\n",
-        "\"data_item_resource_labels\": {\n",
-        "      \"aiplatform.googleapis.com/ml_use\": \"training|test\"\n",
-        "    }\n",
-        "\n",
-        "*Note*: The dictionary key fields may alternatively be in camelCase. For example, 'video_gcs_uri' can also be 'videoGcsUri'."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {

notebooks/community/ml_ops/stage2/get_started_bqml_training.ipynb

@@ -40,11 +40,11 @@
         "  </td>\n",
         "    <td>\n",
         "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_bqml_training.ipynb\">\n",
-        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\\\" alt=\"Colab logo\"> Run in Colab\n",
         "        </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage2/get_started_bqml_training.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_bqml_training.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
@@ -65,33 +65,6 @@
         "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 2 : experimentation: get started with BigQuery ML Training."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "objective:mlops,stage2,get_started_bqml_training"
-      },
-      "source": [
-        "### Objective\n",
-        "\n",
-        "In this tutorial, you learn how to use `BigQueryML` for training with `Vertex AI`.\n",
-        "\n",
-        "This tutorial uses the following Google Cloud ML services:\n",
-        "\n",
-        "- `BigQueryML Training`\n",
-        "- `Vertex AI Model resource`\n",
-        "- `Vertex AI Vizier`\n",
-        "\n",
-        "The steps performed include:\n",
-        "\n",
-        "- Create a local BigQuery table in your project\n",
-        "- Train a BigQuery ML model\n",
-        "- Evaluate the BigQuery ML model\n",
-        "- Export the BigQuery ML model as a cloud model\n",
-        "- Upload the exported model as a `Vertex AI Model` resource\n",
-        "- Hyperparameter tune a BigQuery ML model with `Vertex AI Vizier`\n",
-        "- Automatically register a BigQuery ML model to `Vertex AI Model Registry`"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -106,9 +79,29 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "81c777b8ad32"
+        "id": "objective:mlops,stage2,get_started_bqml_training"
       },
       "source": [
+        "### Objective\n",
+        "\n",
+        "In this tutorial, you learn how to use `BigQueryML` (BQML) for training with `Vertex AI`.\n",
+        "\n",
+        "This tutorial uses the following Google Cloud ML services:\n",
+        "\n",
+        "- `BigQueryML Training`\n",
+        "- `Vertex AI Model resource`\n",
+        "- `Vertex AI Vizier`\n",
+        "\n",
+        "The steps performed include:\n",
+        "\n",
+        "- Create a local BigQuery table in your project\n",
+        "- Train a BQML model\n",
+        "- Evaluate the BQML model\n",
+        "- Export the BQML model as a cloud model\n",
+        "- Upload the exported model as a `Vertex AI Model` resource\n",
+        "- Hyperparameter tune a BQML model with `Vertex AI Vizier`\n",
+        "- Automatically register a BQML model to `Vertex AI Model Registry`\n",
+        "\n",
         "### Costs\n",
         "This tutorial uses billable components of Google Cloud:\n",
         "\n",
@@ -141,7 +134,7 @@
         "import os\n",
         "\n",
         "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
         "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
         "\n",
         "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
@@ -191,8 +184,6 @@
         "id": "project_id"
       },
       "source": [
-        "## Before you begin\n",
-        "\n",
         "### Set up your Google Cloud project\n",
         "\n",
         "**The following steps are required, regardless of your notebook environment.**\n",
@@ -208,15 +199,8 @@
         "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
         "Cloud SDK uses the right project for all the commands in this notebook.\n",
         "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "56d591439df1"
-      },
-      "source": [
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands.\n",
+        "\n",
         "#### Set your project ID\n",
         "\n",
         "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
@@ -458,54 +442,6 @@
         "! gsutil ls -al $BUCKET_URI"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "set_service_account"
-      },
-      "source": [
-        "#### Service Account\n",
-        "\n",
-        "You use a service account to create Vertex AI Pipeline jobs. If you do not want to use your project's Compute Engine service account, set `SERVICE_ACCOUNT` to another service account ID."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "set_service_account"
-      },
-      "outputs": [],
-      "source": [
-        "SERVICE_ACCOUNT = \"[your-service-account]\"  # @param {type:\"string\"}"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "autoset_service_account"
-      },
-      "outputs": [],
-      "source": [
-        "if (\n",
-        "    SERVICE_ACCOUNT == \"\"\n",
-        "    or SERVICE_ACCOUNT is None\n",
-        "    or SERVICE_ACCOUNT == \"[your-service-account]\"\n",
-        "):\n",
-        "    # Get your service account from gcloud\n",
-        "    if not IS_COLAB:\n",
-        "        shell_output = !gcloud auth list 2>/dev/null\n",
-        "        SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
-        "\n",
-        "    if IS_COLAB:\n",
-        "        shell_output = ! gcloud projects describe  $PROJECT_ID\n",
-        "        project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n",
-        "        SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n",
-        "\n",
-        "    print(\"Service Account:\", SERVICE_ACCOUNT)"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -749,9 +685,9 @@
         "id": "bqml_create_model"
       },
       "source": [
-        "### Train BigQuery ML model\n",
+        "### Train BQML model\n",
         "\n",
-        "Next, you create and train a BigQuery ML tabular classification model from the public dataset penguins and store the model in your project using the `CREATE MODEL` statement. The model configuration is specified in the `OPTIONS` statement as follows:\n",
+        "Next, you create and train a BQML tabular classification model from the public dataset penguins and store the model in your project using the `CREATE MODEL` statement. The model configuration is specified in the `OPTIONS` statement as follows:\n",
         "\n",
         "- `model_type`: The type and archictecture of tabular model to train, e.g., DNN classification.\n",
         "- `labels`: The column which are the labels.\n",
@@ -794,15 +730,41 @@
         "print(\"{} created in {}\".format(tblname, job.ended - job.started))"
       ]
     },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3b3aa4481cd7"
+      },
+      "outputs": [],
+      "source": [
+        "MODEL_QUERY = f\"\"\"\n",
+        "DROP MODEL `{BQ_DATASET_NAME}.{MODEL_NAME}`\n",
+        "\"\"\"\n",
+        "\n",
+        "job = bqclient.query(MODEL_QUERY)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ef9e14b91475"
+      },
+      "outputs": [],
+      "source": [
+        "job"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "bqml_eval_model"
       },
       "source": [
-        "### Evaluate the trained BigQuery ML model\n",
+        "### Evaluate the trained BQML model\n",
         "\n",
-        "Next, retrieve the model evaluation for the trained BigQuery ML model.\n",
+        "Next, retrieve the model evaluation for the trained BQML model.\n",
         "\n",
         "Learn more about [The ML.EVALUATE function](https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-evaluate)."
       ]
@@ -833,9 +795,9 @@
         "id": "bqml_export_model"
       },
       "source": [
-        "### Export the model from BigQuery ML\n",
+        "### Export the model from BQML\n",
         "\n",
-        "The model you trained in BigQuery ML is a TensorFlow model. Next, you export the TensorFlow model artifacts in TF.SavedModel format."
+        "The model you trained in BQML is a TensorFlow model. Next, you export the TensorFlow model artifacts in TF.SavedModel format."
       ]
     },
     {
@@ -1028,9 +990,9 @@
         "id": "bqml_create_model:vizier"
       },
       "source": [
-        "### Hyperparameter Tune and train a BigQuery ML model\n",
+        "### Hyperparameter Tune and train a BQML model\n",
         "\n",
-        "Next, you train a BigQuery ML tabular classification model with hyperparameter tuning using the `Vertex AI Vizier` service. The hyperparameter settings are specified in the `OPTIONS` statement as follows:\n",
+        "Next, you train a BQML tabular classification model with hyperparameter tuning using the `Vertex AI Vizier` service. The hyperparameter settings are specified in the `OPTIONS` statement as follows:\n",
         "\n",
         "- `HPARAM_TUNING_ALGORITHM`: The algorithm for selecting the next trial parameters.\n",
         "- `num_trials`: The number of trials.\n",
@@ -1083,9 +1045,9 @@
         "id": "bqml_eval_model"
       },
       "source": [
-        "### Evaluate the BigQuery ML trained model\n",
+        "### Evaluate the BQML trained model\n",
         "\n",
-        "Next, retrieve the model evaluation results for the trained BigQuery ML model.\n",
+        "Next, retrieve the model evaluation results for the trained BQML model.\n",
         "\n",
         "Learn more about [The ML.EVALUATE function](https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-evaluate)."
       ]
@@ -1142,9 +1104,9 @@
         "id": "bqml_create_model:xai"
       },
       "source": [
-        "### Train a BigQuery ML model with Explainability\n",
+        "### Train a BQML model with Explainability\n",
         "\n",
-        "Next, you train the same BigQuery ML model, but this time you enable Vertex AI Explainability on the model predictions by adding the option:\n",
+        "Next, you train the same BQML model, but this time you enable Vertex AI Explainability on the model predictions by adding the option:\n",
         "\n",
         "- `ENABLE_GLOBAL_EXPLAIN`"
       ]
@@ -1223,21 +1185,24 @@
         "\n",
         "### Setting permissions to automatically register the model\n",
         "\n",
-        "You need to set some additional IAM permissions for BigQuery ML to automatically upload and register the model after training. Depending on your service account, the setting of the permissions below may fail. In this case, we recommend executing the permissions in a Cloud Shell.\n",
-        "\n",
-        "Learn more about [Setting permissions for Model Registry](https://cloud.google.com/bigquery-ml/docs/managing-models-vertex)\n"
+        "You need to set some additional IAM permissions for BigQuery ML to automatically upload and register the model after training. Depending on your service account, the setting of the permissions below may fail. In this case, we recommend executing the permissions in a Cloud Shell."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "29229f72d13d"
+        "id": "0472e888105e"
       },
       "outputs": [],
       "source": [
         "! gcloud projects add-iam-policy-binding $PROJECT_ID \\\n",
-        "    --member=serviceAccount:$SERVICE_ACCOUNT --role=roles/aiplatform.admin --condition=None"
+        "      --member='serviceAccount:cloud-dataengine@system.gserviceaccount.com' \\\n",
+        "      --role='roles/aiplatform.admin'\n",
+        "\n",
+        "! gcloud projects add-iam-policy-binding $PROJECT_ID \\\n",
+        "      --member='user:cloud-dataengine@prod.google.com' \\\n",
+        "      --role='roles/aiplatform.admin'"
       ]
     },
     {
@@ -1318,20 +1283,6 @@
         "print(model.gca_resource)"
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "48e6ef5d5ffa"
-      },
-      "outputs": [],
-      "source": [
-        "models = aiplatform.Model.list()\n",
-        "for model in models:\n",
-        "    if model.gca_resource.display_name.startswith(\"bqml\"):\n",
-        "        print(model.gca_resource.display_name)"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {

notebooks/community/ml_ops/stage2/get_started_vertex_distributed_training.ipynb

@@ -29,7 +29,7 @@
         "id": "title:generic,gcp"
       },
       "source": [
-        "# E2E ML on GCP: MLOps stage 2 : experimentation: get started with Vertex AI Distributed Training\n",
+        "# E2E ML on GCP: MLOps stage 2 : experimentation: get started with Vertex Distributed Training\n",
         "\n",
         "<table align=\"left\">\n",
         "      <td>\n",
@@ -65,6 +65,17 @@
         "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 2 : experimentation: get started with Vertex Distributed Training. Please note: There are incompatibilities between Colab and Docker and the Docker section may not work until resolved by the platform."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dataset:custom,boston,lrg"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the [Boston Housing Prices dataset](https://www.cs.toronto.edu/~delve/data/boston/bostonDetail.html). The version of the dataset you will use in this tutorial is built into TensorFlow. The trained model predicts the median price of a house in units of 1K USD."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -95,6 +106,15 @@
         "id": "recommendation:mlops,stage2,vertex,distributed_training"
       },
       "source": [
+        "### Costs\n",
+        " \n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "Vertex AI\n",
+        "Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage pricing](https://cloud.google.com/storage/pricing), and use the [Pricing Calculator](https://cloud.google.com/products/calculator/),\n",
+        "        to generate a cost estimate based on your projected usage.\n",
         "### Recommendations\n",
         "\n",
         "When doing E2E MLOps on Google Cloud, the following are best practices for when to use Vertex AI Distributed Training:\n",
@@ -118,41 +138,13 @@
         "While training across a large number of VMs and the model parameters updates to sync is very large."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "dataset:custom,boston,lrg"
-      },
-      "source": [
-        "### Dataset\n",
-        "\n",
-        "The dataset used for this tutorial is the [Boston Housing Prices dataset](https://www.cs.toronto.edu/~delve/data/boston/bostonDetail.html). The version of the dataset you use in this tutorial is built into TensorFlow. The trained model predicts the median price of a house in units of 1K USD."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "d10166df7141"
-      },
-      "source": [
-        "### Costs\n",
-        " \n",
-        "This tutorial uses billable components of Google Cloud:\n",
-        "\n",
-        "Vertex AI\n",
-        "Cloud Storage\n",
-        "\n",
-        "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage pricing](https://cloud.google.com/storage/pricing), and use the [Pricing Calculator](https://cloud.google.com/products/calculator/),\n",
-        "        to generate a cost estimate based on your projected usage."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "XkYpRvOQyVYb"
       },
       "source": [
-        "## Installation\n",
+        "### Install additional packages\n",
         "\n",
         "Install the packages required for executing this notebook."
       ]
@@ -168,7 +160,7 @@
         "import os\n",
         "\n",
         "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
         "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
         "    \"/opt/deeplearning/metadata/env_version\"\n",
         ")\n",
@@ -178,7 +170,7 @@
         "if IS_WORKBENCH_NOTEBOOK:\n",
         "    USER_FLAG = \"--user\"\n",
         "\n",
-        "! pip3 install {USER_FLAG} --upgrade google-cloud-aiplatform -q"
+        "! pip3 install {USER_FLAG} --upgrade google-cloud-aiplatform"
       ]
     },
     {
@@ -256,6 +248,8 @@
       },
       "outputs": [],
       "source": [
+        "import os\n",
+        "\n",
         "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
       ]
     },

notebooks/community/ml_ops/stage2/get_started_vertex_feature_store.ipynb

@@ -29,7 +29,7 @@
         "id": "title:generic,gcp"
       },
       "source": [
-        "# E2E ML on GCP: MLOps stage 2 : experimentation: get started with Vertex AI Feature Store\n",
+        "# E2E ML on GCP: MLOps stage 2 : experimentation: get started with Feature Store\n",
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
@@ -41,7 +41,7 @@
         "    \n",
         "  <td>\n",
         "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_feature_store.ipynb\">\n",
-        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\\\" alt=\"Colab logo\"> Run in Colab\n",
         "        </a>\n",
         "  </td>\n",
         "    \n",
@@ -68,6 +68,19 @@
         "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 2 : experimentation: get started with Feature Store."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dataset:movies,lbn,avro"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the `Movie Recommendations` dataset. The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket, in Avro format.\n",
+        "\n",
+        "This dataset is used to predict whether a person will watch a movie or not."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -94,19 +107,6 @@
         "- Perform batch serving from a `Featurestore` resource."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "dataset:movies,lbn,avro"
-      },
-      "source": [
-        "### Dataset\n",
-        "\n",
-        "The dataset used for this tutorial is the `Movie Recommendations` dataset. The version of the dataset you use in this tutorial is stored in a public Cloud Storage bucket, in Avro format.\n",
-        "\n",
-        "This dataset is used to predict whether a person watches a movie or not."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -145,7 +145,7 @@
         "import os\n",
         "\n",
         "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
         "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
         "    \"/opt/deeplearning/metadata/env_version\"\n",
         ")\n",
@@ -194,8 +194,6 @@
         "id": "project_id"
       },
       "source": [
-        "## Before you begin\n",
-        "\n",
         "### Set up your Google Cloud project\n",
         "\n",
         "**The following steps are required, regardless of your notebook environment.**\n",
@@ -364,11 +362,12 @@
         "import sys\n",
         "\n",
         "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = \"google.colab\" in sys.modules\n",
+        "IS_COLAB = False\n",
         "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
         "    \"DL_ANACONDA_HOME\"\n",
         "):\n",
         "    if \"google.colab\" in sys.modules:\n",
+        "        IS_COLAB = True\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
         "        google_auth.authenticate_user()\n",
@@ -491,7 +490,7 @@
       "outputs": [],
       "source": [
         "# Represents featurestore resource path.\n",
-        "FEATURESTORE_NAME = \"movies_\" + TIMESTAMP\n",
+        "FEATURESTORE_NAME = \"movies\"\n",
         "\n",
         "featurestore = aiplatform.Featurestore.create(\n",
         "    featurestore_id=FEATURESTORE_NAME,\n",