friday updates

Co-authored-by: Karl Weinmeister <11586922+kweinmeister@users.noreply.github.com>

.cloud-build/ExecuteChangedNotebooks.py

@@ -14,18 +14,22 @@
 # limitations under the License.
 
 import argparse
+import concurrent
 import dataclasses
 import datetime
 import functools
-import pathlib
 import os
+import pathlib
+import nbformat
+import re
 import subprocess
-from pathlib import Path
 from typing import List, Optional
-import concurrent
 from tabulate import tabulate
+import operator
 
-import ExecuteNotebook
+import execute_notebook_remote
+from utils import util, NotebookProcessors
+from google.cloud.devtools.cloudbuild_v1.types import BuildOperationMetadata
 
 
 def str2bool(v):
@@ -62,49 +66,135 @@ def format_timedelta(delta: datetime.timedelta) -> str:
 
 @dataclasses.dataclass
 class NotebookExecutionResult:
-    notebook: str
+    name: str
     duration: datetime.timedelta
     is_pass: bool
+    log_url: str
+    output_uri: str
+    build_id: str
     error_message: Optional[str]
 
 
+def _process_notebook(
+    notebook_path: str,
+    variable_project_id: str,
+    variable_region: str,
+):
+    # Read notebook
+    with open(notebook_path) as f:
+        nb = nbformat.read(f, as_version=4)
+
+    # Create preprocessors
+    remove_no_execute_cells_preprocessor = NotebookProcessors.RemoveNoExecuteCells()
+    update_variables_preprocessor = NotebookProcessors.UpdateVariablesPreprocessor(
+        replacement_map={
+            "PROJECT_ID": variable_project_id,
+            "REGION": variable_region,
+        },
+    )
+
+    # Use no-execute preprocessor
+    (
+        nb,
+        resources,
+    ) = remove_no_execute_cells_preprocessor.preprocess(nb)
+
+    (nb, resources) = update_variables_preprocessor.preprocess(nb, resources)
+
+    with open(notebook_path, mode="w", encoding="utf-8") as new_file:
+        nbformat.write(nb, new_file)
+
+
+def _create_tag(filepath: str) -> str:
+    tag = os.path.basename(os.path.normpath(filepath))
+    tag = re.sub("[^0-9a-zA-Z_.-]+", "-", tag)
+
+    if tag.startswith(".") or tag.startswith("-"):
+        tag = tag[1:]
+
+    return tag
+
+
 def execute_notebook(
-    artifacts_path: str,
+    container_uri: str,
+    staging_bucket: str,
+    artifacts_bucket: str,
     variable_project_id: str,
     variable_region: str,
-    should_log_output: bool,
-    should_use_new_kernel: bool,
     notebook: str,
+    should_get_tail_logs: bool = False,
 ) -> NotebookExecutionResult:
     print(f"Running notebook: {notebook}")
 
+    # Create paths
+    notebook_output_uri = "/".join([artifacts_bucket, pathlib.Path(notebook).name])
+
+    # Create tag from notebook
+    tag = _create_tag(filepath=notebook)
+
     result = NotebookExecutionResult(
-        notebook=notebook,
+        name=tag,
         duration=datetime.timedelta(seconds=0),
         is_pass=False,
+        output_uri=notebook_output_uri,
+        log_url="",
+        build_id="",
         error_message=None,
     )
 
     # TODO: Handle cases where multiple notebooks have the same name
     time_start = datetime.datetime.now()
+    operation = None
     try:
-        ExecuteNotebook.execute_notebook(
-            notebook_file_path=notebook,
-            output_file_folder=artifacts_path,
-            replacement_map={
-                "PROJECT_ID": variable_project_id,
-                "REGION": variable_region,
-            },
-            should_log_output=should_log_output,
-            should_use_new_kernel=should_use_new_kernel,
+        # Pre-process notebook by substituting variable names
+        _process_notebook(
+            notebook_path=notebook,
+            variable_project_id=variable_project_id,
+            variable_region=variable_region,
         )
+
+        # Upload the pre-processed code to a GCS bucket
+        code_archive_uri = util.archive_code_and_upload(staging_bucket=staging_bucket)
+
+        operation = execute_notebook_remote.execute_notebook_remote(
+            code_archive_uri=code_archive_uri,
+            notebook_uri=notebook,
+            notebook_output_uri=notebook_output_uri,
+            container_uri=container_uri,
+            tag=tag,
+        )
+
+        operation_metadata = BuildOperationMetadata(mapping=operation.metadata)
+        result.build_id = operation_metadata.build.id
+        result.log_url = operation_metadata.build.log_url
+
+        # Block and wait for the result
+        operation_result = operation.result()
+
         result.duration = datetime.datetime.now() - time_start
         result.is_pass = True
         print(f"{notebook} PASSED in {format_timedelta(result.duration)}.")
     except Exception as error:
+        result.error_message = str(error)
+
+        if operation and should_get_tail_logs:
+            # Extract the logs
+            logs_bucket = operation_metadata.build.logs_bucket
+
+            # Download tail end of logs file
+            log_file_uri = f"{logs_bucket}/log-{result.build_id}.txt"
+
+            # Use gcloud to get tail
+            try:
+                result.error_message = subprocess.check_output(
+                    ["gsutil", "cat", "-r", "-1000", log_file_uri], encoding="UTF-8"
+                )
+            except Exception as error:
+                result.error_message = str(error)
+
         result.duration = datetime.datetime.now() - time_start
         result.is_pass = False
-        result.error_message = str(error)
+
         print(
             f"{notebook} FAILED in {format_timedelta(result.duration)}: {result.error_message}"
         )
@@ -114,18 +204,19 @@ def execute_notebook(
 
 def run_changed_notebooks(
     test_paths_file: str,
-    base_branch: Optional[str],
-    output_folder: str,
+    container_uri: str,
+    staging_bucket: str,
+    artifacts_bucket: str,
     variable_project_id: str,
     variable_region: str,
     should_parallelize: bool,
-    should_use_separate_kernels: bool,
+    base_branch: Optional[str] = None,
 ):
     """
     Run the notebooks that exist under the folders defined in the test_paths_file.
     It only runs notebooks that have differences from the Git base_branch.
 
-    The executed notebooks are saved in the output_folder.
+    The executed notebooks are saved in the artifacts_bucket.
 
     Variables are also injected into the notebooks such as the variable_project_id and variable_region.
 
@@ -136,19 +227,16 @@ def run_changed_notebooks(
         base_branch (str):
             Optional. If provided, only the files that have changed from the base_branch will be checked.
             If not provided, all files will be checked.
-        output_folder (str):
-            Required. The folder to write executed notebooks to.
+        staging_bucket (str):
+            Required. The GCS staging bucket to write source code to.
+        artifacts_bucket (str):
+            Required. The GCS staging bucket to write executed notebooks to.
         variable_project_id (str):
             Required. The value for PROJECT_ID to inject into notebooks.
         variable_region (str):
             Required. The value for REGION to inject into notebooks.
         should_parallelize (bool):
             Required. Should run notebooks in parallel using a thread pool as opposed to in sequence.
-        should_use_separate_kernels (bool):
-            Note: Dependencies don't install correctly when this is set to True
-            See https://github.com/nteract/papermill/issues/625
-
-            Required. Should run each notebook in a separate and independent virtual environment.
     """
 
     test_paths = []
@@ -176,13 +264,7 @@ def run_changed_notebooks(
     notebooks = notebooks.decode("utf-8").split("\n")
     notebooks = [notebook for notebook in notebooks if notebook.endswith(".ipynb")]
     notebooks = [notebook for notebook in notebooks if len(notebook) > 0]
-    notebooks = [notebook for notebook in notebooks if Path(notebook).exists()]
-
-    # Create paths
-    artifacts_path = Path(output_folder)
-    artifacts_path.mkdir(parents=True, exist_ok=True)
-    artifacts_path.joinpath("success").mkdir(parents=True, exist_ok=True)
-    artifacts_path.joinpath("failure").mkdir(parents=True, exist_ok=True)
+    notebooks = [notebook for notebook in notebooks if pathlib.Path(notebook).exists()]
 
     notebook_execution_results: List[NotebookExecutionResult] = []
 
@@ -198,11 +280,11 @@ def run_changed_notebooks(
                     executor.map(
                         functools.partial(
                             execute_notebook,
-                            artifacts_path,
+                            container_uri,
+                            staging_bucket,
+                            artifacts_bucket,
                             variable_project_id,
                             variable_region,
-                            False,
-                            should_use_separate_kernels,
                         ),
                         notebooks,
                     )
@@ -210,12 +292,12 @@ def run_changed_notebooks(
         else:
             notebook_execution_results = [
                 execute_notebook(
-                    artifacts_path=artifacts_path,
+                    container_uri=container_uri,
+                    staging_bucket=staging_bucket,
+                    artifacts_bucket=artifacts_bucket,
                     variable_project_id=variable_project_id,
                     variable_region=variable_region,
                     notebook=notebook,
-                    should_log_output=True,
-                    should_use_new_kernel=should_use_separate_kernels,
                 )
                 for notebook in notebooks
             ]
@@ -224,29 +306,42 @@ def run_changed_notebooks(
 
     print("\n=== RESULTS ===\n")
 
-    notebooks_sorted = sorted(
+    results_sorted = sorted(
         notebook_execution_results,
         key=lambda result: result.is_pass,
         reverse=True,
     )
+
     # Print results
     print(
         tabulate(
             [
                 [
-                    os.path.basename(os.path.normpath(result.notebook)),
+                    result.name,
                     "PASSED" if result.is_pass else "FAILED",
                     format_timedelta(result.duration),
-                    result.error_message or "--",
+                    result.log_url,
                 ]
-                for result in notebooks_sorted
+                for result in results_sorted
             ],
-            headers=["file", "status", "duration", "error"],
+            headers=["build_tag", "status", "duration", "log_url"],
         )
     )
 
     print("\n=== END RESULTS===\n")
 
+    total_notebook_duration = functools.reduce(
+        operator.add,
+        [datetime.timedelta(seconds=0)]
+        + [result.duration for result in results_sorted],
+    )
+
+    print(f"Cumulative notebook duration: {format_timedelta(total_notebook_duration)}")
+
+    # Raise error if any notebooks failed
+    if not all([result.is_pass for result in results_sorted]):
+        raise RuntimeError("Notebook failures detected. See logs for details")
+
 
 parser = argparse.ArgumentParser(description="Run changed notebooks.")
 parser.add_argument(
@@ -261,9 +356,9 @@ parser.add_argument(
     required=False,
 )
 parser.add_argument(
-    "--output_folder",
-    type=pathlib.Path,
-    help="The path to the folder to store executed notebooks.",
+    "--container_uri",
+    type=str,
+    help="The container uri to run each notebook in.",
     required=True,
 )
 parser.add_argument(
@@ -278,35 +373,35 @@ parser.add_argument(
     help="The GCP region. This is used to inject a variable value into the notebook before running.",
     required=True,
 )
-
-# Note: Dependencies don't install correctly when this is set to True
+parser.add_argument(
+    "--staging_bucket",
+    type=str,
+    help="The GCP directory for staging temporary files.",
+    required=True,
+)
+parser.add_argument(
+    "--artifacts_bucket",
+    type=str,
+    help="The GCP directory for storing executed notebooks.",
+    required=True,
+)
 parser.add_argument(
     "--should_parallelize",
     type=str2bool,
     nargs="?",
     const=True,
-    default=False,
+    default=True,
     help="Should run notebooks in parallel.",
 )
 
-# Note: This isn't guaranteed to work correctly due to existing Papermill issue
-# See https://github.com/nteract/papermill/issues/625
-parser.add_argument(
-    "--should_use_separate_kernels",
-    type=str2bool,
-    nargs="?",
-    const=True,
-    default=False,
-    help="(Experimental) Should run each notebook in a separate and independent virtual environment.",
-)
-
 args = parser.parse_args()
 run_changed_notebooks(
     test_paths_file=args.test_paths_file,
-    base_branch=args.base_branch,
-    output_folder=args.output_folder,
+    container_uri=args.container_uri,
+    staging_bucket=args.staging_bucket,
+    artifacts_bucket=args.artifacts_bucket,
     variable_project_id=args.variable_project_id,
     variable_region=args.variable_region,
     should_parallelize=args.should_parallelize,
-    should_use_separate_kernels=args.should_use_separate_kernels,
+    base_branch=args.base_branch,
 )

.cloud-build/ExecuteNotebook.py

@@ -13,163 +13,76 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import json
 import sys
-import nbformat
 import os
 import errno
-from NotebookProcessors import RemoveNoExecuteCells, UpdateVariablesPreprocessor
-from typing import Dict, Tuple
 import papermill as pm
 import shutil
-import virtualenv
-import uuid
-from jupyter_client.kernelspecapp import KernelSpecManager
+
+from utils import util
+from google.cloud.aiplatform import utils
 
 # This script is used to execute a notebook and write out the output notebook.
-# The replaces calling the nbconvert via command-line, which doesn't write the output notebook correctly when there are errors during execution.
-
-STAGING_FOLDER = "staging"
-ENVIRONMENTS_PATH = "environments"
-KERNELS_SPECS_PATH = "kernel_specs"
-
-
-def create_and_install_kernel() -> Tuple[str, str]:
-    # Create environment
-    kernel_name = str(uuid.uuid4())
-    env_name = f"{ENVIRONMENTS_PATH}/{kernel_name}"
-    # venv.create(env_name, system_site_packages=True, with_pip=True)
-    virtualenv.cli_run([env_name, "--system-site-packages"])
-
-    # Create kernel spec
-    kernel_spec = {
-        "argv": [
-            f"{env_name}/bin/python",
-            "-m",
-            "ipykernel_launcher",
-            "-f",
-            "{connection_file}",
-        ],
-        "display_name": "Python 3",
-        "language": "python",
-    }
-    kernel_spec_folder = os.path.join(KERNELS_SPECS_PATH, kernel_name)
-    kernel_spec_file = os.path.join(kernel_spec_folder, "kernel.json")
-
-    # Create kernel spec folder
-    if not os.path.exists(os.path.dirname(kernel_spec_file)):
-        try:
-            os.makedirs(os.path.dirname(kernel_spec_file))
-        except OSError as exc:  # Guard against race condition
-            if exc.errno != errno.EEXIST:
-                raise
-
-    with open(kernel_spec_file, mode="w", encoding="utf-8") as f:
-        json.dump(kernel_spec, f)
-
-    # Install kernel
-    kernel_spec_manager = KernelSpecManager()
-    kernel_spec_manager.install_kernel_spec(
-        source_dir=kernel_spec_folder, kernel_name=kernel_name
-    )
-
-    return kernel_name, env_name
 
 
 def execute_notebook(
-    notebook_file_path: str,
-    output_file_folder: str,
-    replacement_map: Dict[str, str],
+    notebook_source: str,
+    output_file_or_uri: str,
     should_log_output: bool,
-    should_use_new_kernel: bool,
 ):
-    # Create staging directory if it doesn't exist
-    staging_file_path = f"{STAGING_FOLDER}/{notebook_file_path}"
-    if not os.path.exists(os.path.dirname(staging_file_path)):
-        try:
-            os.makedirs(os.path.dirname(staging_file_path))
-        except OSError as exc:  # Guard against race condition
-            if exc.errno != errno.EEXIST:
-                raise
+    file_name = os.path.basename(os.path.normpath(notebook_source))
 
-    file_name = os.path.basename(os.path.normpath(notebook_file_path))
+    # Download notebook if it's a GCS URI
+    if notebook_source.startswith("gs://"):
+        # Extract uri components
+        bucket_name, prefix = utils.extract_bucket_and_prefix_from_gcs_path(
+            notebook_source
+        )
 
-    # Create environments folder
-    if not os.path.exists(ENVIRONMENTS_PATH):
-        try:
-            os.makedirs(ENVIRONMENTS_PATH)
-        except OSError as exc:  # Guard against race condition
-            if exc.errno != errno.EEXIST:
-                raise
+        # Download remote notebook to local file system
+        notebook_source = file_name
+        util.download_file(
+            bucket_name=bucket_name, blob_name=prefix, destination_file=notebook_source
+        )
 
-    # Create and install kernel
-    kernel_name = next(
-        iter(KernelSpecManager().find_kernel_specs().keys()), None
-    )  # Find first existing kernel and use as default
-    env_name = None
-    if should_use_new_kernel:
-        kernel_name, env_name = create_and_install_kernel()
-
-    # Read notebook
-    with open(notebook_file_path) as f:
-        nb = nbformat.read(f, as_version=4)
-
-    has_error = False
+    execution_exception = None
 
     # Execute notebook
     try:
-        # Create preprocessors
-        remove_no_execute_cells_preprocessor = RemoveNoExecuteCells()
-        update_variables_preprocessor = UpdateVariablesPreprocessor(
-            replacement_map=replacement_map
-        )
-
-        # Use no-execute preprocessor
-        (
-            nb,
-            resources,
-        ) = remove_no_execute_cells_preprocessor.preprocess(nb)
-
-        (nb, resources) = update_variables_preprocessor.preprocess(nb, resources)
-
-        # print(f"Staging modified notebook to: {staging_file_path}")
-        with open(staging_file_path, mode="w", encoding="utf-8") as f:
-            nbformat.write(nb, f)
-
         # Execute notebook
         pm.execute_notebook(
-            input_path=staging_file_path,
-            output_path=staging_file_path,
-            kernel_name=kernel_name,
+            input_path=notebook_source,
+            output_path=notebook_source,
             progress_bar=should_log_output,
             request_save_on_cell_execute=should_log_output,
             log_output=should_log_output,
             stdout_file=sys.stdout if should_log_output else None,
             stderr_file=sys.stderr if should_log_output else None,
         )
-    except Exception:
-        # print(f"Error executing the notebook: {notebook_file_path}.\n\n")
-        has_error = True
-
-        raise
-
+    except Exception as exception:
+        execution_exception = exception
     finally:
-        # Clear env
-        if env_name is not None:
-            shutil.rmtree(path=env_name)
+        # Copy executed notebook
+        if output_file_or_uri.startswith("gs://"):
+            # Upload to GCS path
+            util.upload_file(notebook_source, remote_file_path=output_file_or_uri)
 
-        # Copy execute notebook
-        output_file_path = os.path.join(
-            output_file_folder, "failure" if has_error else "success", file_name
-        )
+            print("\n=== EXECUTION FINISHED ===\n")
+            print(
+                f"Please debug the executed notebook by downloading: {output_file_or_uri}"
+            )
+            print("\n======\n")
+        else:
+            # Create directories if they don't exist
+            if not os.path.exists(os.path.dirname(output_file_or_uri)):
+                try:
+                    os.makedirs(os.path.dirname(output_file_or_uri))
+                except OSError as exc:  # Guard against race condition
+                    if exc.errno != errno.EEXIST:
+                        raise
 
-        # Create directories if they don't exist
-        if not os.path.exists(os.path.dirname(output_file_path)):
-            try:
-                os.makedirs(os.path.dirname(output_file_path))
-            except OSError as exc:  # Guard against race condition
-                if exc.errno != errno.EEXIST:
-                    raise
+            print(f"Writing output to: {output_file_or_uri}")
+            shutil.move(notebook_source, output_file_or_uri)
 
-        # print(f"Writing output to: {output_file_path}")
-        shutil.move(staging_file_path, output_file_path)
+        if execution_exception:
+            raise execution_exception

.cloud-build/execute_notebook_cli.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import ExecuteNotebook
+
+parser = argparse.ArgumentParser(description="Run changed notebooks.")
+parser.add_argument(
+    "--notebook_source",
+    type=str,
+    help="Local filepath or GCS URI to notebook.",
+    required=True,
+)
+parser.add_argument(
+    "--output_file_or_uri",
+    type=str,
+    help="Local file or GCS URI to save executed notebook to.",
+    required=True,
+)
+
+args = parser.parse_args()
+ExecuteNotebook.execute_notebook(
+    notebook_source=args.notebook_source,
+    output_file_or_uri=args.output_file_or_uri,
+    should_log_output=True,
+)

.cloud-build/execute_notebook_remote.py

@@ -0,0 +1,72 @@
+from google.protobuf import duration_pb2
+from yaml.loader import FullLoader
+
+import google.auth
+from google.cloud.devtools import cloudbuild_v1
+from google.cloud.devtools.cloudbuild_v1.types import Source, StorageSource
+
+from typing import Optional
+import yaml
+
+from google.cloud.aiplatform import utils
+from google.api_core import operation
+
+CLOUD_BUILD_FILEPATH = ".cloud-build/notebook-execution-test-cloudbuild-single.yaml"
+TIMEOUT_IN_SECONDS = 86400
+
+
+def execute_notebook_remote(
+    code_archive_uri: str,
+    notebook_uri: str,
+    notebook_output_uri: str,
+    container_uri: str,
+    tag: Optional[str],
+) -> operation.Operation:
+    """Create and execute a simple Google Cloud Build configuration,
+    print the in-progress status and print the completed status."""
+
+    # Authorize the client with Google defaults
+    credentials, project_id = google.auth.default()
+    client = cloudbuild_v1.services.cloud_build.CloudBuildClient()
+
+    build = cloudbuild_v1.Build()
+
+    # The following build steps will output "hello world"
+    # For more information on build configuration, see
+    # https://cloud.google.com/build/docs/configuring-builds/create-basic-configuration
+    cloudbuild_config = yaml.load(open(CLOUD_BUILD_FILEPATH), Loader=FullLoader)
+
+    substitutions = {
+        "_PYTHON_IMAGE": container_uri,
+        "_NOTEBOOK_GCS_URI": notebook_uri,
+        "_NOTEBOOK_OUTPUT_GCS_URI": notebook_output_uri,
+    }
+
+    (
+        source_archived_file_gcs_bucket,
+        source_archived_file_gcs_object,
+    ) = utils.extract_bucket_and_prefix_from_gcs_path(code_archive_uri)
+
+    build.source = Source(
+        storage_source=StorageSource(
+            bucket=source_archived_file_gcs_bucket,
+            object_=source_archived_file_gcs_object,
+        )
+    )
+
+    build.steps = cloudbuild_config["steps"]
+    build.substitutions = substitutions
+    build.timeout = duration_pb2.Duration(seconds=TIMEOUT_IN_SECONDS)
+    build.queue_ttl = duration_pb2.Duration(seconds=TIMEOUT_IN_SECONDS)
+
+    if tag:
+        build.tags = [tag]
+
+    operation = client.create_build(project_id=project_id, build=build)
+    # Print the in-progress operation
+    # print("IN PROGRESS:")
+    # print(operation.metadata)
+
+    # Print the completed status
+    # print("RESULT:", result.status)
+    return operation

.cloud-build/notebook-execution-test-cloudbuild-single.yaml

@@ -0,0 +1,28 @@
+steps:
+  # Show the gcloud info and check if gcloud exists
+  - name: ${_PYTHON_IMAGE}
+    entrypoint: /bin/sh
+    args:
+    - -c
+    - 'gcloud config list'
+  # Check the Python version
+  - name: ${_PYTHON_IMAGE}
+    entrypoint: /bin/sh
+    args:
+    - -c
+    - 'python3 .cloud-build/CheckPythonVersion.py'
+  # Install Python dependencies
+  - name: ${_PYTHON_IMAGE}
+    entrypoint: /bin/sh
+    args:
+    - -c
+    - 'python3 -m pip install -U pip && python3 -m pip install -U --user -r .cloud-build/requirements.txt'
+  # Install Python dependencies and run testing script
+  - name: ${_PYTHON_IMAGE}
+    entrypoint: /bin/sh
+    args:
+    - -c
+    - 'python3 -m pip install -U pip && python3 -m pip freeze && python3 .cloud-build/execute_notebook_cli.py --notebook_source "${_NOTEBOOK_GCS_URI}" --output_file_or_uri "${_NOTEBOOK_OUTPUT_GCS_URI}"'
+    env:
+    - 'IS_TESTING=1'
+timeout: 86400s

.cloud-build/notebook-execution-test-cloudbuild.yaml

@@ -5,6 +5,10 @@ steps:
     args:
     - -c
     - 'gcloud config list'
+  # # Clone the Git repo
+  # - name: ${_PYTHON_IMAGE}
+  #   entrypoint: git
+  #   args: ['clone', "${_GIT_REPO}", "--branch", "${_GIT_BRANCH_NAME}", "."]    
   # Check the Python version
   - name: ${_PYTHON_IMAGE}
     entrypoint: /bin/sh
@@ -19,26 +23,16 @@ steps:
     - 'if [ -n "${_BASE_BRANCH}" ]; then git fetch origin "${_BASE_BRANCH}":refs/remotes/origin/"${_BASE_BRANCH}"; else echo "Skipping fetch."; fi'
   # Install Python dependencies
   - name: ${_PYTHON_IMAGE}
-    entrypoint: pip
-    args: ['install', '--upgrade',  '--user', '--requirement', '.cloud-build/requirements.txt']
+    entrypoint: /bin/sh
+    args:
+    - -c
+    - 'python3 -m pip install -U pip && python3 -m pip install -U --user -r .cloud-build/requirements.txt'
   # Install Python dependencies and run testing script
   - name: ${_PYTHON_IMAGE}
     entrypoint: /bin/sh
     args:
     - -c
-    - 'python3 -m pip freeze && python3 .cloud-build/ExecuteChangedNotebooks.py --test_paths_file "${_TEST_PATHS_FILE}" --base_branch "${_FORCED_BASE_BRANCH}" --output_folder ${BUILD_ID} --variable_project_id ${PROJECT_ID} --variable_region ${_GCP_REGION}'
+    - 'python3 -m pip install -U pip && python3 -m pip freeze && python3 .cloud-build/ExecuteChangedNotebooks.py --test_paths_file "${_TEST_PATHS_FILE}" --base_branch "${_FORCED_BASE_BRANCH}" --container_uri ${_PYTHON_IMAGE} --staging_bucket ${_GCS_STAGING_BUCKET}  --artifacts_bucket ${_GCS_STAGING_BUCKET}/executed_notebooks/PR_${_PR_NUMBER}/BUILD_${BUILD_ID} --variable_project_id ${PROJECT_ID} --variable_region ${_GCP_REGION}'
     env:
     - 'IS_TESTING=1'
-  # Manually copy artifacts to GCS
-  - name: gcr.io/cloud-builders/gsutil
-    entrypoint: /bin/sh
-    args: 
-    - -c
-    - 'if [ $(ls -pR "/workspace/${BUILD_ID}"  | grep -v / | grep -v ^$ | wc -l) -ne 0 ]; then gsutil -m -q rsync -r "/workspace/${BUILD_ID}" "gs://${_GCS_ARTIFACTS_BUCKET}/test-artifacts/PR_${_PR_NUMBER}/BUILD_${BUILD_ID}/"; else echo "No artifacts to copy."; fi'
-  # Fail if there is anything in the failure folder
-  - name: ${_PYTHON_IMAGE}
-    entrypoint: /bin/sh
-    args: 
-    - -c
-    - 'echo "Download executed notebooks with this command: \"mkdir -p artifacts && gsutil rsync -r gs://${_GCS_ARTIFACTS_BUCKET}/test-artifacts/PR_${_PR_NUMBER}/BUILD_${BUILD_ID} artifacts/\"" && if [ "$(ls -A /workspace/${BUILD_ID}/failure | wc -l)" -ne 0 ]; then exit 1; else exit 0; fi' 
 timeout: 86400s

.cloud-build/requirements.txt

@@ -1,8 +1,12 @@
-ipython>=7.0
-jupyter>=1.0
-nbconvert>=6.0
-papermill>=2.3
-numpy>=1.19
-pandas>=1.2
-matplotlib>=3.4
+ipython==7.30.1
+jupyter==1.0.0
+nbconvert==6.3.0
+papermill==2.3.3
+numpy==1.21.4
+pandas==1.3.5
+matplotlib==3.5.1
 tabulate
+google-cloud-aiplatform
+google-cloud-storage
+google-cloud-build
+gcloud

.cloud-build/utils/NotebookProcessors.py

@@ -15,7 +15,7 @@
 
 from nbconvert.preprocessors import Preprocessor
 from typing import Dict
-import UpdateNotebookVariables
+from . import UpdateNotebookVariables as update_notebook_variables
 
 
 class RemoveNoExecuteCells(Preprocessor):
@@ -41,7 +41,7 @@ class UpdateVariablesPreprocessor(Preprocessor):
         # VARIABLE_NAME = '[description]'
 
         for variable_name, variable_value in replacement_map.items():
-            content = UpdateNotebookVariables.get_updated_value(
+            content = update_notebook_variables.get_updated_value(
                 content=content,
                 variable_name=variable_name,
                 variable_value=variable_value,

.cloud-build/utils/util.py

@@ -0,0 +1,60 @@
+from datetime import datetime
+from typing import Optional
+from google.cloud import storage
+from google.cloud.aiplatform import utils
+from google.auth import credentials as auth_credentials
+import os
+
+import subprocess
+import tarfile
+import uuid
+
+
+def download_file(bucket_name: str, blob_name: str, destination_file: str) -> str:
+    """Copies a remote GCS file to a local path."""
+    remote_file_path = "".join(["gs://", "/".join([bucket_name, blob_name])])
+
+    subprocess.check_output(
+        ["gsutil", "cp", remote_file_path, destination_file], encoding="UTF-8"
+    )
+
+    return destination_file
+
+
+def upload_file(
+    local_file_path: str,
+    remote_file_path: str,
+) -> str:
+    """Copies a local file to a GCS path."""
+    subprocess.check_output(
+        ["gsutil", "cp", local_file_path, remote_file_path], encoding="UTF-8"
+    )
+
+    return remote_file_path
+
+
+def archive_code_and_upload(staging_bucket: str):
+    # Archive all source in current directory
+    unique_id = uuid.uuid4()
+    source_archived_file = f"source_archived_{unique_id}.tar.gz"
+
+    git_files = subprocess.check_output(
+        ["git", "ls-tree", "-r", "HEAD", "--name-only"], encoding="UTF-8"
+    ).split("\n")
+
+    with tarfile.open(source_archived_file, "w:gz") as tar:
+        for file in git_files:
+            if len(file) > 0 and os.path.exists(file):
+                tar.add(file)
+
+    # Upload archive to GCS bucket
+    source_archived_file_gcs = upload_file(
+        local_file_path=f"{source_archived_file}",
+        remote_file_path="/".join(
+            [staging_bucket, "code_archives", source_archived_file]
+        ),
+    )
+
+    print(f"Uploaded source code archive to {source_archived_file_gcs}")
+
+    return source_archived_file_gcs

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,17 +1,18 @@
-If you are opening a PR for `Official Notebooks` under the [notebooks/official](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/master/notebooks/official) folder, follow this mandatory checklist:
-- [ ] Use the [notebook template](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/notebook_template.ipynb) as a starting point.
+If you are opening a PR for `Official Notebooks` under the [notebooks/official](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/notebooks/official) folder, follow this mandatory checklist:
+- [ ] Use the [notebook template](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/notebook_template.ipynb) as a starting point.
 - [ ] Follow the style and grammar rules outlined in the above notebook template.
 - [ ] Verify the notebook runs successfully in Colab since the automated tests cannot guarantee this even when it passes.
-- [ ] Passes all the required automated checks
+- [ ] Passes all the required automated checks. You can locally test for formatting and linting with these [instructions](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/docs/contributing.md#code-quality-checks).
 - [ ] You have consulted with a tech writer to see if tech writer review is necessary. If so, the notebook has been reviewed by a tech writer, and they have approved it.
-- [ ] This notebook has been added to the [CODEOWNERS](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/docs/CODEOWNERS) file under `# Official Notebooks` section, pointing to the author or the author's team.
+- [ ] This notebook has been added to the [CODEOWNERS](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/docs/CODEOWNERS) file under `# Official Notebooks` section, pointing to the author or the author's team.
 - [ ] The Jupyter notebook cleans up any artifacts it has created (datasets, ML models, endpoints, etc) so as not to eat up unnecessary resources.
 
 
-If you are opening a PR for `Community Notebooks` under the [notebooks/community](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/master/notebooks/community) folder:
-- [ ] This notebook has been added to the [CODEOWNERS](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/docs/CODEOWNERS) file under the `# Community Notebooks` section, pointing to the author or the author's team.
+If you are opening a PR for `Community Notebooks` under the [notebooks/community](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/notebooks/community) folder:
+- [ ] This notebook has been added to the [CODEOWNERS](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/docs/CODEOWNERS) file under the `# Community Notebooks` section, pointing to the author or the author's team.
+- [ ] Passes all the required formatting and linting checks. You can locally test with these [instructions](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/docs/contributing.md#code-quality-checks).
 
-
-If you are opening a PR for `Community Content` under the [community-content](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/master/community-content) folder:
+If you are opening a PR for `Community Content` under the [community-content](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/community-content) folder:
 - [ ] Make sure your main `Content Directory Name` is descriptive, informative, and includes some of the key products and attributes of your content, so that it is differentiable from other content
-- [ ] The main content directory has been added to the [CODEOWNERS](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/docs/CODEOWNERS) file under the `# Community Content` section, pointing to the author or the author's team.
+- [ ] The main content directory has been added to the [CODEOWNERS](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/docs/CODEOWNERS) file under the `# Community Content` section, pointing to the author or the author's team.
+- [ ] Passes all the required formatting and linting checks. You can locally test with these [instructions](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/docs/contributing.md#code-quality-checks).

.github/workflows/ci.yaml

@@ -12,8 +12,8 @@ jobs:
       uses: actions/checkout@v2
       with:
         fetch-depth: 0      
-    - name: Fetch base master branch
-      run: git fetch -u "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY" master:master
+    - name: Fetch base main branch
+      run: git fetch -u "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY" main:main
     - name: Install requirements
       run: python3 -m pip install -U -r .github/workflows/linter/requirements.txt
     - name: Format and lint notebooks

.github/workflows/linter/requirements.txt

@@ -2,8 +2,8 @@ git+https://github.com/tensorflow/docs
 ipython
 jupyter
 nbconvert
-black==20.8b1
-pyupgrade==2.7.3
-isort==5.6.4
-flake8==3.9.0
-nbqa==0.6.0
+black==21.10b0
+pyupgrade==2.29.1
+isort==5.10.1
+flake8==4.0.1
+nbqa==1.2.2

.github/workflows/linter/run_linter.sh

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# This script automatically formats and lints all notebooks that have changed from the head of the master branch.
+# This script automatically formats and lints all notebooks that have changed from the head of the main branch.
 #
 # Options:
 # -t: Test-mode. Only test if format and linting are required but make no changes to files.
@@ -52,7 +52,7 @@ echo "Test mode: $is_test"
 notebooks=()
 while read -r file || [ -n "$line" ]; do
     notebooks+=("$file")
-done < <(git diff --name-only master... | grep '\.ipynb$')
+done < <(git diff --name-only main... | grep '\.ipynb$')
 
 problematic_notebooks=()
 if [ ${#notebooks[@]} -gt 0 ]; then
@@ -80,7 +80,7 @@ if [ ${#notebooks[@]} -gt 0 ]; then
                 python3 -m nbqa isort "$notebook" --check
                 ISORT_RTN=$?
                 echo "Running flake8..."
-                python3 -m nbqa flake8 "$notebook" --show-source --extend-ignore=W391,E501,F821,E402,F404,W503,E203,E722
+                python3 -m nbqa flake8 "$notebook" --show-source --extend-ignore=W391,E501,F821,E402,F404,W503,E203,E722,W293,W291
                 FLAKE8_RTN=$?
             else
                 echo "Running black..."
@@ -96,7 +96,7 @@ if [ ${#notebooks[@]} -gt 0 ]; then
                 python3 -m tensorflow_docs.tools.nbfmt --remove_outputs "$notebook"
                 NBFMT_RTN=$?
                 echo "Running flake8..."
-                python3 -m nbqa flake8 "$notebook" --show-source --extend-ignore=W391,E501,F821,E402,F404,W503,E203,E722 --nbqa-mutate
+                python3 -m nbqa flake8 "$notebook" --show-source --extend-ignore=W391,E501,F821,E402,F404,W503,E203,E722,W293,W291 --nbqa-mutate
                 FLAKE8_RTN=$?
             fi
 

CODEOWNERS

@@ -0,0 +1,6 @@
+# See https://help.github.com/en/articles/about-code-owners
+# for more info about CODEOWNERS file.
+
+# These owners will be the default owners for everything in
+# the repo. Unless a later match takes precedence.
+*  @GoogleCloudPlatform/vertex-ai-samples-owners

CODE_OF_CONDUCT.md

@@ -0,0 +1,43 @@
+# Contributor Code of Conduct
+
+As contributors and maintainers of this project,
+and in the interest of fostering an open and welcoming community,
+we pledge to respect all people who contribute through reporting issues,
+posting feature requests, updating documentation,
+submitting pull requests or patches, and other activities.
+
+We are committed to making participation in this project
+a harassment-free experience for everyone,
+regardless of level of experience, gender, gender identity and expression,
+sexual orientation, disability, personal appearance,
+body size, race, ethnicity, age, religion, or nationality.
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery
+* Personal attacks
+* Trolling or insulting/derogatory comments
+* Public or private harassment
+* Publishing other's private information,
+such as physical or electronic
+addresses, without explicit permission
+* Other unethical or unprofessional conduct.
+
+Project maintainers have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct.
+By adopting this Code of Conduct,
+project maintainers commit themselves to fairly and consistently
+applying these principles to every aspect of managing this project.
+Project maintainers who do not follow or enforce the Code of Conduct
+may be permanently removed from the project team.
+
+This code of conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community.
+
+Instances of abusive, harassing, or otherwise unacceptable behavior
+may be reported by opening an issue
+or contacting one or more of the project maintainers.
+
+This Code of Conduct is adapted from the [Contributor Covenant](http://contributor-covenant.org), version 1.2.0,
+available at [http://contributor-covenant.org/version/1/2/0/](http://contributor-covenant.org/version/1/2/0/)

CONTRIBUTING.md

@@ -0,0 +1,65 @@
+# How to Contribute
+
+We'd love to accept your patches and contributions to this project. There are
+just a few small guidelines you need to follow.
+
+## Contributor License Agreement
+
+Contributions to this project must be accompanied by a Contributor License
+Agreement. You (or your employer) retain the copyright to your contribution;
+this simply gives us permission to use and redistribute your contributions as
+part of the project. Head over to <https://cla.developers.google.com/> to see
+your current agreements on file or to sign a new one.
+
+You generally only need to submit a CLA once, so if you've already submitted one
+(even if it was for a different project), you probably don't need to do it
+again.
+
+## Code Quality Checks
+
+All notebooks in this project are checked for formatting and style, to ensure a
+consistent experience. To test notebooks prior to submitting a pull request,
+you can follow these steps.
+
+From a command-line terminal (e.g. from Vertex Workbench or locally), install
+the code analysis tools:
+
+```shell
+pip3 install --user -U nbqa black flake8 isort pyupgrade git+https://github.com/tensorflow/docs
+```
+
+You'll likely need to add the directory where these were installed to your PATH:
+
+```shell
+export PATH=“$HOME/.local/bin:$PATH"
+```
+
+Then, set an environment variable for your notebook (or directory):
+
+```shell
+export notebook="your-notebook.ipynb"
+```
+
+Finally, run this code block to check for errors. Each step will attempt to
+automatically fix any issues. If the fixes can't be performed automatically,
+then you will need to manually address them before submitting your PR.
+
+```shell
+nbqa black "$notebook"
+nbqa pyupgrade "$notebook"
+nbqa isort "$notebook"
+python3 -m tensorflow_docs.tools.nbfmt --remove_outputs "$notebook"
+nbqa flake8 "$notebook" --extend-ignore=W391,E501,F821,E402,F404,W503,E203,E722,W293,W291
+```
+
+## Code Reviews
+
+All submissions, including submissions by project members, require review. We
+use GitHub pull requests for this purpose. Consult
+[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more
+information on using pull requests.
+
+## Community Guidelines
+
+This project follows [Google's Open Source Community
+Guidelines](https://opensource.google/conduct/).

README.md

@@ -6,15 +6,16 @@ Welcome to the Google Cloud [Vertex AI](https://cloud.google.com/vertex-ai/docs/
 
 ## Overview
 
-The repository contains [Notebooks](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/notebooks) and [Tutorials](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/tutorials) that demonstrate how to develop and manage ML workflow using Google Cloud Vertex AI. 
+The repository contains [Notebooks](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/master/notebooks) and [Community Content](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/master/community-content) that demonstrate how to develop and manage ML workflows using Google Cloud Vertex AI.
 
 ## Contributing
 
-Contributions welcome! See the [Contributing Guide](Contributions welcome! See the [Contributing Guide](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/docs/contributing.md).
+Contributions welcome! See the [Contributing Guide](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/CONTRIBUTING.md).
 
 ## Getting help
 
 Please use the [issues page](https://github.com/GoogleCloudPlatform/vertex-ai-samples/issues) to provide feedback or submit a bug report.
 
 ## Disclaimer
+
 This is not an officially supported Google product. The code in this repository is for demonstrative purposes only.

SECURITY.md

@@ -0,0 +1,7 @@
+# Security Policy
+
+To report a security issue, please use [g.co/vulnz](https://g.co/vulnz).
+
+The Google Security Team will respond within 5 working days of your report on g.co/vulnz.
+
+We use g.co/vulnz for our intake, and do coordination and disclosure here using GitHub Security Advisory to privately discuss and fix the issue.

community-content/CODEOWNERS

@@ -0,0 +1,4 @@
+*  @vertex-ai-samples-contributors  @GoogleCloudPlatform/cloudml-samples-owners
+/tf_agents_bandits_movie_recommendation_with_kfp_and_vertex_sdk	@yinghsienwu
+/pytorch_text_classification_using_vertex_sdk_and_gcloud		@RajeshThallam @ultrons
+/sklearn_text_classification_from_script_using_vertex_sdk       @maxhardt

community-content/pytorch_image_classification_distributed_data_parallel_training_with_vertex_sdk/multi_node_ddp_nccl_vertex_training_with_custom_container.ipynb

@@ -1,391 +1,347 @@
 {
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "# Copyright 2021 Google LLC\n",
-    "#\n",
-    "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
-    "# you may not use this file except in compliance with the License.\n",
-    "# You may obtain a copy of the License at\n",
-    "#\n",
-    "#     https://www.apache.org/licenses/LICENSE-2.0\n",
-    "#\n",
-    "# Unless required by applicable law or agreed to in writing, software\n",
-    "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
-    "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
-    "# See the License for the specific language governing permissions and\n",
-    "# limitations under the License."
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "a6b56b1c7b76"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2021 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "20a5ea0081d0"
+      },
+      "source": [
+        "# PyTorch Image Classification Multi-Node Distributed Data Parallel Training on GPU using Vertex Training with Custom Container"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8752d4a255fb"
+      },
+      "source": [
+        "<table align=\"left\">\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/community-content/pytorch_image_classification_distributed_data_parallel_training_with_vertex_sdk/multi_node_ddp_nccl_vertex_training_with_custom_container.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "03d216c7f7b1"
+      },
+      "source": [
+        "## Setup"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "c5ac73516218"
+      },
+      "outputs": [],
+      "source": [
+        "PROJECT_ID = \"YOUR PROJECT ID\"\n",
+        "BUCKET_NAME = \"gs://YOUR BUCKET NAME\"\n",
+        "REGION = \"YOUR REGION\"\n",
+        "SERVICE_ACCOUNT = \"YOUR SERVICE ACCOUNT\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "0b5ae674177e"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil ls -al $BUCKET_NAME"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "19a9b3bdd553"
+      },
+      "outputs": [],
+      "source": [
+        "content_name = \"pt-img-cls-multi-node-ddp-cust-cont\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5307fe28b633"
+      },
+      "source": [
+        "## Vertex Training using Vertex SDK and Custom Container"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "46cb58c7fbf9"
+      },
+      "source": [
+        "### Built Custom Container"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "97e66e9f9bab"
+      },
+      "outputs": [],
+      "source": [
+        "hostname = \"gcr.io\"\n",
+        "image_name = content_name\n",
+        "tag = \"latest\"\n",
+        "\n",
+        "custom_container_image_uri = f\"{hostname}/{PROJECT_ID}/{image_name}:{tag}\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ae9b29c4773f"
+      },
+      "source": [
+        "### Initialize Vertex SDK"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "dc1e84d5dec2"
+      },
+      "outputs": [],
+      "source": [
+        "! pip install -r requirements.txt"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6964be27b98e"
+      },
+      "outputs": [],
+      "source": [
+        "from google.cloud import aiplatform\n",
+        "\n",
+        "aiplatform.init(\n",
+        "    project=PROJECT_ID,\n",
+        "    staging_bucket=BUCKET_NAME,\n",
+        "    location=REGION,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "594a91f438f2"
+      },
+      "source": [
+        "### Create a Vertex Tensorboard Instance"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "93134273261e"
+      },
+      "outputs": [],
+      "source": [
+        "content_name = content_name + \"-gpu\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "c2bd82dbcd9b"
+      },
+      "outputs": [],
+      "source": [
+        "tensorboard = aiplatform.Tensorboard.create(\n",
+        "    display_name=content_name,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ebc593c6472e"
+      },
+      "source": [
+        "#### Option: Use a Previously Created Vertex Tensorboard Instance\n",
+        "\n",
+        "```\n",
+        "tensorboard_name = \"Your Tensorboard Resource Name or Tensorboard ID\"\n",
+        "tensorboard = aiplatform.Tensorboard(tensorboard_name=tensorboard_name)\n",
+        "```"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0769e8e34c2f"
+      },
+      "source": [
+        "### Run a Vertex SDK CustomContainerTrainingJob"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "023f33ece826"
+      },
+      "outputs": [],
+      "source": [
+        "display_name = content_name\n",
+        "gcs_output_uri_prefix = f\"{BUCKET_NAME}/{display_name}\"\n",
+        "\n",
+        "replica_count = 1\n",
+        "machine_type = \"n1-standard-4\"\n",
+        "accelerator_count = 4\n",
+        "accelerator_type = \"NVIDIA_TESLA_K80\"\n",
+        "\n",
+        "args = [\n",
+        "    \"--backend\",\n",
+        "    \"nccl\",\n",
+        "    \"--batch-size\",\n",
+        "    \"128\",\n",
+        "    \"--epochs\",\n",
+        "    \"25\",\n",
+        "]"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "d4b599e726ef"
+      },
+      "outputs": [],
+      "source": [
+        "custom_container_training_job = aiplatform.CustomContainerTrainingJob(\n",
+        "    display_name=display_name,\n",
+        "    container_uri=custom_container_image_uri,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "81321e3bdf7f"
+      },
+      "outputs": [],
+      "source": [
+        "custom_container_training_job.run(\n",
+        "    args=args,\n",
+        "    base_output_dir=gcs_output_uri_prefix,\n",
+        "    replica_count=replica_count,\n",
+        "    machine_type=machine_type,\n",
+        "    accelerator_count=accelerator_count,\n",
+        "    accelerator_type=accelerator_type,\n",
+        "    tensorboard=tensorboard.resource_name,\n",
+        "    service_account=SERVICE_ACCOUNT,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "5100712c2c4c"
+      },
+      "outputs": [],
+      "source": [
+        "print(f\"Custom Training Job Name: {custom_container_training_job.resource_name}\")\n",
+        "print(f\"GCS Output URI Prefix: {gcs_output_uri_prefix}\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "f9b77676e5a6"
+      },
+      "source": [
+        "### Training Output Artifact"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "0e171ce95ace"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil ls $gcs_output_uri_prefix"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "cf1b74a12b87"
+      },
+      "source": [
+        "## Clean Up Artifact"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "a0b15089c341"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil rm -rf $gcs_output_uri_prefix"
+      ]
     }
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "# PyTorch Image Classification Multi-Node Distributed Data Parallel Training on GPU using Vertex Training with Custom Container"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "<table align=\"left\">\n",
-    "  <td>\n",
-    "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/community-content/pytorch_image_classification_distributed_data_parallel_training_with_vertex_sdk/multi_node_ddp_nccl_vertex_training_with_custom_container.ipynb\">\n",
-    "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
-    "      View on GitHub\n",
-    "    </a>\n",
-    "  </td>\n",
-    "</table>"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "## Setup"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "PROJECT_ID = \"YOUR PROJECT ID\"\n",
-    "BUCKET_NAME = \"gs://YOUR BUCKET NAME\"\n",
-    "REGION = \"YOUR REGION\"\n",
-    "SERVICE_ACCOUNT = \"YOUR SERVICE ACCOUNT\""
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
+  ],
+  "metadata": {
+    "colab": {
+      "name": "multi_node_ddp_nccl_vertex_training_with_custom_container.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
     }
-   }
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "! gsutil ls -al $BUCKET_NAME"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "content_name = \"pt-img-cls-multi-node-ddp-cust-cont\""
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "## Vertex Training using Vertex SDK and Custom Container"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "### Built Custom Container"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "hostname = \"gcr.io\"\n",
-    "image_name = content_name\n",
-    "tag = \"latest\"\n",
-    "\n",
-    "custom_container_image_uri=f\"{hostname}/{PROJECT_ID}/{image_name}:{tag}\""
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "### Initialize Vertex SDK"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%% md\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "! pip install -r requirements.txt"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "source": [
-    "from google.cloud import aiplatform\n",
-    "\n",
-    "aiplatform.init(\n",
-    "    project=PROJECT_ID,\n",
-    "    staging_bucket=BUCKET_NAME,\n",
-    "    location=REGION,\n",
-    ")"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   },
-   "execution_count": null,
-   "outputs": []
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "### Create a Vertex Tensorboard Instance"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%% md\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "content_name = content_name + \"-gpu\"\n",
-    "\n",
-    "tensorboard = aiplatform.Tensorboard.create(\n",
-    "    display_name=content_name,\n",
-    ")"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "#### Option: Use a Previously Created Vertex Tensorboard Instance\n",
-    "\n",
-    "```\n",
-    "tensorboard_name = \"Your Tensorboard Resource Name or Tensorboard ID\"\n",
-    "tensorboard = aiplatform.Tensorboard(tensorboard_name=tensorboard_name)\n",
-    "```"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "### Run a Vertex SDK CustomContainerTrainingJob"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "display_name = content_name\n",
-    "gcs_output_uri_prefix = f\"{BUCKET_NAME}/{display_name}\"\n",
-    "\n",
-    "replica_count = 4\n",
-    "machine_type = \"n1-standard-4\"\n",
-    "accelerator_count = 1\n",
-    "accelerator_type = \"NVIDIA_TESLA_K80\"\n",
-    "\n",
-    "args = [\n",
-    "    '--backend', 'nccl',\n",
-    "    '--batch-size', '128',\n",
-    "    '--epochs', '25',\n",
-    "]"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "source": [
-    "custom_container_training_job = aiplatform.CustomContainerTrainingJob(\n",
-    "    display_name=display_name,\n",
-    "    container_uri=custom_container_image_uri,\n",
-    ")"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   },
-   "execution_count": null,
-   "outputs": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "custom_container_training_job.run(\n",
-    "    args=args,\n",
-    "    base_output_dir=gcs_output_uri_prefix,\n",
-    "    machine_type=machine_type,\n",
-    "    accelerator_count=accelerator_count,\n",
-    "    accelerator_type=accelerator_type,\n",
-    "    tensorboard=tensorboard.resource_name,\n",
-    "    service_account=SERVICE_ACCOUNT,\n",
-    ")"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "print(f'Custom Training Job Name: {custom_container_training_job.resource_name}')\n",
-    "print(f'GCS Output URI Prefix: {gcs_output_uri_prefix}')"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "### Training Output Artifact"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "! gsutil ls $gcs_output_uri_prefix"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "## Clean Up Artifact"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "! gsutil rm -rf $gcs_output_uri_prefix"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 2
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython2",
-   "version": "2.7.6"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 0
-}
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

community-content/pytorch_image_classification_distributed_data_parallel_training_with_vertex_sdk/trainer/task.py

@@ -19,7 +19,7 @@ Adapted from: https://github.com/narumiruna/pytorch-distributed-example
 
 import argparse
 import os
-import tempfile
+import shutil
 
 import torch
 from torch import distributed
@@ -29,8 +29,6 @@ from torch.utils.tensorboard import SummaryWriter
 
 from torchvision import datasets, transforms
 
-import utils
-
 def parse_args():
 
   parser = argparse.ArgumentParser()
@@ -43,6 +41,9 @@ def parse_args():
   parser.add_argument(
       '--tensorboard-log-dir', default=os.getenv('AIP_TENSORBOARD_LOG_DIR'), type=str,
       help='a Cloud Storage URI of a directory intended for saving TensorBoard')
+  parser.add_argument(
+      '--checkpoint-dir', default=os.getenv('AIP_CHECKPOINT_DIR'), type=str,
+      help='a Cloud Storage URI of a directory intended for saving checkpoints')
 
   parser.add_argument(
       '--backend', type=str, default='gloo',
@@ -68,7 +69,6 @@ def parse_args():
       '-lr', '--learning-rate', type=float, default=1e-3)
   parser.add_argument(
       '--batch-size', type=int, default=128)
-
   parser.add_argument(
       '--local-mode', action='store_true', help='use local mode when running on your local machine')
 
@@ -76,6 +76,12 @@ def parse_args():
 
   return args
 
+def makedirs(model_dir):
+  if os.path.exists(model_dir) and os.path.isdir(model_dir):
+    shutil.rmtree(model_dir)
+  os.makedirs(model_dir)
+  return
+
 def distributed_is_initialized():
   if distributed.is_available():
     if distributed.is_initialized():
@@ -176,7 +182,7 @@ class Trainer(object):
 
     for epoch in range(1, epochs + 1):
 
-      print("Epoch: {}, Training ...".format(epoch))
+      print('Epoch: {}, Training ...'.format(epoch))
       train_loss, train_acc = self.train()
 
       if is_chief:
@@ -242,16 +248,30 @@ def main():
   local_data_dir = './tmp/data'
   local_model_dir = './tmp/model'
   local_tensorboard_log_dir = './tmp/logs'
+  local_checkpoint_dir = './tmp/checkpoints'
 
-  #TODO: update when gcsfuse ready
-  gcsfuse_ready = False
-  model_dir = (gcsfuse_ready and args.model_dir) or local_model_dir
-  tensorboard_log_dir = (gcsfuse_ready and
-                         args.tensorboard_log_dir) or local_tensorboard_log_dir
+  model_dir = args.model_dir or local_model_dir
+  tensorboard_log_dir = args.tensorboard_log_dir or local_tensorboard_log_dir
+  checkpoint_dir = args.checkpoint_dir or local_checkpoint_dir
+
+  gs_prefix = 'gs://'
+  gcsfuse_prefix = '/gcs/'
+  if model_dir and model_dir.startswith(gs_prefix):
+    model_dir = model_dir.replace(gs_prefix, gcsfuse_prefix)
+  if tensorboard_log_dir and tensorboard_log_dir.startswith(gs_prefix):
+    tensorboard_log_dir = tensorboard_log_dir.replace(gs_prefix, gcsfuse_prefix)
+  if checkpoint_dir and checkpoint_dir.startswith(gs_prefix):
+    checkpoint_dir = checkpoint_dir.replace(gs_prefix, gcsfuse_prefix)
 
   writer = SummaryWriter(tensorboard_log_dir)
 
   is_chief = args.rank == 0
+  if is_chief:
+    makedirs(checkpoint_dir)
+    print(f'Checkpoints will be saved to {checkpoint_dir}')
+
+  checkpoint_path = os.path.join(checkpoint_dir, 'checkpoint.pt')
+  print(f'checkpoint_path is {checkpoint_path}')
 
   if args.world_size > 1:
     print('Initializing distributed backend with {} nodes'.format(args.world_size))
@@ -261,36 +281,38 @@ def main():
           world_size=args.world_size,
           rank=args.rank,
       )
-    print(f"[{os.getpid()}]: "
-          f"world_size = {distributed.get_world_size()}, "
-          f"rank = {distributed.get_rank()}, "
-          f"backend={distributed.get_backend()} \n", end='')
+    print(f'[{os.getpid()}]: '
+          f'world_size = {distributed.get_world_size()}, '
+          f'rank = {distributed.get_rank()}, '
+          f'backend={distributed.get_backend()} \n', end='')
 
   if torch.cuda.is_available() and not args.no_cuda:
-    device = torch.device("cuda:{}".format(args.rank))
+    device = torch.device('cuda:{}'.format(args.rank))
   else:
-    device = torch.device("cpu")
+    device = torch.device('cpu')
 
   model = Net(device=device)
   if distributed_is_initialized():
+    model.to(device)
     model = DistributedDataParallel(model)
 
-  checkpoint_path = tempfile.gettempdir() + "/model.checkpoint"
   if is_chief:
     # All processes should see same parameters as they all start from same
     # random parameters and gradients are synchronized in backward passes.
     # Therefore, saving it in one process is sufficient.
     torch.save(model.state_dict(), checkpoint_path)
+    print(f'Initial chief checkpoint is saved to {checkpoint_path}')
 
   # Use a barrier() to make sure that process 1 loads the model after process
   # 0 saves it.
   if distributed_is_initialized():
     distributed.barrier()
     # configure map_location properly
-    map_location = {'cuda:%d' % 0: 'cuda:%d' % args.rank}
-    model.load_state_dict(torch.load(checkpoint_path, map_location=map_location))
+    model.load_state_dict(torch.load(checkpoint_path, map_location=device))
+    print(f'Initial chief checkpoint is saved to {checkpoint_path} with map_location {device}')
   else:
     model.load_state_dict(torch.load(checkpoint_path))
+    print(f'Initial chief checkpoint is loaded from {checkpoint_path}')
 
   optimizer = torch.optim.Adam(model.parameters(), lr=args.learning_rate)
 
@@ -310,29 +332,12 @@ def main():
   )
   trainer.fit(args.epochs, is_chief, writer)
 
-  if is_chief and model_dir == local_model_dir:
-    utils.makedirs(model_dir)
+  if model_dir == local_model_dir:
+    makedirs(model_dir)
     trainer.save(model_dir)
     print(f'Model is saved to {model_dir}')
 
-  if is_chief and not args.local_mode:
-    utils.gcs_upload(
-      dir=model_dir,
-      local_dir=local_model_dir,
-      gcs_dir=args.model_dir,
-      gcsfuse_ready=gcsfuse_ready,
-      local_mode=args.local_mode,
-  )
-
   print(f'Tensorboard logs are saved to: {tensorboard_log_dir}')
-  if is_chief and not args.local_mode:
-    utils.gcs_upload(
-      dir=tensorboard_log_dir,
-      local_dir=local_tensorboard_log_dir,
-      gcs_dir=args.tensorboard_log_dir,
-      gcsfuse_ready=gcsfuse_ready,
-      local_mode=args.local_mode,
-  )
 
   writer.close()
 

community-content/pytorch_image_classification_distributed_data_parallel_training_with_vertex_sdk/trainer/utils.py

@@ -1,31 +0,0 @@
-# Copyright 2021 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the \"License\");
-# you may not use this file except in compliance with the License.\n",
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an \"AS IS\" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-import shutil
-import subprocess
-
-def makedirs(model_dir):
-  if os.path.exists(model_dir) and os.path.isdir(model_dir):
-    shutil.rmtree(model_dir)
-  os.makedirs(model_dir)
-  return
-
-def gcs_upload(dir, local_dir, gcs_dir, gcsfuse_ready, local_mode):
-  if not local_mode and dir == local_dir and not gcsfuse_ready:
-    subprocess.run(['gsutil', 'cp', '-r',
-                    local_dir,
-                    os.path.dirname(gcs_dir)])
-    print(f'{local_dir} is uploaded to {gcs_dir}')
-  return

community-content/pytorch_image_classification_single_gpu_with_vertex_sdk_and_torchserve/trainer/task.py

@@ -16,6 +16,7 @@ import argparse
 import copy
 import os
 import pathlib
+import shutil
 import time
 
 import torch
@@ -24,8 +25,6 @@ from torch.utils.tensorboard import SummaryWriter
 
 from torchvision import datasets, models, transforms
 
-import utils
-
 def parse_args():
 
   parser = argparse.ArgumentParser()
@@ -62,6 +61,12 @@ def parse_args():
 
   return args
 
+def makedirs(model_dir):
+  if os.path.exists(model_dir) and os.path.isdir(model_dir):
+    shutil.rmtree(model_dir)
+  os.makedirs(model_dir)
+  return
+
 def download_data(data_dir):
 
   dataset_url = 'https://download.pytorch.org/tutorial/hymenoptera_data.zip'
@@ -207,13 +212,17 @@ def main():
   local_model_dir = './tmp/model'
   local_tensorboard_log_dir = './tmp/logs'
 
-  #TODO: update when gcsfuse ready
-  gcsfuse_ready = False
+  model_dir = args.model_dir or local_model_dir
+  tensorboard_log_dir = args.tensorboard_log_dir or local_tensorboard_log_dir
 
-  model_dir = (gcsfuse_ready and args.model_dir) or local_model_dir
-  tensorboard_log_dir = (gcsfuse_ready and
-                         args.tensorboard_log_dir) or local_tensorboard_log_dir
+  gs_prefix = 'gs://'
+  gcsfuse_prefix = '/gcs/'
+  if model_dir and model_dir.startswith(gs_prefix):
+    model_dir = model_dir.replace(gs_prefix, gcsfuse_prefix)
+  if tensorboard_log_dir and tensorboard_log_dir.startswith(gs_prefix):
+    tensorboard_log_dir = tensorboard_log_dir.replace(gs_prefix, gcsfuse_prefix)
 
+  makedirs(model_dir)
   writer = SummaryWriter(tensorboard_log_dir)
 
   device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
@@ -225,9 +234,6 @@ def main():
   dataset_sizes = {x: len(image_datasets[x]) for x in ['train', 'val']}
   print(f'Dataset sizes: {dataset_sizes}')
 
-  if model_dir == local_model_dir:
-    utils.makedirs(model_dir)
-
   dataloaders = {
       x: torch.utils.data.DataLoader(
           image_datasets[x],
@@ -266,22 +272,8 @@ def main():
 
   torch.save(model.state_dict(), model_path)
   print(f'Model is saved to {model_dir}')
-  utils.gcs_upload(
-      dir=model_dir,
-      local_dir=local_model_dir,
-      gcs_dir=args.model_dir,
-      gcsfuse_ready=gcsfuse_ready,
-      local_mode=args.local_mode
-  )
 
   print(f'Tensorboard logs are saved to: {tensorboard_log_dir}')
-  utils.gcs_upload(
-      dir=tensorboard_log_dir,
-      local_dir=local_tensorboard_log_dir,
-      gcs_dir=args.tensorboard_log_dir,
-      gcsfuse_ready=gcsfuse_ready,
-      local_mode=args.local_mode
-  )
 
   writer.close()
 

community-content/pytorch_image_classification_single_gpu_with_vertex_sdk_and_torchserve/trainer/utils.py

@@ -1,31 +0,0 @@
-# Copyright 2021 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the \"License\");
-# you may not use this file except in compliance with the License.\n",
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an \"AS IS\" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-import shutil
-import subprocess
-
-def makedirs(model_dir):
-  if os.path.exists(model_dir) and os.path.isdir(model_dir):
-    shutil.rmtree(model_dir)
-  os.makedirs(model_dir)
-  return
-
-def gcs_upload(dir, local_dir, gcs_dir, gcsfuse_ready, local_mode):
-  if not local_mode and dir == local_dir and not gcsfuse_ready:
-    subprocess.run(['gsutil', 'cp', '-r',
-                    local_dir,
-                    os.path.dirname(gcs_dir)])
-    print(f'{local_dir} is uploaded to {gcs_dir}')
-  return

community-content/pytorch_image_classification_single_gpu_with_vertex_sdk_and_torchserve/vertex_training_with_custom_container.ipynb

@@ -1,507 +1,431 @@
 {
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "# Copyright 2021 Google LLC\n",
-    "#\n",
-    "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
-    "# you may not use this file except in compliance with the License.\n",
-    "# You may obtain a copy of the License at\n",
-    "#\n",
-    "#     https://www.apache.org/licenses/LICENSE-2.0\n",
-    "#\n",
-    "# Unless required by applicable law or agreed to in writing, software\n",
-    "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
-    "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
-    "# See the License for the specific language governing permissions and\n",
-    "# limitations under the License."
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "a6b56b1c7b76"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2021 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d229c51d7335"
+      },
+      "source": [
+        "# PyTorch Image Classification Single GPU using Vertex Training with Custom Container"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "08b6ccb2de0f"
+      },
+      "source": [
+        "<table align=\"left\">\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/community-content/pytorch_image_classification_single_gpu_with_vertex_sdk_and_torchserve/vertex_training_with_custom_container.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "03d216c7f7b1"
+      },
+      "source": [
+        "## Setup"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "c5ac73516218"
+      },
+      "outputs": [],
+      "source": [
+        "PROJECT_ID = \"YOUR PROJECT ID\"\n",
+        "BUCKET_NAME = \"gs://YOUR BUCKET NAME\"\n",
+        "REGION = \"YOUR REGION\"\n",
+        "SERVICE_ACCOUNT = \"YOUR SERVICE ACCOUNT\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "410ef4707617"
+      },
+      "outputs": [],
+      "source": [
+        "content_name = \"pt-img-cls-gpu-cust-cont-torchserve\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ed102448ae75"
+      },
+      "source": [
+        "## Local Training"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "47dfe4e1dfe6"
+      },
+      "outputs": [],
+      "source": [
+        "! ls trainer"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "be6fcb587414"
+      },
+      "outputs": [],
+      "source": [
+        "! cat trainer/requirements.txt"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "060178a8ad71"
+      },
+      "outputs": [],
+      "source": [
+        "! pip install -r trainer/requirements.txt"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "4421e56c4ee3"
+      },
+      "outputs": [],
+      "source": [
+        "! cat trainer/task.py"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "4bb2f6fc75cc"
+      },
+      "outputs": [],
+      "source": [
+        "%run trainer/task.py --epochs 5 --local-mode"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cb748af8f2b8"
+      },
+      "outputs": [],
+      "source": [
+        "! ls ./tmp"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2ca1626c5dbd"
+      },
+      "outputs": [],
+      "source": [
+        "! rm -rf ./tmp"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "4eb55916219c"
+      },
+      "source": [
+        "## Vertex Training using Vertex SDK and Custom Container"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e61fc5c1b9e0"
+      },
+      "source": [
+        "### Build Custom Container"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "c70a580cc6d4"
+      },
+      "outputs": [],
+      "source": [
+        "hostname = \"gcr.io\"\n",
+        "image_name_train = content_name\n",
+        "tag = \"latest\"\n",
+        "\n",
+        "custom_container_image_uri_train = f\"{hostname}/{PROJECT_ID}/{image_name_train}:{tag}\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "e58c75158872"
+      },
+      "outputs": [],
+      "source": [
+        "! cd trainer && docker build -t $custom_container_image_uri_train -f Dockerfile ."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "bde55966086a"
+      },
+      "outputs": [],
+      "source": [
+        "! docker run --rm $custom_container_image_uri_train --epochs 5 --local-mode"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "f4a3cdb78045"
+      },
+      "outputs": [],
+      "source": [
+        "! docker push $custom_container_image_uri_train"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "526115eabf35"
+      },
+      "outputs": [],
+      "source": [
+        "! gcloud container images list --repository $hostname/$PROJECT_ID"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "f026b57d265e"
+      },
+      "source": [
+        "### Initialize Vertex SDK"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "19c02e9e9b8d"
+      },
+      "outputs": [],
+      "source": [
+        "! pip install -r requirements.txt"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "a0deece1086e"
+      },
+      "outputs": [],
+      "source": [
+        "from google.cloud import aiplatform\n",
+        "\n",
+        "aiplatform.init(\n",
+        "    project=PROJECT_ID,\n",
+        "    staging_bucket=BUCKET_NAME,\n",
+        "    location=REGION,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "1ec68e279ed1"
+      },
+      "source": [
+        "### Create a Vertex Tensorboard Instance"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "d945f0e32b02"
+      },
+      "outputs": [],
+      "source": [
+        "tensorboard = aiplatform.Tensorboard.create(\n",
+        "    display_name=content_name,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "cb13cc82acc5"
+      },
+      "source": [
+        "#### Option: Use a Previously Created Vertex Tensorboard Instance\n",
+        "\n",
+        "```\n",
+        "tensorboard_name = \"Your Tensorboard Resource Name or Tensorboard ID\"\n",
+        "tensorboard = aiplatform.Tensorboard(tensorboard_name=tensorboard_name)\n",
+        "```"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d04453c214d2"
+      },
+      "source": [
+        "### Run a Vertex SDK CustomContainerTrainingJob"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "4b5b5dea03c8"
+      },
+      "outputs": [],
+      "source": [
+        "display_name = content_name\n",
+        "gcs_output_uri_prefix = f\"{BUCKET_NAME}/{display_name}\"\n",
+        "\n",
+        "machine_type = \"n1-standard-4\"\n",
+        "accelerator_count = 1\n",
+        "accelerator_type = \"NVIDIA_TESLA_K80\"\n",
+        "\n",
+        "container_args = [\n",
+        "    \"--batch-size\",\n",
+        "    \"256\",\n",
+        "    \"--epochs\",\n",
+        "    \"100\",\n",
+        "]"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b2073fb5824e"
+      },
+      "outputs": [],
+      "source": [
+        "custom_container_training_job = aiplatform.CustomContainerTrainingJob(\n",
+        "    display_name=display_name,\n",
+        "    container_uri=custom_container_image_uri_train,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "d442a02878d1"
+      },
+      "outputs": [],
+      "source": [
+        "custom_container_training_job.run(\n",
+        "    args=container_args,\n",
+        "    base_output_dir=gcs_output_uri_prefix,\n",
+        "    machine_type=machine_type,\n",
+        "    accelerator_type=accelerator_type,\n",
+        "    accelerator_count=accelerator_count,\n",
+        "    tensorboard=tensorboard.resource_name,\n",
+        "    service_account=SERVICE_ACCOUNT,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "29b14f2289f3"
+      },
+      "outputs": [],
+      "source": [
+        "print(f\"Custom Training Job Name: {custom_container_training_job.resource_name}\")\n",
+        "print(f\"GCS Output URI Prefix: {gcs_output_uri_prefix}\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3ab127e5f2b2"
+      },
+      "source": [
+        "### Training Artifact"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "8e0c74a00dee"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil ls $gcs_output_uri_prefix"
+      ]
     }
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "# PyTorch Image Classification Single GPU using Vertex Training with Custom Container"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "<table align=\"left\">\n",
-    "  <td>\n",
-    "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/community-content/pytorch_image_classification_single_gpu_with_vertex_sdk_and_torchserve/vertex_training_with_custom_container.ipynb\">\n",
-    "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
-    "      View on GitHub\n",
-    "    </a>\n",
-    "  </td>\n",
-    "</table>"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "## Setup"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "PROJECT_ID = \"YOUR PROJECT ID\"\n",
-    "BUCKET_NAME = \"gs://YOUR BUCKET NAME\"\n",
-    "REGION = \"YOUR REGION\"\n",
-    "SERVICE_ACCOUNT = \"YOUR SERVICE ACCOUNT\""
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
+  ],
+  "metadata": {
+    "colab": {
+      "name": "vertex_training_with_custom_container.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
     }
-   }
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "content_name = \"pt-img-cls-gpu-cust-cont-torchserve\""
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "## Local Training"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "! ls trainer"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "! cat trainer/requirements.txt"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "! pip install -r trainer/requirements.txt"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "! cat trainer/task.py"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "%run trainer/task.py --epochs 5 --local-mode"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "! ls ./tmp"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "! rm -rf ./tmp"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "## Vertex Training using Vertex SDK and Custom Container"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "### Build Custom Container"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "hostname = \"gcr.io\"\n",
-    "image_name_train = content_name\n",
-    "tag = \"latest\"\n",
-    "\n",
-    "custom_container_image_uri_train=f\"{hostname}/{PROJECT_ID}/{image_name_train}:{tag}\""
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "! cd trainer && docker build -t $custom_container_image_uri_train -f Dockerfile ."
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "! docker run --rm $custom_container_image_uri_train --epochs 5 --local-mode"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "! docker push $custom_container_image_uri_train"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "! gcloud container images list --repository $hostname/$PROJECT_ID"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "### Initialize Vertex SDK"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "! pip install -r requirements.txt"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "from google.cloud import aiplatform\n",
-    "\n",
-    "aiplatform.init(\n",
-    "    project=PROJECT_ID,\n",
-    "    staging_bucket=BUCKET_NAME,\n",
-    "    location=REGION,\n",
-    ")"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "### Create a Vertex Tensorboard Instance"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "tensorboard = aiplatform.Tensorboard.create(\n",
-    "    display_name=content_name,\n",
-    ")"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "#### Option: Use a Previously Created Vertex Tensorboard Instance\n",
-    "\n",
-    "```\n",
-    "tensorboard_name = \"Your Tensorboard Resource Name or Tensorboard ID\"\n",
-    "tensorboard = aiplatform.Tensorboard(tensorboard_name=tensorboard_name)\n",
-    "```"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "### Run a Vertex SDK CustomContainerTrainingJob"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "display_name = content_name\n",
-    "gcs_output_uri_prefix = f\"{BUCKET_NAME}/{display_name}\"\n",
-    "\n",
-    "machine_type = \"n1-standard-4\"\n",
-    "accelerator_count = 1\n",
-    "accelerator_type = \"NVIDIA_TESLA_K80\"\n",
-    "\n",
-    "container_args = [\n",
-    "    '--batch-size', '256',\n",
-    "    '--epochs', '100',\n",
-    "]"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "custom_container_training_job = aiplatform.CustomContainerTrainingJob(\n",
-    "    display_name=display_name,\n",
-    "    container_uri=custom_container_image_uri_train,\n",
-    ")"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "custom_container_training_job.run(\n",
-    "    args=container_args,\n",
-    "    base_output_dir=gcs_output_uri_prefix,\n",
-    "    machine_type=machine_type,\n",
-    "    accelerator_type=accelerator_type,\n",
-    "    accelerator_count=accelerator_count,\n",
-    "    tensorboard=tensorboard.resource_name,\n",
-    "    service_account=SERVICE_ACCOUNT,\n",
-    ")"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "print(f'Custom Training Job Name: {custom_container_training_job.resource_name}')\n",
-    "print(f'GCS Output URI Prefix: {gcs_output_uri_prefix}')"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "### Training Artifact"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "! gsutil ls $gcs_output_uri_prefix\n"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 2
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython2",
-   "version": "2.7.6"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 0
-}
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

community-content/sklearn_text_classification_from_script_using_vertex_sdk/README.md

@@ -0,0 +1,28 @@
+# Train and deploy a scikit-learn model with Vertex AI
+
+This repository shows how to train and deploy a text classifier using scikit-learn and Vertex AI.
+
+The main used Vertex AI features are:
+- Vertex AI Custom Training
+- Vertex AI Model
+- Vertex AI Endpoint
+
+Further used GCP services are:
+- Google Cloud Logging
+- Google Cloud Storage
+
+## Repository
+
+    ├── README.md
+    ├── create_job.ipynb    # <-- creates the training job and deploys the model
+    ├── requirements.txt    # <-- requirements for deploying the job
+    └── task.py             # <-- contains the training application
+
+## Training job overview
+
+The training job performs the following steps:
+
+1. Downloads the `NewsAggregator` dataset from the UCI Machine Learning Repository
+2. Trains and evaluates a classifier using scikit-learn
+3. Exports model and evaluation artifacts to GCS
+4. Deploys the model as a `Vertex AI Endpoint`

community-content/sklearn_text_classification_from_script_using_vertex_sdk/create_job.ipynb

@@ -0,0 +1,290 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "72b875d67303"
+      },
+      "source": [
+        "# Create and run a custom Vertex AI Training Job from a local script"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "398b976f501e"
+      },
+      "source": [
+        "## Install Vertex AI Python Client"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2162adc1d8fe"
+      },
+      "outputs": [],
+      "source": [
+        "!pip install -r requirements.txt --upgrade"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d4bf4ab70e65"
+      },
+      "source": [
+        "## GCP authentication"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "41084de2e96a"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "os.environ[\n",
+        "    \"GOOGLE_APPLICATION_CREDENTIALS\"\n",
+        "] = \"\"  # TODO: path to credentials .json file"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "1ce7efb99a95"
+      },
+      "source": [
+        "## Create the custom Vertex AI Training Job\n",
+        "\n",
+        "1. Define the custom job parameters\n",
+        "2. Submit the job to create a `Vertex AI Model`"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "c04b9efb5eb3"
+      },
+      "outputs": [],
+      "source": [
+        "# Import the Vertex AI SDK (Python Client)\n",
+        "from google.cloud import aiplatform"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b56f5fdcefd6"
+      },
+      "outputs": [],
+      "source": [
+        "# Project meta data\n",
+        "PROJECT_ID = \"\"  # TODO\n",
+        "REGION = \"\"  # TODO e.g. europe\n",
+        "ZONE = \"\"  # TODO e.g. west4\n",
+        "LOCATION = f\"{REGION}-{ZONE}\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "4af8cfb1e1a1"
+      },
+      "outputs": [],
+      "source": [
+        "aiplatform.init(project=PROJECT_ID, location=LOCATION)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "e23d0161b489"
+      },
+      "outputs": [],
+      "source": [
+        "# Variables for specifying the job\n",
+        "DISPLAY_NAME = (\n",
+        "    \"news-classifier-training\"  # TODO: How the job is displayed on Vertex AI GUI\n",
+        ")\n",
+        "SCRIPT_PATH = \"./task.py\"  # Path to local training script\n",
+        "STAGING_BUCKET = (\n",
+        "    \"\"  # TODO GCS URI where meta data and artifacts are stored for this job\n",
+        ")\n",
+        "MODEL_TRAINING_IMAGE = f\"{REGION}-docker.pkg.dev/vertex-ai/training/scikit-learn-cpu.0-23:latest\"  # Pre-built training image\n",
+        "REQUIREMENTS = [\"wget\"]  # Additional requirements not already part of the base image\n",
+        "# !Required if the Training Pipeline produces a managed Vertex AI Model!\n",
+        "MODEL_SERVING_IMAGE = f\"{REGION}-docker.pkg.dev/vertex-ai/prediction/sklearn-cpu.0-23:latest\"  # Pre-built serving image"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "61565ec3e6de"
+      },
+      "outputs": [],
+      "source": [
+        "# Job definition\n",
+        "custom_training_job = aiplatform.CustomTrainingJob(\n",
+        "    project=PROJECT_ID,\n",
+        "    location=LOCATION,\n",
+        "    display_name=DISPLAY_NAME,\n",
+        "    script_path=SCRIPT_PATH,\n",
+        "    staging_bucket=STAGING_BUCKET,\n",
+        "    container_uri=MODEL_TRAINING_IMAGE,\n",
+        "    requirements=REQUIREMENTS,\n",
+        "    model_serving_container_image_uri=MODEL_SERVING_IMAGE,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "8c8d4bd78688"
+      },
+      "outputs": [],
+      "source": [
+        "# Variables for running the job\n",
+        "MACHINE_TYPE = \"n1-standard-4\"  # Standard VM with 4 CPUs\n",
+        "# !Required if the Training Pipeline produces a managed Vertex AI Model!\n",
+        "MODEL_DISPLAY_NAME = (\n",
+        "    \"news-classifier-model\"  # TODO: Name for the resulting managed Vertex AI Model.\n",
+        ")\n",
+        "# Note that a single job may produce multiple models (e.g. one per run).\n",
+        "# The url to download the training data from.\n",
+        "DATASET_URL = \"https://archive.ics.uci.edu/ml/machine-learning-databases/00359/NewsAggregatorDataset.zip\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "13d31a7d0bb5"
+      },
+      "outputs": [],
+      "source": [
+        "# Run the job\n",
+        "model = custom_training_job.run(\n",
+        "    machine_type=MACHINE_TYPE,\n",
+        "    model_display_name=MODEL_DISPLAY_NAME,\n",
+        "    args=[f\"--dataset_url={DATASET_URL}\", f\"--project_id={PROJECT_ID}\"],\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "8026ac119722"
+      },
+      "outputs": [],
+      "source": [
+        "MODEL_RESOURCE_NAME = model.resource_name"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2d985fca37f3"
+      },
+      "source": [
+        "## Deploy model to Vertex AI Endpoint\n",
+        "\n",
+        "1. Retrieve the registered `Vertex AI Model`\n",
+        "2. Deploy the model to a new `Vertex AI Endpoint`\n",
+        "3. Get some test predictions from the endpoint"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "58c08631f94c"
+      },
+      "outputs": [],
+      "source": [
+        "ENDPOINT_DISPLAY_NAME = \"news-classifier-endpoint\"  # TODO\n",
+        "MACHINE_TYPE_SERVING = \"n1-standard-2\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b54867240880"
+      },
+      "outputs": [],
+      "source": [
+        "endpoint = aiplatform.Endpoint.create(\n",
+        "    display_name=ENDPOINT_DISPLAY_NAME,\n",
+        "    location=LOCATION,\n",
+        "    project=PROJECT_ID,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "a76e7c721b88"
+      },
+      "outputs": [],
+      "source": [
+        "model = aiplatform.Model(model_name=MODEL_RESOURCE_NAME)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "d06714302f81"
+      },
+      "outputs": [],
+      "source": [
+        "model.deploy(\n",
+        "    endpoint=endpoint,\n",
+        "    deployed_model_display_name=MODEL_DISPLAY_NAME,\n",
+        "    machine_type=MACHINE_TYPE,\n",
+        "    traffic_percentage=100,\n",
+        "    min_replica_count=1,\n",
+        "    max_replica_count=1,\n",
+        "    accelerator_type=None,\n",
+        "    accelerator_count=None,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "5b7eca31d80e"
+      },
+      "outputs": [],
+      "source": [
+        "endpoint.predict(instances={\"instances\": [\"A news headline to be classified\"]})"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "create_job.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

community-content/sklearn_text_classification_from_script_using_vertex_sdk/requirements.txt

@@ -0,0 +1,2 @@
+google-cloud-aiplatform
+ipykernel

community-content/sklearn_text_classification_from_script_using_vertex_sdk/task.py

@@ -0,0 +1,167 @@
+import argparse
+import logging
+import os
+import pickle
+import zipfile
+from typing import List, Tuple
+
+import pandas as pd
+import wget
+from google.cloud import storage
+from google.cloud.logging import Client as LogClient
+from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
+from sklearn.model_selection import train_test_split
+from sklearn.naive_bayes import MultinomialNB
+from sklearn.pipeline import Pipeline
+
+
+def download_dataset_from_url(url: str) -> pd.DataFrame:
+    """Downloads and unzips the dataset from `url` and reads it with pandas.
+
+    Args:
+        url (str, optional): URL to the dataset.
+    """
+
+    zip_filepath = wget.download(url, out=".")
+
+    with zipfile.ZipFile(zip_filepath, "r") as zf:
+        zf.extract(path=".", member="newsCorpora.csv")
+
+    COLUMN_NAMES = ["id", "title", "url", "publisher",
+                    "category", "story", "hostname", "timestamp"]
+
+    return pd.read_csv(
+        "newsCorpora.csv", delimiter="\t", names=COLUMN_NAMES, index_col=0
+    )
+
+
+def get_train_test_data(dataframe: pd.DataFrame, test_size: float = 0.2
+) -> Tuple[List, List, List, List]:
+    """Splits the news dataset into train and test features and labels.
+
+    Args:
+        news (pd.DataFrame): The dataset as pandas DataFrame.
+        test_size (float): The size in percent of the test data.
+
+    Returns:
+        Tuple[List, List, List, List]: Tuple with train and test data
+    """
+
+    train, test = train_test_split(dataframe, test_size=test_size)
+
+    x_train, y_train = train["title"].values, train["category"].values
+    x_test, y_test = test["title"].values, test["category"].values
+
+    return x_train, y_train, x_test, y_test
+
+
+def export_model_to_gcs(fitted_pipeline: Pipeline, gcs_uri: str) -> str:
+    """Exports trained pipeline to GCS
+
+    Parameters:
+            fitted_pipeline (sklearn.pipelines.Pipeline): the Pipeline object
+                with data already fitted (trained pipeline object).
+            gcs_uri (str): GCS path to store the trained pipeline
+                i.e gs://example_bucket/training-job.
+    Returns:
+            export_path (str): Model GCS location
+    """
+
+    artifact_filename = 'model.pkl'
+
+    # Save model artifact to local filesystem (doesn't persist)
+    local_path = artifact_filename
+    with open(local_path, 'wb') as model_file:
+        pickle.dump(fitted_pipeline, model_file)
+
+    # Upload model artifact to Cloud Storage
+    storage_path = os.path.join(gcs_uri, artifact_filename)
+    blob = storage.blob.Blob.from_string(storage_path, client=storage.Client())
+    blob.upload_from_filename(local_path)
+
+
+def export_evaluation_report_to_gcs(report: str, gcs_uri: str) -> None:
+    """
+    Exports training job report to GCS
+
+    Parameters:
+        report (str): Full report in text to sent to GCS
+        gcs_uri (str): GCS path to store the report
+            i.e gs://example_bucket/training-job
+    """
+
+    artifact_filename = 'report.txt'
+
+    # Upload model artifact to Cloud Storage
+    storage_path = os.path.join(gcs_uri, artifact_filename)
+    blob = storage.blob.Blob.from_string(storage_path, client=storage.Client())
+    blob.upload_from_string(report)
+
+
+def train_and_score(X_train: List, y_train: List, X_test: List, y_test: List
+) -> Tuple[Pipeline, float]:
+    """Trains and cross-validates a text classifier pipeline.
+
+    Args:
+        X_train (List): Train features as list of strings.
+        y_train (List): Train labels as list of strings.
+        X_test (List): Test labels as list of strings.
+        y_test (List): Test labels as list of strings.
+
+    Returns:
+        Tuple[Pipeline, float]: Fitted pipeline and mean accuracy.
+    """
+
+    pipeline = Pipeline([
+                ("vectorizer", CountVectorizer()),
+                ("tfidf", TfidfTransformer()),
+                ("naivebayes", MultinomialNB()),
+        ])
+
+    pipeline.fit(X_train, y_train)
+    score = pipeline.score(X_test, y_test)
+
+    return pipeline, score
+
+
+# Define all the command line arguments your model can accept for training
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument(
+        "--dataset_url",
+        help="Download url for the training data.",
+        type=str
+    )
+
+    parser.add_argument(
+        "--project_id",
+        help="GCP project id for cloud logging.",
+        type=str
+    )
+
+    args = parser.parse_args()
+    arguments = args.__dict__
+
+    # set up the GCP logger
+    client = LogClient(project=arguments["project_id"])
+    client.setup_logging(log_level=logging.INFO)
+    logging.info("Starting custom training job.")
+
+    # download the data from url
+    logging.info("Downloading training data from: {}".format(arguments["dataset_url"]))
+    dataframe = download_dataset_from_url(arguments["dataset_url"])
+    train_test_data = get_train_test_data(dataframe)
+
+    # train and cross validate
+    logging.info("Training started ...")
+    model, score = train_and_score(*train_test_data)
+    logging.info(f"Training completed with model score: {score}")
+
+    # export model to gcs
+    _gcs_uri = os.environ["AIP_MODEL_DIR"]
+    logging.info("Exporting model artifacts ...")
+    export_model_to_gcs(model, _gcs_uri)
+    export_evaluation_report_to_gcs(str(score), _gcs_uri)
+    logging.info(f"Exported model artifacts to GCS bucket: {_gcs_uri}")

community-content/tf_agents_bandits_movie_recommendation_with_kfp_and_vertex_sdk/mlops_pipeline_tf_agents_bandits_movie_recommendation/src/logger/requirements.txt

@@ -1,3 +1,3 @@
 google-cloud-bigquery==2.20.0
-tensorflow==2.5.0
+tensorflow==2.5.2
 tf-agents==0.8.0

community-content/tf_agents_bandits_movie_recommendation_with_kfp_and_vertex_sdk/mlops_pipeline_tf_agents_bandits_movie_recommendation/src/prediction_container/requirements.txt

@@ -1,2 +1,2 @@
 tf-agents==0.8.0
-tensorflow==2.5.0
+tensorflow==2.5.2

community-content/tf_keras_image_classification_distributed_multi_worker_with_vertex_sdk/multi_worker_vertex_training_on_cpu_with_custom_container.ipynb

@@ -1,423 +1,363 @@
 {
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "# Copyright 2021 Google LLC\n",
-    "#\n",
-    "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
-    "# you may not use this file except in compliance with the License.\n",
-    "# You may obtain a copy of the License at\n",
-    "#\n",
-    "#     https://www.apache.org/licenses/LICENSE-2.0\n",
-    "#\n",
-    "# Unless required by applicable law or agreed to in writing, software\n",
-    "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
-    "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
-    "# See the License for the specific language governing permissions and\n",
-    "# limitations under the License."
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "a6b56b1c7b76"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2021 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ac14831de120"
+      },
+      "source": [
+        "# TF-Keras Image Classification Distributed Multi-Worker Training on CPU using Vertex Training with Custom Container"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "1f2615ea7930"
+      },
+      "source": [
+        "<table align=\"left\">\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/community-content/tf_keras_image_classification_distributed_multi_worker_with_vertex_sdk/multi_worker_vertex_training_on_cpu_with_custom_container.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "03d216c7f7b1"
+      },
+      "source": [
+        "## Setup"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "c5ac73516218"
+      },
+      "outputs": [],
+      "source": [
+        "PROJECT_ID = \"YOUR PROJECT ID\"\n",
+        "BUCKET_NAME = \"gs://YOUR BUCKET NAME\"\n",
+        "REGION = \"YOUR REGION\"\n",
+        "SERVICE_ACCOUNT = \"YOUR SERVICE ACCOUNT\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b9b9229d7bc4"
+      },
+      "outputs": [],
+      "source": [
+        "content_name = \"tf-keras-img-cls-dist-multi-worker-cpu-cust-cont\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "070e602ded80"
+      },
+      "source": [
+        "## Vertex Training using Vertex SDK and Custom Container"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "99b9c8546a46"
+      },
+      "source": [
+        "### Build Custom Container"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "f5196e294db6"
+      },
+      "outputs": [],
+      "source": [
+        "hostname = \"gcr.io\"\n",
+        "image_name = content_name\n",
+        "tag = \"latest\"\n",
+        "\n",
+        "custom_container_image_uri = f\"{hostname}/{PROJECT_ID}/{image_name}:{tag}\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "723478af31e6"
+      },
+      "outputs": [],
+      "source": [
+        "! cd trainer && docker build -t $custom_container_image_uri -f cpu.Dockerfile ."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "66ac893a871b"
+      },
+      "outputs": [],
+      "source": [
+        "! docker run --rm $custom_container_image_uri --epochs 2 --local-mode"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "d131ee61bfca"
+      },
+      "outputs": [],
+      "source": [
+        "! docker push $custom_container_image_uri"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "bc5682081ee8"
+      },
+      "outputs": [],
+      "source": [
+        "! gcloud container images list --repository $hostname/$PROJECT_ID"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8b9c18d51ddf"
+      },
+      "source": [
+        "### Initialize Vertex SDK"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cfee2f165779"
+      },
+      "outputs": [],
+      "source": [
+        "! pip install -r requirements.txt"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b542a33f1782"
+      },
+      "outputs": [],
+      "source": [
+        "from google.cloud import aiplatform\n",
+        "\n",
+        "aiplatform.init(\n",
+        "    project=PROJECT_ID,\n",
+        "    staging_bucket=BUCKET_NAME,\n",
+        "    location=REGION,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "a0f25261a58c"
+      },
+      "source": [
+        "### Create a Vertex Tensorboard Instance"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "abacc13094c9"
+      },
+      "outputs": [],
+      "source": [
+        "tensorboard = aiplatform.Tensorboard.create(\n",
+        "    display_name=content_name,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "77ca3177b71e"
+      },
+      "source": [
+        "#### Option: Use a Previously Created Vertex Tensorboard Instance\n",
+        "\n",
+        "```\n",
+        "tensorboard_name = \"Your Tensorboard Resource Name or Tensorboard ID\"\n",
+        "tensorboard = aiplatform.Tensorboard(tensorboard_name=tensorboard_name)\n",
+        "```"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "f1733451a790"
+      },
+      "source": [
+        "### Run a Vertex SDK CustomContainerTrainingJob"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "0d5e38fade0a"
+      },
+      "outputs": [],
+      "source": [
+        "display_name = content_name\n",
+        "gcs_output_uri_prefix = f\"{BUCKET_NAME}/{display_name}\"\n",
+        "\n",
+        "replica_count = 4\n",
+        "machine_type = \"n1-standard-4\"\n",
+        "\n",
+        "container_args = [\n",
+        "    \"--epochs\",\n",
+        "    \"50\",\n",
+        "    \"--batch-size\",\n",
+        "    \"32\",\n",
+        "]"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "23ceccc746f5"
+      },
+      "outputs": [],
+      "source": [
+        "custom_container_training_job = aiplatform.CustomContainerTrainingJob(\n",
+        "    display_name=display_name,\n",
+        "    container_uri=custom_container_image_uri,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "469599f8b676"
+      },
+      "outputs": [],
+      "source": [
+        "custom_container_training_job.run(\n",
+        "    args=container_args,\n",
+        "    base_output_dir=gcs_output_uri_prefix,\n",
+        "    replica_count=replica_count,\n",
+        "    machine_type=machine_type,\n",
+        "    tensorboard=tensorboard.resource_name,\n",
+        "    service_account=SERVICE_ACCOUNT,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "e35aa8a39a11"
+      },
+      "outputs": [],
+      "source": [
+        "print(f\"Custom Training Job Name: {custom_container_training_job.resource_name}\")\n",
+        "print(f\"GCS Output URI Prefix: {gcs_output_uri_prefix}\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "40118a4bb0de"
+      },
+      "source": [
+        "### Training Output Artifact"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6d414dfc4585"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil ls $gcs_output_uri_prefix"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "77cd165390f9"
+      },
+      "source": [
+        "## Clean Up Artifact"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9b73b8cb646d"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil rm -rf $gcs_output_uri_prefix"
+      ]
     }
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "# TF-Keras Image Classification Distributed Multi-Worker Training on CPU using Vertex Training with Custom Container"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "<table align=\"left\">\n",
-    "  <td>\n",
-    "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/community-content/tf_keras_image_classification_distributed_multi_worker_with_vertex_sdk/multi_worker_vertex_training_on_cpu_with_custom_container.ipynb\">\n",
-    "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
-    "      View on GitHub\n",
-    "    </a>\n",
-    "  </td>\n",
-    "</table>"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "## Setup"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "PROJECT_ID = \"YOUR PROJECT ID\"\n",
-    "BUCKET_NAME = \"gs://YOUR BUCKET NAME\"\n",
-    "REGION = \"YOUR REGION\"\n",
-    "SERVICE_ACCOUNT = \"YOUR SERVICE ACCOUNT\""
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
+  ],
+  "metadata": {
+    "colab": {
+      "name": "multi_worker_vertex_training_on_cpu_with_custom_container.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
     }
-   }
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "content_name = \"tf-keras-img-cls-dist-multi-worker-cpu-cust-cont\""
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "## Vertex Training using Vertex SDK and Custom Container"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "### Build Custom Container"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "hostname = \"gcr.io\"\n",
-    "image_name = content_name\n",
-    "tag = \"latest\"\n",
-    "\n",
-    "custom_container_image_uri=f\"{hostname}/{PROJECT_ID}/{image_name}:{tag}\""
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "! cd trainer && docker build -t $custom_container_image_uri -f cpu.Dockerfile ."
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "! docker run --rm $custom_container_image_uri --epochs 2 --local-mode"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "! docker push $custom_container_image_uri"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "! gcloud container images list --repository $hostname/$PROJECT_ID"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "### Initialize Vertex SDK"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "! pip install -r requirements.txt"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "from google.cloud import aiplatform\n",
-    "\n",
-    "aiplatform.init(\n",
-    "    project=PROJECT_ID,\n",
-    "    staging_bucket=BUCKET_NAME,\n",
-    "    location=REGION,\n",
-    ")"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "### Create a Vertex Tensorboard Instance"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "tensorboard = aiplatform.Tensorboard.create(\n",
-    "    display_name=content_name,\n",
-    ")"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "#### Option: Use a Previously Created Vertex Tensorboard Instance\n",
-    "\n",
-    "```\n",
-    "tensorboard_name = \"Your Tensorboard Resource Name or Tensorboard ID\"\n",
-    "tensorboard = aiplatform.Tensorboard(tensorboard_name=tensorboard_name)\n",
-    "```"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%% md\n"
-    }
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "### Run a Vertex SDK CustomContainerTrainingJob"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "display_name = content_name\n",
-    "gcs_output_uri_prefix = f\"{BUCKET_NAME}/{display_name}\"\n",
-    "\n",
-    "replica_count = 4\n",
-    "machine_type = \"n1-standard-4\"\n",
-    "\n",
-    "container_args = [\n",
-    "    '--epochs', '50',\n",
-    "    '--batch-size', '32',\n",
-    "]"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "custom_container_training_job = aiplatform.CustomContainerTrainingJob(\n",
-    "    display_name=display_name,\n",
-    "    container_uri=custom_container_image_uri,\n",
-    ")"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "custom_container_training_job.run(\n",
-    "    args=container_args,\n",
-    "    base_output_dir=gcs_output_uri_prefix,\n",
-    "    machine_type=machine_type,\n",
-    "    tensorboard=tensorboard.resource_name,\n",
-    "    service_account=SERVICE_ACCOUNT,\n",
-    ")"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "print(f'Custom Training Job Name: {custom_container_training_job.resource_name}')\n",
-    "print(f'GCS Output URI Prefix: {gcs_output_uri_prefix}')"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "### Training Output Artifact"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "! gsutil ls $gcs_output_uri_prefix"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "## Clean Up Artifact"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "! gsutil rm -rf $gcs_output_uri_prefix"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 2
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython2",
-   "version": "2.7.6"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 0
-}
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

community-content/tf_keras_image_classification_distributed_multi_worker_with_vertex_sdk/multi_worker_vertex_training_on_gpu_with_custom_container.ipynb

@@ -1,427 +1,367 @@
 {
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "# Copyright 2021 Google LLC\n",
-    "#\n",
-    "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
-    "# you may not use this file except in compliance with the License.\n",
-    "# You may obtain a copy of the License at\n",
-    "#\n",
-    "#     https://www.apache.org/licenses/LICENSE-2.0\n",
-    "#\n",
-    "# Unless required by applicable law or agreed to in writing, software\n",
-    "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
-    "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
-    "# See the License for the specific language governing permissions and\n",
-    "# limitations under the License."
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "a6b56b1c7b76"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2021 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e8fc31cbf9f6"
+      },
+      "source": [
+        "# TF-Keras Image Classification Distributed Multi-Worker Training on GPU using Vertex Training with Custom Container"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5825c076e0f8"
+      },
+      "source": [
+        "<table align=\"left\">\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/community-content/tf_keras_image_classification_distributed_multi_worker_with_vertex_sdk/multi_worker_vertex_training_on_gpu_with_custom_container.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "03d216c7f7b1"
+      },
+      "source": [
+        "## Setup"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "c5ac73516218"
+      },
+      "outputs": [],
+      "source": [
+        "PROJECT_ID = \"YOUR PROJECT ID\"\n",
+        "BUCKET_NAME = \"gs://YOUR BUCKET NAME\"\n",
+        "REGION = \"YOUR REGION\"\n",
+        "SERVICE_ACCOUNT = \"YOUR SERVICE ACCOUNT\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "a5f0c3700b8e"
+      },
+      "outputs": [],
+      "source": [
+        "content_name = \"tf-keras-img-cls-dist-multi-worker-gpu-cust-cont\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "070e602ded80"
+      },
+      "source": [
+        "## Vertex Training using Vertex SDK and Custom Container"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "99b9c8546a46"
+      },
+      "source": [
+        "### Build Custom Container"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "f5196e294db6"
+      },
+      "outputs": [],
+      "source": [
+        "hostname = \"gcr.io\"\n",
+        "image_name = content_name\n",
+        "tag = \"latest\"\n",
+        "\n",
+        "custom_container_image_uri = f\"{hostname}/{PROJECT_ID}/{image_name}:{tag}\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "c24b9a97df06"
+      },
+      "outputs": [],
+      "source": [
+        "! cd trainer && docker build -t $custom_container_image_uri -f gpu.Dockerfile ."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "66ac893a871b"
+      },
+      "outputs": [],
+      "source": [
+        "! docker run --rm $custom_container_image_uri --epochs 2 --local-mode"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "d131ee61bfca"
+      },
+      "outputs": [],
+      "source": [
+        "! docker push $custom_container_image_uri"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "bc5682081ee8"
+      },
+      "outputs": [],
+      "source": [
+        "! gcloud container images list --repository $hostname/$PROJECT_ID"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8b9c18d51ddf"
+      },
+      "source": [
+        "### Initialize Vertex SDK"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cfee2f165779"
+      },
+      "outputs": [],
+      "source": [
+        "! pip install -r requirements.txt"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b542a33f1782"
+      },
+      "outputs": [],
+      "source": [
+        "from google.cloud import aiplatform\n",
+        "\n",
+        "aiplatform.init(\n",
+        "    project=PROJECT_ID,\n",
+        "    staging_bucket=BUCKET_NAME,\n",
+        "    location=REGION,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "a0f25261a58c"
+      },
+      "source": [
+        "### Create a Vertex Tensorboard Instance"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "abacc13094c9"
+      },
+      "outputs": [],
+      "source": [
+        "tensorboard = aiplatform.Tensorboard.create(\n",
+        "    display_name=content_name,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "77ca3177b71e"
+      },
+      "source": [
+        "#### Option: Use a Previously Created Vertex Tensorboard Instance\n",
+        "\n",
+        "```\n",
+        "tensorboard_name = \"Your Tensorboard Resource Name or Tensorboard ID\"\n",
+        "tensorboard = aiplatform.Tensorboard(tensorboard_name=tensorboard_name)\n",
+        "```"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "f1733451a790"
+      },
+      "source": [
+        "### Run a Vertex SDK CustomContainerTrainingJob"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3e29b50a65d3"
+      },
+      "outputs": [],
+      "source": [
+        "display_name = content_name\n",
+        "gcs_output_uri_prefix = f\"{BUCKET_NAME}/{display_name}\"\n",
+        "\n",
+        "replica_count = 4\n",
+        "machine_type = \"n1-standard-4\"\n",
+        "accelerator_count = 1\n",
+        "accelerator_type = \"NVIDIA_TESLA_K80\"\n",
+        "\n",
+        "container_args = [\n",
+        "    \"--epochs\",\n",
+        "    \"50\",\n",
+        "    \"--batch-size\",\n",
+        "    \"32\",\n",
+        "]"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "23ceccc746f5"
+      },
+      "outputs": [],
+      "source": [
+        "custom_container_training_job = aiplatform.CustomContainerTrainingJob(\n",
+        "    display_name=display_name,\n",
+        "    container_uri=custom_container_image_uri,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "239e58fe141d"
+      },
+      "outputs": [],
+      "source": [
+        "custom_container_training_job.run(\n",
+        "    args=container_args,\n",
+        "    base_output_dir=gcs_output_uri_prefix,\n",
+        "    replica_count=replica_count,\n",
+        "    machine_type=machine_type,\n",
+        "    accelerator_type=accelerator_type,\n",
+        "    accelerator_count=accelerator_count,\n",
+        "    tensorboard=tensorboard.resource_name,\n",
+        "    service_account=SERVICE_ACCOUNT,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "e35aa8a39a11"
+      },
+      "outputs": [],
+      "source": [
+        "print(f\"Custom Training Job Name: {custom_container_training_job.resource_name}\")\n",
+        "print(f\"GCS Output URI Prefix: {gcs_output_uri_prefix}\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "40118a4bb0de"
+      },
+      "source": [
+        "### Training Output Artifact"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6d414dfc4585"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil ls $gcs_output_uri_prefix"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "77cd165390f9"
+      },
+      "source": [
+        "## Clean Up Artifact"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9b73b8cb646d"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil rm -rf $gcs_output_uri_prefix"
+      ]
     }
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "# TF-Keras Image Classification Distributed Multi-Worker Training on GPU using Vertex Training with Custom Container"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "<table align=\"left\">\n",
-    "  <td>\n",
-    "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/community-content/tf_keras_image_classification_distributed_multi_worker_with_vertex_sdk/multi_worker_vertex_training_on_gpu_with_custom_container.ipynb\">\n",
-    "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
-    "      View on GitHub\n",
-    "    </a>\n",
-    "  </td>\n",
-    "</table>"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "## Setup"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "PROJECT_ID = \"YOUR PROJECT ID\"\n",
-    "BUCKET_NAME = \"gs://YOUR BUCKET NAME\"\n",
-    "REGION = \"YOUR REGION\"\n",
-    "SERVICE_ACCOUNT = \"YOUR SERVICE ACCOUNT\""
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
+  ],
+  "metadata": {
+    "colab": {
+      "name": "multi_worker_vertex_training_on_gpu_with_custom_container.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
     }
-   }
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "content_name = \"tf-keras-img-cls-dist-multi-worker-gpu-cust-cont\""
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "## Vertex Training using Vertex SDK and Custom Container"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "### Build Custom Container"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "hostname = \"gcr.io\"\n",
-    "image_name = content_name\n",
-    "tag = \"latest\"\n",
-    "\n",
-    "custom_container_image_uri=f\"{hostname}/{PROJECT_ID}/{image_name}:{tag}\""
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "! cd trainer && docker build -t $custom_container_image_uri -f gpu.Dockerfile ."
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "! docker run --rm $custom_container_image_uri --epochs 2 --local-mode"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "! docker push $custom_container_image_uri"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "! gcloud container images list --repository $hostname/$PROJECT_ID"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "### Initialize Vertex SDK"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "! pip install -r requirements.txt"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "from google.cloud import aiplatform\n",
-    "\n",
-    "aiplatform.init(\n",
-    "    project=PROJECT_ID,\n",
-    "    staging_bucket=BUCKET_NAME,\n",
-    "    location=REGION,\n",
-    ")"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "### Create a Vertex Tensorboard Instance"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "tensorboard = aiplatform.Tensorboard.create(\n",
-    "    display_name=content_name,\n",
-    ")"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "#### Option: Use a Previously Created Vertex Tensorboard Instance\n",
-    "\n",
-    "```\n",
-    "tensorboard_name = \"Your Tensorboard Resource Name or Tensorboard ID\"\n",
-    "tensorboard = aiplatform.Tensorboard(tensorboard_name=tensorboard_name)\n",
-    "```"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%% md\n"
-    }
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "### Run a Vertex SDK CustomContainerTrainingJob"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "display_name = content_name\n",
-    "gcs_output_uri_prefix = f\"{BUCKET_NAME}/{display_name}\"\n",
-    "\n",
-    "replica_count = 4\n",
-    "machine_type = \"n1-standard-4\"\n",
-    "accelerator_count = 1\n",
-    "accelerator_type = \"NVIDIA_TESLA_K80\"\n",
-    "\n",
-    "container_args = [\n",
-    "    '--epochs', '50',\n",
-    "    '--batch-size', '32',\n",
-    "]"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "custom_container_training_job = aiplatform.CustomContainerTrainingJob(\n",
-    "    display_name=display_name,\n",
-    "    container_uri=custom_container_image_uri,\n",
-    ")"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "custom_container_training_job.run(\n",
-    "    args=container_args,\n",
-    "    base_output_dir=gcs_output_uri_prefix,\n",
-    "    machine_type=machine_type,\n",
-    "    accelerator_type=accelerator_type,\n",
-    "    accelerator_count=accelerator_count,\n",
-    "    tensorboard=tensorboard.resource_name,\n",
-    "    service_account=SERVICE_ACCOUNT,\n",
-    ")"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "print(f'Custom Training Job Name: {custom_container_training_job.resource_name}')\n",
-    "print(f'GCS Output URI Prefix: {gcs_output_uri_prefix}')"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "### Training Output Artifact"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "! gsutil ls $gcs_output_uri_prefix"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "## Clean Up Artifact"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "! gsutil rm -rf $gcs_output_uri_prefix"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 2
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython2",
-   "version": "2.7.6"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 0
-}
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

community-content/tf_keras_image_classification_distributed_multi_worker_with_vertex_sdk/trainer/requirements.txt

@@ -1 +1 @@
-tensorflow==2.4.1
+tensorflow==2.5.2

community-content/tf_keras_image_classification_distributed_multi_worker_with_vertex_sdk/trainer/task.py

@@ -123,7 +123,16 @@ def main():
 
   model_dir = args.model_dir or local_model_dir
   tensorboard_log_dir = args.tensorboard_log_dir or local_tensorboard_log_dir
-  checkpoint_dir = local_checkpoint_dir
+  checkpoint_dir = args.checkpoint_dir or local_checkpoint_dir
+
+  gs_prefix = 'gs://'
+  gcsfuse_prefix = '/gcs/'
+  if model_dir and model_dir.startswith(gs_prefix):
+    model_dir = model_dir.replace(gs_prefix, gcsfuse_prefix)
+  if tensorboard_log_dir and tensorboard_log_dir.startswith(gs_prefix):
+    tensorboard_log_dir = tensorboard_log_dir.replace(gs_prefix, gcsfuse_prefix)
+  if checkpoint_dir and checkpoint_dir.startswith(gs_prefix):
+    checkpoint_dir = checkpoint_dir.replace(gs_prefix, gcsfuse_prefix)
 
   num_worker, task_type, task_id = distribution_utils.setup()
   print(f'task_type: {task_type}, '

community-content/tf_keras_text_classification_distributed_single_worker_gpus_with_gcloud_local_run_and_vertex_sdk/trainer/requirements.txt

@@ -1 +1 @@
-tensorflow==2.5.1
+tensorflow==2.5.2

community-content/tf_keras_text_classification_distributed_single_worker_gpus_with_gcloud_local_run_and_vertex_sdk/trainer/task.py

@@ -21,7 +21,6 @@ from tensorflow.keras import layers, losses
 from tensorflow.keras.layers.experimental.preprocessing import TextVectorization
 
 import distribution_utils
-import utils
 
 VOCAB_SIZE = 10000
 MAX_SEQUENCE_LENGTH = 250
@@ -175,13 +174,18 @@ def main():
   local_checkpoint_dir = './tmp/checkpoints'
   local_tensorboard_log_dir = './tmp/logs'
 
-  #TODO: update when gcsfuse ready
-  gcsfuse_ready = False
-
   model_dir = args.model_dir or local_model_dir
-  checkpoint_dir = (gcsfuse_ready and
-                         args.checkpoint_dir) or local_checkpoint_dir
   tensorboard_log_dir = args.tensorboard_log_dir or local_tensorboard_log_dir
+  checkpoint_dir = args.checkpoint_dir or local_checkpoint_dir
+
+  gs_prefix = 'gs://'
+  gcsfuse_prefix = '/gcs/'
+  if model_dir and model_dir.startswith(gs_prefix):
+    model_dir = model_dir.replace(gs_prefix, gcsfuse_prefix)
+  if tensorboard_log_dir and tensorboard_log_dir.startswith(gs_prefix):
+    tensorboard_log_dir = tensorboard_log_dir.replace(gs_prefix, gcsfuse_prefix)
+  if checkpoint_dir and checkpoint_dir.startswith(gs_prefix):
+    checkpoint_dir = checkpoint_dir.replace(gs_prefix, gcsfuse_prefix)
 
   class_names = ['csharp', 'java', 'javascript', 'python']
   class_indices = dict(zip(class_names, range(len(class_names))))
@@ -278,13 +282,6 @@ def main():
   print(f'Tensorboard logs are saved to: {tensorboard_log_dir}')
 
   print(f'Checkpoints are saved to: {checkpoint_dir}')
-  utils.gcs_upload(
-      dir=checkpoint_dir,
-      local_dir=local_checkpoint_dir,
-      gcs_dir=args.checkpoint_dir,
-      gcsfuse_ready=gcsfuse_ready,
-      local_mode=args.local_mode
-  )
 
   return
 

community-content/tf_keras_text_classification_distributed_single_worker_gpus_with_gcloud_local_run_and_vertex_sdk/trainer/utils.py

@@ -1,31 +0,0 @@
-# Copyright 2021 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-import shutil
-import subprocess
-
-def makedirs(model_dir):
-  if os.path.exists(model_dir) and os.path.isdir(model_dir):
-    shutil.rmtree(model_dir)
-  os.makedirs(model_dir)
-  return
-
-def gcs_upload(dir, local_dir, gcs_dir, gcsfuse_ready, local_mode):
-  if not local_mode and dir == local_dir and not gcsfuse_ready:
-    subprocess.run(['gsutil', 'cp', '-r',
-                    local_dir,
-                    os.path.dirname(gcs_dir)])
-    print(f'{local_dir} is uploaded to {gcs_dir}')
-  return

community-content/tf_keras_text_classification_distributed_single_worker_gpus_with_gcloud_local_run_and_vertex_sdk/vertex_training_with_local_mode_container.ipynb

@@ -1,437 +1,380 @@
 {
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "# Copyright 2021 Google LLC\n",
-    "#\n",
-    "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
-    "# you may not use this file except in compliance with the License.\n",
-    "# You may obtain a copy of the License at\n",
-    "#\n",
-    "#     https://www.apache.org/licenses/LICENSE-2.0\n",
-    "#\n",
-    "# Unless required by applicable law or agreed to in writing, software\n",
-    "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
-    "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
-    "# See the License for the specific language governing permissions and\n",
-    "# limitations under the License."
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "a6b56b1c7b76"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2021 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "745ff8aea147"
+      },
+      "source": [
+        "# TF-Keras Text Classification Distributed Single Worker GPUs using Vertex Training with Local Mode Container"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "a20c6440f341"
+      },
+      "source": [
+        "<table align=\"left\">\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/community-content/tf_keras_text_classification_distributed_single_worker_gpus_with_gcloud_local_run_and_vertex_sdk/vertex_training_with_local_mode_container.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "03d216c7f7b1"
+      },
+      "source": [
+        "## Setup"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "c5ac73516218"
+      },
+      "outputs": [],
+      "source": [
+        "PROJECT_ID = \"YOUR PROJECT ID\"\n",
+        "BUCKET_NAME = \"gs://YOUR BUCKET NAME\"\n",
+        "REGION = \"YOUR REGION\"\n",
+        "SERVICE_ACCOUNT = \"YOUR SERVICE ACCOUNT\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "1c71d07625f5"
+      },
+      "outputs": [],
+      "source": [
+        "content_name = \"tf-keras-txt-cls-dist-single-worker-gpus-local-mode-cont\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "378392e0adb9"
+      },
+      "source": [
+        "## Local Training with Vertex Local Mode and Auto Packaging"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "05f8e4885312"
+      },
+      "outputs": [],
+      "source": [
+        "BASE_IMAGE_URI = \"us-docker.pkg.dev/vertex-ai/training/tf-gpu.2-5:latest\"\n",
+        "SCRIPT_PATH = \"trainer/task.py\"\n",
+        "OUTPUT_IMAGE_NAME = \"gcr.io/{}/{}:latest\".format(PROJECT_ID, content_name)\n",
+        "ARGS = \"--epochs 5 --batch-size 16 --local-mode\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b45a98212ef5"
+      },
+      "outputs": [],
+      "source": [
+        "! gcloud beta ai custom-jobs local-run \\\n",
+        "  --base-image=$BASE_IMAGE_URI \\\n",
+        "  --script=$SCRIPT_PATH \\\n",
+        "  --output-image-uri=$OUTPUT_IMAGE_NAME \\\n",
+        "  -- \\\n",
+        "  $ARGS"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "cbd77cb4193c"
+      },
+      "source": [
+        "## Vertex Training using Vertex SDK and Vertex Local Mode Container"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "c8b69c5e3dd4"
+      },
+      "source": [
+        "### Container Built by Vertex Local Mode"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "4869e0d1657b"
+      },
+      "outputs": [],
+      "source": [
+        "custom_container_image_uri = OUTPUT_IMAGE_NAME"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "d53296526bde"
+      },
+      "outputs": [],
+      "source": [
+        "! docker push $custom_container_image_uri"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "eb193581654a"
+      },
+      "outputs": [],
+      "source": [
+        "! gcloud container images list --repository \"gcr.io\"/$PROJECT_ID"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "f5b907c91ea6"
+      },
+      "source": [
+        "### Initialize Vertex SDK"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "fbfec07eaea4"
+      },
+      "outputs": [],
+      "source": [
+        "! pip install -r requirements.txt"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "8fcf24c75bcd"
+      },
+      "outputs": [],
+      "source": [
+        "from google.cloud import aiplatform\n",
+        "\n",
+        "aiplatform.init(\n",
+        "    project=PROJECT_ID,\n",
+        "    staging_bucket=BUCKET_NAME,\n",
+        "    location=REGION,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "b68b79268025"
+      },
+      "source": [
+        "### Create a Vertex Tensorboard Instance"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7577c86757ab"
+      },
+      "outputs": [],
+      "source": [
+        "tensorboard = aiplatform.Tensorboard.create(\n",
+        "    display_name=content_name,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2a327cb1722d"
+      },
+      "source": [
+        "#### Option: Use a Previously Created Vertex Tensorboard Instance\n",
+        "\n",
+        "```\n",
+        "tensorboard_name = \"Your Tensorboard Resource Name or Tensorboard ID\"\n",
+        "tensorboard = aiplatform.Tensorboard(tensorboard_name=tensorboard_name)\n",
+        "```"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2e1069a8f7e2"
+      },
+      "source": [
+        "### Run a Vertex SDK CustomContainerTrainingJob"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "d7f0043d3e9b"
+      },
+      "outputs": [],
+      "source": [
+        "display_name = content_name\n",
+        "gcs_output_uri_prefix = f\"{BUCKET_NAME}/{display_name}\"\n",
+        "\n",
+        "machine_type = \"n1-standard-8\"\n",
+        "accelerator_count = 4\n",
+        "accelerator_type = \"NVIDIA_TESLA_P100\"\n",
+        "\n",
+        "args = [\n",
+        "    \"--epochs\",\n",
+        "    \"100\",\n",
+        "    \"--batch-size\",\n",
+        "    \"128\",\n",
+        "    \"--num-gpus\",\n",
+        "    f\"{accelerator_count}\",\n",
+        "]"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "69c88c73bb5d"
+      },
+      "outputs": [],
+      "source": [
+        "custom_container_training_job = aiplatform.CustomContainerTrainingJob(\n",
+        "    display_name=display_name,\n",
+        "    container_uri=custom_container_image_uri,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "947dd755cdc8"
+      },
+      "outputs": [],
+      "source": [
+        "custom_container_training_job.run(\n",
+        "    args=args,\n",
+        "    base_output_dir=gcs_output_uri_prefix,\n",
+        "    machine_type=machine_type,\n",
+        "    accelerator_type=accelerator_type,\n",
+        "    accelerator_count=accelerator_count,\n",
+        "    tensorboard=tensorboard.resource_name,\n",
+        "    service_account=SERVICE_ACCOUNT,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "703e297f83fe"
+      },
+      "outputs": [],
+      "source": [
+        "print(f\"Custom Training Job Name: {custom_container_training_job.resource_name}\")\n",
+        "print(f\"GCS Output URI Prefix: {gcs_output_uri_prefix}\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d459452eca40"
+      },
+      "source": [
+        "### Training Output Artifact"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "d02a5890a4af"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil ls $gcs_output_uri_prefix"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "545fe0dbe7c4"
+      },
+      "source": [
+        "## Clean Up Artifact"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b86c0fe8028e"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil rm -rf $gcs_output_uri_prefix"
+      ]
     }
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "# TF-Keras Text Classification Distributed Single Worker GPUs using Vertex Training with Local Mode Container"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "<table align=\"left\">\n",
-    "  <td>\n",
-    "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/community-content/tf_keras_text_classification_distributed_single_worker_gpus_with_gcloud_local_run_and_vertex_sdk/vertex_training_with_local_mode_container.ipynb\">\n",
-    "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
-    "      View on GitHub\n",
-    "    </a>\n",
-    "  </td>\n",
-    "</table>"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "## Setup"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "PROJECT_ID = \"YOUR PROJECT ID\"\n",
-    "BUCKET_NAME = \"gs://YOUR BUCKET NAME\"\n",
-    "REGION = \"YOUR REGION\"\n",
-    "SERVICE_ACCOUNT = \"YOUR SERVICE ACCOUNT\""
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
+  ],
+  "metadata": {
+    "colab": {
+      "name": "vertex_training_with_local_mode_container.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
     }
-   }
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "content_name = \"tf-keras-txt-cls-dist-single-worker-gpus-local-mode-cont\""
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "## Local Training with Vertex Local Mode and Auto Packaging"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "BASE_IMAGE_URI = 'us-docker.pkg.dev/vertex-ai/training/tf-gpu.2-5:latest'\n",
-    "SCRIPT_PATH = 'trainer/task.py'\n",
-    "OUTPUT_IMAGE_NAME = 'gcr.io/{}/{}:latest'.format(PROJECT_ID, content_name)\n",
-    "ARGS = '--epochs 5 --batch-size 16 --local-mode'"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "! gcloud beta ai custom-jobs local-run \\\n",
-    "  --base-image=$BASE_IMAGE_URI \\\n",
-    "  --script=$SCRIPT_PATH \\\n",
-    "  --output-image-uri=$OUTPUT_IMAGE_NAME \\\n",
-    "  -- \\\n",
-    "  $ARGS"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "## Vertex Training using Vertex SDK and Vertex Local Mode Container"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "### Container Built by Vertex Local Mode"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "custom_container_image_uri = OUTPUT_IMAGE_NAME"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "! docker push $custom_container_image_uri"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "! gcloud container images list --repository \"gcr.io\"/$PROJECT_ID"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "### Initialize Vertex SDK"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "! pip install -r requirements.txt"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "from google.cloud import aiplatform\n",
-    "\n",
-    "aiplatform.init(\n",
-    "    project=PROJECT_ID,\n",
-    "    staging_bucket=BUCKET_NAME,\n",
-    "    location=REGION,\n",
-    ")"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "### Create a Vertex Tensorboard Instance"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "tensorboard = aiplatform.Tensorboard.create(\n",
-    "    display_name=content_name,\n",
-    ")"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "#### Option: Use a Previously Created Vertex Tensorboard Instance\n",
-    "\n",
-    "```\n",
-    "tensorboard_name = \"Your Tensorboard Resource Name or Tensorboard ID\"\n",
-    "tensorboard = aiplatform.Tensorboard(tensorboard_name=tensorboard_name)\n",
-    "```"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "### Run a Vertex SDK CustomContainerTrainingJob"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "display_name = content_name\n",
-    "gcs_output_uri_prefix = f\"{BUCKET_NAME}/{display_name}\"\n",
-    "\n",
-    "machine_type = \"n1-standard-8\"\n",
-    "accelerator_count = 4\n",
-    "accelerator_type = \"NVIDIA_TESLA_P100\"\n",
-    "\n",
-    "args = [\n",
-    "    '--epochs', '100',\n",
-    "    '--batch-size', '128',\n",
-    "    '--num-gpus', f'{accelerator_count}',\n",
-    "]"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "custom_container_training_job = aiplatform.CustomContainerTrainingJob(\n",
-    "    display_name=display_name,\n",
-    "    container_uri=custom_container_image_uri,\n",
-    ")"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "custom_container_training_job.run(\n",
-    "    args=args,\n",
-    "    base_output_dir=gcs_output_uri_prefix,\n",
-    "    machine_type=machine_type,\n",
-    "    accelerator_type=accelerator_type,\n",
-    "    accelerator_count=accelerator_count,\n",
-    "    tensorboard=tensorboard.resource_name,\n",
-    "    service_account=SERVICE_ACCOUNT,\n",
-    ")"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "print(f'Custom Training Job Name: {custom_container_training_job.resource_name}')\n",
-    "print(f'GCS Output URI Prefix: {gcs_output_uri_prefix}')"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "### Training Output Artifact"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "! gsutil ls $gcs_output_uri_prefix"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "## Clean Up Artifact"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "! gsutil rm -rf $gcs_output_uri_prefix"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 2
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython2",
-   "version": "2.7.6"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 0
-}
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

docs/CODEOWNERS

@@ -1,28 +0,0 @@
-# These owners will be the default owners for everything in
-# the repo. Unless a later match takes precedence,
-# Our Global Owners: @andrewferlitsch @aribray @dizcology @ivanmkc @morgandu @telpirion @vinnysenthil   
-*       @GoogleCloudPlatform/cloudml-samples-owners
-
-# Notebooks
-/notebooks					@ivanmkc
-
-# Official Notebooks
-
-notebooks/official/model_monitoring/model_monitoring.ipynb  @mco-gh
-notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb  @jialuzh
-notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-locally-trained-models.ipynb  @jialuzh
-notebooks/official/vizier/gapic-vizier-multi-objective-optimization.ipynb  @halio-g
-notebooks/official/feature_store/gapic-feature-store.ipynb  @protorganizer
-
-# Community Notebooks
-/notebooks/community/sdk/SDK_FBProphet_Forecasting_Online.ipynb			@brianchunkang
-
-
-# Community Content
-/community-content			@morgandu
-/community-content/tf_agents_bandits_movie_recommendation_with_kfp_and_vertex_sdk	@yinghsienwu
-/community-content/pytorch_text_classification_using_vertex_sdk_and_gcloud		@RajeshThallam @ultrons
-
-
-
-# Official Community Content

docs/code-of-conduct.md

@@ -1,93 +0,0 @@
-# Code of Conduct
-
-## Our Pledge
-
-In the interest of fostering an open and welcoming environment, we as
-contributors and maintainers pledge to making participation in our project and
-our community a harassment-free experience for everyone, regardless of age, body
-size, disability, ethnicity, gender identity and expression, level of
-experience, education, socio-economic status, nationality, personal appearance,
-race, religion, or sexual identity and orientation.
-
-## Our Standards
-
-Examples of behavior that contributes to creating a positive environment
-include:
-
-*   Using welcoming and inclusive language
-*   Being respectful of differing viewpoints and experiences
-*   Gracefully accepting constructive criticism
-*   Focusing on what is best for the community
-*   Showing empathy towards other community members
-
-Examples of unacceptable behavior by participants include:
-
-*   The use of sexualized language or imagery and unwelcome sexual attention or
-    advances
-*   Trolling, insulting/derogatory comments, and personal or political attacks
-*   Public or private harassment
-*   Publishing others' private information, such as a physical or electronic
-    address, without explicit permission
-*   Other conduct which could reasonably be considered inappropriate in a
-    professional setting
-
-## Our Responsibilities
-
-Project maintainers are responsible for clarifying the standards of acceptable
-behavior and are expected to take appropriate and fair corrective action in
-response to any instances of unacceptable behavior.
-
-Project maintainers have the right and responsibility to remove, edit, or reject
-comments, commits, code, wiki edits, issues, and other contributions that are
-not aligned to this Code of Conduct, or to ban temporarily or permanently any
-contributor for other behaviors that they deem inappropriate, threatening,
-offensive, or harmful.
-
-## Scope
-
-This Code of Conduct applies both within project spaces and in public spaces
-when an individual is representing the project or its community. Examples of
-representing a project or community include using an official project e-mail
-address, posting via an official social media account, or acting as an appointed
-representative at an online or offline event. Representation of a project may be
-further defined and clarified by project maintainers.
-
-This Code of Conduct also applies outside the project spaces when the Project
-Steward has a reasonable belief that an individual's behavior may have a
-negative impact on the project or its community.
-
-## Conflict Resolution
-
-We do not believe that all conflict is bad; healthy debate and disagreement
-often yield positive results. However, it is never okay to be disrespectful or
-to engage in behavior that violates the project’s code of conduct.
-
-If you see someone violating the code of conduct, you are encouraged to address
-the behavior directly with those involved. Many issues can be resolved quickly
-and easily, and this gives people more control over the outcome of their
-dispute. If you are unable to resolve the matter for any reason, or if the
-behavior is threatening or harassing, report it. We are dedicated to providing
-an environment where participants feel welcome and safe.
-
-Reports should be directed to *[PROJECT STEWARD NAME(s) AND EMAIL(s)]*, the
-Project Steward(s) for *[PROJECT NAME]*. It is the Project Steward’s duty to
-receive and address reported violations of the code of conduct. They will then
-work with a committee consisting of representatives from the Open Source
-Programs Office and the Google Open Source Strategy team. If for any reason you
-are uncomfortable reaching out to the Project Steward, please email
-opensource@google.com.
-
-We will investigate every complaint, but you may not receive a direct response.
-We will use our discretion in determining when and how to follow up on reported
-incidents, which may range from not taking action to permanent expulsion from
-the project and project-sponsored spaces. We will notify the accused of the
-report and provide them an opportunity to discuss it before any action is taken.
-The identity of the reporter will be omitted from the details of the report
-supplied to the accused. In potentially harmful situations, such as ongoing
-harassment or threats to anyone's safety, we may take action without notice.
-
-## Attribution
-
-This Code of Conduct is adapted from the Contributor Covenant, version 1.4,
-available at
-https://www.contributor-covenant.org/version/1/4/code-of-conduct.html

docs/contributing.md

@@ -1,28 +0,0 @@
-# How to Contribute
-
-We'd love to accept your patches and contributions to this project. There are
-just a few small guidelines you need to follow.
-
-## Contributor License Agreement
-
-Contributions to this project must be accompanied by a Contributor License
-Agreement. You (or your employer) retain the copyright to your contribution;
-this simply gives us permission to use and redistribute your contributions as
-part of the project. Head over to <https://cla.developers.google.com/> to see
-your current agreements on file or to sign a new one.
-
-You generally only need to submit a CLA once, so if you've already submitted one
-(even if it was for a different project), you probably don't need to do it
-again.
-
-## Code Reviews
-
-All submissions, including submissions by project members, require review. We
-use GitHub pull requests for this purpose. Consult
-[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more
-information on using pull requests.
-
-## Community Guidelines
-
-This project follows [Google's Open Source Community
-Guidelines](https://opensource.google/conduct/).

notebooks/community/CODEOWNERS

@@ -0,0 +1,18 @@
+# These owners will be the default owners for everything in
+# the repo. Unless a later match takes precedence,
+# @global-owner1 and @global-owner2 will be requested for
+# review when someone opens a pull request.
+
+/sdk/sdk_* @aferlitsch
+/gapic @aferlitsch
+/ml_ops @aferlitsch
+/model_monitoring/* @mco
+/structured_data/rapid_prototyping_* @rafael-carvalho
+
+/managed_notebooks/ @notebooks-team
+/sdk/SDK_FBProphet_Forecasting_Online.ipynb			@brianchunkang
+/pipelines/google_cloud_pipeline_components_TPU_model_train_upload_deploy.ipynb @brianchunkang
+/sdk/SDK_AutoML_Forecasting_Model_Training_Example.ipynb @thehardikv
+/sdk/sdk_automl_forecasting_evaluating_a_model.ipynb @thehardikv
+/matching_engine @yinghsienwu
+/neo4j @benofben @htappen

notebooks/community/gapic/custom/showcase_custom_image_classification_batch.ipynb

@@ -480,8 +480,7 @@
         "\n",
         "from google.cloud.aiplatform import gapic as aip\n",
         "from google.protobuf import json_format\n",
-        "from google.protobuf.json_format import MessageToJson, ParseDict\n",
-        "from google.protobuf.struct_pb2 import Struct, Value"
+        "from google.protobuf.struct_pb2 import Value"
       ]
     },
     {
@@ -1496,9 +1495,8 @@
         "\n",
         "When you send a prediction or explanation request, the content of the request is base 64 decoded into a Tensorflow string (`tf.string`), which is passed to the serving function (`serving_fn`). The serving function preprocesses the `tf.string` into raw (uncompressed) numpy bytes (`preprocess_fn`) to match the input requirements of the model:\n",
         "- `io.decode_jpeg`- Decompresses the JPG image which is returned as a Tensorflow tensor with three channels (RGB).\n",
-        "- `image.convert_image_dtype` - Changes integer pixel values to float 32.\n",
+        "- `image.convert_image_dtype` - Changes integer pixel values to float 32, and rescales pixel data between 0 and 1.\n",
         "- `image.resize` - Resizes the image to match the input shape for the model.\n",
-        "- `resized / 255.0` - Rescales (normalization) the pixel data between 0 and 1.\n",
         "\n",
         "At this point, the data can be passed to the model (`m_call`)."
       ]
@@ -1518,8 +1516,7 @@
         "    decoded = tf.io.decode_jpeg(bytes_input, channels=3)\n",
         "    decoded = tf.image.convert_image_dtype(decoded, tf.float32)\n",
         "    resized = tf.image.resize(decoded, size=(32, 32))\n",
-        "    rescale = tf.cast(resized / 255.0, tf.float32)\n",
-        "    return rescale\n",
+        "    return resized\n",
         "\n",
         "\n",
         "@tf.function(input_signature=[tf.TensorSpec([None], tf.string)])\n",

notebooks/community/gapic/custom/showcase_custom_image_classification_batch_explain.ipynb

@@ -505,8 +505,7 @@
         "\n",
         "import google.cloud.aiplatform_v1beta1 as aip\n",
         "from google.protobuf import json_format\n",
-        "from google.protobuf.json_format import MessageToJson, ParseDict\n",
-        "from google.protobuf.struct_pb2 import Struct, Value"
+        "from google.protobuf.struct_pb2 import Value"
       ]
     },
     {
@@ -1521,9 +1520,8 @@
         "\n",
         "When you send a prediction or explanation request, the content of the request is base 64 decoded into a Tensorflow string (`tf.string`), which is passed to the serving function (`serving_fn`). The serving function preprocesses the `tf.string` into raw (uncompressed) numpy bytes (`preprocess_fn`) to match the input requirements of the model:\n",
         "- `io.decode_jpeg`- Decompresses the JPG image which is returned as a Tensorflow tensor with three channels (RGB).\n",
-        "- `image.convert_image_dtype` - Changes integer pixel values to float 32.\n",
+        "- `image.convert_image_dtype` - Changes integer pixel values to float 32, and rescales pixel data between 0 and 1.\n",
         "- `image.resize` - Resizes the image to match the input shape for the model.\n",
-        "- `resized / 255.0` - Rescales (normalization) the pixel data between 0 and 1.\n",
         "\n",
         "At this point, the data can be passed to the model (`m_call`).\n",
         "\n",
@@ -1552,8 +1550,7 @@
         "    decoded = tf.io.decode_jpeg(bytes_input, channels=3)\n",
         "    decoded = tf.image.convert_image_dtype(decoded, tf.float32)\n",
         "    resized = tf.image.resize(decoded, size=(32, 32))\n",
-        "    rescale = tf.cast(resized / 255.0, tf.float32)\n",
-        "    return rescale\n",
+        "    return resized\n",
         "\n",
         "\n",
         "@tf.function(input_signature=[tf.TensorSpec([None], tf.string)])\n",

notebooks/community/gapic/custom/showcase_custom_image_classification_online.ipynb

@@ -482,8 +482,7 @@
         "\n",
         "from google.cloud.aiplatform import gapic as aip\n",
         "from google.protobuf import json_format\n",
-        "from google.protobuf.json_format import MessageToJson, ParseDict\n",
-        "from google.protobuf.struct_pb2 import Struct, Value"
+        "from google.protobuf.struct_pb2 import Value"
       ]
     },
     {
@@ -1498,9 +1497,8 @@
         "\n",
         "When you send a prediction or explanation request, the content of the request is base 64 decoded into a Tensorflow string (`tf.string`), which is passed to the serving function (`serving_fn`). The serving function preprocesses the `tf.string` into raw (uncompressed) numpy bytes (`preprocess_fn`) to match the input requirements of the model:\n",
         "- `io.decode_jpeg`- Decompresses the JPG image which is returned as a Tensorflow tensor with three channels (RGB).\n",
-        "- `image.convert_image_dtype` - Changes integer pixel values to float 32.\n",
+        "- `image.convert_image_dtype` - Changes integer pixel values to float 32, and rescales pixel data between 0 and 1.\n",
         "- `image.resize` - Resizes the image to match the input shape for the model.\n",
-        "- `resized / 255.0` - Rescales (normalization) the pixel data between 0 and 1.\n",
         "\n",
         "At this point, the data can be passed to the model (`m_call`)."
       ]
@@ -1520,8 +1518,7 @@
         "    decoded = tf.io.decode_jpeg(bytes_input, channels=3)\n",
         "    decoded = tf.image.convert_image_dtype(decoded, tf.float32)\n",
         "    resized = tf.image.resize(decoded, size=(32, 32))\n",
-        "    rescale = tf.cast(resized / 255.0, tf.float32)\n",
-        "    return rescale\n",
+        "    return resized\n",
         "\n",
         "\n",
         "@tf.function(input_signature=[tf.TensorSpec([None], tf.string)])\n",

notebooks/community/gapic/custom/showcase_custom_image_classification_online_ab_testing.ipynb

@@ -483,8 +483,7 @@
         "\n",
         "from google.cloud.aiplatform import gapic as aip\n",
         "from google.protobuf import json_format\n",
-        "from google.protobuf.json_format import MessageToJson, ParseDict\n",
-        "from google.protobuf.struct_pb2 import Struct, Value"
+        "from google.protobuf.struct_pb2 import Value"
       ]
     },
     {
@@ -1710,7 +1709,7 @@
       "outputs": [],
       "source": [
         "import tensorflow as tf\n",
-        "from tensorflow.keras import Input, Model\n",
+        "from tensorflow.keras import Model\n",
         "from tensorflow.keras.layers import Lambda\n",
         "\n",
         "softmax = model_A.outputs[0]\n",
@@ -1761,9 +1760,8 @@
         "\n",
         "When you send a prediction or explanation request, the content of the request is base 64 decoded into a Tensorflow string (`tf.string`), which is passed to the serving function (`serving_fn`). The serving function preprocesses the `tf.string` into raw (uncompressed) numpy bytes (`preprocess_fn`) to match the input requirements of the model:\n",
         "- `io.decode_jpeg`- Decompresses the JPG image which is returned as a Tensorflow tensor with three channels (RGB).\n",
-        "- `image.convert_image_dtype` - Changes integer pixel values to float 32.\n",
+        "- `image.convert_image_dtype` - Changes integer pixel values to float 32, and rescales pixel data between 0 and 1.\n",
         "- `image.resize` - Resizes the image to match the input shape for the model.\n",
-        "- `resized / 255.0` - Rescales (normalization) the pixel data between 0 and 1.\n",
         "\n",
         "At this point, the data can be passed to the model (`m_call`)."
       ]
@@ -1783,8 +1781,7 @@
         "    decoded = tf.io.decode_jpeg(bytes_input, channels=3)\n",
         "    decoded = tf.image.convert_image_dtype(decoded, tf.float32)\n",
         "    resized = tf.image.resize(decoded, size=(32, 32))\n",
-        "    rescale = tf.cast(resized / 255.0, tf.float32)\n",
-        "    return rescale\n",
+        "    return resized\n",
         "\n",
         "\n",
         "@tf.function(input_signature=[tf.TensorSpec([None], tf.string)])\n",
@@ -1824,16 +1821,15 @@
         "CONCRETE_INPUT = \"numpy_inputs\"\n",
         "\n",
         "\n",
-        "def _preprocess(bytes_input):\n",
+        "def _preprocess(bytes_input):  # noqa: 811\n",
         "    decoded = tf.io.decode_jpeg(bytes_input, channels=3)\n",
         "    decoded = tf.image.convert_image_dtype(decoded, tf.float32)\n",
         "    resized = tf.image.resize(decoded, size=(32, 32))\n",
-        "    rescale = tf.cast(resized / 255.0, tf.float32)\n",
-        "    return rescale\n",
+        "    return resized\n",
         "\n",
         "\n",
         "@tf.function(input_signature=[tf.TensorSpec([None], tf.string)])\n",
-        "def preprocess_fn(bytes_inputs):\n",
+        "def preprocess_fn(bytes_inputs):  # noqa: 811\n",
         "    decoded_images = tf.map_fn(\n",
         "        _preprocess, bytes_inputs, dtype=tf.float32, back_prop=False\n",
         "    )\n",

notebooks/community/gapic/custom/showcase_custom_image_classification_online_container.ipynb

@@ -482,8 +482,7 @@
         "\n",
         "from google.cloud.aiplatform import gapic as aip\n",
         "from google.protobuf import json_format\n",
-        "from google.protobuf.json_format import MessageToJson, ParseDict\n",
-        "from google.protobuf.struct_pb2 import Struct, Value"
+        "from google.protobuf.struct_pb2 import Value"
       ]
     },
     {
@@ -1600,9 +1599,8 @@
         "\n",
         "When you send a prediction or explanation request, the content of the request is base 64 decoded into a Tensorflow string (`tf.string`), which is passed to the serving function (`serving_fn`). The serving function preprocesses the `tf.string` into raw (uncompressed) numpy bytes (`preprocess_fn`) to match the input requirements of the model:\n",
         "- `io.decode_jpeg`- Decompresses the JPG image which is returned as a Tensorflow tensor with three channels (RGB).\n",
-        "- `image.convert_image_dtype` - Changes integer pixel values to float 32.\n",
+        "- `image.convert_image_dtype` - Changes integer pixel values to float 32, and rescales pixel data between 0 and 1.\n",
         "- `image.resize` - Resizes the image to match the input shape for the model.\n",
-        "- `resized / 255.0` - Rescales (normalization) the pixel data between 0 and 1.\n",
         "\n",
         "At this point, the data can be passed to the model (`m_call`)."
       ]
@@ -1622,8 +1620,7 @@
         "    decoded = tf.io.decode_jpeg(bytes_input, channels=3)\n",
         "    decoded = tf.image.convert_image_dtype(decoded, tf.float32)\n",
         "    resized = tf.image.resize(decoded, size=(32, 32))\n",
-        "    rescale = tf.cast(resized / 255.0, tf.float32)\n",
-        "    return rescale\n",
+        "    return resized\n",
         "\n",
         "\n",
         "@tf.function(input_signature=[tf.TensorSpec([None], tf.string)])\n",

notebooks/community/gapic/custom/showcase_custom_image_classification_online_explain.ipynb

@@ -507,8 +507,7 @@
         "\n",
         "import google.cloud.aiplatform_v1beta1 as aip\n",
         "from google.protobuf import json_format\n",
-        "from google.protobuf.json_format import MessageToJson, ParseDict\n",
-        "from google.protobuf.struct_pb2 import Struct, Value"
+        "from google.protobuf.struct_pb2 import Value"
       ]
     },
     {
@@ -1523,9 +1522,8 @@
         "\n",
         "When you send a prediction or explanation request, the content of the request is base 64 decoded into a Tensorflow string (`tf.string`), which is passed to the serving function (`serving_fn`). The serving function preprocesses the `tf.string` into raw (uncompressed) numpy bytes (`preprocess_fn`) to match the input requirements of the model:\n",
         "- `io.decode_jpeg`- Decompresses the JPG image which is returned as a Tensorflow tensor with three channels (RGB).\n",
-        "- `image.convert_image_dtype` - Changes integer pixel values to float 32.\n",
+        "- `image.convert_image_dtype` - Changes integer pixel values to float 32, and rescales pixel data between 0 and 1.\n",
         "- `image.resize` - Resizes the image to match the input shape for the model.\n",
-        "- `resized / 255.0` - Rescales (normalization) the pixel data between 0 and 1.\n",
         "\n",
         "At this point, the data can be passed to the model (`m_call`).\n",
         "\n",
@@ -1554,8 +1552,7 @@
         "    decoded = tf.io.decode_jpeg(bytes_input, channels=3)\n",
         "    decoded = tf.image.convert_image_dtype(decoded, tf.float32)\n",
         "    resized = tf.image.resize(decoded, size=(32, 32))\n",
-        "    rescale = tf.cast(resized / 255.0, tf.float32)\n",
-        "    return rescale\n",
+        "    return resized\n",
         "\n",
         "\n",
         "@tf.function(input_signature=[tf.TensorSpec([None], tf.string)])\n",

notebooks/community/gapic/custom/showcase_custom_image_classification_online_exported_ds.ipynb

@@ -484,8 +484,7 @@
         "\n",
         "from google.cloud.aiplatform import gapic as aip\n",
         "from google.protobuf import json_format\n",
-        "from google.protobuf.json_format import MessageToJson, ParseDict\n",
-        "from google.protobuf.struct_pb2 import Struct, Value"
+        "from google.protobuf.struct_pb2 import Value"
       ]
     },
     {
@@ -2103,9 +2102,8 @@
         "\n",
         "When you send a prediction or explanation request, the content of the request is base 64 decoded into a Tensorflow string (`tf.string`), which is passed to the serving function (`serving_fn`). The serving function preprocesses the `tf.string` into raw (uncompressed) numpy bytes (`preprocess_fn`) to match the input requirements of the model:\n",
         "- `io.decode_jpeg`- Decompresses the JPG image which is returned as a Tensorflow tensor with three channels (RGB).\n",
-        "- `image.convert_image_dtype` - Changes integer pixel values to float 32.\n",
+        "- `image.convert_image_dtype` - Changes integer pixel values to float 32, and rescales pixel data between 0 and 1.\n",
         "- `image.resize` - Resizes the image to match the input shape for the model.\n",
-        "- `resized / 255.0` - Rescales (normalization) the pixel data between 0 and 1.\n",
         "\n",
         "At this point, the data can be passed to the model (`m_call`)."
       ]
@@ -2125,8 +2123,7 @@
         "    decoded = tf.io.decode_jpeg(bytes_input, channels=3)\n",
         "    decoded = tf.image.convert_image_dtype(decoded, tf.float32)\n",
         "    resized = tf.image.resize(decoded, size=(128, 128))\n",
-        "    rescale = tf.cast(resized / 255.0, tf.float32)\n",
-        "    return rescale\n",
+        "    return resized\n",
         "\n",
         "\n",
         "@tf.function(input_signature=[tf.TensorSpec([None], tf.string)])\n",

notebooks/community/gapic/custom/showcase_custom_image_classification_online_pipeline.ipynb

@@ -483,8 +483,7 @@
         "\n",
         "from google.cloud.aiplatform import gapic as aip\n",
         "from google.protobuf import json_format\n",
-        "from google.protobuf.json_format import MessageToJson, ParseDict\n",
-        "from google.protobuf.struct_pb2 import Struct, Value"
+        "from google.protobuf.struct_pb2 import Value"
       ]
     },
     {
@@ -1247,9 +1246,6 @@
       },
       "outputs": [],
       "source": [
-        "from google.protobuf import json_format\n",
-        "from google.protobuf.struct_pb2 import Value\n",
-        "\n",
         "MODEL_NAME = \"custom_pipeline-\" + TIMESTAMP\n",
         "PIPELINE_DISPLAY_NAME = \"custom-training-pipeline\" + TIMESTAMP\n",
         "\n",
@@ -1567,9 +1563,8 @@
         "\n",
         "When you send a prediction or explanation request, the content of the request is base 64 decoded into a Tensorflow string (`tf.string`), which is passed to the serving function (`serving_fn`). The serving function preprocesses the `tf.string` into raw (uncompressed) numpy bytes (`preprocess_fn`) to match the input requirements of the model:\n",
         "- `io.decode_jpeg`- Decompresses the JPG image which is returned as a Tensorflow tensor with three channels (RGB).\n",
-        "- `image.convert_image_dtype` - Changes integer pixel values to float 32.\n",
+        "- `image.convert_image_dtype` - Changes integer pixel values to float 32, and rescales pixel data between 0 and 1.\n",
         "- `image.resize` - Resizes the image to match the input shape for the model.\n",
-        "- `resized / 255.0` - Rescales (normalization) the pixel data between 0 and 1.\n",
         "\n",
         "At this point, the data can be passed to the model (`m_call`)."
       ]
@@ -1589,8 +1584,7 @@
         "    decoded = tf.io.decode_jpeg(bytes_input, channels=3)\n",
         "    decoded = tf.image.convert_image_dtype(decoded, tf.float32)\n",
         "    resized = tf.image.resize(decoded, size=(32, 32))\n",
-        "    rescale = tf.cast(resized / 255.0, tf.float32)\n",
-        "    return rescale\n",
+        "    return resized\n",
         "\n",
         "\n",
         "@tf.function(input_signature=[tf.TensorSpec([None], tf.string)])\n",

notebooks/community/gapic/custom/showcase_custom_image_classification_online_tfserving.ipynb

@@ -482,8 +482,7 @@
         "\n",
         "from google.cloud.aiplatform import gapic as aip\n",
         "from google.protobuf import json_format\n",
-        "from google.protobuf.json_format import MessageToJson, ParseDict\n",
-        "from google.protobuf.struct_pb2 import Struct, Value"
+        "from google.protobuf.struct_pb2 import Value"
       ]
     },
     {
@@ -1559,9 +1558,8 @@
         "\n",
         "When you send a prediction or explanation request, the content of the request is base 64 decoded into a Tensorflow string (`tf.string`), which is passed to the serving function (`serving_fn`). The serving function preprocesses the `tf.string` into raw (uncompressed) numpy bytes (`preprocess_fn`) to match the input requirements of the model:\n",
         "- `io.decode_jpeg`- Decompresses the JPG image which is returned as a Tensorflow tensor with three channels (RGB).\n",
-        "- `image.convert_image_dtype` - Changes integer pixel values to float 32.\n",
+        "- `image.convert_image_dtype` - Changes integer pixel values to float 32, and rescales pixel data between 0 and 1.\n",
         "- `image.resize` - Resizes the image to match the input shape for the model.\n",
-        "- `resized / 255.0` - Rescales (normalization) the pixel data between 0 and 1.\n",
         "\n",
         "At this point, the data can be passed to the model (`m_call`)."
       ]
@@ -1581,8 +1579,7 @@
         "    decoded = tf.io.decode_jpeg(bytes_input, channels=3)\n",
         "    decoded = tf.image.convert_image_dtype(decoded, tf.float32)\n",
         "    resized = tf.image.resize(decoded, size=(32, 32))\n",
-        "    rescale = tf.cast(resized / 255.0, tf.float32)\n",
-        "    return rescale\n",
+        "    return resized\n",
         "\n",
         "\n",
         "@tf.function(input_signature=[tf.TensorSpec([None], tf.string)])\n",

notebooks/community/gapic/custom/showcase_custom_image_super_resolution_online_post.ipynb

@@ -483,8 +483,7 @@
         "\n",
         "from google.cloud.aiplatform import gapic as aip\n",
         "from google.protobuf import json_format\n",
-        "from google.protobuf.json_format import MessageToJson, ParseDict\n",
-        "from google.protobuf.struct_pb2 import Struct, Value"
+        "from google.protobuf.struct_pb2 import Value"
       ]
     },
     {
@@ -1501,9 +1500,8 @@
         "\n",
         "When you send a prediction or explanation request, the content of the request is base 64 decoded into a Tensorflow string (`tf.string`), which is passed to the serving function (`serving_fn`). The serving function preprocesses the `tf.string` into raw (uncompressed) numpy bytes (`preprocess_fn`) to match the input requirements of the model:\n",
         "- `io.decode_jpeg`- Decompresses the JPG image which is returned as a Tensorflow tensor with three channels (RGB).\n",
-        "- `image.convert_image_dtype` - Changes integer pixel values to float 32.\n",
+        "- `image.convert_image_dtype` - Changes integer pixel values to float 32, and rescales pixel data between 0 and 1.\n",
         "- `image.resize` - Resizes the image to match the input shape for the model.\n",
-        "- `resized / 255.0` - Rescales (normalization) the pixel data between 0 and 1.\n",
         "\n",
         "At this point, the data can be passed to the model (`m_call`).\n",
         "\n",
@@ -1529,8 +1527,7 @@
         "    decoded = tf.io.decode_jpeg(bytes_input, channels=3)\n",
         "    decoded = tf.image.convert_image_dtype(decoded, tf.float32)\n",
         "    resized = tf.image.resize(decoded, size=(16, 16))\n",
-        "    rescale = tf.cast(resized / 255.0, tf.float32)\n",
-        "    return rescale\n",
+        "    return resized\n",
         "\n",
         "\n",
         "@tf.function(input_signature=[tf.TensorSpec([None], tf.string)])\n",

notebooks/community/gapic/custom/showcase_hyperparmeter_tuning_image_classification.ipynb

@@ -1706,11 +1706,11 @@
       "outputs": [],
       "source": [
         "# The full unique ID for the custom job\n",
-        "job_id = response.name\n",
+        "hpt_job_id = response.name\n",
         "# The short numeric ID for the custom job\n",
-        "job_short_id = job_id.split(\"/\")[-1]\n",
+        "hpt_job_short_id = hpt_job_id.split(\"/\")[-1]\n",
         "\n",
-        "print(job_id)"
+        "print(hpt_job_id)"
       ]
     },
     {

notebooks/community/gapic/custom/showcase_hyperparmeter_tuning_tabular_regression.ipynb

@@ -1715,11 +1715,11 @@
       "outputs": [],
       "source": [
         "# The full unique ID for the custom job\n",
-        "job_id = response.name\n",
+        "hpt_job_id = response.name\n",
         "# The short numeric ID for the custom job\n",
-        "job_short_id = job_id.split(\"/\")[-1]\n",
+        "hpt_job_short_id = hpt_job_id.split(\"/\")[-1]\n",
         "\n",
-        "print(job_id)"
+        "print(hpt_job_id)"
       ]
     },
     {

notebooks/community/gapic/custom/showcase_hyperparmeter_tuning_text_binary_classification.ipynb

@@ -1707,11 +1707,11 @@
       "outputs": [],
       "source": [
         "# The full unique ID for the custom job\n",
-        "job_id = response.name\n",
+        "hpt_job_id = response.name\n",
         "# The short numeric ID for the custom job\n",
-        "job_short_id = job_id.split(\"/\")[-1]\n",
+        "hpt_job_short_id = hpt_job_id.split(\"/\")[-1]\n",
         "\n",
-        "print(job_id)"
+        "print(hpt_job_id)"
       ]
     },
     {

notebooks/community/managed_notebooks/ads_targetting/training-multi-class-classification-model-for-ads-targeting-usecase.ipynb

@@ -0,0 +1,821 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "503077811e70"
+      },
+      "source": [
+        "# Train a multi-class classification model for ads-targeting\n",
+        "\n",
+        "## Table of contents\n",
+        "\n",
+        "* [Overview](#section-1)\n",
+        "* [Dataset](#section-2)\n",
+        "* [Objective](#section-3)\n",
+        "* [Costs](#section-4)\n",
+        "* [Tutorial](#section-5)\n",
+        "\t- [Fetch the data from BigQuery](#section-5)\n",
+        "    - [Preprocess the data](#section-6)\n",
+        "    - [Train a TensorFlow model](#section-7)\n",
+        "    - [Run the model on test data](#section-8)\n",
+        "    - [Automating the execution of the notebook using executor](#section-9)\n",
+        "    - [Scheduled runs on executor](#section-10)\n",
+        "    - [Parameterizing the variables](#section-11)\n",
+        "* [Save the model to a Cloud Storage path](#section-12)\n",
+        "* [Clean up](#section-13)\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "00f095e91b2d"
+      },
+      "source": [
+        "## Overview\n",
+        "<a name=\"section-1\"></a>\n",
+        "\n",
+        "This tutorial demonstrates how to build a machine learning model for an ads-targeting use case. Ads-targeting is an advertisement technique where chosen or tailor-made ads are shown to the customers based on their past behavior and preferences. Targeted ads are meant to reach specific customers based on demographics, psychographics, behavior, and other second-order activities that are learned usually through data collected from the customers.\n",
+        "\n",
+        "*Note: This notebook file was designed to run in a [Vertex AI Workbench managed notebooks](https://cloud.google.com/vertex-ai/docs/workbench/managed/create-instance) instance using the `TensorFlow 2 (Local)` kernel. Some components of this notebook may not work in other notebook environments.*\n",
+        "\n",
+        "## Dataset\n",
+        "<a name=\"section-2\"></a>\n",
+        "\n",
+        "This tutorial uses the `looker-private-demo.ecomm` dataset in BigQuery. The dataset consists of information about various advertisement campaigns including the demographics of users who have clicked and made some purchases after seeing the ads. For this tutorial, the top three campaigns from the USA are selected from this dataset and user information for those who have made purchases shall be used to train a model with the campaigns as the classes. The idea is to see if the advertisement and the user data can be used to identify which campaign is best-suited for the user.\n",
+        "\n",
+        "The dataset can be accessed by pinning the `looker-private-demo` project in BigQuery. Instead of going to the BigQuery user interface, this process can be performed from the JupyterLab user interface on a Vertex AI Workbench managed notebooks instance. Vertex AI Workbench managed notebooks instances support browsing through the datasets and tables from BigQuery through its BigQuery integration. \n",
+        "\n",
+        "<img src=\"images/Bigquery_UI_new.PNG\"></img>\n",
+        "\n",
+        "## Objective\n",
+        "<a name=\"section-3\"></a>\n",
+        "\n",
+        "This tutorial demonstrates how to collect data from BigQuery, preprocess it, and train a multi-class classification model on an E-commerce dataset. The steps performed include the following:\n",
+        "\n",
+        "- Fetch the required data from BigQuery\n",
+        "- Preprocess the data\n",
+        "- Train a TensorFlow (>=2.4) classification model\n",
+        "- Evaluate the loss for the trained model\n",
+        "- Automate the notebook execution using the executor feature\n",
+        "- Save the model to a Cloud Storage path\n",
+        "- Clean up the created resources\n",
+        "\n",
+        "### Costs \n",
+        "<a name=\"section-4\"></a>\n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* BigQuery\n",
+        "* Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI\n",
+        "pricing](https://cloud.google.com/vertex-ai/pricing), [BigQuery\n",
+        "pricing](https://cloud.google.com/bigquery/pricing) and [Cloud Storage\n",
+        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
+        "Calculator](https://cloud.google.com/products/calculator/)\n",
+        "to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "a7be962cec10"
+      },
+      "source": [
+        "#### Set your project ID\n",
+        "\n",
+        "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "684595f229b3"
+      },
+      "outputs": [],
+      "source": [
+        "PROJECT_ID = \"\"\n",
+        "\n",
+        "# Get your Google Cloud project ID from gcloud\n",
+        "if not os.getenv(\"IS_TESTING\"):\n",
+        "    shell_output=!gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "    PROJECT_ID = shell_output[0]\n",
+        "    print(\"Project ID: \", PROJECT_ID)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d0058f55f8cf"
+      },
+      "source": [
+        "Otherwise, set your project ID here."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "19579640c063"
+      },
+      "outputs": [],
+      "source": [
+        "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
+        "    PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "b2b04f364669"
+      },
+      "source": [
+        "#### Timestamp\n",
+        "\n",
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append it onto the name of resources you create in this tutorial."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "a9ee95826661"
+      },
+      "outputs": [],
+      "source": [
+        "from datetime import datetime\n",
+        "\n",
+        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "7f0d924a47aa"
+      },
+      "source": [
+        "### Create a Cloud Storage bucket\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "When you submit a training job using the Cloud SDK, you upload a Python package\n",
+        "containing your training code to a Cloud Storage bucket. Vertex AI runs\n",
+        "the code from this package. In this tutorial, Vertex AI also saves the\n",
+        "trained model that results from your job in the same bucket. Using this model artifact, you can then\n",
+        "create Vertex AI model and endpoint resources in order to serve\n",
+        "online predictions.\n",
+        "\n",
+        "Set the name of your Cloud Storage bucket below. It must be unique across all\n",
+        "Cloud Storage buckets.\n",
+        "\n",
+        "You may also change the `REGION` variable, which is used for operations\n",
+        "throughout the rest of this notebook. Make sure to [choose a region where Vertex AI services are\n",
+        "available](https://cloud.google.com/vertex-ai/docs/general/locations#available_regions). You may\n",
+        "not use a Multi-Regional Storage bucket for training with Vertex AI."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "59ebce2a16fa"
+      },
+      "outputs": [],
+      "source": [
+        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "REGION = \"[your-region]\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "d0caa7b3c8b8"
+      },
+      "outputs": [],
+      "source": [
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "7a7c87b6f171"
+      },
+      "source": [
+        "**Only if your bucket doesn't already exist**: Run the following cell to create your Cloud Storage bucket."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "76f7e3e043f5"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil mb -l $REGION $BUCKET_NAME"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "f4353e94da5f"
+      },
+      "source": [
+        "**Finally**, validate access to your Cloud Storage bucket by examining its contents:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6f092d6ebc92"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil ls -al $BUCKET_NAME"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9abd63bb1a85"
+      },
+      "source": [
+        "## Tutorial\n",
+        "\n",
+        "### Fetch the data from BigQuery \n",
+        "<a name=\"section-5\"></a>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "4a25861cf833"
+      },
+      "source": [
+        "#@bigquery\n",
+        "\n",
+        "WITH\n",
+        "  traindata AS (\n",
+        "  SELECT\n",
+        "    b.* EXCEPT(ad_event_id,\n",
+        "      user_id),\n",
+        "    c.* EXCEPT(id),\n",
+        "    d.* EXCEPT(keyword_id,\n",
+        "      ad_id),\n",
+        "    a.amount,\n",
+        "    a.device_type,\n",
+        "    e.name\n",
+        "  FROM\n",
+        "    `looker-private-demo.ecomm.ad_events` a\n",
+        "  JOIN (\n",
+        "    SELECT\n",
+        "      ad_event_id,\n",
+        "      user_id,\n",
+        "      state,\n",
+        "      os,\n",
+        "      browser\n",
+        "    FROM\n",
+        "      `looker-private-demo.ecomm.events`\n",
+        "    WHERE\n",
+        "      event_type=\"Purchase\"\n",
+        "      AND country=\"USA\") b\n",
+        "  ON\n",
+        "    a.id = b.ad_event_id\n",
+        "  JOIN (\n",
+        "    SELECT\n",
+        "      id,\n",
+        "      gender,\n",
+        "      age\n",
+        "    FROM\n",
+        "      `looker-private-demo.ecomm.users`) c\n",
+        "  ON\n",
+        "    b.user_id = c.id\n",
+        "  JOIN (\n",
+        "    SELECT\n",
+        "      keyword_id,\n",
+        "      ad_id,\n",
+        "      cpc_bid_amount,\n",
+        "      bidding_strategy_type,\n",
+        "      quality_score,\n",
+        "      keyword_match_type\n",
+        "    FROM\n",
+        "      `looker-private-demo.ecomm.keywords`\n",
+        "    WHERE\n",
+        "      cpc_bid_amount <= 3000) d\n",
+        "  ON\n",
+        "    a.keyword_id = d.keyword_id\n",
+        "  JOIN (\n",
+        "    SELECT\n",
+        "      ad_id,\n",
+        "      name\n",
+        "    FROM\n",
+        "      `looker-private-demo.ecomm.ad_groups`) e\n",
+        "  ON\n",
+        "    d.ad_id = e.ad_id )\n",
+        "SELECT\n",
+        "  *\n",
+        "FROM\n",
+        "  traindata"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "923fdd823683"
+      },
+      "source": [
+        "Once the results from BigQuery are displayed in the above cell, click the **Query and load as DataFrame** button and execute the generated code stub to fetch the data into the current notebook as a dataframe.\n",
+        "\n",
+        "*Note: By default the data is loaded into a `df` variable, though this can be changed before executing the cell if required.*"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "f8b5112f231f"
+      },
+      "outputs": [],
+      "source": [
+        "# The following two lines are only necessary to run once.\n",
+        "# Comment out otherwise for speed-up.\n",
+        "from google.cloud.bigquery import Client\n",
+        "\n",
+        "client = Client()\n",
+        "\n",
+        "query = \"\"\"WITH traindata AS (\n",
+        "SELECT b.* except(ad_event_id, user_id), c.* except(id), d.* except(keyword_id, ad_id), a.amount, a.device_type, e.name\n",
+        "FROM `looker-private-demo.ecomm.ad_events` a\n",
+        "JOIN\n",
+        "(SELECT ad_event_id, user_id, state, os, browser from `looker-private-demo.ecomm.events` WHERE event_type=\"Purchase\" AND country=\"USA\") b\n",
+        "ON a.id = b.ad_event_id\n",
+        "JOIN\n",
+        "(SELECT id, gender, age FROM `looker-private-demo.ecomm.users`) c\n",
+        "ON b.user_id = c.id\n",
+        "JOIN\n",
+        "(SELECT keyword_id, ad_id, cpc_bid_amount, bidding_strategy_type, quality_score, keyword_match_type FROM `looker-private-demo.ecomm.keywords`\n",
+        "WHERE cpc_bid_amount <= 3000) d\n",
+        "ON a.keyword_id = d.keyword_id\n",
+        "JOIN\n",
+        "(SELECT ad_id, name FROM `looker-private-demo.ecomm.ad_groups`) e\n",
+        "ON d.ad_id = e.ad_id\n",
+        ")\n",
+        "SELECT * FROM traindata\"\"\"\n",
+        "job = client.query(query)\n",
+        "df = job.to_dataframe()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "7f69bea65019"
+      },
+      "source": [
+        "### Preprocess the data\n",
+        "<a name=\"section-6\"></a>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e8503e799eec"
+      },
+      "source": [
+        "Import the required libraries."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "5b11973ccf76"
+      },
+      "outputs": [],
+      "source": [
+        "import warnings\n",
+        "\n",
+        "import pandas as pd\n",
+        "from sklearn.model_selection import train_test_split\n",
+        "from sklearn.preprocessing import StandardScaler\n",
+        "from tensorflow.keras import Sequential\n",
+        "from tensorflow.keras.layers import Dense\n",
+        "from tensorflow.keras.utils import to_categorical\n",
+        "\n",
+        "warnings.filterwarnings(\"ignore\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e48d156d8bb6"
+      },
+      "source": [
+        "Select the necessary columns from the E-commerce data and divide them based on their type (numerical/categorical)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7cea2c44c50b"
+      },
+      "outputs": [],
+      "source": [
+        "target = \"name\"\n",
+        "categ_cols = [\n",
+        "    \"state\",\n",
+        "    \"os\",\n",
+        "    \"browser\",\n",
+        "    \"gender\",\n",
+        "    \"bidding_strategy_type\",\n",
+        "    \"keyword_match_type\",\n",
+        "    \"device_type\",\n",
+        "]\n",
+        "num_cols = [\"age\", \"cpc_bid_amount\", \"quality_score\", \"amount\"]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ace612851261"
+      },
+      "source": [
+        "From the current dataset, only the top three camapigns will be chosen to target the users. All the relevant information about the advertisement and the user who purchased an item after seeing the advertisement is available in the dataframe already. "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7282fbab4586"
+      },
+      "outputs": [],
+      "source": [
+        "df = df[df[\"name\"].isin([\"Tops & Tees\", \"Active\", \"Accessories\"])]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "f89106348ffe"
+      },
+      "source": [
+        "Encode the target variable."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "d6cd256f455a"
+      },
+      "outputs": [],
+      "source": [
+        "df[\"name\"] = df[\"name\"].map({\"Tops & Tees\": 0, \"Active\": 1, \"Accessories\": 2})"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8902f763d1ca"
+      },
+      "source": [
+        "One-hot encode the categorical variables. After one-hot encoding, the first level-column is dropped to avoid the [dummy-variable trap](https://en.wikipedia.org/wiki/Dummy_variable_(statistics)) scenario. This process is called *dummy-encoding*."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "d57706df2441"
+      },
+      "outputs": [],
+      "source": [
+        "def encode_cols(data, col):\n",
+        "    # Creating a dummy variable for the variable 'CategoryID' and dropping the first one.\n",
+        "    categ = pd.get_dummies(data[col], prefix=col, drop_first=True)\n",
+        "    # Adding the results to the master dataframe\n",
+        "    data = pd.concat([data, categ], axis=1)\n",
+        "    return data\n",
+        "\n",
+        "\n",
+        "# dummy-encode the categorical fields\n",
+        "for i in categ_cols:\n",
+        "    df = encode_cols(df, i)\n",
+        "    df.drop(columns=[i], inplace=True)\n",
+        "\n",
+        "# check the data's shape\n",
+        "df.shape"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3abf027eda2d"
+      },
+      "source": [
+        "Split the data into train and test."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "0072d44b6163"
+      },
+      "outputs": [],
+      "source": [
+        "X = df[[i for i in df.columns if i != target]].copy()\n",
+        "y = df[target].copy()\n",
+        "X_train, X_test, y_train, y_test = train_test_split(\n",
+        "    X, y, train_size=0.8, random_state=36\n",
+        ")\n",
+        "print(X_train.shape, X_test.shape)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d1a32b9d9640"
+      },
+      "source": [
+        "Scale the data."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9620e04d8db2"
+      },
+      "outputs": [],
+      "source": [
+        "scaler = StandardScaler()\n",
+        "X_train.loc[:, num_cols] = scaler.fit_transform(X_train[num_cols])\n",
+        "X_test.loc[:, num_cols] = scaler.transform(X_test[num_cols])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5b9f3ca04f91"
+      },
+      "source": [
+        "### Train a TensorFlow model\n",
+        "<a name=\"section-7\"></a>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3e7656556a48"
+      },
+      "source": [
+        "Convert the target column to a categorical encoded colum (one-hot encoded)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ebc87650ae13"
+      },
+      "outputs": [],
+      "source": [
+        "y_train_categ = to_categorical(y_train)\n",
+        "y_test_categ = to_categorical(y_test)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3dd0014a7e1d"
+      },
+      "source": [
+        "Define hyperparameters for model training. \n",
+        "\n",
+        "*Note: Comment or remove the parameters from the following cell if they are provided already as an input parameter through the executor feature.*"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ec020b36af20"
+      },
+      "outputs": [],
+      "source": [
+        "optimizer = \"sgd\"\n",
+        "num_hidden_layers = 3\n",
+        "num_neurons = [64, 128, 256]\n",
+        "activ_func = [\"relu\", \"relu\", \"relu\"]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "406b731f576b"
+      },
+      "source": [
+        "Define the architecture and compile the model."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "57839a187cf0"
+      },
+      "outputs": [],
+      "source": [
+        "model = Sequential()\n",
+        "# construct the neural network as per the defined parameters\n",
+        "for i in range(num_hidden_layers):\n",
+        "    if i == 0:\n",
+        "        # add the input layer\n",
+        "        model.add(\n",
+        "            Dense(\n",
+        "                num_neurons[i],\n",
+        "                activation=activ_func[i],\n",
+        "                input_shape=(X_train.shape[1],),\n",
+        "            )\n",
+        "        )\n",
+        "    else:\n",
+        "        # add the hidden layers\n",
+        "        model.add(Dense(num_neurons[i], activation=activ_func[i]))\n",
+        "\n",
+        "# add the output layer\n",
+        "model.add(Dense(3, activation=\"softmax\"))\n",
+        "# compile the model\n",
+        "model.compile(loss=\"categorical_crossentropy\", optimizer=optimizer)\n",
+        "model.summary()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "4ab12c34f258"
+      },
+      "source": [
+        "Fit the model."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9321005e55ae"
+      },
+      "outputs": [],
+      "source": [
+        "history = model.fit(X_train, y_train_categ, epochs=50, verbose=1)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "51a2d0b52df3"
+      },
+      "source": [
+        "### Run the model on test data\n",
+        "<a name=\"section-8\"></a>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "f08445f2cd02"
+      },
+      "source": [
+        "Evaluate the model on test data."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "599df6d2b9a4"
+      },
+      "outputs": [],
+      "source": [
+        "test_results = model.evaluate(X_test, y_test_categ, verbose=1)\n",
+        "print(f\"Test results - Loss: {test_results}\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9769168778e8"
+      },
+      "source": [
+        "### Automating the execution of the notebook using executor\n",
+        "<a name=\"section-9\"></a>\n",
+        "\n",
+        "The executor can help you run a notebook file from start to end, with your choice of the environment, machine type, input parameters, and other characteristics. After setting up an execution, the notebook is executed as a job in Vertex AI custom training. Your jobs can be monitored from the <b>Notebook Executor</b> pane in the menu on the left.\n",
+        "\n",
+        "<img src=\"images/executor.png\"></img>\n",
+        "\n",
+        "Executor lets you choose the environment and machine type while automating the runs similar to Vertex AI training jobs without switching to the training jobs UI. Apart from the custom container that replicates the existing kernel by default, pre-built environments like TensorFlow Enterprise, PyTorch, and others can also be selected to run the notebook. Furthermore the required compute power can be specified by choosing from the list of machine types available, including GPUs."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "cf486c351581"
+      },
+      "source": [
+        "### Scheduled runs on executor\n",
+        "<a name=\"section-10\"></a>\n",
+        "\n",
+        "Notebook runs can also be scheduled recurringly with the executor. To do so, select <b>Schedule-based recurring executions</b> as the run type instead of <b>One-time execution</b>. The frequency of the job and the time when it executes is provided when you create the execution.\n",
+        "\n",
+        "<img src=\"images/executor_scheduled_runs2.png\"></img>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6561007ac7f2"
+      },
+      "source": [
+        "### Parameterizing the variables\n",
+        "<a name=\"section-11\"></a>\n",
+        "\n",
+        "Executor lets you run a notebook with different sets of input parameters. If required, constants in the notebook can be treated as arguments to a function, and when you submit the execution, you can provide those constants as input parameters.\n",
+        "\n",
+        "<img src=\"images/executor_input_parameters.png\"></img>\n",
+        "\n",
+        "The hyperparameters defined during the model training step can be passed as arguments while submitting an execution. However, the values defined in the notebook itself should be removed or commented out before submitting the execution. Otherwise, the input parameters would just be overwritten by the values in the notebook."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "c81db97fa3e8"
+      },
+      "source": [
+        "### Save the model to a Cloud Storage path\n",
+        "<a name=\"section-12\"></a>\n",
+        "\n",
+        "TensorFlow's `model.save()` method supports Cloud Storage paths as well as the local file paths while writing the model object to a file. It needs to be ensured that the service account being used to run this notebook has `write` permissions to the specified Cloud Storage path."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2dcefeb6a2d8"
+      },
+      "outputs": [],
+      "source": [
+        "GCS_PATH = \"gs://\" + BUCKET_NAME + \"/[path-to-save]/\"\n",
+        "model.save(GCS_PATH)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "29c0ca2a517a"
+      },
+      "source": [
+        "## Clean up\n",
+        "<a name=\"section-13\"></a>\n",
+        "\n",
+        "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
+        "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
+        "\n",
+        "Otherwise, you can delete the individual resources you created in this tutorial:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "4ab69210d5a8"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil -m rm -r [cloud-storage-folder-path-to-delete]"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "training-multi-class-classification-model-for-ads-targeting-usecase.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/community/managed_notebooks/recommender_systems/recommender-system-on-retail-data.ipynb

@@ -0,0 +1,963 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "aa9846856f05"
+      },
+      "source": [
+        "# Build a recommender system with retail data on Vertex AI using PySpark\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "1caa36b69240"
+      },
+      "source": [
+        "## Table of contents\n",
+        "\n",
+        "* [Overview](#section-1)\n",
+        "* [Dataset](#section-2)\n",
+        "* [Objective](#section-3)\n",
+        "* [Costs](#section-4)\n",
+        "* [Create a Dataproc cluster with component gateway enabled and JupyterLab extension](#section-5)\n",
+        "* [Connect to the cluster from the notebook](#section-6)\n",
+        "* [Explore the data](#section-7)\n",
+        "* [Define the ALS Model](#section-8)\n",
+        "* [Evaluate the model](#section-9)\n",
+        "* [Save the ALS model to Cloud Storage](#section-10)  \n",
+        "* [Write the recommendations to BigQuery](#section-11)\n",
+        "* [Clean up](#section-12)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5ec25b247838"
+      },
+      "source": [
+        "## Overview\n",
+        "<a name=\"section-1\"></a>\n",
+        "\n",
+        "Recommender systems are powerful tools that model existing customer behavior to generate recommendations. These models generally build complex matrices and map out existing customer preferences in order to find intersecting interests and offer recommendations. These matrices can be very large and will benefit from distributed computing and large memory pools. In a Vertex AI Workbench managed notebooks instance, you can use distributed computing by processing your data in PySpark on a Dataproc cluster.\n",
+        "\n",
+        "*Note: This notebook file was designed to run in a [Vertex AI Workbench managed notebooks](https://cloud.google.com/vertex-ai/docs/workbench/managed/create-instance) instance using a `Python 3` kernel generated by a Dataproc runtime. Some components of this notebook may not work in other notebook environments.*"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8ccfcd2355f3"
+      },
+      "source": [
+        "## Dataset\n",
+        "<a name=\"section-2\"></a>\n",
+        "\n",
+        "This notebook uses the `looker-private-demo.retail` dataset in BigQuery. The dataset can be accessed by pinning the `looker-private-demo` project in BigQuery. Instead of going to the BigQuery user interface, this process can be performed from the JupyterLab user interface on a Vertex AI Workbench managed notebooks instance. Vertex AI Workbench managed notebooks instances support browsing through the datasets and tables from BigQuery through its BigQuery integration. \n",
+        "\n",
+        "<img src=\"images/Bigquery_UI_new.PNG\"></img>\n",
+        "\n",
+        "In this dataset, the `retail.order_items` table will be used to train the recommendation system using PySpark. This table contains information on various orders related to the users and items (products) in the dataset.\n",
+        "\n",
+        "## Objective\n",
+        "<a name=\"section-3\"></a>\n",
+        "\n",
+        "This tutorial builds a recommendation model with a [collaborative filtering](https://en.wikipedia.org/wiki/Collaborative_filtering) approach using the interactive PySpark features offered by the Vertex AI Workbench's managed notebooks instances. You'll set up a remotely-connected Dataproc cluster and use the <a href=\"http://dl.acm.org/citation.cfm?id=1608614\">Alternating Least Squares(ALS)</a> method implemented in PySpark's MLlib library.\n",
+        "\n",
+        "The steps performed in this notebook are:\n",
+        "\n",
+        "1. Connect your managed notebooks instance to a Dataproc cluster with PySpark.\n",
+        "2. Explore the dataset in BigQuery from within the notebook.\n",
+        "3. Preprocess the data.\n",
+        "4. Train a PySpark ALS model on the data.\n",
+        "5. Evaluate the ALS model.\n",
+        "6. Generate recommendations.\n",
+        "7. Save the recommendations to a BigQuery table using the PySpark-BigQuery connector.\n",
+        "8. Save the ALS model to a Cloud Storage bucket.\n",
+        "9. Clean up the resources.\n",
+        "\n",
+        "## Costs\n",
+        "<a name=\"section-4\"></a>\n",
+        "This tutorial uses the following billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Dataproc\n",
+        "* BigQuery\n",
+        "* Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI\n",
+        "pricing](https://cloud.google.com/vertex-ai/pricing), [Dataproc pricing](https://cloud.google.com/dataproc/pricing), [BigQuery\n",
+        "pricing](https://cloud.google.com/bigquery/pricing) and [Cloud Storage\n",
+        "pricing](https://cloud.google.com/storage/pricing) and use the [Pricing\n",
+        "Calculator](https://cloud.google.com/products/calculator/)\n",
+        "to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "4ae6fd90ce34"
+      },
+      "source": [
+        "#### Set your project ID\n",
+        "\n",
+        "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "0b690120659f"
+      },
+      "outputs": [],
+      "source": [
+        "PROJECT_ID = \"\"\n",
+        "\n",
+        "# Get your Google Cloud project ID from gcloud\n",
+        "if not os.getenv(\"IS_TESTING\"):\n",
+        "    shell_output=!gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "    PROJECT_ID = shell_output[0]\n",
+        "    print(\"Project ID: \", PROJECT_ID)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "c23bde1f51c5"
+      },
+      "source": [
+        "Otherwise, set your project ID here."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "a258c0ac1442"
+      },
+      "outputs": [],
+      "source": [
+        "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
+        "    PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "826232e3a213"
+      },
+      "source": [
+        "#### Timestamp\n",
+        "\n",
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append it onto the name of resources you create in this tutorial."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "47aebdd88e9e"
+      },
+      "outputs": [],
+      "source": [
+        "from datetime import datetime\n",
+        "\n",
+        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "98ca9d3da7e6"
+      },
+      "source": [
+        "### Create a Cloud Storage bucket\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "When you submit a training job using the Cloud SDK, you upload a Python package\n",
+        "containing your training code to a Cloud Storage bucket. Vertex AI runs\n",
+        "the code from this package. In this tutorial, Vertex AI also saves the\n",
+        "trained model that results from your job in the same bucket. Using this model artifact, you can then\n",
+        "create Vertex AI model and endpoint resources in order to serve\n",
+        "online predictions.\n",
+        "\n",
+        "Set the name of your Cloud Storage bucket below. It must be unique across all\n",
+        "Cloud Storage buckets.\n",
+        "\n",
+        "You may also change the `REGION` variable, which is used for operations\n",
+        "throughout the rest of this notebook. Make sure to [choose a region where Vertex AI services are\n",
+        "available](https://cloud.google.com/vertex-ai/docs/general/locations#available_regions). You may\n",
+        "not use a Multi-Regional Storage bucket for training with Vertex AI."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "5ead6150209c"
+      },
+      "outputs": [],
+      "source": [
+        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "REGION = \"[your-region]\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "933efba860bc"
+      },
+      "outputs": [],
+      "source": [
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "26fd8d22e96f"
+      },
+      "source": [
+        "**Only if your bucket doesn't already exist**: Run the following cell to create your Cloud Storage bucket."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "d201bf34f895"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil mb -l $REGION $BUCKET_NAME"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3f62553e5054"
+      },
+      "source": [
+        "**Finally**, validate access to your Cloud Storage bucket by examining its contents:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "73f53d17ae47"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil ls -al $BUCKET_NAME"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "7ef78f6261a0"
+      },
+      "source": [
+        "## Before you begin\n",
+        "\n",
+        "The ALS model approach is compute-intensive and could take a lot of time to train on a regular notebook environment, so this tutorial uses a Dataproc cluster with PySpark environment.\n",
+        "\n",
+        "### Create a Dataproc cluster with component gateway enabled and JupyterLab extension\n",
+        "<a name=\"section-5\"></a>\n",
+        "\n",
+        "Create the cluster using the following `gcloud` command."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "950554272656"
+      },
+      "outputs": [],
+      "source": [
+        "CLUSTER_NAME = \"[your-cluster-name]\"\n",
+        "CLUSTER_REGION = \"[your-cluster-region]\"\n",
+        "CLUSTER_ZONE = \"[your-cluster-zone]\"\n",
+        "MACHINE_TYPE = \"[your=machine-type]\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "1e3e719e27f6"
+      },
+      "outputs": [],
+      "source": [
+        "! gcloud dataproc clusters create $CLUSTER_NAME \\\n",
+        "--enable-component-gateway \\\n",
+        "--region $CLUSTER_REGION \\\n",
+        "--zone $CLUSTER_ZONE \\\n",
+        "--single-node \\\n",
+        "--master-machine-type $MACHINE_TYPE \\\n",
+        "--master-boot-disk-size 100 \\\n",
+        "--image-version 2.0-debian10 \\\n",
+        "--optional-components JUPYTER \\\n",
+        "--project $PROJECT_ID"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2961b0a96176"
+      },
+      "source": [
+        "Alternatively, the cluster can be created through the Dataproc console as well. Additional settings like network configuratons and service-accounts can be configured there if required. While configuring the cluster, make sure you complete the following:\n",
+        "\n",
+        "- Provide a name for the cluster.\n",
+        "- Select a region and zone for the cluster.\n",
+        "- Select the cluster type as single-node. For small and proof-of-concept use-cases, a single-node cluster is recommended.\n",
+        "- Enable the component gateway.\n",
+        "- In the optional components, select Jupyter Notebook.\n",
+        "- (Optional) Select the machine-type (preferably a high-mem machine type).\n",
+        "- Create the cluster."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "b864144904fb"
+      },
+      "source": [
+        "## Connect to the cluster from the notebook\n",
+        "<a name=\"section-6\"></a>\n",
+        "\n",
+        "When the new Dataproc cluster is running, the corresponding runtime appears as a kernel in the notebook. The created cluster's name will appear in the list of kernels that can be selected for this notebook. In the top right corner of this notebook file, click the current kernel name, **Python (local)**, and then select the Python 3 kernel that is running on your Dataproc cluster.\n",
+        "\n",
+        "<img src=\"images/cluster_kernel_selection.png\"></img>\n",
+        "\n",
+        "Note the following:\n",
+        "\n",
+        "- Your Dataproc kernel might take a few minutes to show up in the list of kernels.\n",
+        "- PySpark code in this tutorial can be run on either a PySpark or Python 3 kernel on the Dataproc cluster, but to run the optional code that saves recommendations to a BigQuery table, the Python 3 kernel is recommended."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "fc8d6f0cc21b"
+      },
+      "source": [
+        "## Tutorial "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "03c8dc1024f7"
+      },
+      "source": [
+        "## Explore the data\n",
+        "<a name=\"section-7\"></a>\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ffc1113d6504"
+      },
+      "source": [
+        "Vertex AI Workbench managed notebooks instances let you explore the BigQuery content from within the managed notebooks instance using a BigQuery integration. This feature lets you look at the metadata and preview of table content, query tables, and get a description of the data in the tables.\n",
+        "\n",
+        "<img src=\"images/BQ_view_table_new.PNG\"></img>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "1fb418e82451"
+      },
+      "source": [
+        "Check the distribution of the `STATUS` field."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dfebfe18e127"
+      },
+      "source": [
+        "#@bigquery\n",
+        "SELECT STATUS, COUNT(*) order_count FROM looker-private-demo.retail.order_items GROUP BY 1"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3dd73d32bc80"
+      },
+      "source": [
+        "Join the `order_items` table with the `inventory_items` table from the same dataset to retrieve the product IDs for the orders."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "4abc4fb13e25"
+      },
+      "source": [
+        "#@bigquery\n",
+        "WITH user_prod_table AS (\n",
+        "SELECT USER_ID, PRODUCT_ID, STATUS FROM looker-private-demo.retail.order_items AS a\n",
+        "join\n",
+        "(SELECT ID, PRODUCT_ID FROM looker-private-demo.retail.inventory_items) AS b\n",
+        "on a.inventory_item_id = b.ID )\n",
+        "\n",
+        "SELECT USER_ID, PRODUCT_ID, STATUS from user_prod_table"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5072aded172b"
+      },
+      "source": [
+        "Once the results from BigQuery are displayed in the above cell, click the **Query and load as DataFrame** button and execute the generated code stub to fetch the data into the current notebook as a dataframe.\n",
+        "\n",
+        "*Note: By default the data is loaded into a `df` variable, though this can be changed before executing the cell if required.*"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ba1a54b04d9a"
+      },
+      "outputs": [],
+      "source": [
+        "# The following two lines are only necessary to run once.\n",
+        "# Comment out otherwise for speed-up.\n",
+        "from google.cloud.bigquery import Client\n",
+        "\n",
+        "client = Client()\n",
+        "\n",
+        "query = \"\"\"WITH user_prod_table AS (\n",
+        "SELECT USER_ID, PRODUCT_ID, STATUS FROM looker-private-demo.retail.order_items AS a\n",
+        "join\n",
+        "(SELECT ID, PRODUCT_ID FROM looker-private-demo.retail.inventory_items) AS b\n",
+        "on a.inventory_item_id = b.ID )\n",
+        "\n",
+        "SELECT USER_ID, PRODUCT_ID, STATUS from user_prod_table\"\"\"\n",
+        "job = client.query(query)\n",
+        "df = job.to_dataframe()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "acb1077cbf1e"
+      },
+      "source": [
+        "### Preprocess the Data\n",
+        "\n",
+        "To run PySpark's ALS method on the existing data, there must be some fields to quantify the relationship between a `USER_ID` and a `PRODUCT_ID`, such as *ratings given by the user*. If such fields already exist in the data, they can be treated as an *explicit feedback* for the ALS model. Otherwise, the fields indicative of a relationship can be given as an *implicit feedback*. Learn more about [feedback for PySpark's ALS method](https://spark.apache.org/docs/2.2.0/ml-collaborative-filtering.html#explicit-vs-implicit-feedback).\n",
+        "\n",
+        "In the current dataset, as there are no such numerical fields, the `STATUS` field is further used to quantify the association between a `USER_ID` and  a `PRODUCT_ID`. Based on when they occur during an order lifecycle and how likely the user is going to like the order, the `STATUS` field is assigned one of the following ratings:\n",
+        "\n",
+        "- Cancelled - 1\n",
+        "- Returned - 2\n",
+        "- Processing - 3\n",
+        "- Shipped - 4\n",
+        "- Complete - 5\n",
+        "\n",
+        "The ratings given are subjective and can be modified according to the use case."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "365dca51641f"
+      },
+      "outputs": [],
+      "source": [
+        "score_mapping = {\n",
+        "    \"Cancelled\": 1,\n",
+        "    \"Returned\": 2,\n",
+        "    \"Processing\": 3,\n",
+        "    \"Shipped\": 4,\n",
+        "    \"Complete\": 5,\n",
+        "}\n",
+        "df[\"RATING\"] = df[\"STATUS\"].map(score_mapping)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "a88eb70c6dec"
+      },
+      "source": [
+        "Check the distribution of the newly generated `RATING` field."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cfe29a1f9f16"
+      },
+      "outputs": [],
+      "source": [
+        "df[\"RATING\"].plot(kind=\"hist\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "367bc4912977"
+      },
+      "source": [
+        "Load the required methods and classes from PySpark MLlib."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b674b037dce0"
+      },
+      "outputs": [],
+      "source": [
+        "from pyspark.ml.evaluation import RegressionEvaluator\n",
+        "from pyspark.ml.recommendation import ALS\n",
+        "from pyspark.ml.tuning import CrossValidator, ParamGridBuilder\n",
+        "from pyspark.sql import SparkSession"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "1c3c74870bcf"
+      },
+      "source": [
+        "Generate a Spark session with the BigQuery-Spark connector configured.\n",
+        "\n",
+        "*Note: If the notebook is connected to a Dataproc cluster, the session object would show `yarn` as the Master.*"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2e55770bb894"
+      },
+      "outputs": [],
+      "source": [
+        "spark = (\n",
+        "    SparkSession.builder.appName(\"Recommendations\")\n",
+        "    .config(\"spark.jars\", \"gs://spark-lib/bigquery/spark-bigquery-latest_2.12.jar\")\n",
+        "    .getOrCreate()\n",
+        ")\n",
+        "\n",
+        "spark"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "89b6a2146b7c"
+      },
+      "source": [
+        "Convert the pandas dataframe to a spark dataframe for further processing."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "0d1d5b5e6a56"
+      },
+      "outputs": [],
+      "source": [
+        "spark_df = spark.createDataFrame(df[[\"USER_ID\", \"PRODUCT_ID\", \"RATING\"]])\n",
+        "spark_df.printSchema()\n",
+        "spark_df.show()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5b159af5c085"
+      },
+      "source": [
+        "### Split the data into train and test"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "951e7c2bf8fe"
+      },
+      "outputs": [],
+      "source": [
+        "(train, test) = spark_df.randomSplit([0.8, 0.2], seed=36)\n",
+        "train.count(), test.count()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dea0589dd474"
+      },
+      "source": [
+        "## Define the ALS Model\n",
+        "<a name=\"section-8\"></a>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "99e0365f32be"
+      },
+      "source": [
+        "The PySpark ALS recommender, Alternating Least Squares, is a matrix factorization algorithm. The idea is to build a matrix that maps users to actions. The actions can be reviews, purchases, various options taken, and more. Due to the complexity and size of the matrix, PySpark can run the algorithm in parallel.\n",
+        "\n",
+        "ALS will attempt to estimate the rating matrix R as the product of two lower-rank matrices, X and Y. Typically these approximations are called \"factor\" matrices. During each iteration, one of the factor matrices is held constant, while the other is solved for using least squares. The newly-solved factor matrix is then held constant while solving for the other factor matrix.\n",
+        "\n",
+        "PySpark uses a blocked implementation of the ALS factorization algorithm that groups the two sets of factors (referred to as “users” and “products”) into blocks and reduces communication by only sending one copy of each user vector to each product block on each iteration, and only for the product blocks that need that user’s feature vector.\n",
+        "\n",
+        "Essentially instead of finding the low-rank approximations to the rating matrix R, this finds the approximations for a preference matrix P where the elements of P are 1 if r > 0 and 0 if r <= 0. The ratings then act as confidence values related to the strength of indicated user preferences rather than explicit ratings given to items. Learn more [about PySpark's ALS algorithm](https://spark.apache.org/docs/latest/api/python/reference/api/pyspark.ml.recommendation.ALS.html)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b47e5bfe0c30"
+      },
+      "outputs": [],
+      "source": [
+        "als = ALS(\n",
+        "    userCol=\"USER_ID\",\n",
+        "    itemCol=\"PRODUCT_ID\",\n",
+        "    ratingCol=\"RATING\",\n",
+        "    nonnegative=True,\n",
+        "    implicitPrefs=False,\n",
+        "    coldStartStrategy=\"drop\",\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9726f220acb5"
+      },
+      "source": [
+        "The ALS model tries to predict the ratings between users and items and so RMSE can be used for evaluating the model."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "50ae46e7db7c"
+      },
+      "outputs": [],
+      "source": [
+        "evaluator = RegressionEvaluator(\n",
+        "    metricName=\"rmse\", labelCol=\"RATING\", predictionCol=\"prediction\"\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "01d3b2a2a801"
+      },
+      "source": [
+        "Define a hyperparameter grid for cross-validation."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "d0397bf82ba6"
+      },
+      "outputs": [],
+      "source": [
+        "param_grid = (\n",
+        "    ParamGridBuilder()\n",
+        "    .addGrid(als.rank, [10, 50])\n",
+        "    .addGrid(als.regParam, [0.01, 0.1, 0.2])\n",
+        "    .build()\n",
+        ")\n",
+        "print(\"No. of settings to be tested: \", len(param_grid))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "98f3c23ff0a1"
+      },
+      "source": [
+        "Perform cross-validation and save the best model."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "13617284cdb3"
+      },
+      "outputs": [],
+      "source": [
+        "cv = CrossValidator(\n",
+        "    estimator=als, estimatorParamMaps=param_grid, evaluator=evaluator, numFolds=3\n",
+        ")\n",
+        "model = cv.fit(train)\n",
+        "best_model = model.bestModel"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "66bc83eb4adb"
+      },
+      "outputs": [],
+      "source": [
+        "print(\"##Parameters for the Best Model##\")\n",
+        "print(\"Rank:\", best_model._java_obj.parent().getRank())\n",
+        "print(\"MaxIter:\", best_model._java_obj.parent().getMaxIter())\n",
+        "print(\"RegParam:\", best_model._java_obj.parent().getRegParam())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "12a02668017e"
+      },
+      "source": [
+        "## Evaluate the model\n",
+        "<a name=\"section-9\"></a>\n",
+        "Evaluate the model by computing the RMSE on the train and test data."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "04d7e30661fa"
+      },
+      "outputs": [],
+      "source": [
+        "# View the rating predictions by the model on train and test sets\n",
+        "train_predictions = best_model.transform(train)\n",
+        "train_RMSE = evaluator.evaluate(train_predictions)\n",
+        "\n",
+        "test_predictions = best_model.transform(test)\n",
+        "test_RMSE = evaluator.evaluate(test_predictions)\n",
+        "\n",
+        "print(\"Train RMSE \", train_RMSE)\n",
+        "print(\"Test RMSE \", test_RMSE)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "521dc4a5a688"
+      },
+      "source": [
+        "### Generate recommendations for all users\n",
+        "\n",
+        "The required number of recommendations for the users can be generated using the ALS model's `recommendForAllUsers()` method."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "c7ddfd001900"
+      },
+      "outputs": [],
+      "source": [
+        "# Generate 10 product recommendations for all users\n",
+        "nrecommendations = best_model.recommendForAllUsers(10)\n",
+        "nrecommendations.limit(10).show()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "219e986f5c23"
+      },
+      "source": [
+        "### Generate recommendations for a specific user\n",
+        "\n",
+        "The earlier step already generated and stored the specified number of product recommendations for all users in the `nrecommendations` dataframe object. To obtain recommendations for a single user, this dataframe object can be queried."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "782b17eb3621"
+      },
+      "outputs": [],
+      "source": [
+        "# get product recommendations for the selected user (USER_ID = 1)\n",
+        "nrecommendations.where(nrecommendations.USER_ID == 1).select(\n",
+        "    \"recommendations.PRODUCT_ID\", \"recommendations.rating\"\n",
+        ").collect()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "964c4d610d31"
+      },
+      "source": [
+        "## Save the ALS model to Cloud Storage (optional)\n",
+        "<a name=\"section-10\"></a>\n",
+        "\n",
+        "PySpark's `ALS.save()` method creates a folder at the specified path where it saves the trained model. A Cloud Storage file browser is available in the managed notebooks instance's environment, which you can use to save the model to a Cloud Storage bucket."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "73db311320d0"
+      },
+      "source": [
+        "Use the ALS object's `.save()` function to write the model to the Cloud Storage bucket. "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9f910addca92"
+      },
+      "outputs": [],
+      "source": [
+        "# Save the trained model\n",
+        "GCS_MODEL_PATH = \"gs://\" + BUCKET_NAME + \"/recommender_systems/\"\n",
+        "best_model.save(GCS_MODEL_PATH + \"rcmd_model\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "665a05361fbd"
+      },
+      "source": [
+        "## Write the recommendations to BigQuery (optional)\n",
+        "<a name=\"section-11\"></a>\n",
+        "\n",
+        "In order to serve the recommendations to the end-users or any applications, the output from the `recommendForAllUsers()` method can be saved to a BigQuery table using Spark's BigQuery connector.\n",
+        "\n",
+        "### Create a Dataset in BigQuery\n",
+        "\n",
+        "The following cell creates a new dataset in BigQuery."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "22e136731378"
+      },
+      "source": [
+        "#@bigquery\n",
+        "-- create a dataset in BigQuery\n",
+        "CREATE SCHEMA recommender_sys\n",
+        "OPTIONS(\n",
+        "  location=\"us\"\n",
+        "  )"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "4910f961001f"
+      },
+      "source": [
+        "### Write the Recommendations to BigQuery\n",
+        "\n",
+        "PySpark's BigQuery connector requires two necessary fields: a *BigQuery Table name* and a *Cloud Storage path to write the temporary files* while saving the model. These two fields can be provided while writing the recommendations to BigQuery."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "63b76c37123d"
+      },
+      "outputs": [],
+      "source": [
+        "DATASET = \"[your-dataset-name]\"\n",
+        "TABLE = \"[your-bigquery-table-name]\"\n",
+        "GCS_TEMP_PATH = \"[your-cloud-storage-path]\"\n",
+        "\n",
+        "nrecommendations.write.format(\"bigquery\").option(\n",
+        "    \"table\", \"{}.{}\".format(DATASET, TABLE)\n",
+        ").option(\"temporaryGcsBucket\", GCS_TEMP_PATH).mode(\"overwrite\").save()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6c3a6d501600"
+      },
+      "source": [
+        "## Clean up\n",
+        "<a name=\"section-12\"></a>\n",
+        "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
+        "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
+        "\n",
+        "Otherwise, you can delete the individual resources you created in this tutorial:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "72488a2999a6"
+      },
+      "outputs": [],
+      "source": [
+        "# remove the BigQuery dataset created for storing the recommendations and all of its tables\n",
+        "! bq rm -r -f -d $PROJECT:$DATASET\n",
+        "\n",
+        "# remove the Cloud Storage bucket created and all of its tables\n",
+        "! gsutil rm -r gs://$BUCKET_NAME"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "75cfc772a3f4"
+      },
+      "outputs": [],
+      "source": [
+        "# delete the created Dataproc cluster\n",
+        "! gcloud dataproc clusters delete $CLUSTER_NAME --region=$CLUSTER_REGION"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "recommender-system-on-retail-data.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/community/matching_engine/matching_engine_for_indexing.ipynb

@@ -541,7 +541,7 @@
       "source": [
         "# NOTE: Everything in this GCS DIR will be DELETED before uploading the data.\n",
         "\n",
-        "! gsutil rm -rf {BUCKET_NAME}"
+        "! gsutil rm -rf {BUCKET_NAME}/*"
       ]
     },
     {

notebooks/community/migration/UJ1 AutoML for vision with Vertex AI Image Classification.ipynb

@@ -431,7 +431,7 @@
         "\n",
         "- Dataset Schemas: Tells the managed dataset service which type of dataset it is.\n",
         "- Data Labeling (Annotations) Schemas: Tells the managed dataset service how the data is labeled (annotated).\n",
-        "- Dataset Training Schemas: Tells the managed pipelines service the task (e.g., classification) to train the model for.\n"
+        "- Dataset Training Schemas: Tells the Vertex AI Pipelines service the task (e.g., classification) to train the model for.\n"
       ]
     },
     {
@@ -3261,7 +3261,7 @@
         "aip_constants",
         "automl_constants:automl"
       ],
-      "name": "UJ1 unified AutoML Vision Image Classification.ipynb",
+      "name": "UJ1 unified AutoML for vision with Vertex AI Image Classification.ipynb",
       "toc_visible": true
     },
     "kernelspec": {

notebooks/community/migration/UJ13 Data Labeling task.ipynb

@@ -431,7 +431,7 @@
         "\n",
         "- Dataset Schemas: Tells the managed dataset service which type of dataset it is.\n",
         "- Data Labeling (Annotations) Schemas: Tells the managed dataset service how the data is labeled (annotated).\n",
-        "- Dataset Training Schemas: Tells the managed pipelines service the task (e.g., classification) to train the model for.\n"
+        "- Dataset Training Schemas: Tells the Vertex AI Pipelines service the task (e.g., classification) to train the model for.\n"
       ]
     },
     {

notebooks/community/migration/UJ14 AutoML for vision with Vertex AI Video Classification.ipynb

@@ -430,7 +430,7 @@
         "\n",
         "- Dataset Schemas: Tells the managed dataset service which type of dataset it is.\n",
         "- Data Labeling (Annotations) Schemas: Tells the managed dataset service how the data is labeled (annotated).\n",
-        "- Dataset Training Schemas: Tells the managed pipelines service the task (e.g., classification) to train the model for.\n"
+        "- Dataset Training Schemas: Tells the Vertex AI Pipelines service the task (e.g., classification) to train the model for.\n"
       ]
     },
     {
@@ -1983,7 +1983,7 @@
         "7RdKtPhlSnkw",
         "Hgt8RcLLSnkw"
       ],
-      "name": "UJ14 unified AutoML Vision Video Classification.ipynb",
+      "name": "UJ14 unified AutoML for vision with Vertex AI Video Classification.ipynb",
       "toc_visible": true
     },
     "kernelspec": {

notebooks/community/migration/UJ15 AutoML for vision with Vertex AI Video Object Tracking.ipynb

@@ -430,7 +430,7 @@
         "\n",
         "- Dataset Schemas: Tells the managed dataset service which type of dataset it is.\n",
         "- Data Labeling (Annotations) Schemas: Tells the managed dataset service how the data is labeled (annotated).\n",
-        "- Dataset Training Schemas: Tells the managed pipelines service the task (e.g., classification) to train the model for.\n"
+        "- Dataset Training Schemas: Tells the Vertex AI Pipelines service the task (e.g., classification) to train the model for.\n"
       ]
     },
     {
@@ -1889,7 +1889,7 @@
   "metadata": {
     "colab": {
       "collapsed_sections": [],
-      "name": "UJ15 unified AutoML Vision Video Object Tracking.ipynb",
+      "name": "UJ15 unified AutoML for vision with Vertex AI Video Object Tracking.ipynb",
       "toc_visible": true
     },
     "kernelspec": {

notebooks/community/migration/UJ4 AutoML for structured data with Vertex AI Regression.ipynb

@@ -431,7 +431,7 @@
         "\n",
         "- Dataset Schemas: Tells the managed dataset service which type of dataset it is.\n",
         "- Data Labeling (Annotations) Schemas: Tells the managed dataset service how the data is labeled (annotated).\n",
-        "- Dataset Training Schemas: Tells the managed pipelines service the task (e.g., classification) to train the model for.\n"
+        "- Dataset Training Schemas: Tells the Vertex AI Pipelines service the task (e.g., classification) to train the model for.\n"
       ]
     },
     {
@@ -2469,7 +2469,7 @@
         "call:migration",
         "response:migration"
       ],
-      "name": "UJ4 unified AutoML Tables Regression.ipynb",
+      "name": "UJ4 unified AutoML for structured data with Vertex AI Regression.ipynb",
       "toc_visible": true
     },
     "kernelspec": {

notebooks/community/migration/UJ5 AutoML for vision with Vertex AI Video Classification.ipynb

@@ -29,7 +29,7 @@
         "id": "title:migration,new"
       },
       "source": [
-        "# Vertex AI AutoML Vision Image Object Detection\n"
+        "# Vertex AI AutoML Image Object Detection\n"
       ]
     },
     {
@@ -430,7 +430,7 @@
         "\n",
         "- Dataset Schemas: Tells the managed dataset service which type of dataset it is.\n",
         "- Data Labeling (Annotations) Schemas: Tells the managed dataset service how the data is labeled (annotated).\n",
-        "- Dataset Training Schemas: Tells the managed pipelines service the task (e.g., classification) to train the model for.\n"
+        "- Dataset Training Schemas: Tells the Vertex AI Pipelines service the task (e.g., classification) to train the model for.\n"
       ]
     },
     {
@@ -2467,7 +2467,7 @@
         "_RXG0aaSV2HS",
         "EDuJAyzbV2HW"
       ],
-      "name": "UJ5 unified AutoML Vision Video Classification.ipynb",
+      "name": "UJ5 unified AutoML for vision with Vertex AI Video Classification.ipynb",
       "toc_visible": true
     },
     "kernelspec": {

notebooks/community/migration/UJ6 AutoML for natural language with Vertex AI Text Classification.ipynb

@@ -432,7 +432,7 @@
         "\n",
         "- Dataset Schemas: Tells the managed dataset service which type of dataset it is.\n",
         "- Data Labeling (Annotations) Schemas: Tells the managed dataset service how the data is labeled (annotated).\n",
-        "- Dataset Training Schemas: Tells the managed pipelines service the task (e.g., classification) to train the model for."
+        "- Dataset Training Schemas: Tells the Vertex AI Pipelines service the task (e.g., classification) to train the model for."
       ]
     },
     {
@@ -2522,7 +2522,7 @@
         "hIHTX-pkJjkO",
         "4x_t-MWnJjkQ"
       ],
-      "name": "UJ6 unified AutoML Natural Language Text Classification.ipynb",
+      "name": "UJ6 unified AutoML for natural language with Vertex AI Text Classification.ipynb",
       "toc_visible": true
     },
     "kernelspec": {

notebooks/community/migration/UJ7 AutoML for natural language with Vertex AI Text Entity Extraction.ipynb

@@ -442,7 +442,7 @@
         "\n",
         "- Dataset Schemas: Tells the managed dataset service which type of dataset it is.\n",
         "- Data Labeling (Annotations) Schemas: Tells the managed dataset service how the data is labeled (annotated).\n",
-        "- Dataset Training Schemas: Tells the managed pipelines service the task (e.g., classification) to train the model for."
+        "- Dataset Training Schemas: Tells the Vertex AI Pipelines service the task (e.g., classification) to train the model for."
       ]
     },
     {
@@ -2404,7 +2404,7 @@
   ],
   "metadata": {
     "colab": {
-      "name": "UJ7 unified AutoML Natural Language Text Entity Extraction.ipynb",
+      "name": "UJ7 unified AutoML for natural language with Vertex AI Text Entity Extraction.ipynb",
       "toc_visible": true
     },
     "kernelspec": {

notebooks/community/migration/UJ8 AutoML for natural language with Vertex AI - Text Sentiment Analysis.ipynb

@@ -424,7 +424,7 @@
         "\n",
         "- Dataset Schemas: Tells the managed dataset service which type of dataset it is.\n",
         "- Data Labeling (Annotations) Schemas: Tells the managed dataset service how the data is labeled (annotated).\n",
-        "- Dataset Training Schemas: Tells the managed pipelines service the task (e.g., classification) to train the model for."
+        "- Dataset Training Schemas: Tells the Vertex AI Pipelines service the task (e.g., classification) to train the model for."
       ]
     },
     {
@@ -2342,7 +2342,7 @@
   ],
   "metadata": {
     "colab": {
-      "name": "UJ8 unified AutoML Natural Language - Text Sentiment Analysis.ipynb",
+      "name": "UJ8 unified AutoML for natural language with Vertex AI - Text Sentiment Analysis.ipynb",
       "toc_visible": true
     },
     "kernelspec": {

notebooks/community/ml_metadata/sdk-metric-parameter-tracking-for-locally-trained-models.ipynb

@@ -688,7 +688,7 @@
         "id": "O8XJZB3gR8eL"
       },
       "source": [
-        "### Initialize the Model Builder SDK and create an Experiment\n",
+        "### Initialize the Vertex AI SDK for Python and create an Experiment\n",
         "\n",
         "Initialize the *client* for Vertex AI and create an experiment."
       ]

notebooks/community/ml_ops/README.md

@@ -0,0 +1,18 @@
+
+# E2E MLOps on Google Cloud (Work in Progress)
+## Best Practices
+
+“Practicing MLOps means that you advocate for automation and monitoring at all steps of ML system construction, including integration, testing, releasing, deployment and infrastructure management” – Google
+
+The purpose of this set of notebooks and markdown files is to demonstrate Google Cloud AI’s best practices for MLOps.
+
+## MLOps Stages
+
+1. [Data Management](stage1)
+2. [Experimentation](stage2)
+3. [Formalization](stage3)
+4. Evaluation
+5. Deployment
+6. Serving
+7. Monitoring
+8. Continuous Training

notebooks/community/ml_ops/stage1/README.md

@@ -0,0 +1,39 @@
+# Stage 1: Data Management
+
+## Purpose
+
+Progressively improve the quality of the data and the performance in delivering the data for model training.
+
+## Recommendations  
+
+The first stage in MLOps is the collection and preparation for the purpose of delivering data (model feeding). We recommend:
+
+- Data is stored in GCS buckets and/or BigQuery tables. 
+- Data sources partitioned by level of data wrangling. Typically these are:
+  - Curated data – real-world data for training.
+  - Synthetic data - synthetically generated data for pre-training. 
+  - Unlabeled data - data that needs to be labeled, which optionally can be labeled using Vertex Data Labeling service. Some of this data may come from production.
+  - Noisy - labeled data that needs cleansing.
+- Data examples are searchable by feature conditions, labeling and metadata – such as the labeling source.
+- Data examples can be tagged for train/eval slicing.
+- Statistics are collected and stored on the data distribution.
+- Data sources are under version control.
+- Features that dynamically change per example (e.g., bank balance) are stored in Vertex Feature Store.
+- Data is preprocessed for training and evaluation using Dataflow.
+- Data augmentation is performed on-the-fly and is coupled with model feeding.
+
+<img src='stage1.jpg'>
+
+## Notebooks
+
+### Get Started
+
+[Get Started with BQ datasets](get_started_bq_datasets.ipynb)
+
+[Get Started with Vertex datasets](get_started_vertex_datasets.ipynb)
+
+[Get Started with Dataflow](get_started_dataflow.ipynb)
+
+### E2E Stage Example
+
+[Stage 1: Data Management](mlops_data_management.ipynb)