sync with ai-platform-samples (#25)

* sync with ai-platform-samples

* fix: update links

.cloud-build/CheckPythonVersion.py

@@ -1,3 +1,18 @@
+#!/usr/bin/env python
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import sys
 
 MINIMUM_MAJOR_VERSION = 3

.cloud-build/ExecuteChangedNotebooks.py

@@ -1,5 +1,5 @@
-#!/bin/bash
-# Copyright 2019 Google LLC
+#!/usr/bin/env python
+# Copyright 2021 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,21 +12,114 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 import argparse
+import dataclasses
+import datetime
+import functools
 import pathlib
+import os
 import subprocess
 from pathlib import Path
-from typing import Dict, List, Optional
+from typing import List, Optional
+import concurrent
+from tabulate import tabulate
 
 import ExecuteNotebook
 
 
+def str2bool(v):
+    if isinstance(v, bool):
+        return v
+    if v.lower() in ("yes", "true", "t", "y", "1"):
+        return True
+    elif v.lower() in ("no", "false", "f", "n", "0"):
+        return False
+    else:
+        raise argparse.ArgumentTypeError("Boolean value expected.")
+
+
+def format_timedelta(delta: datetime.timedelta) -> str:
+    """Formats a timedelta duration to [N days] %H:%M:%S format"""
+    seconds = int(delta.total_seconds())
+
+    secs_in_a_day = 86400
+    secs_in_a_hour = 3600
+    secs_in_a_min = 60
+
+    days, seconds = divmod(seconds, secs_in_a_day)
+    hours, seconds = divmod(seconds, secs_in_a_hour)
+    minutes, seconds = divmod(seconds, secs_in_a_min)
+
+    time_fmt = f"{hours:02d}:{minutes:02d}:{seconds:02d}"
+
+    if days > 0:
+        suffix = "s" if days > 1 else ""
+        return f"{days} day{suffix} {time_fmt}"
+
+    return time_fmt
+
+
+@dataclasses.dataclass
+class NotebookExecutionResult:
+    notebook: str
+    duration: datetime.timedelta
+    is_pass: bool
+    error_message: Optional[str]
+
+
+def execute_notebook(
+    artifacts_path: str,
+    variable_project_id: str,
+    variable_region: str,
+    should_log_output: bool,
+    should_use_new_kernel: bool,
+    notebook: str,
+) -> NotebookExecutionResult:
+    print(f"Running notebook: {notebook}")
+
+    result = NotebookExecutionResult(
+        notebook=notebook,
+        duration=datetime.timedelta(seconds=0),
+        is_pass=False,
+        error_message=None,
+    )
+
+    # TODO: Handle cases where multiple notebooks have the same name
+    time_start = datetime.datetime.now()
+    try:
+        ExecuteNotebook.execute_notebook(
+            notebook_file_path=notebook,
+            output_file_folder=artifacts_path,
+            replacement_map={
+                "PROJECT_ID": variable_project_id,
+                "REGION": variable_region,
+            },
+            should_log_output=should_log_output,
+            should_use_new_kernel=should_use_new_kernel,
+        )
+        result.duration = datetime.datetime.now() - time_start
+        result.is_pass = True
+        print(f"{notebook} PASSED in {format_timedelta(result.duration)}.")
+    except Exception as error:
+        result.duration = datetime.datetime.now() - time_start
+        result.is_pass = False
+        result.error_message = str(error)
+        print(
+            f"{notebook} FAILED in {format_timedelta(result.duration)}: {result.error_message}"
+        )
+
+    return result
+
+
 def run_changed_notebooks(
     test_paths_file: str,
+    base_branch: Optional[str],
     output_folder: str,
     variable_project_id: str,
     variable_region: str,
-    base_branch: Optional[str],
+    should_parallelize: bool,
+    should_use_separate_kernels: bool,
 ):
     """
     Run the notebooks that exist under the folders defined in the test_paths_file.
@@ -49,6 +142,13 @@ def run_changed_notebooks(
             Required. The value for PROJECT_ID to inject into notebooks.
         variable_region (str):
             Required. The value for REGION to inject into notebooks.
+        should_parallelize (bool):
+            Required. Should run notebooks in parallel using a thread pool as opposed to in sequence.
+        should_use_separate_kernels (bool):
+            Note: Dependencies don't install correctly when this is set to True
+            See https://github.com/nteract/papermill/issues/625
+
+            Required. Should run each notebook in a separate and independent virtual environment.
     """
 
     test_paths = []
@@ -84,41 +184,68 @@ def run_changed_notebooks(
     artifacts_path.joinpath("success").mkdir(parents=True, exist_ok=True)
     artifacts_path.joinpath("failure").mkdir(parents=True, exist_ok=True)
 
-    passed_notebooks: List[str] = []
-    failed_notebooks: List[str] = []
+    notebook_execution_results: List[NotebookExecutionResult] = []
 
     if len(notebooks) > 0:
         print(f"Found {len(notebooks)} modified notebooks: {notebooks}")
 
-        for notebook in notebooks:
-            print(f"Running notebook: {notebook}")
-
-            # TODO: Handle cases where multiple notebooks have the same name
-            try:
-                ExecuteNotebook.execute_notebook(
-                    notebook_file_path=notebook,
-                    output_file_folder=artifacts_path,
-                    replacement_map={
-                        "PROJECT_ID": variable_project_id,
-                        "REGION": variable_region,
-                    },
+        if should_parallelize and len(notebooks) > 1:
+            print(
+                "Running notebooks in parallel, so no logs will be displayed. Please wait..."
+            )
+            with concurrent.futures.ThreadPoolExecutor(max_workers=None) as executor:
+                notebook_execution_results = list(
+                    executor.map(
+                        functools.partial(
+                            execute_notebook,
+                            artifacts_path,
+                            variable_project_id,
+                            variable_region,
+                            False,
+                            should_use_separate_kernels,
+                        ),
+                        notebooks,
+                    )
                 )
-                print(f"Notebook finished successfully.")
-                passed_notebooks.append(notebook)
-            except Exception as error:
-                print(f"Notebook finished with failure: {error}")
-                failed_notebooks.append(notebook)
+        else:
+            notebook_execution_results = [
+                execute_notebook(
+                    artifacts_path=artifacts_path,
+                    variable_project_id=variable_project_id,
+                    variable_region=variable_region,
+                    notebook=notebook,
+                    should_log_output=True,
+                    should_use_new_kernel=should_use_separate_kernels,
+                )
+                for notebook in notebooks
+            ]
     else:
         print("No notebooks modified in this pull request.")
 
-    if len(failed_notebooks) > 0:
-        print(f"{len(failed_notebooks)} notebooks failed:")
-        print(failed_notebooks)
-        print(f"{len(passed_notebooks)} notebooks passed:")
-        print(passed_notebooks)
-    elif len(passed_notebooks) > 0:
-        print("All notebooks executed successfully:")
-        print(passed_notebooks)
+    print("\n=== RESULTS ===\n")
+
+    notebooks_sorted = sorted(
+        notebook_execution_results,
+        key=lambda result: result.is_pass,
+        reverse=True,
+    )
+    # Print results
+    print(
+        tabulate(
+            [
+                [
+                    os.path.basename(os.path.normpath(result.notebook)),
+                    "PASSED" if result.is_pass else "FAILED",
+                    format_timedelta(result.duration),
+                    result.error_message or "--",
+                ]
+                for result in notebooks_sorted
+            ],
+            headers=["file", "status", "duration", "error"],
+        )
+    )
+
+    print("\n=== END RESULTS===\n")
 
 
 parser = argparse.ArgumentParser(description="Run changed notebooks.")
@@ -152,6 +279,27 @@ parser.add_argument(
     required=True,
 )
 
+# Note: Dependencies don't install correctly when this is set to True
+parser.add_argument(
+    "--should_parallelize",
+    type=str2bool,
+    nargs="?",
+    const=True,
+    default=False,
+    help="Should run notebooks in parallel.",
+)
+
+# Note: This isn't guaranteed to work correctly due to existing Papermill issue
+# See https://github.com/nteract/papermill/issues/625
+parser.add_argument(
+    "--should_use_separate_kernels",
+    type=str2bool,
+    nargs="?",
+    const=True,
+    default=False,
+    help="(Experimental) Should run each notebook in a separate and independent virtual environment.",
+)
+
 args = parser.parse_args()
 run_changed_notebooks(
     test_paths_file=args.test_paths_file,
@@ -159,4 +307,6 @@ run_changed_notebooks(
     output_folder=args.output_folder,
     variable_project_id=args.variable_project_id,
     variable_region=args.variable_region,
+    should_parallelize=args.should_parallelize,
+    should_use_separate_kernels=args.should_use_separate_kernels,
 )

.cloud-build/ExecuteNotebook.py

@@ -1,20 +1,115 @@
+#!/usr/bin/env python
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
 import sys
 import nbformat
-from nbconvert.preprocessors import ExecutePreprocessor, CellExecutionError
 import os
 import errno
 from NotebookProcessors import RemoveNoExecuteCells, UpdateVariablesPreprocessor
-from typing import Dict
+from typing import Dict, Tuple
+import papermill as pm
+import shutil
+import virtualenv
+import uuid
+from jupyter_client.kernelspecapp import KernelSpecManager
 
 # This script is used to execute a notebook and write out the output notebook.
 # The replaces calling the nbconvert via command-line, which doesn't write the output notebook correctly when there are errors during execution.
 
+STAGING_FOLDER = "staging"
+ENVIRONMENTS_PATH = "environments"
+KERNELS_SPECS_PATH = "kernel_specs"
+
+
+def create_and_install_kernel() -> Tuple[str, str]:
+    # Create environment
+    kernel_name = str(uuid.uuid4())
+    env_name = f"{ENVIRONMENTS_PATH}/{kernel_name}"
+    # venv.create(env_name, system_site_packages=True, with_pip=True)
+    virtualenv.cli_run([env_name, "--system-site-packages"])
+
+    # Create kernel spec
+    kernel_spec = {
+        "argv": [
+            f"{env_name}/bin/python",
+            "-m",
+            "ipykernel_launcher",
+            "-f",
+            "{connection_file}",
+        ],
+        "display_name": "Python 3",
+        "language": "python",
+    }
+    kernel_spec_folder = os.path.join(KERNELS_SPECS_PATH, kernel_name)
+    kernel_spec_file = os.path.join(kernel_spec_folder, "kernel.json")
+
+    # Create kernel spec folder
+    if not os.path.exists(os.path.dirname(kernel_spec_file)):
+        try:
+            os.makedirs(os.path.dirname(kernel_spec_file))
+        except OSError as exc:  # Guard against race condition
+            if exc.errno != errno.EEXIST:
+                raise
+
+    with open(kernel_spec_file, mode="w", encoding="utf-8") as f:
+        json.dump(kernel_spec, f)
+
+    # Install kernel
+    kernel_spec_manager = KernelSpecManager()
+    kernel_spec_manager.install_kernel_spec(
+        source_dir=kernel_spec_folder, kernel_name=kernel_name
+    )
+
+    return kernel_name, env_name
+
 
 def execute_notebook(
-    notebook_file_path: str, output_file_folder: str, replacement_map: Dict[str, str]
+    notebook_file_path: str,
+    output_file_folder: str,
+    replacement_map: Dict[str, str],
+    should_log_output: bool,
+    should_use_new_kernel: bool,
 ):
+    # Create staging directory if it doesn't exist
+    staging_file_path = f"{STAGING_FOLDER}/{notebook_file_path}"
+    if not os.path.exists(os.path.dirname(staging_file_path)):
+        try:
+            os.makedirs(os.path.dirname(staging_file_path))
+        except OSError as exc:  # Guard against race condition
+            if exc.errno != errno.EEXIST:
+                raise
+
     file_name = os.path.basename(os.path.normpath(notebook_file_path))
 
+    # Create environments folder
+    if not os.path.exists(ENVIRONMENTS_PATH):
+        try:
+            os.makedirs(ENVIRONMENTS_PATH)
+        except OSError as exc:  # Guard against race condition
+            if exc.errno != errno.EEXIST:
+                raise
+
+    # Create and install kernel
+    kernel_name = next(
+        iter(KernelSpecManager().find_kernel_specs().keys()), None
+    )  # Find first existing kernel and use as default
+    env_name = None
+    if should_use_new_kernel:
+        kernel_name, env_name = create_and_install_kernel()
+
     # Read notebook
     with open(notebook_file_path) as f:
         nb = nbformat.read(f, as_version=4)
@@ -28,7 +123,6 @@ def execute_notebook(
         update_variables_preprocessor = UpdateVariablesPreprocessor(
             replacement_map=replacement_map
         )
-        execute_preprocessor = ExecutePreprocessor(timeout=-1, kernel_name="python3")
 
         # Use no-execute preprocessor
         (
@@ -38,16 +132,33 @@ def execute_notebook(
 
         (nb, resources) = update_variables_preprocessor.preprocess(nb, resources)
 
-        # Execute notebook
-        out = execute_preprocessor.preprocess(nb, resources)
+        # print(f"Staging modified notebook to: {staging_file_path}")
+        with open(staging_file_path, mode="w", encoding="utf-8") as f:
+            nbformat.write(nb, f)
 
-    except Exception as error:
-        out = None
-        print(f"Error executing the notebook: {notebook_file_path}.\n\n")
+        # Execute notebook
+        pm.execute_notebook(
+            input_path=staging_file_path,
+            output_path=staging_file_path,
+            kernel_name=kernel_name,
+            progress_bar=should_log_output,
+            request_save_on_cell_execute=should_log_output,
+            log_output=should_log_output,
+            stdout_file=sys.stdout if should_log_output else None,
+            stderr_file=sys.stderr if should_log_output else None,
+        )
+    except Exception:
+        # print(f"Error executing the notebook: {notebook_file_path}.\n\n")
         has_error = True
 
         raise
+
     finally:
+        # Clear env
+        if env_name is not None:
+            shutil.rmtree(path=env_name)
+
+        # Copy execute notebook
         output_file_path = os.path.join(
             output_file_folder, "failure" if has_error else "success", file_name
         )
@@ -60,6 +171,5 @@ def execute_notebook(
                 if exc.errno != errno.EEXIST:
                     raise
 
-        print(f"Writing output to: {output_file_path}")
-        with open(output_file_path, mode="w", encoding="utf-8") as f:
-            nbformat.write(nb, f)
+        # print(f"Writing output to: {output_file_path}")
+        shutil.move(staging_file_path, output_file_path)

.cloud-build/NotebookProcessors.py

@@ -1,3 +1,18 @@
+#!/usr/bin/env python
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 from nbconvert.preprocessors import Preprocessor
 from typing import Dict
 import UpdateNotebookVariables

.cloud-build/UpdateNotebookVariables.py

@@ -1,3 +1,18 @@
+#!/usr/bin/env python
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import re
 
 """

.cloud-build/cleanup/cleanup-cloudbuild.yaml

@@ -0,0 +1,8 @@
+steps:  
+  # Install Python dependencies and run cleanup script
+  - name: ${_PYTHON_IMAGE}
+    entrypoint: /bin/sh
+    args:
+    - -c
+    - 'python3 -m pip install -U -r .cloud-build/cleanup/cleanup-requirements.txt && python3 .cloud-build/cleanup/cleanup.py'
+timeout: 86400s

.cloud-build/cleanup/cleanup-requirements.txt

@@ -0,0 +1 @@
+google-cloud-aiplatform

.cloud-build/cleanup/cleanup.py

@@ -0,0 +1,42 @@
+from typing import List
+from resource_cleanup_manager import (
+    ResourceCleanupManager,
+    DatasetResourceCleanupManager,
+    EndpointResourceCleanupManager,
+    ModelResourceCleanupManager,
+)
+
+
+def run_cleanup_managers(managers: List[ResourceCleanupManager], is_dry_run: bool):
+    for manager in managers:
+        type_name = manager.type_name
+
+        print(f"Fetching {type_name}'s...")
+        resources = manager.list()
+        print(f"Found {len(resources)} {type_name}'s")
+        for resource in resources:
+            if not manager.is_deletable(resource):
+                continue
+
+            if is_dry_run:
+                resource_name = manager.resource_name(resource)
+                print(f"Will delete '{type_name}': {resource_name}")
+            else:
+                manager.delete(resource)
+
+        print("")
+
+
+is_dry_run = False
+
+if is_dry_run:
+    print("Starting cleanup in dry run mode...")
+
+# List of all cleanup managers
+managers = [
+    DatasetResourceCleanupManager(),
+    EndpointResourceCleanupManager(),
+    ModelResourceCleanupManager(),
+]
+
+run_cleanup_managers(managers=managers, is_dry_run=is_dry_run)

.cloud-build/cleanup/resource_cleanup_manager.py

@@ -0,0 +1,87 @@
+import abc
+from google.cloud import aiplatform
+from typing import Any
+from proto.datetime_helpers import DatetimeWithNanoseconds
+from google.cloud.aiplatform import base
+
+# If a resource was updated within this number of seconds, do not delete.
+RESOURCE_UPDATE_BUFFER_IN_SECONDS = 60 * 60 * 8
+
+
+class ResourceCleanupManager(abc.ABC):
+    @property
+    @abc.abstractmethod
+    def type_name(str) -> str:
+        pass
+
+    @abc.abstractmethod
+    def list(self) -> Any:
+        pass
+
+    @abc.abstractmethod
+    def resource_name(self, resource: Any) -> str:
+        pass
+
+    @abc.abstractmethod
+    def delete(self, resource: Any):
+        pass
+
+    @abc.abstractmethod
+    def get_seconds_since_modification(self, resource: Any) -> float:
+        pass
+
+    def is_deletable(self, resource: Any) -> bool:
+        time_difference = self.get_seconds_since_modification(resource)
+
+        if self.resource_name(resource).startswith("perm"):
+            print(f"Skipping '{resource}' due to name starting with 'perm'.")
+            return False
+
+        # Check that it wasn't created too recently, to prevent race conditions
+        if time_difference <= RESOURCE_UPDATE_BUFFER_IN_SECONDS:
+            print(
+                f"Skipping '{resource}' due update_time being '{time_difference}', which is less than '{RESOURCE_UPDATE_BUFFER_IN_SECONDS}'."
+            )
+            return False
+
+        return True
+
+
+class VertexAIResourceCleanupManager(ResourceCleanupManager):
+    @property
+    @abc.abstractmethod
+    def vertex_ai_resource(self) -> base.VertexAiResourceNounWithFutureManager:
+        pass
+
+    @property
+    def type_name(self) -> str:
+        return self.vertex_ai_resource._resource_noun
+
+    def list(self) -> Any:
+        return self.vertex_ai_resource.list()
+
+    def resource_name(self, resource: Any) -> str:
+        return resource.display_name
+
+    def delete(self, resource):
+        resource.delete()
+
+    def get_seconds_since_modification(self, resource: Any) -> bool:
+        update_time = resource.update_time
+        current_time = DatetimeWithNanoseconds.now(tz=update_time.tzinfo)
+        return (current_time - update_time).total_seconds()
+
+
+class DatasetResourceCleanupManager(VertexAIResourceCleanupManager):
+    vertex_ai_resource = aiplatform.datasets._Dataset
+
+
+class EndpointResourceCleanupManager(VertexAIResourceCleanupManager):
+    vertex_ai_resource = aiplatform.Endpoint
+
+    def delete(self, resource):
+        resource.delete(force=True)
+
+
+class ModelResourceCleanupManager(VertexAIResourceCleanupManager):
+    vertex_ai_resource = aiplatform.Model

.cloud-build/notebook-execution-test-cloudbuild.yaml

@@ -17,12 +17,16 @@ steps:
     args:
     - -c
     - 'if [ -n "${_BASE_BRANCH}" ]; then git fetch origin "${_BASE_BRANCH}":refs/remotes/origin/"${_BASE_BRANCH}"; else echo "Skipping fetch."; fi'
+  # Install Python dependencies
+  - name: ${_PYTHON_IMAGE}
+    entrypoint: pip
+    args: ['install', '--upgrade',  '--user', '--requirement', '.cloud-build/requirements.txt']
   # Install Python dependencies and run testing script
   - name: ${_PYTHON_IMAGE}
     entrypoint: /bin/sh
     args:
     - -c
-    - 'python3 -m pip install -U -r .cloud-build/requirements.txt && python3 -m pip freeze && python3 .cloud-build/ExecuteChangedNotebooks.py --test_paths_file .cloud-build/test_folders.txt --base_branch "${_FORCED_BASE_BRANCH}" --output_folder ${BUILD_ID} --variable_project_id ${PROJECT_ID} --variable_region ${_GCP_REGION}'
+    - 'python3 -m pip freeze && python3 .cloud-build/ExecuteChangedNotebooks.py --test_paths_file "${_TEST_PATHS_FILE}" --base_branch "${_FORCED_BASE_BRANCH}" --output_folder ${BUILD_ID} --variable_project_id ${PROJECT_ID} --variable_region ${_GCP_REGION}'
     env:
     - 'IS_TESTING=1'
   # Manually copy artifacts to GCS
@@ -30,7 +34,7 @@ steps:
     entrypoint: /bin/sh
     args: 
     - -c
-    - 'if [ $(ls -pR "/workspace/${BUILD_ID}"  | grep -v / | grep -v ^$ | wc -l) -ne 0 ]; then gsutil rsync -r "/workspace/${BUILD_ID}" "gs://${_GCS_ARTIFACTS_BUCKET}/test-artifacts/PR_${_PR_NUMBER}/BUILD_${BUILD_ID}/"; else echo "No artifacts to copy."; fi'
+    - 'if [ $(ls -pR "/workspace/${BUILD_ID}"  | grep -v / | grep -v ^$ | wc -l) -ne 0 ]; then gsutil -m -q rsync -r "/workspace/${BUILD_ID}" "gs://${_GCS_ARTIFACTS_BUCKET}/test-artifacts/PR_${_PR_NUMBER}/BUILD_${BUILD_ID}/"; else echo "No artifacts to copy."; fi'
   # Fail if there is anything in the failure folder
   - name: ${_PYTHON_IMAGE}
     entrypoint: /bin/sh

.cloud-build/requirements.txt

@@ -1,5 +1,8 @@
-ipython 
+ipython>=7.0
 jupyter>=1.0
 nbconvert>=6.0
-numpy
-pandas
+papermill>=2.3
+numpy>=1.19
+pandas>=1.2
+matplotlib>=3.4
+tabulate

.cloud-build/test_folders.txt

@@ -1,3 +1,2 @@
 notebooks/official
-tutorials/official
 notebooks/notebook_template.ipynb

.github/workflows/ci.yaml

@@ -10,6 +10,8 @@ jobs:
       uses: actions/setup-python@v2
     - name: Fetch pull request branch
       uses: actions/checkout@v2
+      with:
+        fetch-depth: 0      
     - name: Fetch base master branch
       run: git fetch -u "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY" master:master
     - name: Install requirements

.github/workflows/linter/run_linter.sh

@@ -35,13 +35,13 @@ is_test=false
 # Process all options supplied on the command line
 while getopts 'tc' arg; do
     case $arg in
-        't')
-            is_test=true
-            ;;
-        *)
-            echo "Unimplemented flag"
-            exit 1
-            ;;
+    't')
+        is_test=true
+        ;;
+    *)
+        echo "Unimplemented flag"
+        exit 1
+        ;;
     esac
 done
 
@@ -50,15 +50,13 @@ echo "Test mode: $is_test"
 # Only check notebooks in test folders modified in this pull request.
 # Note: Use process substitution to persist the data in the array
 notebooks=()
-while read -r file || [ -n "$line" ]; 
-do
+while read -r file || [ -n "$line" ]; do
     notebooks+=("$file")
 done < <(git diff --name-only master... | grep '\.ipynb$')
 
 problematic_notebooks=()
 if [ ${#notebooks[@]} -gt 0 ]; then
-    for notebook in "${notebooks[@]}"
-    do    
+    for notebook in "${notebooks[@]}"; do
         if [ -f "$notebook" ]; then
             echo "Checking notebook: ${notebook}"
 
@@ -68,7 +66,7 @@ if [ ${#notebooks[@]} -gt 0 ]; then
             ISORT_RTN="0"
             FLAKE8_RTN="0"
 
-            if [ "$is_test" = true ] ; then
+            if [ "$is_test" = true ]; then
                 echo "Running nbfmt..."
                 python3 -m tensorflow_docs.tools.nbfmt --remove_outputs --test "$notebook"
                 NBFMT_RTN=$?
@@ -82,7 +80,7 @@ if [ ${#notebooks[@]} -gt 0 ]; then
                 python3 -m nbqa isort "$notebook" --check
                 ISORT_RTN=$?
                 echo "Running flake8..."
-                python3 -m nbqa flake8 "$notebook" --show-source --extend-ignore=W391,E501,F821,E402,F404,W503,E203
+                python3 -m nbqa flake8 "$notebook" --show-source --extend-ignore=W391,E501,F821,E402,F404,W503,E203,E722
                 FLAKE8_RTN=$?
             else
                 echo "Running black..."
@@ -98,7 +96,7 @@ if [ ${#notebooks[@]} -gt 0 ]; then
                 python3 -m tensorflow_docs.tools.nbfmt --remove_outputs "$notebook"
                 NBFMT_RTN=$?
                 echo "Running flake8..."
-                python3 -m nbqa flake8 "$notebook" --show-source --extend-ignore=W391,E501,F821,E402,F404,W503,E203 --nbqa-mutate
+                python3 -m nbqa flake8 "$notebook" --show-source --extend-ignore=W391,E501,F821,E402,F404,W503,E203,E722 --nbqa-mutate
                 FLAKE8_RTN=$?
             fi
 
@@ -131,8 +129,7 @@ if [ ${#notebooks[@]} -gt 0 ]; then
 
             echo "Notebook lint finished with return code = $NOTEBOOK_RTN"
             echo ""
-            if [ "$NOTEBOOK_RTN" != "0" ]
-            then                    
+            if [ "$NOTEBOOK_RTN" != "0" ]; then
                 problematic_notebooks+=("$notebook")
                 RTN=$NOTEBOOK_RTN
             fi

notebooks/official/automl/automl-text-classification.ipynb

@@ -0,0 +1,963 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ur8xi4C7S06n"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2021 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "JAPoU8Sm5E6e"
+      },
+      "source": [
+        "<table align=\"left\">\n",
+        "\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/official/automl/automl-text-classification.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/official/automl/automl-text-classification.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/raw/master/notebooks/official/automl/automl-text-classification.ipynb\">\n",
+        "      Open in Google Cloud Notebooks\n",
+        "    </a>\n",
+        "  </td>    \n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0259a7ce8120"
+      },
+      "source": [
+        "# Vertex AI: Create, train, and deploy an AutoML text classification model\n",
+        "\n",
+        "## Overview\n",
+        "\n",
+        "This notebook walks you through the major phases of building and using a text classification model on [Vertex AI](https://cloud.google.com/vertex-ai/docs/). In this notebook, you use the \"Happy Moments\" sample dataset to train a model. The resulting model classifies happy moments into categores that reflect the causes of happiness. \n",
+        "\n",
+        "### Objective\n",
+        "\n",
+        "In this notebook, you learn how to:\n",
+        "\n",
+        "* Set up your development environment\n",
+        "* Create a dataset and import data\n",
+        "* Train an AutoML model\n",
+        "* Get and review evaluations for the model\n",
+        "* Deploy a model to an endpoint\n",
+        "* Get online predictions\n",
+        "* Get batch predictions\n",
+        "\n",
+        "### Costs\n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI Training and Serving\n",
+        "* Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI\n",
+        "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
+        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
+        "Calculator](https://cloud.google.com/products/calculator/)\n",
+        "to generate a cost estimate based on your projected usage"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "lWEdiXsJg0XY"
+      },
+      "source": [
+        "## Before you begin\n",
+        "\n",
+        "**Note:** This notebook does not require a GPU runtime."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "a5cb73702a9b"
+      },
+      "source": [
+        "### Set up your local development environment\n",
+        "\n",
+        "**If you are using Colab or Google Cloud Notebooks**, your environment already meets\n",
+        "all the requirements to run this notebook. You can skip this step."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gCuSR8GkAgzl"
+      },
+      "source": [
+        "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
+        "You need the following:\n",
+        "\n",
+        "* The Google Cloud SDK\n",
+        "* Git\n",
+        "* Python 3\n",
+        "* virtualenv\n",
+        "* Jupyter notebook running in a virtual environment with Python 3\n",
+        "\n",
+        "The Google Cloud guide to [Setting up a Python development\n",
+        "environment](https://cloud.google.com/python/setup) and the [Jupyter\n",
+        "installation guide](https://jupyter.org/install) provide detailed instructions\n",
+        "for meeting these requirements. The following steps provide a condensed set of\n",
+        "instructions:\n",
+        "\n",
+        "1. [Install and initialize the Cloud SDK.](https://cloud.google.com/sdk/docs/)\n",
+        "\n",
+        "1. [Install Python 3.](https://cloud.google.com/python/setup#installing_python)\n",
+        "\n",
+        "1. [Install\n",
+        "   virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv)\n",
+        "   and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
+        "\n",
+        "1. To install Jupyter, run `pip install jupyter` on the\n",
+        "command-line in a terminal shell.\n",
+        "\n",
+        "1. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
+        "\n",
+        "1. Open this notebook in the Jupyter Notebook Dashboard."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "i1VRlEu-l0BW"
+      },
+      "source": [
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI, Cloud Storage, and Compute Engine APIs](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component,storage-component.googleapis.com). \n",
+        "\n",
+        "1. Follow the \"**Configuring your project**\" instructions from the Vertex Pipelines documentation.\n",
+        "\n",
+        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "db52a0a61fca"
+      },
+      "source": [
+        "### Install additional packages\n",
+        "\n",
+        "This notebook uses the Python SDK for Vertex AI, which is contained in the `python-aiplatform` package. You must first install the package into your development environment."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b75757581291"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "# The Google Cloud Notebook product has specific requirements\n",
+        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
+        "\n",
+        "# Google Cloud Notebook requires dependencies to be installed with '--user'\n",
+        "USER_FLAG = \"\"\n",
+        "if IS_GOOGLE_CLOUD_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\"\n",
+        "\n",
+        "! pip install {USER_FLAG} --upgrade google-cloud-aiplatform google-cloud-storage jsonlines"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WReHDGG5g0XY"
+      },
+      "source": [
+        "### Set your project ID\n",
+        "\n",
+        "Finally, you must initialize the client library before you can send requests to the Vertex AI service. With the Python SDK, you initialize the client library as shown in the following cell. This tutorial also uses the Cloud Storage Python library for accessing batch prediction results.\n",
+        "\n",
+        "Be sure to provide the ID for your Google Cloud project in the `project` variable. This notebook uses the `us-central1` region, although you can change it to another region. \n",
+        "\n",
+        "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "oM1iC_MfAts1"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "from datetime import datetime\n",
+        "\n",
+        "import jsonlines\n",
+        "from google.cloud import aiplatform, storage\n",
+        "from google.protobuf import json_format\n",
+        "\n",
+        "PROJECT_ID = \"[your-project-id]\"\n",
+        "REGION = \"us-central1\"\n",
+        "\n",
+        "# Get your Google Cloud project ID from gcloud\n",
+        "if not os.getenv(\"IS_TESTING\"):\n",
+        "    shell_output=!gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "    PROJECT_ID = shell_output[0]\n",
+        "    print(\"Project ID: \", PROJECT_ID)\n",
+        "\n",
+        "aiplatform.init(project=PROJECT_ID, location=REGION)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "32c971919605"
+      },
+      "source": [
+        "## Create a dataset and import your data\n",
+        "\n",
+        "The notebook uses the 'Happy Moments' dataset for demonstration purposes. You can change it to another text classification dataset that [conforms to the data preparation requirements](https://cloud.google.com/vertex-ai/docs/datasets/prepare-text#classification).\n",
+        "\n",
+        "Using the Python SDK, you can create a dataset and import the dataset in one call to `TextDataset.create()`, as shown in the following cell.\n",
+        "\n",
+        "Creating and importing data is a long-running operation. This next step can take a while. The sample waits for the operation to complete, outputting statements as the operation progresses. The statements contain the full name of the dataset that you will use in the following section.\n",
+        "\n",
+        "**Note**: You can close the noteboook while you wait for this operation to complete. "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6caf82e5e84e"
+      },
+      "outputs": [],
+      "source": [
+        "# Use a timestamp to ensure unique resources\n",
+        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")\n",
+        "\n",
+        "src_uris = \"gs://cloud-ml-data/NL-classification/happiness.csv\"\n",
+        "display_name = f\"e2e-text-dataset-{TIMESTAMP}\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "d35b8b6b94ae"
+      },
+      "outputs": [],
+      "source": [
+        "ds = aiplatform.TextDataset.create(\n",
+        "    display_name=display_name,\n",
+        "    gcs_source=src_uris,\n",
+        "    import_schema_uri=aiplatform.schema.dataset.ioformat.text.single_label_classification,\n",
+        "    sync=True,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5b3cc427353a"
+      },
+      "source": [
+        "## Train your text classification model\n",
+        "\n",
+        "Once your dataset has finished importing data, you are ready to train your model. To do this, you first need the full resource name of your dataset, where the full name has the format `projects/[YOUR_PROJECT]/locations/us-central1/datasets/[YOUR_DATASET_ID]`. If you don't have the resource name handy, you can list all of the datasets in your project using `TextDataset.list()`. \n",
+        "\n",
+        "As shown in the following code block, you can pass in the display name of your dataset in the call to `list()` to filter the results.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "52cf56f1c8a9"
+      },
+      "outputs": [],
+      "source": [
+        "datasets = aiplatform.TextDataset.list(filter=f'display_name=\"{display_name}\"')\n",
+        "print(datasets)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "58df3e02df82"
+      },
+      "source": [
+        "When you create a new model, you need a reference to the `TextDataset` object that corresponds to your dataset. You can use the `ds` variable you created previously when you created the dataset or you can also list all of your datasets to get a reference to your dataset. Each item returned from `TextDataset.list()` is an instance of `TextDataset`.\n",
+        "\n",
+        "The following code block shows how to instantiate a `TextDataset` object using a dataset ID. Note that this code is intentionally verbose for demonstration purposes."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "aa667203da03"
+      },
+      "outputs": [],
+      "source": [
+        "# Get the dataset ID if it's not available\n",
+        "dataset_id = \"[your-dataset-id]\"\n",
+        "\n",
+        "if dataset_id == \"[your-dataset-id]\":\n",
+        "    # Use the reference to the new dataset captured when we created it\n",
+        "    dataset_id = ds.resource_name.split(\"/\")[-1]\n",
+        "    print(f\"Dataset ID: {dataset_id}\")\n",
+        "\n",
+        "text_dataset = aiplatform.TextDataset(dataset_id)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "68f10356cab9"
+      },
+      "source": [
+        "Now you can begin training your model. Training the model is a two part process:\n",
+        "\n",
+        "1. **Define the training job.** You must provide a display name and the type of training you want when you define the training job.\n",
+        "2. **Run the training job.** When you run the training job, you need to supply a reference to the dataset to use for training. At this step, you can also configure the data split percentages.\n",
+        "\n",
+        "You do not need to specify [data splits](https://cloud.google.com/vertex-ai/docs/general/ml-use). The training job has a default setting of  training 80%/ testing 10%/ validate 10% if you don't provide these values.\n",
+        "\n",
+        "To train your model, you call `AutoMLTextTrainingJob.run()` as shown in the following snippets. The method returns a reference to your new `Model` object.\n",
+        "\n",
+        "As with importing data into the dataset, training your model can take a substantial amount of time. The client library prints out operation status messages while the training pipeline operation processes. You must wait for the training process to complete before you can get the resource name and ID of your new model, which is required for model evaluation and model deployment.\n",
+        "\n",
+        "**Note**: You can close the notebook while you wait for the operation to complete."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "0aa0f01805ea"
+      },
+      "outputs": [],
+      "source": [
+        "# Define the training job\n",
+        "training_job_display_name = f\"e2e-text-training-job-{TIMESTAMP}\"\n",
+        "job = aiplatform.AutoMLTextTrainingJob(\n",
+        "    display_name=training_job_display_name,\n",
+        "    prediction_type=\"classification\",\n",
+        "    multi_label=False,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "1ec60baf2c51"
+      },
+      "outputs": [],
+      "source": [
+        "model_display_name = f\"e2e-text-classification-model-{TIMESTAMP}\"\n",
+        "\n",
+        "# Run the training job\n",
+        "model = job.run(\n",
+        "    dataset=text_dataset,\n",
+        "    model_display_name=model_display_name,\n",
+        "    training_fraction_split=0.7,\n",
+        "    validation_fraction_split=0.2,\n",
+        "    test_fraction_split=0.1,\n",
+        "    sync=True,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "caaa3f32b12e"
+      },
+      "source": [
+        "## Get and review model evaluation scores\n",
+        "\n",
+        "After your model has finished training, you can review the evaluation scores for it.\n",
+        "\n",
+        "First, you need to get a reference to the new model.  As with datasets, you can either use the reference to the `model` variable you created when deployed the model or you can list all of the models in your project. When listing your models, you can provide filter criteria to narrow down your search."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b0bb6be8621a"
+      },
+      "outputs": [],
+      "source": [
+        "models = aiplatform.Model.list(filter=f'display_name=\"{model_display_name}\"')\n",
+        "print(models)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8481b6878ed2"
+      },
+      "source": [
+        "Using the model name (in the format `projects/[PROJECT_NAME]/locations/us-central1/models/[MODEL_ID]`), you can get its model evaluations. To get model evaluations, you must use the underlying service client.\n",
+        "\n",
+        "Building a service client requires that you provide the name of the regionalized hostname used for your model. In this tutorial, the hostname is `us-central1-aiplatform.googleapis.com` because the model was created in the `us-central1` location."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "a8443fc8861f"
+      },
+      "outputs": [],
+      "source": [
+        "# Get the ID of the model\n",
+        "model_name = \"[your-model-resource-name]\"\n",
+        "if model_name == \"[your-model-resource-name]\":\n",
+        "    # Use the `resource_name` of the Model instance you created previously\n",
+        "    model_name = model.resource_name\n",
+        "    print(f\"Model name: {model_name}\")\n",
+        "\n",
+        "\n",
+        "# Get a reference to the Model Service client\n",
+        "client_options = {\"api_endpoint\": \"us-central1-aiplatform.googleapis.com\"}\n",
+        "model_service_client = aiplatform.gapic.ModelServiceClient(\n",
+        "    client_options=client_options\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "b8a788593609"
+      },
+      "source": [
+        "Before you can view the model evaluation you must first list all of the evaluations for that model. Each model can have multiple evaluations, although a new model is likely to only have one. "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "fdcb045e29f2"
+      },
+      "outputs": [],
+      "source": [
+        "model_evaluations = model_service_client.list_model_evaluations(parent=model_name)\n",
+        "model_evaluation = list(model_evaluations)[0]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "cd7d3afae05c"
+      },
+      "source": [
+        "Now that you have the model evaluation, you can look at your model's scores. If you have questions about what the scores mean, review the [public documentation](https://cloud.google.com/vertex-ai/docs/training/evaluating-automl-models#text).\n",
+        "\n",
+        "The results returned from the service are formatted as [`google.protobuf.Value`](https://googleapis.dev/python/protobuf/latest/google/protobuf/struct_pb2.html) objects. You can transform the return object as a `dict` for easier reading and parsing."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6eb9ccb0a0a0"
+      },
+      "outputs": [],
+      "source": [
+        "model_eval_dict = json_format.MessageToDict(model_evaluation._pb)\n",
+        "metrics = model_eval_dict[\"metrics\"]\n",
+        "confidence_metrics = metrics[\"confidenceMetrics\"]\n",
+        "\n",
+        "print(f'Area under precision-recall curve (AuPRC): {metrics[\"auPrc\"]}')\n",
+        "for confidence_scores in confidence_metrics:\n",
+        "    metrics = confidence_scores.keys()\n",
+        "    print(\"\\n\")\n",
+        "    for metric in metrics:\n",
+        "        print(f\"\\t{metric}: {confidence_scores[metric]}\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "b5dbe4dbaa60"
+      },
+      "source": [
+        "## Deploy your text classification model\n",
+        "\n",
+        "Once your model has completed training, you must deploy it to an _endpoint_ to get online predictions from it. When you deploy the model to an endpoint, a copy of the model is made on the endpoint with a new resource name and display name.\n",
+        "\n",
+        "You can deploy multiple models to the same endpoint and split traffic between the various models assigned to the endpoint. However, you must deploy one model at a time to the endpoint. To change the traffic split percentages, you must assign new values on your second (and subsequent) models each time you deploy a new model.\n",
+        "\n",
+        "The following code block demonstrates how to deploy a model. The code snippet relies on the Python SDK to create a new endpoint for deployment. The call to `modely.deploy()` returns a reference to an `Endpoint` object--you need this reference for online predictions in the next section."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "19bc4a55ccfe"
+      },
+      "outputs": [],
+      "source": [
+        "deployed_model_display_name = f\"e2e-deployed-text-classification-model-{TIMESTAMP}\"\n",
+        "\n",
+        "endpoint = model.deploy(\n",
+        "    deployed_model_display_name=deployed_model_display_name, sync=True\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "531da446035b"
+      },
+      "source": [
+        "In case you didn't record the name of the new endpoint, you can get a list of all your endpoints as you did before with datasets and models. For each endpoint, you can list the models deployed to that endpoint. To get a reference to the model that you just deployed, you can check the `display_name` of each model deployed to the endpoint against the model you're looking for."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "f61fb44181b4"
+      },
+      "outputs": [],
+      "source": [
+        "endpoints = aiplatform.Endpoint.list()\n",
+        "\n",
+        "endpoint_with_deployed_model = []\n",
+        "\n",
+        "for endpoint_ in endpoints:\n",
+        "    for model in endpoint_.list_models():\n",
+        "        if model.display_name.find(deployed_model_display_name) == 0:\n",
+        "            endpoint_with_deployed_model.append(endpoint_)\n",
+        "\n",
+        "print(endpoint_with_deployed_model)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "351a6e8be3a5"
+      },
+      "source": [
+        "## Get online predictions from your model\n",
+        "\n",
+        "Now that you have your endpoint's resource name, you can get online predictions from the text classification model. To get the online prediction, you send a prediction request to your endpoint."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "953b333fc0fc"
+      },
+      "outputs": [],
+      "source": [
+        "endpoint_name = \"[your-endpoint-name]\"\n",
+        "if endpoint_name == \"[your-endpoint-name]\":\n",
+        "    endpoint_name = endpoint.resource_name\n",
+        "\n",
+        "print(f\"Endpoint name: {endpoint_name}\")\n",
+        "\n",
+        "endpoint = aiplatform.Endpoint(endpoint_name)\n",
+        "content = \"I got a high score on my math final!\"\n",
+        "\n",
+        "response = endpoint.predict(instances=[{\"content\": content}])\n",
+        "\n",
+        "for prediction_ in response.predictions:\n",
+        "    ids = prediction_[\"ids\"]\n",
+        "    display_names = prediction_[\"displayNames\"]\n",
+        "    confidence_scores = prediction_[\"confidences\"]\n",
+        "    for count, id in enumerate(ids):\n",
+        "        print(f\"Prediction ID: {id}\")\n",
+        "        print(f\"Prediction display name: {display_names[count]}\")\n",
+        "        print(f\"Prediction confidence score: {confidence_scores[count]}\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "f18811cd0477"
+      },
+      "source": [
+        "## Get batch predictions from your model\n",
+        "\n",
+        "You can get batch predictions from a text classification model without deploying it. You must first format all of your prediction instances (prediction input) in JSONL format and you must store the JSONL file in a Google Cloud Storage bucket. You must also provide a Google Cloud Storage bucket to hold your prediction output.\n",
+        "\n",
+        "To start, you must first create your predictions input file in JSONL format. Each line in the JSONL document needs to be formatted like so:\n",
+        "\n",
+        "```\n",
+        "{ \"content\": \"gs://sourcebucket/datasets/texts/source_text.txt\", \"mimeType\": \"text/plain\"}\n",
+        "```\n",
+        "\n",
+        "The `content` field in the JSON structure must be a Google Cloud Storage URI to another document that contains the text input for prediction.\n",
+        "[See the documentation for more information.](https://cloud.google.com/ai-platform-unified/docs/predictions/batch-predictions#text)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "e4b838cbcd99"
+      },
+      "outputs": [],
+      "source": [
+        "instances = [\n",
+        "    \"We hiked through the woods and up the hill to the ice caves\",\n",
+        "    \"My kitten is so cute\",\n",
+        "]\n",
+        "input_file_name = \"batch-prediction-input.jsonl\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "76ac422ab8dd"
+      },
+      "source": [
+        "For batch prediction, you must supply the following:\n",
+        "\n",
+        "+ All of your prediction instances as individual files on Google Cloud Storage, as TXT files for your instances\n",
+        "+ A JSONL file that lists the URIs of all your prediction instances\n",
+        "+ A Google Cloud Storage bucket to hold the output from batch prediction\n",
+        "\n",
+        "For this tutorial, the following cells create a new Storage bucket, upload individual prediction instances as text files to the bucket, and then create the JSONL file with the URIs of your prediction instances."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "1e0759fbb219"
+      },
+      "outputs": [],
+      "source": [
+        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")\n",
+        "BUCKET_NAME = \"[your-bucket-name]\"\n",
+        "\n",
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
+        "    BUCKET_NAME = f\"automl-text-notebook-{TIMESTAMP}\"\n",
+        "\n",
+        "BUCKET_URI = f\"gs://{BUCKET_NAME}\"\n",
+        "\n",
+        "! gsutil mb -l $REGION $BUCKET_URI"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "8b7cabbb86ad"
+      },
+      "outputs": [],
+      "source": [
+        "# Instantiate the Storage client and create the new bucket\n",
+        "storage = storage.Client()\n",
+        "bucket = storage.bucket(BUCKET_NAME)\n",
+        "\n",
+        "# Iterate over the prediction instances, creating a new TXT file\n",
+        "# for each.\n",
+        "input_file_data = []\n",
+        "for count, instance in enumerate(instances):\n",
+        "    instance_name = f\"input_{count}.txt\"\n",
+        "    instance_file_uri = f\"{BUCKET_URI}/{instance_name}\"\n",
+        "\n",
+        "    # Add the data to store in the JSONL input file.\n",
+        "    tmp_data = {\"content\": instance_file_uri, \"mimeType\": \"text/plain\"}\n",
+        "    input_file_data.append(tmp_data)\n",
+        "\n",
+        "    # Create the new instance file\n",
+        "    blob = bucket.blob(instance_name)\n",
+        "    blob.upload_from_string(instance)\n",
+        "\n",
+        "input_str = \"\\n\".join([str(d) for d in input_file_data])\n",
+        "file_blob = bucket.blob(f\"{input_file_name}\")\n",
+        "file_blob.upload_from_string(input_str)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "31c262320610"
+      },
+      "source": [
+        "Now that you have the bucket with the prediction instances ready, you can send a batch prediction request to Vertex AI. When you send a request to the service, you must provide the URI of your JSONL file and your output bucket, including the `gs://` protocols.\n",
+        "\n",
+        "With the Python SDK, you can create a batch prediction job by calling `Model.batch_predict()`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "f5ab2139d52d"
+      },
+      "outputs": [],
+      "source": [
+        "job_display_name = \"e2e-text-classification-batch-prediction-job\"\n",
+        "model = aiplatform.Model(model_name=model_name)\n",
+        "\n",
+        "batch_prediction_job = model.batch_predict(\n",
+        "    job_display_name=job_display_name,\n",
+        "    gcs_source=f\"{BUCKET_URI}/{input_file_name}\",\n",
+        "    gcs_destination_prefix=f\"{BUCKET_URI}/output\",\n",
+        "    sync=True,\n",
+        ")\n",
+        "\n",
+        "batch_prediction_job_name = batch_prediction_job.resource_name"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "11503f2e08a2"
+      },
+      "source": [
+        "Once the batch prediction job completes, the Python SDK prints out the resource name of the batch prediction job in the format `projects/[PROJECT_ID]/locations/[LOCATION]/batchPredictionJobs/[BATCH_PREDICTION_JOB_ID]`. You can query the Vertex AI service for the status of the batch prediction job using its ID.\n",
+        "\n",
+        "The following code snippet demonstrates how to create an instance of the `BatchPredictionJob` class to review its status. Note that you need the full resource name printed out from the Python SDK for this snippet.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "bf6e614723ed"
+      },
+      "outputs": [],
+      "source": [
+        "from google.cloud.aiplatform import jobs\n",
+        "\n",
+        "batch_job = jobs.BatchPredictionJob(batch_prediction_job_name)\n",
+        "print(f\"Batch prediction job state: {str(batch_job.state)}\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "1f9a12dadf6f"
+      },
+      "source": [
+        "After the batch job has completed, you can view the results of the job in your output Storage bucket. You might want to first list all of the files in your output bucket to find the URI of the output file."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "8ff1ec03205c"
+      },
+      "outputs": [],
+      "source": [
+        "BUCKET_OUTPUT = f\"{BUCKET_URI}/output\"\n",
+        "\n",
+        "! gsutil ls -a $BUCKET_OUTPUT"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "52f3f8af2e41"
+      },
+      "source": [
+        "The output from the batch prediction job should be contained in a folder (or _prefix_) that includes the name of the batch prediction job plus a time stamp for when it was created.\n",
+        "\n",
+        "For example, if your batch prediction job name is `my-job` and your bucket name is `my-bucket`, the URI of the folder containing your output might look like the following:\n",
+        "\n",
+        "```\n",
+        "gs://my-bucket/output/prediction-my-job-2021-06-04T19:54:25.889262Z/\n",
+        "```\n",
+        "\n",
+        "To read the batch prediction results, you must download the file locally and open the file. The next cell copies all of the files in the `BUCKET_OUTPUT_FOLDER` into a local folder."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "4bb16e040942"
+      },
+      "outputs": [],
+      "source": [
+        "RESULTS_DIRECTORY = \"prediction_results\"\n",
+        "RESULTS_DIRECTORY_FULL = f\"{RESULTS_DIRECTORY}/output\"\n",
+        "\n",
+        "# Create missing directories\n",
+        "os.makedirs(RESULTS_DIRECTORY, exist_ok=True)\n",
+        "\n",
+        "# Get the Cloud Storage paths for each result\n",
+        "! gsutil -m cp -r $BUCKET_OUTPUT $RESULTS_DIRECTORY\n",
+        "\n",
+        "# Get most recently modified directory\n",
+        "latest_directory = max(\n",
+        "    [\n",
+        "        os.path.join(RESULTS_DIRECTORY_FULL, d)\n",
+        "        for d in os.listdir(RESULTS_DIRECTORY_FULL)\n",
+        "    ],\n",
+        "    key=os.path.getmtime,\n",
+        ")\n",
+        "\n",
+        "print(f\"Local results folder: {latest_directory}\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "f406e1e4d5ec"
+      },
+      "source": [
+        "With all of the results files downloaded locally, you can open them and read the results. In this tutorial, you use the [`jsonlines`](https://jsonlines.readthedocs.io/en/latest/) library to read the output results.\n",
+        "\n",
+        "The following cell opens up the JSONL output file and then prints the predictions for each instance."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "91d7f2a74a7c"
+      },
+      "outputs": [],
+      "source": [
+        "# Get downloaded results in directory\n",
+        "results_files = []\n",
+        "for dirpath, subdirs, files in os.walk(latest_directory):\n",
+        "    for file in files:\n",
+        "        if file.find(\"predictions\") >= 0:\n",
+        "            results_files.append(os.path.join(dirpath, file))\n",
+        "\n",
+        "\n",
+        "# Consolidate all the results into a list\n",
+        "results = []\n",
+        "for results_file in results_files:\n",
+        "    # Open each result\n",
+        "    with jsonlines.open(results_file) as reader:\n",
+        "        for result in reader.iter(type=dict, skip_invalid=True):\n",
+        "            instance = result[\"instance\"]\n",
+        "            prediction = result[\"prediction\"]\n",
+        "            print(f\"\\ninstance: {instance['content']}\")\n",
+        "            for key, output in prediction.items():\n",
+        "                print(f\"\\n{key}: {output}\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "af3874f08502"
+      },
+      "source": [
+        "## Cleaning up\n",
+        "\n",
+        "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
+        "\n",
+        "Otherwise, you can delete the individual resources you created in this tutorial:\n",
+        "\n",
+        "* Dataset\n",
+        "* Training job\n",
+        "* Model\n",
+        "* Endpoint\n",
+        "* Batch prediction\n",
+        "* Batch prediction bucket"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "adce73b48b72"
+      },
+      "outputs": [],
+      "source": [
+        "if os.getenv(\"IS_TESTING\") is True:\n",
+        "    ! gsutil rm -r $BUCKET_URI\n",
+        "\n",
+        "batch_job.delete()\n",
+        "\n",
+        "# `force` parameter ensures that models are undeployed before deletion\n",
+        "endpoint.delete(force=True)\n",
+        "\n",
+        "model.delete()\n",
+        "\n",
+        "text_dataset.delete()\n",
+        "\n",
+        "# Training job\n",
+        "job.delete()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "fa6a8c434c79"
+      },
+      "source": [
+        "## Next Steps\n",
+        "\n",
+        "After completing this tutorial, see the following documentation pages to learn more about Vertex AI:\n",
+        "\n",
+        "* [Preparing text training data](https://cloud.google.com/vertex-ai/docs/datasets/prepare-text)\n",
+        "* [Training an AutoML model using the API](https://cloud.google.com/vertex-ai/docs/training/automl-api#text)\n",
+        "* [Evaluating AutoML models](https://cloud.google.com/vertex-ai/docs/training/evaluating-automl-models#text)\n",
+        "* [Deploying a model using ther Vertex AI API](https://cloud.google.com/vertex-ai/docs/predictions/deploy-model-api#aiplatform_create_endpoint_sample-python)\n",
+        "* [Getting online predictions from AutoML models](https://cloud.google.com/vertex-ai/docs/predictions/deploy-model-api#aiplatform_create_endpoint_sample-python)\n",
+        "* [Getting batch predictions](https://cloud.google.com/vertex-ai/docs/predictions/batch-predictions#text)"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "automl-text-classification.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-locally-trained-models.ipynb

@@ -0,0 +1,890 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ur8xi4C7S06n"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2021 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "JAPoU8Sm5E6e"
+      },
+      "source": [
+        "<table align=\"left\">\n",
+        "\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-locally-trained-models.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-locally-trained-models.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WBFL9LagqmwT"
+      },
+      "source": [
+        "#Vertex AI: Track parameters and metrics for locally trained models"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tvgnzT1CKxrO"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "This notebook demonstrates how to track metrics and parameters for ML training jobs and analyze this metadata using Vertex AI SDK.\n",
+        "\n",
+        "### Dataset\n",
+        "\n",
+        "In this notebook, we will train a simple distributed neural network (DNN) model to predict automobile's miles per gallon (MPG) based on automobile information in the [auto-mpg dataset](https://www.kaggle.com/devanshbesain/exploration-and-analysis-auto-mpg).\n",
+        "\n",
+        "### Objective\n",
+        "\n",
+        "In this notebook, you will learn how to use Vertex AI SDK to:\n",
+        "\n",
+        "    * Track parameters and metrics for a locally trainined model.\n",
+        "    * Extract and perform analysis for all parameters and metrics within an Experiment.\n",
+        "\n",
+        "### Costs \n",
+        "\n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Cloud Storage\n",
+        "\n",
+        "\n",
+        "Learn about [Vertex AI\n",
+        "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
+        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
+        "Calculator](https://cloud.google.com/products/calculator/)\n",
+        "to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ze4-nDLfK4pw"
+      },
+      "source": [
+        "### Set up your local development environment\n",
+        "\n",
+        "**If you are using Colab or AI Platform Notebooks**, your environment already meets\n",
+        "all the requirements to run this notebook. You can skip this step."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gCuSR8GkAgzl"
+      },
+      "source": [
+        "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
+        "You need the following:\n",
+        "\n",
+        "* The Google Cloud SDK\n",
+        "* Git\n",
+        "* Python 3\n",
+        "* virtualenv\n",
+        "* Jupyter notebook running in a virtual environment with Python 3\n",
+        "\n",
+        "The Google Cloud guide to [Setting up a Python development\n",
+        "environment](https://cloud.google.com/python/setup) and the [Jupyter\n",
+        "installation guide](https://jupyter.org/install) provide detailed instructions\n",
+        "for meeting these requirements. The following steps provide a condensed set of\n",
+        "instructions:\n",
+        "\n",
+        "1. [Install and initialize the Cloud SDK.](https://cloud.google.com/sdk/docs/)\n",
+        "\n",
+        "1. [Install Python 3.](https://cloud.google.com/python/setup#installing_python)\n",
+        "\n",
+        "1. [Install\n",
+        "   virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv)\n",
+        "   and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
+        "\n",
+        "1. To install Jupyter, run `pip install jupyter` on the\n",
+        "command-line in a terminal shell.\n",
+        "\n",
+        "1. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
+        "\n",
+        "1. Open this notebook in the Jupyter Notebook Dashboard."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "i7EUnXsZhAGF"
+      },
+      "source": [
+        "### Install additional packages\n",
+        "\n",
+        "Run the following commands to install the Vertex AI SDK and packages used in this notebook."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "IaYsrh0Tc17L"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "# The Google Cloud Notebook product has specific requirements\n",
+        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
+        "\n",
+        "# Google Cloud Notebook requires dependencies to be installed with '--user'\n",
+        "USER_FLAG = \"\"\n",
+        "if IS_GOOGLE_CLOUD_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MCQDRsnE3uzz"
+      },
+      "source": [
+        "Install Vertex AI SDK and tensorflow for training and evaluation model."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "wyy5Lbnzg5fi"
+      },
+      "outputs": [],
+      "source": [
+        "! pip install {USER_FLAG} --upgrade tensorflow\n",
+        "! pip install {USER_FLAG} --upgrade google-cloud-aiplatform"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "hhq5zEbGg0XX"
+      },
+      "source": [
+        "### Restart the kernel\n",
+        "\n",
+        "After you install the additional packages, you need to restart the notebook kernel so it can find the packages."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "EzrelQZ22IZj"
+      },
+      "outputs": [],
+      "source": [
+        "# Automatically restart kernel after installs\n",
+        "import os\n",
+        "\n",
+        "if not os.getenv(\"IS_TESTING\"):\n",
+        "    # Automatically restart kernel after installs\n",
+        "    import IPython\n",
+        "\n",
+        "    app = IPython.Application.instance()\n",
+        "    app.kernel.do_shutdown(True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "lWEdiXsJg0XY"
+      },
+      "source": [
+        "## Before you begin\n",
+        "\n",
+        "### Select a GPU runtime\n",
+        "\n",
+        "**Make sure you're running this notebook in a GPU runtime if you have that option. In Colab, select \"Runtime --> Change runtime type > GPU\"**"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "BF1j6f9HApxa"
+      },
+      "source": [
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com).\n",
+        "\n",
+        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WReHDGG5g0XY"
+      },
+      "source": [
+        "#### Set your project ID\n",
+        "\n",
+        "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "oM1iC_MfAts1"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "PROJECT_ID = \"\"\n",
+        "\n",
+        "# Get your Google Cloud project ID from gcloud\n",
+        "if not os.getenv(\"IS_TESTING\"):\n",
+        "    shell_output=!gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "    PROJECT_ID = shell_output[0]\n",
+        "    print(\"Project ID: \", PROJECT_ID)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "qJYoRfYng0XZ"
+      },
+      "source": [
+        "Otherwise, set your project ID here."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "riG_qUokg0XZ"
+      },
+      "outputs": [],
+      "source": [
+        "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
+        "    PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "06571eb4063b"
+      },
+      "source": [
+        "#### Timestamp\n",
+        "\n",
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append it onto the name of resources you create in this tutorial."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "697568e92bd6"
+      },
+      "outputs": [],
+      "source": [
+        "from datetime import datetime\n",
+        "\n",
+        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dr--iN2kAylZ"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using AI Platform Notebooks**, your environment is already\n",
+        "authenticated. Skip this step."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "sBCra4QMA2wR"
+      },
+      "source": [
+        "**If you are using Colab**, run the cell below and follow the instructions\n",
+        "when prompted to authenticate your account via oAuth.\n",
+        "\n",
+        "**Otherwise**, follow these steps:\n",
+        "\n",
+        "1. In the Cloud Console, go to the [**Create service account key**\n",
+        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
+        "\n",
+        "2. Click **Create service account**.\n",
+        "\n",
+        "3. In the **Service account name** field, enter a name, and\n",
+        "   click **Create**.\n",
+        "\n",
+        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"AI Platform\"\n",
+        "into the filter box, and select\n",
+        "   **AI Platform Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "\n",
+        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
+        "local environment.\n",
+        "\n",
+        "6. Enter the path to your service account key as the\n",
+        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "PyQmSRbKA8r-"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "import sys\n",
+        "\n",
+        "# If you are running this notebook in Colab, run this cell and follow the\n",
+        "# instructions to authenticate your GCP account. This provides access to your\n",
+        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
+        "# requests.\n",
+        "\n",
+        "# If on Google Cloud Notebooks, then don't execute this code\n",
+        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
+        "\n",
+        "if not IS_GOOGLE_CLOUD_NOTEBOOK:\n",
+        "    if \"google.colab\" in sys.modules:\n",
+        "        from google.colab import auth as google_auth\n",
+        "\n",
+        "        google_auth.authenticate_user()\n",
+        "\n",
+        "    # If you are running this notebook locally, replace the string below with the\n",
+        "    # path to your service account key and run this cell to authenticate your GCP\n",
+        "    # account.\n",
+        "    elif not os.getenv(\"IS_TESTING\"):\n",
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "XoEqT2Y4DJmf"
+      },
+      "source": [
+        "### Import libraries and define constants"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Y9Uo3tifg1kx"
+      },
+      "source": [
+        "Import required libraries."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "pRUOFELefqf1"
+      },
+      "outputs": [],
+      "source": [
+        "import matplotlib.pyplot as plt\n",
+        "import pandas as pd\n",
+        "from google.cloud import aiplatform\n",
+        "from tensorflow.python.keras import Sequential, layers\n",
+        "from tensorflow.python.lib.io import file_io"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "xtXZWmYqJ1bh"
+      },
+      "source": [
+        "Define some constants"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "JIOrI-hoJ46P"
+      },
+      "outputs": [],
+      "source": [
+        "EXPERIMENT_NAME = \"\"  # @param {type:\"string\"}\n",
+        "REGION = \"[your-region]\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "jWQLXXNVN4Lv"
+      },
+      "source": [
+        "If EXEPERIMENT_NAME is not set, set a default one below:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Q1QInYWOKsmo"
+      },
+      "outputs": [],
+      "source": [
+        "if EXPERIMENT_NAME == \"\" or EXPERIMENT_NAME is None:\n",
+        "    EXPERIMENT_NAME = \"my-experiment-\" + TIMESTAMP"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Xuny18aMcWDb"
+      },
+      "source": [
+        "## Concepts\n",
+        "\n",
+        "To better understanding how parameters and metrics are stored and organized, we'd like to introduce the following concepts:\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "NThDci5bp0Uw"
+      },
+      "source": [
+        "### Experiment\n",
+        "Experiments describe a context that groups your runs and the artifacts you create into a logical session. For example, in this notebook you create an Experiment and log data to that experiment."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "SAyRR3Ydp4X5"
+      },
+      "source": [
+        "### Run\n",
+        "A run represents a single path/avenue that you executed while performing an experiment. A run includes artifacts that you used as inputs or outputs, and parameters that you used in this execution. An Experiment can contain multiple runs. "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "l1YW2pgyegFP"
+      },
+      "source": [
+        "## Getting started tracking parameters and metrics\n",
+        "\n",
+        "You can use the Vertex AI SDK to track metrics and parameters for models trained locally. \n",
+        "\n",
+        "In the following example, you train a simple distributed neural network (DNN) model to predict automobile's miles per gallon (MPG) based on automobile information in the [auto-mpg dataset](https://www.kaggle.com/devanshbesain/exploration-and-analysis-auto-mpg)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "KPY41M9_AhZU"
+      },
+      "source": [
+        "### Load and process the training dataset"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "bfMQSmRuUuX-"
+      },
+      "source": [
+        "Download and process the dataset."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "RiQuMv4bmpuV"
+      },
+      "outputs": [],
+      "source": [
+        "def read_data(file_path):\n",
+        "    column_names = [\n",
+        "        \"MPG\",\n",
+        "        \"Cylinders\",\n",
+        "        \"Displacement\",\n",
+        "        \"Horsepower\",\n",
+        "        \"Weight\",\n",
+        "        \"Acceleration\",\n",
+        "        \"Model Year\",\n",
+        "        \"Origin\",\n",
+        "    ]\n",
+        "    with file_io.FileIO(file_path, \"r\") as f:\n",
+        "        raw_dataset = pd.read_csv(\n",
+        "            f,\n",
+        "            names=column_names,\n",
+        "            na_values=\"?\",\n",
+        "            comment=\"\\t\",\n",
+        "            sep=\" \",\n",
+        "            skipinitialspace=True,\n",
+        "        )\n",
+        "    dataset = raw_dataset.dropna()\n",
+        "    dataset[\"Origin\"] = dataset[\"Origin\"].map(\n",
+        "        lambda x: {1: \"USA\", 2: \"Europe\", 3: \"Japan\"}.get(x)\n",
+        "    )\n",
+        "    dataset = pd.get_dummies(dataset, prefix=\"\", prefix_sep=\"\")\n",
+        "    return dataset\n",
+        "\n",
+        "\n",
+        "dataset = read_data(\"gs://cloud-samples-data/ai-platform/auto_mpg/auto-mpg.data\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Y06J7A7yU21t"
+      },
+      "source": [
+        "Split dataset for training and testing."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "p5JBCBKyH-NC"
+      },
+      "outputs": [],
+      "source": [
+        "def train_test_split(dataset, split_frac=0.8, random_state=0):\n",
+        "    train_dataset = dataset.sample(frac=split_frac, random_state=random_state)\n",
+        "    test_dataset = dataset.drop(train_dataset.index)\n",
+        "    train_labels = train_dataset.pop(\"MPG\")\n",
+        "    test_labels = test_dataset.pop(\"MPG\")\n",
+        "\n",
+        "    return train_dataset, test_dataset, train_labels, test_labels\n",
+        "\n",
+        "\n",
+        "train_dataset, test_dataset, train_labels, test_labels = train_test_split(dataset)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gaNNTFPaU7KT"
+      },
+      "source": [
+        "Normalize the features in the dataset for better model performance."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "VGq5QCoyIEWJ"
+      },
+      "outputs": [],
+      "source": [
+        "def normalize_dataset(train_dataset, test_dataset):\n",
+        "    train_stats = train_dataset.describe()\n",
+        "    train_stats = train_stats.transpose()\n",
+        "\n",
+        "    def norm(x):\n",
+        "        return (x - train_stats[\"mean\"]) / train_stats[\"std\"]\n",
+        "\n",
+        "    normed_train_data = norm(train_dataset)\n",
+        "    normed_test_data = norm(test_dataset)\n",
+        "\n",
+        "    return normed_train_data, normed_test_data\n",
+        "\n",
+        "\n",
+        "normed_train_data, normed_test_data = normalize_dataset(train_dataset, test_dataset)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "UBXUgxgqA_GB"
+      },
+      "source": [
+        "### Define ML model and training function"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "66odBYKrIN4q"
+      },
+      "outputs": [],
+      "source": [
+        "def train(\n",
+        "    train_data,\n",
+        "    train_labels,\n",
+        "    num_units=64,\n",
+        "    activation=\"relu\",\n",
+        "    dropout_rate=0.0,\n",
+        "    validation_split=0.2,\n",
+        "    epochs=1000,\n",
+        "):\n",
+        "\n",
+        "    model = Sequential(\n",
+        "        [\n",
+        "            layers.Dense(\n",
+        "                num_units,\n",
+        "                activation=activation,\n",
+        "                input_shape=[len(train_dataset.keys())],\n",
+        "            ),\n",
+        "            layers.Dropout(rate=dropout_rate),\n",
+        "            layers.Dense(num_units, activation=activation),\n",
+        "            layers.Dense(1),\n",
+        "        ]\n",
+        "    )\n",
+        "\n",
+        "    model.compile(loss=\"mse\", optimizer=\"adam\", metrics=[\"mae\", \"mse\"])\n",
+        "    print(model.summary())\n",
+        "\n",
+        "    history = model.fit(\n",
+        "        train_data, train_labels, epochs=epochs, validation_split=validation_split\n",
+        "    )\n",
+        "\n",
+        "    return model, history"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "O8XJZB3gR8eL"
+      },
+      "source": [
+        "### Initialize the Model Builder SDK and create an Experiment\n",
+        "\n",
+        "Initialize the *client* for Vertex AI and create an experiment."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "o_wnT10RJ7-W"
+      },
+      "outputs": [],
+      "source": [
+        "aiplatform.init(project=PROJECT_ID, location=REGION, experiment=EXPERIMENT_NAME)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "u-iTnzt3B6Z_"
+      },
+      "source": [
+        "### Start several model training runs\n",
+        "\n",
+        "Training parameters and metrics are logged for each run."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "i2wnpu8_7JfV"
+      },
+      "outputs": [],
+      "source": [
+        "parameters = [\n",
+        "    {\"num_units\": 16, \"epochs\": 3, \"dropout_rate\": 0.1},\n",
+        "    {\"num_units\": 16, \"epochs\": 10, \"dropout_rate\": 0.1},\n",
+        "    {\"num_units\": 16, \"epochs\": 10, \"dropout_rate\": 0.2},\n",
+        "    {\"num_units\": 32, \"epochs\": 10, \"dropout_rate\": 0.1},\n",
+        "    {\"num_units\": 32, \"epochs\": 10, \"dropout_rate\": 0.2},\n",
+        "]\n",
+        "\n",
+        "for i, params in enumerate(parameters):\n",
+        "    aiplatform.start_run(run=f\"auto-mpg-local-run-{i}\")\n",
+        "    aiplatform.log_params(params)\n",
+        "    model, history = train(\n",
+        "        normed_train_data,\n",
+        "        train_labels,\n",
+        "        num_units=params[\"num_units\"],\n",
+        "        activation=\"relu\",\n",
+        "        epochs=params[\"epochs\"],\n",
+        "        dropout_rate=params[\"dropout_rate\"],\n",
+        "    )\n",
+        "    aiplatform.log_metrics(\n",
+        "        {metric: values[-1] for metric, values in history.history.items()}\n",
+        "    )\n",
+        "\n",
+        "    loss, mae, mse = model.evaluate(normed_test_data, test_labels, verbose=2)\n",
+        "    aiplatform.log_metrics({\"eval_loss\": loss, \"eval_mae\": mae, \"eval_mse\": mse})"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "jZLrJZTfL7tE"
+      },
+      "source": [
+        "### Extract parameters and metrics into a dataframe for analysis"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "A1PqKxlpOZa2"
+      },
+      "source": [
+        "We can also extract all parameters and metrics associated with any Experiment into a dataframe for further analysis."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "jbRf1WoH_vbY"
+      },
+      "outputs": [],
+      "source": [
+        "experiment_df = aiplatform.get_experiment_df()\n",
+        "experiment_df"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "EYuYgqVCMKU1"
+      },
+      "source": [
+        "### Visualizing an experiment's parameters and metrics"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "r8orCj8iJuO1"
+      },
+      "outputs": [],
+      "source": [
+        "plt.rcParams[\"figure.figsize\"] = [15, 5]\n",
+        "\n",
+        "ax = pd.plotting.parallel_coordinates(\n",
+        "    experiment_df.reset_index(level=0),\n",
+        "    \"run_name\",\n",
+        "    cols=[\n",
+        "        \"param.num_units\",\n",
+        "        \"param.dropout_rate\",\n",
+        "        \"param.epochs\",\n",
+        "        \"metric.loss\",\n",
+        "        \"metric.val_loss\",\n",
+        "        \"metric.eval_loss\",\n",
+        "    ],\n",
+        "    color=[\"blue\", \"green\", \"pink\", \"red\"],\n",
+        ")\n",
+        "ax.set_yscale(\"symlog\")\n",
+        "ax.legend(bbox_to_anchor=(1.0, 0.5))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WTHvPMweMlP1"
+      },
+      "source": [
+        "## Visualizing experiments in Cloud Console"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "F19_5lw0MqXv"
+      },
+      "source": [
+        "Run the following to get the URL of Vertex AI Experiments for your project.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "GmN9vE9pqqzt"
+      },
+      "outputs": [],
+      "source": [
+        "print(\"Vertex AI Experiments:\")\n",
+        "print(\n",
+        "    f\"https://console.cloud.google.com/ai/platform/experiments/experiments?folder=&organizationId=&project={PROJECT_ID}\"\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "TpV-iwP9qw9c"
+      },
+      "source": [
+        "## Cleaning up\n",
+        "\n",
+        "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
+        "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial."
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "sdk-metric-parameter-tracking-for-locally-trained-models.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/official/model_monitoring/setup_env.md

@@ -0,0 +1,16 @@
+## Setup your environment
+Duration: 10:00
+
+You’ll need a Google Cloud Platform project with billing enabled to run this codelab. To create a project, follow the [instructions here](https://cloud.google.com/resource-manager/docs/creating-managing-projects?utm_source=codelabs&utm_medium=et&utm_campaign=CDR_sar_aiml_vertexio_&utm_content=-).
+
+### Step 1: Enable the Vertex AI API
+
+Navigate to the [Vertex AI section of your Cloud Console](https://console.cloud.google.com/ai/platform?utm_source=codelabs&utm_medium=et&utm_campaign=CDR_sar_aiml_vertexio_&utm_content=-) and click **Enable Vertex AI API**.
+
+
+### Step 2: Open a Cloud Shell Session
+
+Click the "Activate Cloud Shell" button in the upper right corner of the Cloud Console. This will take a minute or so to provision your cloud shell, which is a Linux command line interpreter running on an auto-provisioned virtual machine. This is where you will enter all the commands prescibed in subsequent steps. 
+
+**In subsequent steps in this codelab, wherever you see Python code, you'll want to save that code in a file, using either a text editor or the Cloud Editor, and run the code by typing *python3 file.py* in the Cloud Shell window.**
+

notebooks/official/pipelines/README.md

@@ -0,0 +1,20 @@
+# Vertex Pipeline examples
+
+This directory holds [Vertex Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines) example notebooks.
+
+- [pipelines_intro_kfp.ipynb](./pipelines_intro_kfp.ipynb) introduces some of the Vertex Pipelines features, using the [Kubeflow Pipelines (KFP) SDK](https://www.kubeflow.org/docs/components/pipelines/).
+
+- [control_flow_kfp.ipynb](./control_flow_kfp.ipynb) shows how you can build pipelines that include conditionals and parallel 'for' loops using the KFP SDK.
+- [lightweight_functions_component_io_kfp.ipynb](./lightweight_functions_component_io_kfp.ipynb) shows how to build lightweight Python function-based components, and in particular how to support component I/O using the KFP SDK.
+- [metrics_viz_run_compare_kfp.ipynb](./metrics_viz_run_compare_kfp.ipynb) shows how to use the KFP SDK to build Vertex Pipelines that generate model metrics and metrics visualizations; and how to compare pipeline runs.
+
+The following examples show how to use the components defined in [google_cloud_pipeline_components](https://github.com/kubeflow/pipelines/tree/master/components/google-cloud) to build pipelines that access [Vertex AI](https://cloud.google.com/vertex-ai/) services.
+
+- [google-cloud-pipeline-components_automl_images.ipynb](./google-cloud-pipeline-components_automl_images.ipynb)
+- [google-cloud-pipeline-components_automl_tabular.ipynb](./google-cloud-pipeline-components_automl_tabular.ipynb) (tabular regression model)
+- [automl_tabular_classification_beans.ipynb](./automl_tabular_classification_beans.ipynb) (tabular classification model)
+- [google-cloud-pipeline-components_automl_text.ipynb](.google-cloud-pipeline-components_automl_text.ipynb)
+- (Experimental) [google_cloud_pipeline_components_model_train_upload_deploy.ipynb](./google_cloud_pipeline_components_model_train_upload_deploy.ipynb): includes an experimental component to run a custom training job directly by defining its worker specs
+
+**Note**: Currently, pipelines built using `kfp.v2`, such as these examples, will work only with Vertex Pipelines.
+A 'compatibility mode', which will allow these pipelines to be run on OSS KFP as well, is coming soon.

notebooks/official/pipelines/control_flow_kfp.ipynb

@@ -0,0 +1,820 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ur8xi4C7S06n"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2021 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "JAPoU8Sm5E6e"
+      },
+      "source": [
+        "<table align=\"left\">\n",
+        "\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/official/pipelines/control_flow_kfp.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/official/pipelines/control_flow_kfp.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/raw/master/notebooks/official/pipelines/control_flow_kfp.ipynb\">\n",
+        "      Open in Google Cloud Notebooks\n",
+        "    </a>\n",
+        "  </td>    \n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3yTsQctS8QLd"
+      },
+      "source": [
+        "# Vertex Pipelines: pipeline control structures using the KFP SDK"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tvgnzT1CKxrO"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "\n",
+        "In this notebook, you learn how to use [the Kubeflow Pipelines (KFP) SDK](https://www.kubeflow.org/docs/components/pipelines/) to build [Vertex Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines) that use control structures.\n",
+        "\n",
+        "\n",
+        "### Objective\n",
+        "\n",
+        "In this notebook, you learn how to use the KFP SDK to build pipelines that use loops and conditionals, including nested examples.\n",
+        "\n",
+        "### Costs \n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI Training\n",
+        "* Cloud Storage\n",
+        "\n",
+        "Learn about pricing for [Vertex AI](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
+        "Calculator](https://cloud.google.com/products/calculator/)\n",
+        "to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ze4-nDLfK4pw"
+      },
+      "source": [
+        "### Set up your local development environment\n",
+        "\n",
+        "**If you are using Colab or Google Cloud Notebooks**, your environment already meets\n",
+        "all the requirements to run this notebook. You can skip this step."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gCuSR8GkAgzl"
+      },
+      "source": [
+        "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
+        "You need the following:\n",
+        "\n",
+        "* The Google Cloud SDK\n",
+        "* Git\n",
+        "* Python 3\n",
+        "* virtualenv\n",
+        "* Jupyter notebook running in a virtual environment with Python 3\n",
+        "\n",
+        "The Google Cloud guide to [Setting up a Python development\n",
+        "environment](https://cloud.google.com/python/setup) and the [Jupyter\n",
+        "installation guide](https://jupyter.org/install) provide detailed instructions\n",
+        "for meeting these requirements. The following steps provide a condensed set of\n",
+        "instructions:\n",
+        "\n",
+        "1. [Install and initialize the Cloud SDK.](https://cloud.google.com/sdk/docs/)\n",
+        "\n",
+        "1. [Install Python 3.](https://cloud.google.com/python/setup#installing_python)\n",
+        "\n",
+        "1. [Install\n",
+        "   virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv)\n",
+        "   and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
+        "\n",
+        "1. To install Jupyter, run `pip install jupyter` on the\n",
+        "command-line in a terminal shell.\n",
+        "\n",
+        "1. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
+        "\n",
+        "1. Open this notebook in the Jupyter Notebook Dashboard."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "i7EUnXsZhAGF"
+      },
+      "source": [
+        "### Install additional packages\n",
+        "\n",
+        "Install the KFP SDK."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "IaYsrh0Tc17L"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "# The Google Cloud Notebook product has specific requirements\n",
+        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
+        "\n",
+        "# Google Cloud Notebook requires dependencies to be installed with '--user'\n",
+        "USER_FLAG = \"\"\n",
+        "if IS_GOOGLE_CLOUD_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "aR7LNYMUCVKc"
+      },
+      "outputs": [],
+      "source": [
+        "!python3 -m pip install {USER_FLAG} kfp --upgrade"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "hhq5zEbGg0XX"
+      },
+      "source": [
+        "### Restart the kernel\n",
+        "\n",
+        "After you install the additional packages, you need to restart the notebook kernel so it can find the packages."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "EzrelQZ22IZj"
+      },
+      "outputs": [],
+      "source": [
+        "# Automatically restart kernel after installs\n",
+        "import os\n",
+        "\n",
+        "if not os.getenv(\"IS_TESTING\"):\n",
+        "    # Automatically restart kernel after installs\n",
+        "    import IPython\n",
+        "\n",
+        "    app = IPython.Application.instance()\n",
+        "    app.kernel.do_shutdown(True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6GPgNN7eeX1l"
+      },
+      "source": [
+        "Check the version of the package you installed.  The KFP SDK version should be >=1.6."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "NN0mULkEeb84"
+      },
+      "outputs": [],
+      "source": [
+        "!python3 -c \"import kfp; print('KFP SDK version: {}'.format(kfp.__version__))\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "lWEdiXsJg0XY"
+      },
+      "source": [
+        "## Before you begin\n",
+        "\n",
+        "This notebook does not require a GPU runtime."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "_W-BUaBqlaxd"
+      },
+      "source": [
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI, Cloud Storage, and Compute Engine APIs](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component,storage-component.googleapis.com). \n",
+        "\n",
+        "1. Follow the \"**Configuring your project**\" instructions from the Vertex Pipelines documentation.\n",
+        "\n",
+        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WReHDGG5g0XY"
+      },
+      "source": [
+        "#### Set your project ID\n",
+        "\n",
+        "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "oM1iC_MfAts1"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "PROJECT_ID = \"\"\n",
+        "\n",
+        "# Get your Google Cloud project ID from gcloud\n",
+        "if not os.getenv(\"IS_TESTING\"):\n",
+        "    shell_output=!gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "    PROJECT_ID = shell_output[0]\n",
+        "    print(\"Project ID: \", PROJECT_ID)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "qJYoRfYng0XZ"
+      },
+      "source": [
+        "Otherwise, set your project ID here."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "riG_qUokg0XZ"
+      },
+      "outputs": [],
+      "source": [
+        "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
+        "    PROJECT_ID = \"python-docs-samples-tests\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "06571eb4063b"
+      },
+      "source": [
+        "#### Timestamp\n",
+        "\n",
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append it onto the name of resources you create in this tutorial."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "697568e92bd6"
+      },
+      "outputs": [],
+      "source": [
+        "from datetime import datetime\n",
+        "\n",
+        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dr--iN2kAylZ"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using Google Cloud Notebooks**, your environment is already\n",
+        "authenticated. Skip this step."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "sBCra4QMA2wR"
+      },
+      "source": [
+        "**If you are using Colab**, run the cell below and follow the instructions\n",
+        "when prompted to authenticate your account via oAuth.\n",
+        "\n",
+        "**Otherwise**, follow these steps:\n",
+        "\n",
+        "1. In the Cloud Console, go to the [**Create service account key**\n",
+        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
+        "\n",
+        "2. Click **Create service account**.\n",
+        "\n",
+        "3. In the **Service account name** field, enter a name, and\n",
+        "   click **Create**.\n",
+        "\n",
+        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
+        "into the filter box, and select\n",
+        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "\n",
+        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
+        "local environment.\n",
+        "\n",
+        "6. Enter the path to your service account key as the\n",
+        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "PyQmSRbKA8r-"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "import sys\n",
+        "\n",
+        "# If you are running this notebook in Colab, run this cell and follow the\n",
+        "# instructions to authenticate your GCP account. This provides access to your\n",
+        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
+        "# requests.\n",
+        "\n",
+        "# The Google Cloud Notebook product has specific requirements\n",
+        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
+        "\n",
+        "# If on Google Cloud Notebooks, then don't execute this code\n",
+        "if not IS_GOOGLE_CLOUD_NOTEBOOK:\n",
+        "    if \"google.colab\" in sys.modules:\n",
+        "        from google.colab import auth as google_auth\n",
+        "\n",
+        "        google_auth.authenticate_user()\n",
+        "\n",
+        "    # If you are running this notebook locally, replace the string below with the\n",
+        "    # path to your service account key and run this cell to authenticate your GCP\n",
+        "    # account.\n",
+        "    elif not os.getenv(\"IS_TESTING\"):\n",
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "NxhCPW6e46EF"
+      },
+      "source": [
+        "### Create a Cloud Storage bucket as necessary\n",
+        "\n",
+        "You will need a Cloud Storage bucket for this example.  If you don't have one that you want to use, you can make one now.\n",
+        "\n",
+        "\n",
+        "Set the name of your Cloud Storage bucket below. It must be unique across all\n",
+        "Cloud Storage buckets.\n",
+        "\n",
+        "You can also change the `REGION` variable, which is used for operations\n",
+        "throughout the rest of this notebook. Make sure to [choose a region where Vertex AI services are\n",
+        "available](https://cloud.google.com/vertex-ai/docs/general/locations#available_regions). You may\n",
+        "not use a Multi-Regional Storage bucket for training with Vertex Pipelines."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "MzGDU7TWdts_"
+      },
+      "outputs": [],
+      "source": [
+        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "REGION = \"us-central1\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cf221059d072"
+      },
+      "outputs": [],
+      "source": [
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "-EcIXiGsCePi"
+      },
+      "source": [
+        "**Only if your bucket doesn't already exist**: Run the following cell to create your Cloud Storage bucket."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "NIq7R4HZCfIc"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil mb -l $REGION $BUCKET_NAME"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ucvCsknMCims"
+      },
+      "source": [
+        "Finally, validate access to your Cloud Storage bucket by examining its contents:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "vhOb7YnwClBb"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil ls -al $BUCKET_NAME"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e3EQyqZiEMmf"
+      },
+      "source": [
+        "### Import libraries and define constants"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "lNV3Jd8BEMmj"
+      },
+      "source": [
+        "Define some constants. \n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "bEOpztMJEMmj"
+      },
+      "outputs": [],
+      "source": [
+        "PATH=%env PATH\n",
+        "%env PATH={PATH}:/home/jupyter/.local/bin\n",
+        "\n",
+        "USER = \"your-user-name\"  # <---CHANGE THIS\n",
+        "PIPELINE_ROOT = \"{}/pipeline_root/{}\".format(BUCKET_NAME, USER)\n",
+        "\n",
+        "PIPELINE_ROOT"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "wbY_UZSvEMmk"
+      },
+      "source": [
+        "Do some imports:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "lnz2aQ_EEMmk"
+      },
+      "outputs": [],
+      "source": [
+        "import json\n",
+        "\n",
+        "from kfp import dsl\n",
+        "from kfp.v2 import compiler\n",
+        "from kfp.v2.dsl import component\n",
+        "from kfp.v2.google.client import AIPlatformClient"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "f1dkPTDR5bvx"
+      },
+      "source": [
+        "## Create pipeline components\n",
+        "\n",
+        "The following example defines three simple pipeline components: \n",
+        "\n",
+        "- A component that generates a list of dicts and outputs the stringified json.\n",
+        "(Note: This component requires an `import json` in the component function definition)\n",
+        "- A component that just prints its input string\n",
+        "- A component that does a 'coin flip' and outputs `heads` or `tails`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "8hxUYozoh-BV"
+      },
+      "outputs": [],
+      "source": [
+        "@component\n",
+        "def args_generator_op() -> str:\n",
+        "    import json\n",
+        "\n",
+        "    return json.dumps(\n",
+        "        [{\"cats\": \"1\", \"dogs\": \"2\"}, {\"cats\": \"10\", \"dogs\": \"20\"}],\n",
+        "        sort_keys=True,\n",
+        "    )\n",
+        "\n",
+        "\n",
+        "@component\n",
+        "def print_op(msg: str):\n",
+        "    print(msg)\n",
+        "\n",
+        "\n",
+        "@component\n",
+        "def flip_coin_op() -> str:\n",
+        "    \"\"\"Flip a coin and output heads or tails randomly.\"\"\"\n",
+        "    import random\n",
+        "\n",
+        "    result = \"heads\" if random.randint(0, 1) == 0 else \"tails\"\n",
+        "    return result"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "vSxJnIskvHQj"
+      },
+      "source": [
+        "## Define a pipeline that uses control structures\n",
+        "\n",
+        "The following example defines a pipeline that uses these components and demonstrates the use of  `dsl.Condition` and `dsl.ParallelFor`. \n",
+        "\n",
+        "The `json_string` input's default value is a nested JSON list converted to a string. As the pipeline definition shows, the loop and conditional expressions are able to process this string as a list, and access list items and sub-items.\n",
+        "The same holds for the list output by the `args_generator_op`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "sfsolQvRsuQZ"
+      },
+      "outputs": [],
+      "source": [
+        "@dsl.pipeline(\n",
+        "    name=\"pipeline-with-loops-and-conditions-{}\".format(USER),\n",
+        "    pipeline_root=PIPELINE_ROOT,\n",
+        ")\n",
+        "def my_pipeline(\n",
+        "    json_string: str = json.dumps(\n",
+        "        [\n",
+        "            {\n",
+        "                \"snakes\": \"anaconda\",\n",
+        "                \"lizards\": \"anole\",\n",
+        "                \"bunnies\": [{\"cottontail\": \"bugs\"}, {\"cottontail\": \"thumper\"}],\n",
+        "            },\n",
+        "            {\n",
+        "                \"snakes\": \"cobra\",\n",
+        "                \"lizards\": \"gecko\",\n",
+        "                \"bunnies\": [{\"cottontail\": \"roger\"}],\n",
+        "            },\n",
+        "            {\n",
+        "                \"snakes\": \"boa\",\n",
+        "                \"lizards\": \"iguana\",\n",
+        "                \"bunnies\": [\n",
+        "                    {\"cottontail\": \"fluffy\"},\n",
+        "                    {\"fuzzy_lop\": \"petunia\", \"cottontail\": \"peter\"},\n",
+        "                ],\n",
+        "            },\n",
+        "        ],\n",
+        "        sort_keys=True,\n",
+        "    )\n",
+        "):\n",
+        "\n",
+        "    flip1 = flip_coin_op()\n",
+        "\n",
+        "    with dsl.Condition(\n",
+        "        flip1.output != \"no-such-result\", name=\"alwaystrue\"\n",
+        "    ):  # always true\n",
+        "\n",
+        "        args_generator = args_generator_op()\n",
+        "        with dsl.ParallelFor(args_generator.output) as item:\n",
+        "            print_op(json_string)\n",
+        "\n",
+        "            with dsl.Condition(flip1.output == \"heads\", name=\"heads\"):\n",
+        "                print_op(item.cats)\n",
+        "\n",
+        "            with dsl.Condition(flip1.output == \"tails\", name=\"tails\"):\n",
+        "                print_op(item.dogs)\n",
+        "\n",
+        "    with dsl.ParallelFor(json_string) as item:\n",
+        "        with dsl.Condition(item.snakes == \"boa\", name=\"snakes\"):\n",
+        "            print_op(item.snakes)\n",
+        "            print_op(item.lizards)\n",
+        "            print_op(item.bunnies)\n",
+        "\n",
+        "    # it is possible to access sub-items\n",
+        "    with dsl.ParallelFor(json_string) as item:\n",
+        "        with dsl.ParallelFor(item.bunnies) as item_bunnies:\n",
+        "            print_op(item_bunnies.cottontail)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "eV6uXerEOJfA"
+      },
+      "source": [
+        "## Compile and run the pipeline\n",
+        "\n",
+        "Compile the pipeline:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7PwUFV-MleGs"
+      },
+      "outputs": [],
+      "source": [
+        "from kfp.v2 import compiler  # noqa: F811\n",
+        "\n",
+        "compiler.Compiler().compile(\n",
+        "    pipeline_func=my_pipeline, package_path=\"loops-and-conditions.json\"\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "qfNuzFswBB4g"
+      },
+      "source": [
+        "The pipeline compilation generates the `loops-and-conditions.json` job spec file.\n",
+        "\n",
+        "Next, instantiate an API client object:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Hl5Q74_gkW2c"
+      },
+      "outputs": [],
+      "source": [
+        "from kfp.v2.google.client import AIPlatformClient  # noqa: F811\n",
+        "\n",
+        "api_client = AIPlatformClient(\n",
+        "    project_id=PROJECT_ID,\n",
+        "    region=REGION,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WpVAJmSxOe1a"
+      },
+      "source": [
+        "Then, you run the defined pipeline like this: "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "R4Ha4FoDQpkd"
+      },
+      "outputs": [],
+      "source": [
+        "response = api_client.create_run_from_job_spec(\n",
+        "    job_spec_path=\"loops-and-conditions.json\", pipeline_root=PIPELINE_ROOT\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "D962iwUjOmAa"
+      },
+      "source": [
+        "Click on the generated link to see your run in the Cloud Console.\n",
+        "\n",
+        "In the UI, many of the pipeline DAG nodes will expand or collapse when you click on them. Here is a partially-expanded view of the DAG (click image to see larger version).\n",
+        "\n",
+        "<a href=\"https://storage.googleapis.com/amy-jo/images/mp/control_flow_dag.png\" target=\"_blank\"><img src=\"https://storage.googleapis.com/amy-jo/images/mp/control_flow_dag.png\" width=\"95%\"/></a>\n",
+        "\n",
+        "You can see, for example, that the 'heads' condition passed, and thus the 'tails' condition— as we would expect— did not."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "qU1SpUOC2kFk"
+      },
+      "source": [
+        "## Cleaning up\n",
+        "\n",
+        "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
+        "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
+        "\n",
+        "Otherwise, you can delete the individual resources you created in this tutorial:\n",
+        "- delete Cloud Storage objects that were created.  Uncomment and run the command in the cell below **only if you are not using the `PIPELINE_ROOT` path for any other purpose**.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "BTz1ng4z2kFv"
+      },
+      "outputs": [],
+      "source": [
+        "# Warning: this command will delete ALL Cloud Storage objects under the PIPELINE_ROOT path.\n",
+        "# ! gsutil -m rm -r $PIPELINE_ROOT"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "control_flow_kfp.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/official/pipelines/google-cloud-pipeline-components_automl_images.ipynb

@@ -0,0 +1,751 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ur8xi4C7S06n"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2021 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "JAPoU8Sm5E6e"
+      },
+      "source": [
+        "<table align=\"left\">\n",
+        "\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/official/pipelines/google-cloud-pipeline-components_automl_images.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/official/pipelines/google-cloud-pipeline-components_automl_images.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/raw/master/notebooks/official/pipelines/google-cloud-pipeline-components_automl_images.ipynb\">\n",
+        "      Open in Google Cloud Notebooks\n",
+        "    </a>\n",
+        "  </td>    \n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "zZux4nir8gRo"
+      },
+      "source": [
+        "# Vertex Pipelines: AutoML Images pipelines using google-cloud-pipeline-components\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tvgnzT1CKxrO"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "This notebook shows how to use the components defined in [`google_cloud_pipeline_components`](https://github.com/kubeflow/pipelines/tree/master/components/google-cloud) to build an AutoML Images workflow on [Vertex Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines).\n",
+        "\n",
+        "### Objective\n",
+        "\n",
+        "In this example, you'll learn how to use components from `google_cloud_pipeline_components` to:\n",
+        "- create a _Dataset_\n",
+        "- train an AutoML Images model\n",
+        "- deploy the trained model to an _endpoint_ for serving\n",
+        "\n",
+        "The components are [documented here](https://google-cloud-pipeline-components.readthedocs.io/en/latest/google_cloud_pipeline_components.aiplatform.html#module-google_cloud_pipeline_components.aiplatform).\n",
+        "\n",
+        "### Costs \n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI Training and Serving\n",
+        "* Cloud Storage\n",
+        "\n",
+        "Learn about pricing for [Vertex AI](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
+        "Calculator](https://cloud.google.com/products/calculator/)\n",
+        "to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ze4-nDLfK4pw"
+      },
+      "source": [
+        "### Set up your local development environment\n",
+        "\n",
+        "**If you are using Colab or Google Cloud Notebooks**, your environment already meets\n",
+        "all the requirements to run this notebook. You can skip this step."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gCuSR8GkAgzl"
+      },
+      "source": [
+        "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
+        "You need the following:\n",
+        "\n",
+        "* The Google Cloud SDK\n",
+        "* Git\n",
+        "* Python 3\n",
+        "* virtualenv\n",
+        "* Jupyter notebook running in a virtual environment with Python 3\n",
+        "\n",
+        "The Google Cloud guide to [Setting up a Python development\n",
+        "environment](https://cloud.google.com/python/setup) and the [Jupyter\n",
+        "installation guide](https://jupyter.org/install) provide detailed instructions\n",
+        "for meeting these requirements. The following steps provide a condensed set of\n",
+        "instructions:\n",
+        "\n",
+        "1. [Install and initialize the Cloud SDK.](https://cloud.google.com/sdk/docs/)\n",
+        "\n",
+        "1. [Install Python 3.](https://cloud.google.com/python/setup#installing_python)\n",
+        "\n",
+        "1. [Install\n",
+        "   virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv)\n",
+        "   and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
+        "\n",
+        "1. To install Jupyter, run `pip install jupyter` on the\n",
+        "command-line in a terminal shell.\n",
+        "\n",
+        "1. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
+        "\n",
+        "1. Open this notebook in the Jupyter Notebook Dashboard."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "i7EUnXsZhAGF"
+      },
+      "source": [
+        "### Install additional packages\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "IaYsrh0Tc17L"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "# The Google Cloud Notebook product has specific requirements\n",
+        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
+        "\n",
+        "# Google Cloud Notebook requires dependencies to be installed with '--user'\n",
+        "USER_FLAG = \"\"\n",
+        "if IS_GOOGLE_CLOUD_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "yxtzwPPNZ-SH"
+      },
+      "outputs": [],
+      "source": [
+        "! pip install {USER_FLAG} google-cloud-aiplatform --upgrade\n",
+        "! pip install {USER_FLAG} kfp google-cloud-pipeline-components --upgrade"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "hhq5zEbGg0XX"
+      },
+      "source": [
+        "### Restart the kernel\n",
+        "\n",
+        "After you install the additional packages, you need to restart the notebook kernel so it can find the packages."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "EzrelQZ22IZj"
+      },
+      "outputs": [],
+      "source": [
+        "# Automatically restart kernel after installs\n",
+        "import os\n",
+        "\n",
+        "if not os.getenv(\"IS_TESTING\"):\n",
+        "    # Automatically restart kernel after installs\n",
+        "    import IPython\n",
+        "\n",
+        "    app = IPython.Application.instance()\n",
+        "    app.kernel.do_shutdown(True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6GPgNN7eeX1l"
+      },
+      "source": [
+        "Check the versions of the packages you installed.  The KFP SDK version should be >=1.6."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "NN0mULkEeb84"
+      },
+      "outputs": [],
+      "source": [
+        "!python3 -c \"import kfp; print('KFP SDK version: {}'.format(kfp.__version__))\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "lWEdiXsJg0XY"
+      },
+      "source": [
+        "## Before you begin\n",
+        "\n",
+        "This notebook does not require a GPU runtime."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6EMvLDQYmBfI"
+      },
+      "source": [
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI, Cloud Storage, and Compute Engine APIs](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component,storage-component.googleapis.com). \n",
+        "\n",
+        "1. Follow the \"**Configuring your project**\" instructions from the Vertex Pipelines documentation.\n",
+        "\n",
+        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WReHDGG5g0XY"
+      },
+      "source": [
+        "#### Set your project ID\n",
+        "\n",
+        "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "oM1iC_MfAts1"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "PROJECT_ID = \"\"\n",
+        "\n",
+        "# Get your Google Cloud project ID from gcloud\n",
+        "if not os.getenv(\"IS_TESTING\"):\n",
+        "    shell_output=!gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "    PROJECT_ID = shell_output[0]\n",
+        "    print(\"Project ID: \", PROJECT_ID)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "qJYoRfYng0XZ"
+      },
+      "source": [
+        "Otherwise, set your project ID here."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "riG_qUokg0XZ"
+      },
+      "outputs": [],
+      "source": [
+        "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
+        "    PROJECT_ID = \"python-docs-samples-tests\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "06571eb4063b"
+      },
+      "source": [
+        "#### Timestamp\n",
+        "\n",
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append it onto the name of resources you create in this tutorial."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "697568e92bd6"
+      },
+      "outputs": [],
+      "source": [
+        "from datetime import datetime\n",
+        "\n",
+        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dr--iN2kAylZ"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using Google Cloud Notebooks**, your environment is already\n",
+        "authenticated. Skip this step."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "sBCra4QMA2wR"
+      },
+      "source": [
+        "**If you are using Colab**, run the cell below and follow the instructions\n",
+        "when prompted to authenticate your account via oAuth.\n",
+        "\n",
+        "**Otherwise**, follow these steps:\n",
+        "\n",
+        "1. In the Cloud Console, go to the [**Create service account key**\n",
+        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
+        "\n",
+        "2. Click **Create service account**.\n",
+        "\n",
+        "3. In the **Service account name** field, enter a name, and\n",
+        "   click **Create**.\n",
+        "\n",
+        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
+        "into the filter box, and select\n",
+        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "\n",
+        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
+        "local environment.\n",
+        "\n",
+        "6. Enter the path to your service account key as the\n",
+        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "PyQmSRbKA8r-"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "import sys\n",
+        "\n",
+        "# If you are running this notebook in Colab, run this cell and follow the\n",
+        "# instructions to authenticate your GCP account. This provides access to your\n",
+        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
+        "# requests.\n",
+        "\n",
+        "# The Google Cloud Notebook product has specific requirements\n",
+        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
+        "\n",
+        "# If on Google Cloud Notebooks, then don't execute this code\n",
+        "if not IS_GOOGLE_CLOUD_NOTEBOOK:\n",
+        "    if \"google.colab\" in sys.modules:\n",
+        "        from google.colab import auth as google_auth\n",
+        "\n",
+        "        google_auth.authenticate_user()\n",
+        "\n",
+        "    # If you are running this notebook locally, replace the string below with the\n",
+        "    # path to your service account key and run this cell to authenticate your GCP\n",
+        "    # account.\n",
+        "    elif not os.getenv(\"IS_TESTING\"):\n",
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "NxhCPW6e46EF"
+      },
+      "source": [
+        "### Create a Cloud Storage bucket as necessary\n",
+        "\n",
+        "You need a Cloud Storage bucket for this example.  If you don't have one that you want to use, you can make one now.\n",
+        "\n",
+        "\n",
+        "Set the name of your Cloud Storage bucket below. It must be unique across all\n",
+        "Cloud Storage buckets.\n",
+        "\n",
+        "You can also change the `REGION` variable, which is used for operations\n",
+        "throughout the rest of this notebook. Make sure to [choose a region where Vertex AI services are\n",
+        "available](https://cloud.google.com/vertex-ai/docs/general/locations#available_regions). You may\n",
+        "not use a Multi-Regional Storage bucket for training with Vertex Pipelines."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "MzGDU7TWdts_"
+      },
+      "outputs": [],
+      "source": [
+        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "REGION = \"us-central1\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cf221059d072"
+      },
+      "outputs": [],
+      "source": [
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "-EcIXiGsCePi"
+      },
+      "source": [
+        "**Only if your bucket doesn't already exist**: Run the following cell to create your Cloud Storage bucket."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "NIq7R4HZCfIc"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil mb -l $REGION $BUCKET_NAME"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ucvCsknMCims"
+      },
+      "source": [
+        "Finally, validate access to your Cloud Storage bucket by examining its contents:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "vhOb7YnwClBb"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil ls -al $BUCKET_NAME"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "XoEqT2Y4DJmf"
+      },
+      "source": [
+        "### Import libraries and define constants"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "YYtGjGG45ELJ"
+      },
+      "source": [
+        "Define some constants. \n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "5zmD19ryCre7"
+      },
+      "outputs": [],
+      "source": [
+        "PATH=%env PATH\n",
+        "%env PATH={PATH}:/home/jupyter/.local/bin\n",
+        "\n",
+        "USER = \"your-user-name\"  # <---CHANGE THIS\n",
+        "PIPELINE_ROOT = \"{}/pipeline_root/{}\".format(BUCKET_NAME, USER)\n",
+        "\n",
+        "PIPELINE_ROOT"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "IprQaSI25oSk"
+      },
+      "source": [
+        "Do some imports:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "UFDUBveR5UfJ"
+      },
+      "outputs": [],
+      "source": [
+        "import kfp\n",
+        "from google.cloud import aiplatform\n",
+        "from google_cloud_pipeline_components import aiplatform as gcc_aip\n",
+        "from kfp.v2 import compiler\n",
+        "from kfp.v2.google.client import AIPlatformClient"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Z4MjdglUT3Sw"
+      },
+      "source": [
+        "## Define an AutoML Image classification pipeline that uses components from `google_cloud_pipeline_components`\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Pf0pugbvftD1"
+      },
+      "source": [
+        "Create a managed image dataset from a CSV file and train it using AutoML Image Training.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2fjGiImBezMo"
+      },
+      "source": [
+        "Define the pipeline:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "vEEr62NUftD1"
+      },
+      "outputs": [],
+      "source": [
+        "@kfp.dsl.pipeline(name=\"automl-image-training-v2\")\n",
+        "def pipeline(project: str = PROJECT_ID):\n",
+        "    ds_op = gcc_aip.ImageDatasetCreateOp(\n",
+        "        project=project,\n",
+        "        display_name=\"flowers\",\n",
+        "        gcs_source=\"gs://cloud-samples-data/vision/automl_classification/flowers/all_data_v2.csv\",\n",
+        "        import_schema_uri=aiplatform.schema.dataset.ioformat.image.single_label_classification,\n",
+        "    )\n",
+        "\n",
+        "    training_job_run_op = gcc_aip.AutoMLImageTrainingJobRunOp(\n",
+        "        project=project,\n",
+        "        display_name=\"train-iris-automl-mbsdk-1\",\n",
+        "        prediction_type=\"classification\",\n",
+        "        model_type=\"CLOUD\",\n",
+        "        base_model=None,\n",
+        "        dataset=ds_op.outputs[\"dataset\"],\n",
+        "        model_display_name=\"iris-classification-model-mbsdk\",\n",
+        "        training_fraction_split=0.6,\n",
+        "        validation_fraction_split=0.2,\n",
+        "        test_fraction_split=0.2,\n",
+        "        budget_milli_node_hours=8000,\n",
+        "    )\n",
+        "    endpoint_op = gcc_aip.ModelDeployOp(  # noqa: F841\n",
+        "        project=project, model=training_job_run_op.outputs[\"model\"]\n",
+        "    )"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2Hl1iYEKSzjP"
+      },
+      "source": [
+        "## Compile and run the pipeline\n",
+        "\n",
+        "Compile the pipeline:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ycRc83B6bbfO"
+      },
+      "outputs": [],
+      "source": [
+        "from kfp.v2 import compiler  # noqa: F811\n",
+        "\n",
+        "compiler.Compiler().compile(\n",
+        "    pipeline_func=pipeline, package_path=\"image_classif_pipeline.json\"\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "qfNuzFswBB4g"
+      },
+      "source": [
+        "The pipeline compilation generates the `image_classif_pipeline.json` job spec file.\n",
+        "\n",
+        "Next, instantiate an API client object:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Hl5Q74_gkW2c"
+      },
+      "outputs": [],
+      "source": [
+        "from kfp.v2.google.client import AIPlatformClient  # noqa: F811\n",
+        "\n",
+        "api_client = AIPlatformClient(project_id=PROJECT_ID, region=REGION)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "_jrn6saiQsPh"
+      },
+      "source": [
+        "Then, you run the defined pipeline like this: "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "R4Ha4FoDQpkd"
+      },
+      "outputs": [],
+      "source": [
+        "response = api_client.create_run_from_job_spec(\n",
+        "    \"image_classif_pipeline.json\",\n",
+        "    pipeline_root=PIPELINE_ROOT,\n",
+        "    parameter_values={\"project\": PROJECT_ID},\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "GvBTCP318RKs"
+      },
+      "source": [
+        "Click on the generated link to see your run in the Cloud Console.  It should look something like this as it is running:\n",
+        "\n",
+        "<a href=\"https://storage.googleapis.com/amy-jo/images/mp/automl_image_classif.png\" target=\"_blank\"><img src=\"https://storage.googleapis.com/amy-jo/images/mp/automl_image_classif.png\" width=\"40%\"/></a>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "p9_qWdh83C1N"
+      },
+      "source": [
+        "## Cleaning up\n",
+        "\n",
+        "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
+        "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
+        "\n",
+        "Otherwise, you can delete the individual resources you created in this tutorial:\n",
+        "- Delete Cloud Storage objects that were created.  Uncomment and run the command in the cell below **only if you are not using the `PIPELINE_ROOT` path for any other purpose**.\n",
+        "- Delete your deployed model: first, undeploy it from its *endpoint*, then delete the model and endpoint.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3AEwz0T63C1O"
+      },
+      "outputs": [],
+      "source": [
+        "# Warning: this command will delete ALL Cloud Storage objects under the PIPELINE_ROOT path.\n",
+        "# ! gsutil -m rm -r $PIPELINE_ROOT"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "google-cloud-pipeline-components_automl_images.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/official/pipelines/google-cloud-pipeline-components_automl_tabular.ipynb

@@ -0,0 +1,773 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ur8xi4C7S06n"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2021 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "JAPoU8Sm5E6e"
+      },
+      "source": [
+        "<table align=\"left\">\n",
+        "\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/official/pipelines/google-cloud-pipeline-components_automl_tabular.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/official/pipelines/google-cloud-pipeline-components_automl_tabular.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/raw/master/notebooks/official/pipelines/google-cloud-pipeline-components_automl_tabular.ipynb\">\n",
+        "      Open in Google Cloud Notebooks\n",
+        "    </a>\n",
+        "  </td>\n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "mVX0RV_s8zZ7"
+      },
+      "source": [
+        "# Vertex Pipelines: AutoML Tabular pipelines using google-cloud-pipeline-components\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tvgnzT1CKxrO"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "\n",
+        "This notebook shows how to use the components defined in [`google_cloud_pipeline_components`](https://github.com/kubeflow/pipelines/tree/master/components/google-cloud) to build an AutoML Tabular workflow on [Vertex Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines).\n",
+        "\n",
+        "### Objective\n",
+        "\n",
+        "In this example, you'll learn how to use components from `google_cloud_pipeline_components` to:\n",
+        "- create a _Dataset_\n",
+        "- train an AutoML Tabular model\n",
+        "- deploy the trained model to an _endpoint_ for serving\n",
+        "\n",
+        "The components are [documented here](https://google-cloud-pipeline-components.readthedocs.io/en/latest/google_cloud_pipeline_components.aiplatform.html#module-google_cloud_pipeline_components.aiplatform).\n",
+        "\n",
+        "### Costs \n",
+        "\n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI Training and Serving\n",
+        "* Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
+        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
+        "Calculator](https://cloud.google.com/products/calculator/)\n",
+        "to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ze4-nDLfK4pw"
+      },
+      "source": [
+        "### Set up your local development environment\n",
+        "\n",
+        "**If you are using Colab or Google Cloud Notebooks**, your environment already meets\n",
+        "all the requirements to run this notebook. You can skip this step."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gCuSR8GkAgzl"
+      },
+      "source": [
+        "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
+        "You need the following:\n",
+        "\n",
+        "* The Google Cloud SDK\n",
+        "* Git\n",
+        "* Python 3\n",
+        "* virtualenv\n",
+        "* Jupyter notebook running in a virtual environment with Python 3\n",
+        "\n",
+        "The Google Cloud guide to [Setting up a Python development\n",
+        "environment](https://cloud.google.com/python/setup) and the [Jupyter\n",
+        "installation guide](https://jupyter.org/install) provide detailed instructions\n",
+        "for meeting these requirements. The following steps provide a condensed set of\n",
+        "instructions:\n",
+        "\n",
+        "1. [Install and initialize the Cloud SDK.](https://cloud.google.com/sdk/docs/)\n",
+        "\n",
+        "1. [Install Python 3.](https://cloud.google.com/python/setup#installing_python)\n",
+        "\n",
+        "1. [Install\n",
+        "   virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv)\n",
+        "   and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
+        "\n",
+        "1. To install Jupyter, run `pip install jupyter` on the\n",
+        "command-line in a terminal shell.\n",
+        "\n",
+        "1. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
+        "\n",
+        "1. Open this notebook in the Jupyter Notebook Dashboard."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "i7EUnXsZhAGF"
+      },
+      "source": [
+        "### Install additional packages\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "IaYsrh0Tc17L"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "# The Google Cloud Notebook product has specific requirements\n",
+        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
+        "\n",
+        "# Google Cloud Notebook requires dependencies to be installed with '--user'\n",
+        "USER_FLAG = \"\"\n",
+        "if IS_GOOGLE_CLOUD_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "yxtzwPPNZ-SH"
+      },
+      "outputs": [],
+      "source": [
+        "! pip install {USER_FLAG} kfp google-cloud-pipeline-components --upgrade"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "hhq5zEbGg0XX"
+      },
+      "source": [
+        "### Restart the kernel\n",
+        "\n",
+        "After you install the additional packages, you need to restart the notebook kernel so it can find the packages."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "EzrelQZ22IZj"
+      },
+      "outputs": [],
+      "source": [
+        "# Automatically restart kernel after installs\n",
+        "import os\n",
+        "\n",
+        "if not os.getenv(\"IS_TESTING\"):\n",
+        "    # Automatically restart kernel after installs\n",
+        "    import IPython\n",
+        "\n",
+        "    app = IPython.Application.instance()\n",
+        "    app.kernel.do_shutdown(True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6GPgNN7eeX1l"
+      },
+      "source": [
+        "Check the versions of the packages you installed.  The KFP SDK version should be >=1.6."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "NN0mULkEeb84"
+      },
+      "outputs": [],
+      "source": [
+        "!python3 -c \"import kfp; print('KFP SDK version: {}'.format(kfp.__version__))\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "lWEdiXsJg0XY"
+      },
+      "source": [
+        "## Before you begin\n",
+        "\n",
+        "This notebook does not require a GPU runtime."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "O3ysRXrDl91t"
+      },
+      "source": [
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI, Cloud Storage, and Compute Engine APIs](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component,storage-component.googleapis.com). \n",
+        "\n",
+        "1. Follow the \"**Configuring your project**\" instructions from the Vertex Pipelines documentation.\n",
+        "\n",
+        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WReHDGG5g0XY"
+      },
+      "source": [
+        "#### Set your project ID\n",
+        "\n",
+        "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "oM1iC_MfAts1"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "PROJECT_ID = \"\"\n",
+        "\n",
+        "# Get your Google Cloud project ID from gcloud\n",
+        "if not os.getenv(\"IS_TESTING\"):\n",
+        "    shell_output=!gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "    PROJECT_ID = shell_output[0]\n",
+        "    print(\"Project ID: \", PROJECT_ID)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "qJYoRfYng0XZ"
+      },
+      "source": [
+        "Otherwise, set your project ID here."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "riG_qUokg0XZ"
+      },
+      "outputs": [],
+      "source": [
+        "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
+        "    PROJECT_ID = \"python-docs-samples-tests\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "06571eb4063b"
+      },
+      "source": [
+        "#### Timestamp\n",
+        "\n",
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append it onto the name of resources you create in this tutorial."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "697568e92bd6"
+      },
+      "outputs": [],
+      "source": [
+        "from datetime import datetime\n",
+        "\n",
+        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dr--iN2kAylZ"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using Google Cloud Notebooks**, your environment is already\n",
+        "authenticated. Skip this step."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "sBCra4QMA2wR"
+      },
+      "source": [
+        "**If you are using Colab**, run the cell below and follow the instructions\n",
+        "when prompted to authenticate your account via oAuth.\n",
+        "\n",
+        "**Otherwise**, follow these steps:\n",
+        "\n",
+        "1. In the Cloud Console, go to the [**Create service account key**\n",
+        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
+        "\n",
+        "2. Click **Create service account**.\n",
+        "\n",
+        "3. In the **Service account name** field, enter a name, and\n",
+        "   click **Create**.\n",
+        "\n",
+        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
+        "into the filter box, and select\n",
+        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "\n",
+        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
+        "local environment.\n",
+        "\n",
+        "6. Enter the path to your service account key as the\n",
+        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "PyQmSRbKA8r-"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "import sys\n",
+        "\n",
+        "# If you are running this notebook in Colab, run this cell and follow the\n",
+        "# instructions to authenticate your GCP account. This provides access to your\n",
+        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
+        "# requests.\n",
+        "\n",
+        "# The Google Cloud Notebook product has specific requirements\n",
+        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
+        "\n",
+        "# If on Google Cloud Notebooks, then don't execute this code\n",
+        "if not IS_GOOGLE_CLOUD_NOTEBOOK:\n",
+        "    if \"google.colab\" in sys.modules:\n",
+        "        from google.colab import auth as google_auth\n",
+        "\n",
+        "        google_auth.authenticate_user()\n",
+        "\n",
+        "    # If you are running this notebook locally, replace the string below with the\n",
+        "    # path to your service account key and run this cell to authenticate your GCP\n",
+        "    # account.\n",
+        "    elif not os.getenv(\"IS_TESTING\"):\n",
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "NxhCPW6e46EF"
+      },
+      "source": [
+        "### Create a Cloud Storage bucket as necessary\n",
+        "\n",
+        "You will need a Cloud Storage bucket for this example.  If you don't have one that you want to use, you can make one now.\n",
+        "\n",
+        "\n",
+        "Set the name of your Cloud Storage bucket below. It must be unique across all\n",
+        "Cloud Storage buckets.\n",
+        "\n",
+        "You may also change the `REGION` variable, which is used for operations\n",
+        "throughout the rest of this notebook. Make sure to [choose a region where Vertex AI services are\n",
+        "available](https://cloud.google.com/vertex-ai/docs/general/locations#available_regions). You may\n",
+        "not use a Multi-Regional Storage bucket for training with Vertex AI."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "MzGDU7TWdts_"
+      },
+      "outputs": [],
+      "source": [
+        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "REGION = \"us-central1\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cf221059d072"
+      },
+      "outputs": [],
+      "source": [
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "-EcIXiGsCePi"
+      },
+      "source": [
+        "**Only if your bucket doesn't already exist**: Run the following cell to create your Cloud Storage bucket."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "NIq7R4HZCfIc"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil mb -l $REGION $BUCKET_NAME"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ucvCsknMCims"
+      },
+      "source": [
+        "Finally, validate access to your Cloud Storage bucket by examining its contents:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "vhOb7YnwClBb"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil ls -al $BUCKET_NAME"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "XoEqT2Y4DJmf"
+      },
+      "source": [
+        "### Import libraries and define constants"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "YYtGjGG45ELJ"
+      },
+      "source": [
+        "Define some constants.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "5zmD19ryCre7"
+      },
+      "outputs": [],
+      "source": [
+        "PATH=%env PATH\n",
+        "%env PATH={PATH}:/home/jupyter/.local/bin\n",
+        "\n",
+        "USER = \"your-user-name\"  # <---CHANGE THIS\n",
+        "PIPELINE_ROOT = \"{}/pipeline_root/{}\".format(BUCKET_NAME, USER)\n",
+        "\n",
+        "PIPELINE_ROOT"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "IprQaSI25oSk"
+      },
+      "source": [
+        "Do some imports:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "UFDUBveR5UfJ"
+      },
+      "outputs": [],
+      "source": [
+        "import kfp\n",
+        "from google_cloud_pipeline_components import aiplatform as gcc_aip\n",
+        "from kfp.v2 import compiler\n",
+        "from kfp.v2.google.client import AIPlatformClient"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Z4MjdglUT3Sw"
+      },
+      "source": [
+        "## Define an AutoML Tabular regression pipeline that uses components from `google_cloud_pipeline_components`\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Pf0pugbvftD1"
+      },
+      "source": [
+        "Create a managed image dataset from a CSV file and train it using AutoML Tabular Training.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "yYGHro4qFhp5"
+      },
+      "outputs": [],
+      "source": [
+        "TRAIN_FILE_NAME = \"california_housing_train.csv\"\n",
+        "!gsutil cp gs://aju-dev-demos-codelabs/sample_data/california_housing_train.csv {PIPELINE_ROOT}/data/\n",
+        "\n",
+        "gcs_csv_path = f\"{PIPELINE_ROOT}/data/{TRAIN_FILE_NAME}\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2fjGiImBezMo"
+      },
+      "source": [
+        "Define the pipeline:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "vEEr62NUftD1"
+      },
+      "outputs": [],
+      "source": [
+        "@kfp.dsl.pipeline(name=\"automl-tab-training-v2\")\n",
+        "def pipeline(project: str = PROJECT_ID):\n",
+        "\n",
+        "    dataset_create_op = gcc_aip.TabularDatasetCreateOp(\n",
+        "        project=project, display_name=\"housing\", gcs_source=gcs_csv_path\n",
+        "    )\n",
+        "\n",
+        "    training_op = gcc_aip.AutoMLTabularTrainingJobRunOp(\n",
+        "        project=project,\n",
+        "        display_name=\"train-housing-automl_1\",\n",
+        "        optimization_prediction_type=\"regression\",\n",
+        "        optimization_objective=\"minimize-rmse\",\n",
+        "        column_transformations=[\n",
+        "            {\"numeric\": {\"column_name\": \"longitude\"}},\n",
+        "            {\"numeric\": {\"column_name\": \"latitude\"}},\n",
+        "            {\"numeric\": {\"column_name\": \"housing_median_age\"}},\n",
+        "            {\"numeric\": {\"column_name\": \"total_rooms\"}},\n",
+        "            {\"numeric\": {\"column_name\": \"total_bedrooms\"}},\n",
+        "            {\"numeric\": {\"column_name\": \"population\"}},\n",
+        "            {\"numeric\": {\"column_name\": \"households\"}},\n",
+        "            {\"numeric\": {\"column_name\": \"median_income\"}},\n",
+        "            {\"numeric\": {\"column_name\": \"median_house_value\"}},\n",
+        "        ],\n",
+        "        dataset=dataset_create_op.outputs[\"dataset\"],\n",
+        "        target_column=\"median_house_value\",\n",
+        "    )\n",
+        "\n",
+        "    deploy_op = gcc_aip.ModelDeployOp(  # noqa: F841\n",
+        "        model=training_op.outputs[\"model\"],\n",
+        "        project=project,\n",
+        "        machine_type=\"n1-standard-4\",\n",
+        "    )"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2Hl1iYEKSzjP"
+      },
+      "source": [
+        "## Compile and run the pipeline\n",
+        "\n",
+        "Now, you're ready to compile the pipeline:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ycRc83B6bbfO"
+      },
+      "outputs": [],
+      "source": [
+        "from kfp.v2 import compiler  # noqa: F811\n",
+        "\n",
+        "compiler.Compiler().compile(\n",
+        "    pipeline_func=pipeline, package_path=\"tab_regression_pipeline.json\"\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "qfNuzFswBB4g"
+      },
+      "source": [
+        "The pipeline compilation generates the `tab_regression_pipeline.json` job spec file.\n",
+        "\n",
+        "Next, instantiate an API client object:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Hl5Q74_gkW2c"
+      },
+      "outputs": [],
+      "source": [
+        "from kfp.v2.google.client import AIPlatformClient  # noqa: F811\n",
+        "\n",
+        "api_client = AIPlatformClient(project_id=PROJECT_ID, region=REGION)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "_jrn6saiQsPh"
+      },
+      "source": [
+        "Then, you run the defined pipeline like this: "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "R4Ha4FoDQpkd"
+      },
+      "outputs": [],
+      "source": [
+        "response = api_client.create_run_from_job_spec(\n",
+        "    \"tab_regression_pipeline.json\",\n",
+        "    pipeline_root=PIPELINE_ROOT,\n",
+        "    parameter_values={\"project\": PROJECT_ID},\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "GvBTCP318RKs"
+      },
+      "source": [
+        "Click on the generated link to see your run in the Cloud Console.  It should look something like this as it is running:\n",
+        "\n",
+        "<a href=\"https://storage.googleapis.com/amy-jo/images/mp/automl_tabular_classif.png\" target=\"_blank\"><img src=\"https://storage.googleapis.com/amy-jo/images/mp/automl_tabular_classif.png\" width=\"40%\"/></a>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "JNU7IhpB3Hry"
+      },
+      "source": [
+        "## Cleaning up\n",
+        "\n",
+        "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
+        "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
+        "\n",
+        "Otherwise, you can delete the individual resources you created in this tutorial:\n",
+        "- Delete Cloud Storage objects that were created.  Uncomment and run the command in the cell below **only if you are not using the `PIPELINE_ROOT` path for any other purpose**.\n",
+        "- Delete your deployed model: first, undeploy it from its *endpoint*, then delete the model and endpoint.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6vWg3Kly3Hry"
+      },
+      "outputs": [],
+      "source": [
+        "# Warning: this command will delete ALL Cloud Storage objects under the PIPELINE_ROOT path.\n",
+        "# ! gsutil -m rm -r $PIPELINE_ROOT"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "google-cloud-pipeline-components_automl_tabular.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/official/pipelines/google-cloud-pipeline-components_automl_text.ipynb

@@ -0,0 +1,761 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ur8xi4C7S06n"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2021 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "JAPoU8Sm5E6e"
+      },
+      "source": [
+        "<table align=\"left\">\n",
+        "\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/official/pipelines/google-cloud-pipeline-components_automl_text.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/official/pipelines/google-cloud-pipeline-components_automl_text.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/raw/master/notebooks/official/pipelines/google-cloud-pipeline-components_automl_text.ipynb\">\n",
+        "      Open in Google Cloud Notebooks\n",
+        "    </a>\n",
+        "  </td>    \n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "mVX0RV_s8zZ7"
+      },
+      "source": [
+        "# Vertex Pipelines: AutoML Text pipelines using google-cloud-pipeline-components\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tvgnzT1CKxrO"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "\n",
+        "This notebook shows how to use the components defined in [`google_cloud_pipeline_components`](https://github.com/kubeflow/pipelines/tree/master/components/google-cloud) to build an AutoML Text workflow on [Vertex Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines).\n",
+        "\n",
+        "### Objective\n",
+        "\n",
+        "In this example, you'll learn how to use components from `google_cloud_pipeline_components` to:\n",
+        "- create a _Dataset_\n",
+        "- train an AutoML Text model\n",
+        "- deploy the trained model to an _endpoint_ for serving\n",
+        "\n",
+        "The components are [documented here](https://google-cloud-pipeline-components.readthedocs.io/en/latest/google_cloud_pipeline_components.aiplatform.html#module-google_cloud_pipeline_components.aiplatform).\n",
+        "\n",
+        "### Costs \n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI Training and Serving\n",
+        "* Cloud Storage\n",
+        "\n",
+        "\n",
+        "Learn about [Vertex AI\n",
+        "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
+        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
+        "Calculator](https://cloud.google.com/products/calculator/)\n",
+        "to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ze4-nDLfK4pw"
+      },
+      "source": [
+        "### Set up your local development environment\n",
+        "\n",
+        "**If you are using Colab or Google Cloud Notebooks**, your environment already meets\n",
+        "all the requirements to run this notebook. You can skip this step."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gCuSR8GkAgzl"
+      },
+      "source": [
+        "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
+        "You need the following:\n",
+        "\n",
+        "* The Google Cloud SDK\n",
+        "* Git\n",
+        "* Python 3\n",
+        "* virtualenv\n",
+        "* Jupyter notebook running in a virtual environment with Python 3\n",
+        "\n",
+        "The Google Cloud guide to [Setting up a Python development\n",
+        "environment](https://cloud.google.com/python/setup) and the [Jupyter\n",
+        "installation guide](https://jupyter.org/install) provide detailed instructions\n",
+        "for meeting these requirements. The following steps provide a condensed set of\n",
+        "instructions:\n",
+        "\n",
+        "1. [Install and initialize the Cloud SDK.](https://cloud.google.com/sdk/docs/)\n",
+        "\n",
+        "1. [Install Python 3.](https://cloud.google.com/python/setup#installing_python)\n",
+        "\n",
+        "1. [Install\n",
+        "   virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv)\n",
+        "   and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
+        "\n",
+        "1. To install Jupyter, run `pip install jupyter` on the\n",
+        "command-line in a terminal shell.\n",
+        "\n",
+        "1. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
+        "\n",
+        "1. Open this notebook in the Jupyter Notebook Dashboard."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "i7EUnXsZhAGF"
+      },
+      "source": [
+        "### Install additional packages\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "IaYsrh0Tc17L"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "# The Google Cloud Notebook product has specific requirements\n",
+        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
+        "\n",
+        "# Google Cloud Notebook requires dependencies to be installed with '--user'\n",
+        "USER_FLAG = \"\"\n",
+        "if IS_GOOGLE_CLOUD_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "yxtzwPPNZ-SH"
+      },
+      "outputs": [],
+      "source": [
+        "! pip install {USER_FLAG} google-cloud-aiplatform --upgrade\n",
+        "! pip install {USER_FLAG} kfp google-cloud-pipeline-components --upgrade"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "hhq5zEbGg0XX"
+      },
+      "source": [
+        "### Restart the kernel\n",
+        "\n",
+        "After you install the additional packages, you need to restart the notebook kernel so it can find the packages."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "EzrelQZ22IZj"
+      },
+      "outputs": [],
+      "source": [
+        "# Automatically restart kernel after installs\n",
+        "import os\n",
+        "\n",
+        "if not os.getenv(\"IS_TESTING\"):\n",
+        "    # Automatically restart kernel after installs\n",
+        "    import IPython\n",
+        "\n",
+        "    app = IPython.Application.instance()\n",
+        "    app.kernel.do_shutdown(True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6GPgNN7eeX1l"
+      },
+      "source": [
+        "Check the versions of the packages you installed.  The KFP SDK version should be >=1.6."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "NN0mULkEeb84"
+      },
+      "outputs": [],
+      "source": [
+        "!python3 -c \"import kfp; print('KFP SDK version: {}'.format(kfp.__version__))\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "lWEdiXsJg0XY"
+      },
+      "source": [
+        "## Before you begin\n",
+        "\n",
+        "This notebook does not require a GPU runtime."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "i1VRlEu-l0BW"
+      },
+      "source": [
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI, Cloud Storage, and Compute Engine APIs](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component,storage-component.googleapis.com). \n",
+        "\n",
+        "1. Follow the \"**Configuring your project**\" instructions from the Vertex Pipelines documentation.\n",
+        "\n",
+        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WReHDGG5g0XY"
+      },
+      "source": [
+        "#### Set your project ID\n",
+        "\n",
+        "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "oM1iC_MfAts1"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "PROJECT_ID = \"\"\n",
+        "\n",
+        "# Get your Google Cloud project ID from gcloud\n",
+        "if not os.getenv(\"IS_TESTING\"):\n",
+        "    shell_output=!gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "    PROJECT_ID = shell_output[0]\n",
+        "    print(\"Project ID: \", PROJECT_ID)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "qJYoRfYng0XZ"
+      },
+      "source": [
+        "Otherwise, set your project ID here."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "riG_qUokg0XZ"
+      },
+      "outputs": [],
+      "source": [
+        "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
+        "    PROJECT_ID = \"python-docs-samples-tests\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "06571eb4063b"
+      },
+      "source": [
+        "#### Timestamp\n",
+        "\n",
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append it onto the name of resources you create in this tutorial."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "697568e92bd6"
+      },
+      "outputs": [],
+      "source": [
+        "from datetime import datetime\n",
+        "\n",
+        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dr--iN2kAylZ"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using Google Cloud Notebooks**, your environment is already\n",
+        "authenticated. Skip this step."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "sBCra4QMA2wR"
+      },
+      "source": [
+        "**If you are using Colab**, run the cell below and follow the instructions\n",
+        "when prompted to authenticate your account via oAuth.\n",
+        "\n",
+        "**Otherwise**, follow these steps:\n",
+        "\n",
+        "1. In the Cloud Console, go to the [**Create service account key**\n",
+        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
+        "\n",
+        "2. Click **Create service account**.\n",
+        "\n",
+        "3. In the **Service account name** field, enter a name, and\n",
+        "   click **Create**.\n",
+        "\n",
+        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
+        "into the filter box, and select\n",
+        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "\n",
+        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
+        "local environment.\n",
+        "\n",
+        "6. Enter the path to your service account key as the\n",
+        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "PyQmSRbKA8r-"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "import sys\n",
+        "\n",
+        "# If you are running this notebook in Colab, run this cell and follow the\n",
+        "# instructions to authenticate your GCP account. This provides access to your\n",
+        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
+        "# requests.\n",
+        "\n",
+        "# The Google Cloud Notebook product has specific requirements\n",
+        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
+        "\n",
+        "# If on Google Cloud Notebooks, then don't execute this code\n",
+        "if not IS_GOOGLE_CLOUD_NOTEBOOK:\n",
+        "    if \"google.colab\" in sys.modules:\n",
+        "        from google.colab import auth as google_auth\n",
+        "\n",
+        "        google_auth.authenticate_user()\n",
+        "\n",
+        "    # If you are running this notebook locally, replace the string below with the\n",
+        "    # path to your service account key and run this cell to authenticate your GCP\n",
+        "    # account.\n",
+        "    elif not os.getenv(\"IS_TESTING\"):\n",
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "NxhCPW6e46EF"
+      },
+      "source": [
+        "### Create a Cloud Storage bucket as necessary\n",
+        "\n",
+        "You will need a Cloud Storage bucket for this example.  If you don't have one that you want to use, you can make one now.\n",
+        "\n",
+        "\n",
+        "Set the name of your Cloud Storage bucket below. It must be unique across all\n",
+        "Cloud Storage buckets.\n",
+        "\n",
+        "You may also change the `REGION` variable, which is used for operations\n",
+        "throughout the rest of this notebook. Make sure to [choose a region where Vertex AI services are\n",
+        "available](https://cloud.google.com/vertex-ai/docs/general/locations#available_regions). You may\n",
+        "not use a Multi-Regional Storage bucket for training with Vertex AI."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "MzGDU7TWdts_"
+      },
+      "outputs": [],
+      "source": [
+        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "REGION = \"us-central1\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cf221059d072"
+      },
+      "outputs": [],
+      "source": [
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "-EcIXiGsCePi"
+      },
+      "source": [
+        "**Only if your bucket doesn't already exist**: Run the following cell to create your Cloud Storage bucket."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "NIq7R4HZCfIc"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil mb -l $REGION $BUCKET_NAME"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ucvCsknMCims"
+      },
+      "source": [
+        "Finally, validate access to your Cloud Storage bucket by examining its contents:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "vhOb7YnwClBb"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil ls -al $BUCKET_NAME"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "XoEqT2Y4DJmf"
+      },
+      "source": [
+        "### Import libraries and define constants"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "YYtGjGG45ELJ"
+      },
+      "source": [
+        "Define some constants. \n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "5zmD19ryCre7"
+      },
+      "outputs": [],
+      "source": [
+        "PATH=%env PATH\n",
+        "%env PATH={PATH}:/home/jupyter/.local/bin\n",
+        "\n",
+        "USER = \"your-user-name\"  # <---CHANGE THIS\n",
+        "PIPELINE_ROOT = \"{}/pipeline_root/{}\".format(BUCKET_NAME, USER)\n",
+        "\n",
+        "PIPELINE_ROOT"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "IprQaSI25oSk"
+      },
+      "source": [
+        "Do some imports:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "UFDUBveR5UfJ"
+      },
+      "outputs": [],
+      "source": [
+        "import kfp\n",
+        "from google.cloud import aiplatform\n",
+        "from google_cloud_pipeline_components import aiplatform as gcc_aip\n",
+        "from kfp.v2 import compiler\n",
+        "from kfp.v2.google.client import AIPlatformClient"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Z4MjdglUT3Sw"
+      },
+      "source": [
+        "## Define an AutoML Text classification pipeline that uses components from `google_cloud_pipeline_components`\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Pf0pugbvftD1"
+      },
+      "source": [
+        "Create a managed image dataset from a CSV file and train it using AutoML Text Training.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2fjGiImBezMo"
+      },
+      "source": [
+        "Define the pipeline:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "vEEr62NUftD1"
+      },
+      "outputs": [],
+      "source": [
+        "IMPORT_FILE = \"gs://cloud-ml-data/NL-classification/happiness.csv\"\n",
+        "\n",
+        "\n",
+        "@kfp.dsl.pipeline(name=\"automl-text-classification\" + TIMESTAMP)\n",
+        "def pipeline(project: str = PROJECT_ID, import_file: str = IMPORT_FILE):\n",
+        "\n",
+        "    dataset_create_task = gcc_aip.TextDatasetCreateOp(\n",
+        "        display_name=\"happydb\",\n",
+        "        gcs_source=import_file,\n",
+        "        import_schema_uri=aiplatform.schema.dataset.ioformat.text.multi_label_classification,\n",
+        "        project=project,\n",
+        "    )\n",
+        "\n",
+        "    training_run_task = gcc_aip.AutoMLTextTrainingJobRunOp(\n",
+        "        dataset=dataset_create_task.outputs[\"dataset\"],\n",
+        "        display_name=\"train-text-classif\",\n",
+        "        prediction_type=\"classification\",\n",
+        "        multi_label=True,\n",
+        "        training_fraction_split=0.6,\n",
+        "        validation_fraction_split=0.2,\n",
+        "        test_fraction_split=0.2,\n",
+        "        model_display_name=\"happy-model\",\n",
+        "        project=project,\n",
+        "    )\n",
+        "\n",
+        "    model_deploy_op = gcc_aip.ModelDeployOp(  # noqa: F841\n",
+        "        model=training_run_task.outputs[\"model\"], project=project\n",
+        "    )"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2Hl1iYEKSzjP"
+      },
+      "source": [
+        "## Compile and run the pipeline\n",
+        "\n",
+        "Now, you're ready to compile the pipeline:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ycRc83B6bbfO"
+      },
+      "outputs": [],
+      "source": [
+        "from kfp.v2 import compiler  # noqa: F811\n",
+        "\n",
+        "compiler.Compiler().compile(\n",
+        "    pipeline_func=pipeline, package_path=\"text_classsif_pipeline.json\"\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "qfNuzFswBB4g"
+      },
+      "source": [
+        "The pipeline compilation generates the `image_classif_pipeline.json` job spec file.\n",
+        "\n",
+        "Next, instantiate an API client object:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Hl5Q74_gkW2c"
+      },
+      "outputs": [],
+      "source": [
+        "from kfp.v2.google.client import AIPlatformClient  # noqa: F811\n",
+        "\n",
+        "api_client = AIPlatformClient(\n",
+        "    project_id=PROJECT_ID,\n",
+        "    region=REGION,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "_jrn6saiQsPh"
+      },
+      "source": [
+        "Then, you run the defined pipeline like this: "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "R4Ha4FoDQpkd"
+      },
+      "outputs": [],
+      "source": [
+        "response = api_client.create_run_from_job_spec(\n",
+        "    \"text_classsif_pipeline.json\",\n",
+        "    pipeline_root=PIPELINE_ROOT,\n",
+        "    parameter_values={\"project\": PROJECT_ID},\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "GvBTCP318RKs"
+      },
+      "source": [
+        "Click on the generated link to see your run in the Cloud Console.  It should look something like this as it is running:\n",
+        "\n",
+        "<a href=\"https://storage.googleapis.com/amy-jo/images/mp/automl_text_classif.png\" target=\"_blank\"><img src=\"https://storage.googleapis.com/amy-jo/images/mp/automl_text_classif.png\" width=\"40%\"/></a>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "RLqsTQBC2x9O"
+      },
+      "source": [
+        "## Cleaning up\n",
+        "\n",
+        "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
+        "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
+        "\n",
+        "Otherwise, you can delete the individual resources you created in this tutorial:\n",
+        "- Delete Cloud Storage objects that were created.  Uncomment and run the command in the cell below **only if you are not using the `PIPELINE_ROOT` path for any other purpose**.\n",
+        "- Delete your deployed model: first, undeploy it from its *endpoint*, then delete the model and endpoint.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "FbMysa2v2x9P"
+      },
+      "outputs": [],
+      "source": [
+        "# Warning: this command will delete ALL Cloud Storage objects under the PIPELINE_ROOT path.\n",
+        "# ! gsutil -m rm -r $PIPELINE_ROOT"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "google-cloud-pipeline-components_automl_text.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/official/pipelines/google_cloud_pipeline_components_model_train_upload_deploy.ipynb

@@ -0,0 +1,833 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ur8xi4C7S06n"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2021 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "JAPoU8Sm5E6e"
+      },
+      "source": [
+        "<table align=\"left\">\n",
+        "\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/official/pipelines/google_cloud_pipeline_components_model_train_upload_deploy.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/official/pipelines/google_cloud_pipeline_components_model_train_upload_deploy.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/raw/master/notebooks/official/pipelines/google_cloud_pipeline_components_model_train_upload_deploy.ipynb\">\n",
+        "      Open in Google Cloud Notebooks\n",
+        "    </a>\n",
+        "  </td>    \n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "mVX0RV_s8zZ7"
+      },
+      "source": [
+        "# Vertex Pipelines: Model train, upload, and deploy using google-cloud-pipeline-components\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gBRcgrOk7CUf"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "This notebook shows how to use the components defined in [`google_cloud_pipeline_components`](https://github.com/kubeflow/pipelines/tree/master/components/google-cloud) in conjunction with an experimental `run_as_aiplatform_custom_job` method, to build a [Vertex Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines) workflow that trains a [custom model](https://cloud.google.com/vertex-ai/docs/training/containers-overview), uploads the model, creates an *endpoint*, and deploys the model to the endpoint. \n",
+        "\n",
+        "\n",
+        "### Objective\n",
+        "\n",
+        "In this example, you'll learn how to use components from `google_cloud_pipeline_components` to:\n",
+        "- upload a model\n",
+        "- create an *endpoint*\n",
+        "- deploy the trained model to the endpoint for serving\n",
+        "\n",
+        "In addition, you'll use the `kfp.v2.google.experimental.run_as_aiplatform_custom_job` method to train a custom model.\n",
+        "\n",
+        "The components are [documented here](https://google-cloud-pipeline-components.readthedocs.io/en/latest/google_cloud_pipeline_components.aiplatform.html#module-google_cloud_pipeline_components.aiplatform).\n",
+        "(From that page, see also the `CustomPythonPackageTrainingJobRunOp` and `CustomContainerTrainingJobRunOp` components, which similarly run 'custom' training, but as with the related `google.cloud.aiplatform.CustomContainerTrainingJob` and `google.cloud.aiplatform.CustomPythonPackageTrainingJob` methods from the [Vertex AI SDK](https://googleapis.dev/python/aiplatform/latest/aiplatform.html), also upload the trained model).\n",
+        "\n",
+        "### Costs \n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI Training and Serving\n",
+        "* Cloud Storage\n",
+        "\n",
+        "Learn about pricing for [Vertex AI](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
+        "Calculator](https://cloud.google.com/products/calculator/)\n",
+        "to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ze4-nDLfK4pw"
+      },
+      "source": [
+        "### Set up your local development environment\n",
+        "\n",
+        "**If you are using Colab or Google Cloud Notebooks**, your environment already meets\n",
+        "all the requirements to run this notebook. You can skip this step."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gCuSR8GkAgzl"
+      },
+      "source": [
+        "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
+        "You need the following:\n",
+        "\n",
+        "* The Google Cloud SDK\n",
+        "* Git\n",
+        "* Python 3\n",
+        "* virtualenv\n",
+        "* Jupyter notebook running in a virtual environment with Python 3\n",
+        "\n",
+        "The Google Cloud guide to [Setting up a Python development\n",
+        "environment](https://cloud.google.com/python/setup) and the [Jupyter\n",
+        "installation guide](https://jupyter.org/install) provide detailed instructions\n",
+        "for meeting these requirements. The following steps provide a condensed set of\n",
+        "instructions:\n",
+        "\n",
+        "1. [Install and initialize the Cloud SDK.](https://cloud.google.com/sdk/docs/)\n",
+        "\n",
+        "1. [Install Python 3.](https://cloud.google.com/python/setup#installing_python)\n",
+        "\n",
+        "1. [Install\n",
+        "   virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv)\n",
+        "   and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
+        "\n",
+        "1. To install Jupyter, run `pip install jupyter` on the\n",
+        "command-line in a terminal shell.\n",
+        "\n",
+        "1. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
+        "\n",
+        "1. Open this notebook in the Jupyter Notebook Dashboard."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "i7EUnXsZhAGF"
+      },
+      "source": [
+        "### Install additional packages\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "IaYsrh0Tc17L"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "# The Google Cloud Notebook product has specific requirements\n",
+        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
+        "\n",
+        "# Google Cloud Notebook requires dependencies to be installed with '--user'\n",
+        "USER_FLAG = \"\"\n",
+        "if IS_GOOGLE_CLOUD_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "yxtzwPPNZ-SH"
+      },
+      "outputs": [],
+      "source": [
+        "! pip install {USER_FLAG} google-cloud-aiplatform --upgrade\n",
+        "! pip install {USER_FLAG} kfp google-cloud-pipeline-components --upgrade"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "hhq5zEbGg0XX"
+      },
+      "source": [
+        "### Restart the kernel\n",
+        "\n",
+        "After you install the additional packages, you need to restart the notebook kernel so it can find the packages."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "EzrelQZ22IZj"
+      },
+      "outputs": [],
+      "source": [
+        "# Automatically restart kernel after installs\n",
+        "import os\n",
+        "\n",
+        "if not os.getenv(\"IS_TESTING\"):\n",
+        "    # Automatically restart kernel after installs\n",
+        "    import IPython\n",
+        "\n",
+        "    app = IPython.Application.instance()\n",
+        "    app.kernel.do_shutdown(True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6GPgNN7eeX1l"
+      },
+      "source": [
+        "Check the versions of the packages you installed.  The KFP SDK version should be >=1.6."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "NN0mULkEeb84"
+      },
+      "outputs": [],
+      "source": [
+        "!python3 -c \"import kfp; print('KFP SDK version: {}'.format(kfp.__version__))\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "lWEdiXsJg0XY"
+      },
+      "source": [
+        "## Before you begin\n",
+        "\n",
+        "This notebook does not require a GPU runtime."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "kT4WAH38l5et"
+      },
+      "source": [
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI, Cloud Storage, and Compute Engine APIs](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component,storage-component.googleapis.com). \n",
+        "\n",
+        "1. Follow the \"**Configuring your project**\" instructions from the Vertex Pipelines documentation.\n",
+        "\n",
+        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WReHDGG5g0XY"
+      },
+      "source": [
+        "#### Set your project ID\n",
+        "\n",
+        "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "oM1iC_MfAts1"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "PROJECT_ID = \"\"\n",
+        "\n",
+        "# Get your Google Cloud project ID from gcloud\n",
+        "if not os.getenv(\"IS_TESTING\"):\n",
+        "    shell_output=!gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "    PROJECT_ID = shell_output[0]\n",
+        "    print(\"Project ID: \", PROJECT_ID)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "qJYoRfYng0XZ"
+      },
+      "source": [
+        "Otherwise, set your project ID here."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "riG_qUokg0XZ"
+      },
+      "outputs": [],
+      "source": [
+        "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
+        "    PROJECT_ID = \"python-docs-samples-tests\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "06571eb4063b"
+      },
+      "source": [
+        "#### Timestamp\n",
+        "\n",
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append it onto the name of resources you create in this tutorial."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "697568e92bd6"
+      },
+      "outputs": [],
+      "source": [
+        "from datetime import datetime\n",
+        "\n",
+        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dr--iN2kAylZ"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using Google Cloud Notebooks**, your environment is already\n",
+        "authenticated. Skip this step."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "sBCra4QMA2wR"
+      },
+      "source": [
+        "**If you are using Colab**, run the cell below and follow the instructions\n",
+        "when prompted to authenticate your account via oAuth.\n",
+        "\n",
+        "**Otherwise**, follow these steps:\n",
+        "\n",
+        "1. In the Cloud Console, go to the [**Create service account key**\n",
+        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
+        "\n",
+        "2. Click **Create service account**.\n",
+        "\n",
+        "3. In the **Service account name** field, enter a name, and\n",
+        "   click **Create**.\n",
+        "\n",
+        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
+        "into the filter box, and select\n",
+        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "\n",
+        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
+        "local environment.\n",
+        "\n",
+        "6. Enter the path to your service account key as the\n",
+        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "PyQmSRbKA8r-"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "import sys\n",
+        "\n",
+        "# If you are running this notebook in Colab, run this cell and follow the\n",
+        "# instructions to authenticate your GCP account. This provides access to your\n",
+        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
+        "# requests.\n",
+        "\n",
+        "# The Google Cloud Notebook product has specific requirements\n",
+        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
+        "\n",
+        "# If on Google Cloud Notebooks, then don't execute this code\n",
+        "if not IS_GOOGLE_CLOUD_NOTEBOOK:\n",
+        "    if \"google.colab\" in sys.modules:\n",
+        "        from google.colab import auth as google_auth\n",
+        "\n",
+        "        google_auth.authenticate_user()\n",
+        "\n",
+        "    # If you are running this notebook locally, replace the string below with the\n",
+        "    # path to your service account key and run this cell to authenticate your GCP\n",
+        "    # account.\n",
+        "    elif not os.getenv(\"IS_TESTING\"):\n",
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "NxhCPW6e46EF"
+      },
+      "source": [
+        "### Create a Cloud Storage bucket as necessary\n",
+        "\n",
+        "You will need a Cloud Storage bucket for this example.  If you don't have one that you want to use, you can make one now.\n",
+        "\n",
+        "\n",
+        "Set the name of your Cloud Storage bucket below. It must be unique across all\n",
+        "Cloud Storage buckets.\n",
+        "\n",
+        "You may also change the `REGION` variable, which is used for operations\n",
+        "throughout the rest of this notebook. Make sure to [choose a region where Vertex AI services are\n",
+        "available](https://cloud.google.com/vertex-ai/docs/general/locations#available_regions). You may\n",
+        "not use a Multi-Regional Storage bucket for training with Vertex AI."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "MzGDU7TWdts_"
+      },
+      "outputs": [],
+      "source": [
+        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "REGION = \"us-central1\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cf221059d072"
+      },
+      "outputs": [],
+      "source": [
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "-EcIXiGsCePi"
+      },
+      "source": [
+        "**Only if your bucket doesn't already exist**: Run the following cell to create your Cloud Storage bucket."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "NIq7R4HZCfIc"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil mb -l $REGION $BUCKET_NAME"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ucvCsknMCims"
+      },
+      "source": [
+        "Finally, validate access to your Cloud Storage bucket by examining its contents:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "vhOb7YnwClBb"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil ls -al $BUCKET_NAME"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "XoEqT2Y4DJmf"
+      },
+      "source": [
+        "### Import libraries and define constants"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "YYtGjGG45ELJ"
+      },
+      "source": [
+        "Define some constants. "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "5zmD19ryCre7"
+      },
+      "outputs": [],
+      "source": [
+        "PATH=%env PATH\n",
+        "%env PATH={PATH}:/home/jupyter/.local/bin\n",
+        "\n",
+        "USER = \"your-user-name\"  # <---CHANGE THIS\n",
+        "PIPELINE_ROOT = \"{}/pipeline_root/{}\".format(BUCKET_NAME, USER)\n",
+        "\n",
+        "PIPELINE_ROOT"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "IprQaSI25oSk"
+      },
+      "source": [
+        "Do some imports:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "UFDUBveR5UfJ"
+      },
+      "outputs": [],
+      "source": [
+        "import kfp\n",
+        "from google.cloud import aiplatform\n",
+        "from google_cloud_pipeline_components import aiplatform as gcc_aip\n",
+        "from kfp.v2 import compiler\n",
+        "from kfp.v2.dsl import component\n",
+        "from kfp.v2.google import experimental\n",
+        "from kfp.v2.google.client import AIPlatformClient"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Z4MjdglUT3Sw"
+      },
+      "source": [
+        "## Define a pipeline that uses the components\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "65npyM9lYgtr"
+      },
+      "source": [
+        "Set some variables that will be used in constructing the args passed to the custom training job and setting pipeline params."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "f92pOTHtziZW"
+      },
+      "outputs": [],
+      "source": [
+        "# create args list for trainer\n",
+        "\n",
+        "hp_dict: str = '{\"num_hidden_layers\": 3, \"hidden_size\": 32, \"learning_rate\": 0.01, \"epochs\": 1, \"steps_per_epoch\": -1}'\n",
+        "data_dir: str = \"gs://aju-dev-demos-codelabs/bikes_weather/\"\n",
+        "TRAINER_ARGS = [\"--data-dir\", data_dir, \"--hptune-dict\", hp_dict]\n",
+        "\n",
+        "# create working dir to pass to job spec\n",
+        "import time\n",
+        "\n",
+        "ts = int(time.time())\n",
+        "WORKING_DIR = f\"{PIPELINE_ROOT}/{ts}\"\n",
+        "\n",
+        "MODEL_DISPLAY_NAME = f\"train_deploy{ts}\"\n",
+        "print(TRAINER_ARGS, WORKING_DIR, MODEL_DISPLAY_NAME)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "jpaD14TxZyVm"
+      },
+      "source": [
+        "Next, you define a component with which the custom training job is run.  For this example, this component doesn't do anything (but run a print statement)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "5m_ZU0GzBMRi"
+      },
+      "outputs": [],
+      "source": [
+        "@component\n",
+        "def training_op(input1: str):\n",
+        "    print(\"training task: {}\".format(input1))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2fjGiImBezMo"
+      },
+      "source": [
+        "Now, you define the pipeline.  \n",
+        "\n",
+        "The `experimental.run_as_aiplatform_custom_job` method takes as args the component defined above, and the list of `worker_pool_specs`— in this case  one— with which the custom training job is configured. \n",
+        "\n",
+        "Then, [`google_cloud_pipeline_components`](https://github.com/kubeflow/pipelines/tree/master/components/google-cloud) components are used to define the rest of the pipeline: upload the model, create an endpoint, and deploy the model to the endpoint. (While not shown in this example, the model deploy will create an endpoint if one is not provided.)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "lwBLkQygbxjM"
+      },
+      "outputs": [],
+      "source": [
+        "@kfp.dsl.pipeline(name=\"train-endpoint-deploy\" + TIMESTAMP)\n",
+        "def pipeline(\n",
+        "    project: str = PROJECT_ID,\n",
+        "    model_display_name: str = MODEL_DISPLAY_NAME,\n",
+        "    serving_container_image_uri: str = \"us-docker.pkg.dev/cloud-aiplatform/prediction/tf2-cpu.2-3:latest\",\n",
+        "):\n",
+        "\n",
+        "    train_task = training_op(\"model training\")\n",
+        "    experimental.run_as_aiplatform_custom_job(\n",
+        "        train_task,\n",
+        "        worker_pool_specs=[\n",
+        "            {\n",
+        "                \"containerSpec\": {\n",
+        "                    \"args\": TRAINER_ARGS,\n",
+        "                    \"env\": [{\"name\": \"AIP_MODEL_DIR\", \"value\": WORKING_DIR}],\n",
+        "                    \"imageUri\": \"gcr.io/google-samples/bw-cc-train:latest\",\n",
+        "                },\n",
+        "                \"replicaCount\": \"1\",\n",
+        "                \"machineSpec\": {\n",
+        "                    \"machineType\": \"n1-standard-16\",\n",
+        "                    \"accelerator_type\": aiplatform.gapic.AcceleratorType.NVIDIA_TESLA_K80,\n",
+        "                    \"accelerator_count\": 2,\n",
+        "                },\n",
+        "            }\n",
+        "        ],\n",
+        "    )\n",
+        "\n",
+        "    model_upload_op = gcc_aip.ModelUploadOp(\n",
+        "        project=project,\n",
+        "        display_name=model_display_name,\n",
+        "        artifact_uri=WORKING_DIR,\n",
+        "        serving_container_image_uri=serving_container_image_uri,\n",
+        "        serving_container_environment_variables={\"NOT_USED\": \"NO_VALUE\"},\n",
+        "    )\n",
+        "    model_upload_op.after(train_task)\n",
+        "\n",
+        "    endpoint_create_op = gcc_aip.EndpointCreateOp(\n",
+        "        project=project,\n",
+        "        display_name=\"pipelines-created-endpoint\",\n",
+        "    )\n",
+        "\n",
+        "    model_deploy_op = gcc_aip.ModelDeployOp(  # noqa: F841\n",
+        "        project=project,\n",
+        "        endpoint=endpoint_create_op.outputs[\"endpoint\"],\n",
+        "        model=model_upload_op.outputs[\"model\"],\n",
+        "        deployed_model_display_name=model_display_name,\n",
+        "        machine_type=\"n1-standard-4\",\n",
+        "    )"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2Hl1iYEKSzjP"
+      },
+      "source": [
+        "## Compile and run the pipeline\n",
+        "\n",
+        "Now, you're ready to compile the pipeline:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ycRc83B6bbfO"
+      },
+      "outputs": [],
+      "source": [
+        "from kfp.v2 import compiler  # noqa: F811\n",
+        "\n",
+        "compiler.Compiler().compile(\n",
+        "    pipeline_func=pipeline, package_path=\"train_upload_deploy.json\"\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "qfNuzFswBB4g"
+      },
+      "source": [
+        "The pipeline compilation generates the `train_upload_deploy.json` job spec file.\n",
+        "\n",
+        "Next, instantiate an API client object:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Hl5Q74_gkW2c"
+      },
+      "outputs": [],
+      "source": [
+        "from kfp.v2.google.client import AIPlatformClient  # noqa: F811\n",
+        "\n",
+        "api_client = AIPlatformClient(\n",
+        "    project_id=PROJECT_ID,\n",
+        "    region=REGION,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "_jrn6saiQsPh"
+      },
+      "source": [
+        "Then, you run the defined pipeline like this: "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "R4Ha4FoDQpkd"
+      },
+      "outputs": [],
+      "source": [
+        "response = api_client.create_run_from_job_spec(\n",
+        "    \"train_upload_deploy.json\",\n",
+        "    pipeline_root=PIPELINE_ROOT,\n",
+        "    parameter_values={\"project\": PROJECT_ID},\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "GvBTCP318RKs"
+      },
+      "source": [
+        "Click on the generated link to see your run in the Cloud Console.  It should look something like this:\n",
+        "\n",
+        "<a href=\"https://storage.googleapis.com/amy-jo/images/mp/train_endpoint_deploy.png\" target=\"_blank\"><img src=\"https://storage.googleapis.com/amy-jo/images/mp/train_endpoint_deploy.png\" width=\"75%\"/></a>\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "s4jxmfyT26gj"
+      },
+      "source": [
+        "## Cleaning up\n",
+        "\n",
+        "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
+        "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
+        "\n",
+        "Otherwise, you can delete the individual resources you created in this tutorial:\n",
+        "- Delete Cloud Storage objects that were created.  Uncomment and run the command in the cell below **only if you are not using the `PIPELINE_ROOT` path for any other purpose**.\n",
+        "- Delete your deployed model: first, undeploy it from its *endpoint*, then delete the model and endpoint.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "VtZCXIi1aULZ"
+      },
+      "outputs": [],
+      "source": [
+        "# Warning: this command will delete ALL Cloud Storage objects under the PIPELINE_ROOT path.\n",
+        "# ! gsutil -m rm -r $PIPELINE_ROOT"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "google_cloud_pipeline_components_model_train_upload_deploy.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/official/pipelines/lightweight_functions_component_io_kfp.ipynb

@@ -0,0 +1,964 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ur8xi4C7S06n"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2021 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "JAPoU8Sm5E6e"
+      },
+      "source": [
+        "<table align=\"left\">\n",
+        "\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/official/pipelines/lightweight_functions_component_io_kfp.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/official/pipelines/lightweight_functions_component_io_kfp.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/raw/master/notebooks/official/pipelines/lightweight_functions_component_io_kfp.ipynb\">\n",
+        "      Open in Google Cloud Notebooks\n",
+        "    </a>\n",
+        "  </td>    \n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "o2rM_9Ml7-W2"
+      },
+      "source": [
+        "# Vertex Pipelines: lightweight Python function-based components, and component I/O"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tvgnzT1CKxrO"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "\n",
+        "This [Vertex Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines) example shows how to build lightweight Python function-based components using [the Kubeflow Pipelines (KFP) SDK](https://www.kubeflow.org/docs/components/pipelines/), and in particular how to support component I/O using the KFP SDK.\n",
+        "\n",
+        "### Objective\n",
+        "\n",
+        "In this example, you'll learn: \n",
+        "\n",
+        "- How to build Python-function-based components.\n",
+        "- How to pass *Artifacts* and *parameters* between components, both by path reference and by value.\n",
+        "- How to use the `kfp.dsl.importer` method.\n",
+        "\n",
+        "\n",
+        "### Costs \n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI Training\n",
+        "* Cloud Storage\n",
+        "\n",
+        "Learn about pricing for [Vertex AI](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
+        "Calculator](https://cloud.google.com/products/calculator/)\n",
+        "to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Jq63DFPPxAXJ"
+      },
+      "source": [
+        "### KFP Python function-based components\n",
+        "\n",
+        "A Kubeflow pipeline component is a self-contained set of code that performs one step in your ML workflow. A pipeline component is composed of:\n",
+        "\n",
+        "* The component code, which implements the logic needed to perform a step in your ML workflow.\n",
+        "* A component specification, which defines the following:\n",
+        "  * The component’s metadata, its name and description.\n",
+        "  * The component’s interface, the component’s inputs and outputs.\n",
+        "* The component’s implementation, the Docker container image to run, how to pass inputs to your component code, and how to get the component’s outputs.\n",
+        "\n",
+        "Lightweight Python function-based components make it easier to iterate quickly by letting you build your component code as a Python function and generating the component specification for you. This notebook shows how to create Python function-based components for use in [Vertex Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines).\n",
+        "\n",
+        "Python function-based components use the Kubeflow Pipelines SDK to handle the complexity of passing inputs into your component and passing your function’s outputs back to your pipeline.\n",
+        "\n",
+        "There are two categories of inputs/outputs supported in Python function-based components: *artifacts* and *parameters*.\n",
+        "\n",
+        "* Parameters are passed to your component by value and typically contain `int`, `float`, `bool`, or small `string` values.\n",
+        "* Artifacts are passed to your component as a *reference* to a path, to which you can write a file or a subdirectory structure. In addition to the artifact’s data, you can also read and write the artifact’s metadata. This lets you record arbitrary key-value pairs for an artifact such as the accuracy of a trained model, and use metadata in downstream components – for example, you could use metadata to decide if a model is accurate enough to deploy for predictions."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ze4-nDLfK4pw"
+      },
+      "source": [
+        "### Set up your local development environment\n",
+        "\n",
+        "**If you are using Colab or Google Cloud Notebooks**, your environment already meets\n",
+        "all the requirements to run this notebook. You can skip this step."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gCuSR8GkAgzl"
+      },
+      "source": [
+        "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
+        "You need the following:\n",
+        "\n",
+        "* The Google Cloud SDK\n",
+        "* Git\n",
+        "* Python 3\n",
+        "* virtualenv\n",
+        "* Jupyter notebook running in a virtual environment with Python 3\n",
+        "\n",
+        "The Google Cloud guide to [Setting up a Python development\n",
+        "environment](https://cloud.google.com/python/setup) and the [Jupyter\n",
+        "installation guide](https://jupyter.org/install) provide detailed instructions\n",
+        "for meeting these requirements. The following steps provide a condensed set of\n",
+        "instructions:\n",
+        "\n",
+        "1. [Install and initialize the Cloud SDK.](https://cloud.google.com/sdk/docs/)\n",
+        "\n",
+        "1. [Install Python 3.](https://cloud.google.com/python/setup#installing_python)\n",
+        "\n",
+        "1. [Install\n",
+        "   virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv)\n",
+        "   and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
+        "\n",
+        "1. To install Jupyter, run `pip install jupyter` on the\n",
+        "command-line in a terminal shell.\n",
+        "\n",
+        "1. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
+        "\n",
+        "1. Open this notebook in the Jupyter Notebook Dashboard."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "i7EUnXsZhAGF"
+      },
+      "source": [
+        "### Install additional packages\n",
+        "\n",
+        "Install the KFP SDK."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "IaYsrh0Tc17L"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "# The Google Cloud Notebook product has specific requirements\n",
+        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
+        "\n",
+        "# Google Cloud Notebook requires dependencies to be installed with '--user'\n",
+        "USER_FLAG = \"\"\n",
+        "if IS_GOOGLE_CLOUD_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "aR7LNYMUCVKc"
+      },
+      "outputs": [],
+      "source": [
+        "! pip install {USER_FLAG} kfp --upgrade"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "hhq5zEbGg0XX"
+      },
+      "source": [
+        "### Restart the kernel\n",
+        "\n",
+        "After you install the additional packages, you need to restart the notebook kernel so it can find the packages."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "EzrelQZ22IZj"
+      },
+      "outputs": [],
+      "source": [
+        "# Automatically restart kernel after installs\n",
+        "import os\n",
+        "\n",
+        "if not os.getenv(\"IS_TESTING\"):\n",
+        "    # Automatically restart kernel after installs\n",
+        "    import IPython\n",
+        "\n",
+        "    app = IPython.Application.instance()\n",
+        "    app.kernel.do_shutdown(True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6GPgNN7eeX1l"
+      },
+      "source": [
+        "Check the version of the package you installed.  The KFP SDK version should be >=1.6."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "NN0mULkEeb84"
+      },
+      "outputs": [],
+      "source": [
+        "!python3 -c \"import kfp; print('KFP SDK version: {}'.format(kfp.__version__))\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "lWEdiXsJg0XY"
+      },
+      "source": [
+        "## Before you begin\n",
+        "\n",
+        "This notebook does not require a GPU runtime."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Pfmx825FloYH"
+      },
+      "source": [
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI, Cloud Storage, and Compute Engine APIs](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component,storage-component.googleapis.com). \n",
+        "\n",
+        "1. Follow the \"**Configuring your project**\" instructions from the Vertex Pipelines documentation.\n",
+        "\n",
+        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WReHDGG5g0XY"
+      },
+      "source": [
+        "#### Set your project ID\n",
+        "\n",
+        "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "oM1iC_MfAts1"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "PROJECT_ID = \"\"\n",
+        "\n",
+        "# Get your Google Cloud project ID from gcloud\n",
+        "if not os.getenv(\"IS_TESTING\"):\n",
+        "    shell_output=!gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "    PROJECT_ID = shell_output[0]\n",
+        "    print(\"Project ID: \", PROJECT_ID)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "qJYoRfYng0XZ"
+      },
+      "source": [
+        "Otherwise, set your project ID here."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "riG_qUokg0XZ"
+      },
+      "outputs": [],
+      "source": [
+        "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
+        "    PROJECT_ID = \"python-docs-samples-tests\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "06571eb4063b"
+      },
+      "source": [
+        "#### Timestamp\n",
+        "\n",
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append it onto the name of resources you create in this tutorial."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "697568e92bd6"
+      },
+      "outputs": [],
+      "source": [
+        "from datetime import datetime\n",
+        "\n",
+        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dr--iN2kAylZ"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using Google Cloud Notebooks**, your environment is already\n",
+        "authenticated. Skip this step."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "sBCra4QMA2wR"
+      },
+      "source": [
+        "**If you are using Colab**, run the cell below and follow the instructions\n",
+        "when prompted to authenticate your account via oAuth.\n",
+        "\n",
+        "**Otherwise**, follow these steps:\n",
+        "\n",
+        "1. In the Cloud Console, go to the [**Create service account key**\n",
+        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
+        "\n",
+        "2. Click **Create service account**.\n",
+        "\n",
+        "3. In the **Service account name** field, enter a name, and\n",
+        "   click **Create**.\n",
+        "\n",
+        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
+        "into the filter box, and select\n",
+        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "\n",
+        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
+        "local environment.\n",
+        "\n",
+        "6. Enter the path to your service account key as the\n",
+        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "PyQmSRbKA8r-"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "import sys\n",
+        "\n",
+        "# If you are running this notebook in Colab, run this cell and follow the\n",
+        "# instructions to authenticate your GCP account. This provides access to your\n",
+        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
+        "# requests.\n",
+        "\n",
+        "# The Google Cloud Notebook product has specific requirements\n",
+        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
+        "\n",
+        "# If on Google Cloud Notebooks, then don't execute this code\n",
+        "if not IS_GOOGLE_CLOUD_NOTEBOOK:\n",
+        "    if \"google.colab\" in sys.modules:\n",
+        "        from google.colab import auth as google_auth\n",
+        "\n",
+        "        google_auth.authenticate_user()\n",
+        "\n",
+        "    # If you are running this notebook locally, replace the string below with the\n",
+        "    # path to your service account key and run this cell to authenticate your GCP\n",
+        "    # account.\n",
+        "    elif not os.getenv(\"IS_TESTING\"):\n",
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "NxhCPW6e46EF"
+      },
+      "source": [
+        "### Create a Cloud Storage bucket as necessary\n",
+        "\n",
+        "You will need a Cloud Storage bucket for this example.  If you don't have one that you want to use, you can make one now.\n",
+        "\n",
+        "\n",
+        "Set the name of your Cloud Storage bucket below. It must be unique across all\n",
+        "Cloud Storage buckets.\n",
+        "\n",
+        "You may also change the `REGION` variable, which is used for operations\n",
+        "throughout the rest of this notebook. Make sure to [choose a region where Vertex AI services are\n",
+        "available](https://cloud.google.com/vertex-ai/docs/general/locations#available_regions). You may\n",
+        "not use a Multi-Regional Storage bucket for training with Vertex AI."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "MzGDU7TWdts_"
+      },
+      "outputs": [],
+      "source": [
+        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "REGION = \"us-central1\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cf221059d072"
+      },
+      "outputs": [],
+      "source": [
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "-EcIXiGsCePi"
+      },
+      "source": [
+        "**Only if your bucket doesn't already exist**: Run the following cell to create your Cloud Storage bucket."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "NIq7R4HZCfIc"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil mb -l $REGION $BUCKET_NAME"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ucvCsknMCims"
+      },
+      "source": [
+        "Finally, validate access to your Cloud Storage bucket by examining its contents:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "vhOb7YnwClBb"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil ls -al $BUCKET_NAME"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "XoEqT2Y4DJmf"
+      },
+      "source": [
+        "### Import libraries and define constants"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "YYtGjGG45ELJ"
+      },
+      "source": [
+        "Define some constants."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "5zmD19ryCre7"
+      },
+      "outputs": [],
+      "source": [
+        "PATH=%env PATH\n",
+        "%env PATH={PATH}:/home/jupyter/.local/bin\n",
+        "\n",
+        "USER = \"your-user-name\"  # <---CHANGE THIS\n",
+        "PIPELINE_ROOT = \"{}/pipeline_root/{}\".format(BUCKET_NAME, USER)\n",
+        "\n",
+        "PIPELINE_ROOT"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "IprQaSI25oSk"
+      },
+      "source": [
+        "Do some imports:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "TmMeVQ-fUEUM"
+      },
+      "outputs": [],
+      "source": [
+        "from typing import NamedTuple\n",
+        "\n",
+        "import kfp\n",
+        "from kfp.v2 import dsl\n",
+        "from kfp.v2.dsl import (Artifact, Dataset, Input, InputPath, Model, Output,\n",
+        "                        OutputPath, component)\n",
+        "from kfp.v2.google.client import AIPlatformClient"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "IbN_49SUW7b7"
+      },
+      "source": [
+        "## Define a pipeline\n",
+        "\n",
+        "This example defines a pipeline that illustrates component I/O for both *parameters* and *artifacts*.\n",
+        "\n",
+        "In addition, this example demonstrates use of the `kfp.dsl.importer` method in the pipeline.  You can use the importer if you would like to import an existing Artifact. The importer only works for pipelines that are compiled using  `kfp.v2.compiler.Compiler`.\n",
+        "\n",
+        "The first step is to define some pipeline *components*, then define a pipeline that uses them.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "QW9EmIXmYLa6"
+      },
+      "source": [
+        "### Create Python function-based pipeline components\n",
+        "\n",
+        "This example define some function-based components that consume parameters and produce (typed) Artifacts and parameters. Functions can produce Artifacts in three ways:\n",
+        "\n",
+        "* Accept an output local path using `OutputPath` \n",
+        "* Accept an `OutputArtifact` which gives the function a handle to the output artifact's metadata\n",
+        "* Return an `Artifact` (or `Dataset`, `Model`, `Metrics`, etc) in a `NamedTuple` \n",
+        "\n",
+        "These options for producing Artifacts are demonstrated in the examples below."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "RF9bpqJ_iWyr"
+      },
+      "source": [
+        "The first component definition, `preprocess`, shows a component that outputs two `Dataset` Artifacts, as well as an output parameter.  (For this example, the datasets don't reflect real data).\n",
+        "\n",
+        "For the parameter output, you would typically use the approach shown here, using the `OutputPath` type, for \"larger\" data.    \n",
+        "For \"small data\", like a short string, it might be more convenient to use the `NamedTuple` function output as shown in the second component instead.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "GZ_kXbhCUEUN"
+      },
+      "outputs": [],
+      "source": [
+        "@component\n",
+        "def preprocess(\n",
+        "    # An input parameter of type string.\n",
+        "    message: str,\n",
+        "    # Use Output to get a metadata-rich handle to the output artifact\n",
+        "    # of type `Dataset`.\n",
+        "    output_dataset_one: Output[Dataset],\n",
+        "    # A locally accessible filepath for another output artifact of type\n",
+        "    # `Dataset`.\n",
+        "    output_dataset_two_path: OutputPath(\"Dataset\"),\n",
+        "    # A locally accessible filepath for an output parameter of type string.\n",
+        "    output_parameter_path: OutputPath(str),\n",
+        "):\n",
+        "    \"\"\"'Mock' preprocessing step.\n",
+        "    Writes out the passed in message to the output \"Dataset\"s and the output message.\n",
+        "    \"\"\"\n",
+        "    output_dataset_one.metadata[\"hello\"] = \"there\"\n",
+        "    # Use OutputArtifact.path to access a local file path for writing.\n",
+        "    # One can also use OutputArtifact.uri to access the actual URI file path.\n",
+        "    with open(output_dataset_one.path, \"w\") as f:\n",
+        "        f.write(message)\n",
+        "\n",
+        "    # OutputPath is used to just pass the local file path of the output artifact\n",
+        "    # to the function.\n",
+        "    with open(output_dataset_two_path, \"w\") as f:\n",
+        "        f.write(message)\n",
+        "\n",
+        "    with open(output_parameter_path, \"w\") as f:\n",
+        "        f.write(message)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "CskpfNV-RGNO"
+      },
+      "source": [
+        "The second component definition, `train`, defines as input both an `InputPath` of type `Dataset`, and an `InputArtifact` of type `Dataset` (as well as other parameter inputs). It also uses the `NamedTuple` format for function output.  As shown, these outputs can be Artifacts as well as parameters.\n",
+        "\n",
+        "Note that this component also writes some metrics metadata to the `model` output Artifact.  This information is displayed in the Cloud Console user interface when the pipeline runs.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7ZCLrE7IUEUN"
+      },
+      "outputs": [],
+      "source": [
+        "@component(\n",
+        "    base_image=\"python:3.9\",  # Use a different base image.\n",
+        ")\n",
+        "def train(\n",
+        "    # An input parameter of type string.\n",
+        "    message: str,\n",
+        "    # Use InputPath to get a locally accessible path for the input artifact\n",
+        "    # of type `Dataset`.\n",
+        "    dataset_one_path: InputPath(\"Dataset\"),\n",
+        "    # Use InputArtifact to get a metadata-rich handle to the input artifact\n",
+        "    # of type `Dataset`.\n",
+        "    dataset_two: Input[Dataset],\n",
+        "    # Output artifact of type Model.\n",
+        "    imported_dataset: Input[Dataset],\n",
+        "    model: Output[Model],\n",
+        "    # An input parameter of type int with a default value.\n",
+        "    num_steps: int = 3,\n",
+        "    # Use NamedTuple to return either artifacts or parameters.\n",
+        "    # When returning artifacts like this, return the contents of\n",
+        "    # the artifact. The assumption here is that this return value\n",
+        "    # fits in memory.\n",
+        ") -> NamedTuple(\n",
+        "    \"Outputs\",\n",
+        "    [\n",
+        "        (\"output_message\", str),  # Return parameter.\n",
+        "        (\"generic_artifact\", Artifact),  # Return generic Artifact.\n",
+        "    ],\n",
+        "):\n",
+        "    \"\"\"'Mock' Training step.\n",
+        "    Combines the contents of dataset_one and dataset_two into the\n",
+        "    output Model.\n",
+        "    Constructs a new output_message consisting of message repeated num_steps times.\n",
+        "    \"\"\"\n",
+        "\n",
+        "    # Directly access the passed in GCS URI as a local file (uses GCSFuse).\n",
+        "    with open(dataset_one_path, \"r\") as input_file:\n",
+        "        dataset_one_contents = input_file.read()\n",
+        "\n",
+        "    # dataset_two is an Artifact handle. Use dataset_two.path to get a\n",
+        "    # local file path (uses GCSFuse).\n",
+        "    # Alternately, use dataset_two.uri to access the GCS URI directly.\n",
+        "    with open(dataset_two.path, \"r\") as input_file:\n",
+        "        dataset_two_contents = input_file.read()\n",
+        "\n",
+        "    with open(model.path, \"w\") as f:\n",
+        "        f.write(\"My Model\")\n",
+        "\n",
+        "    with open(imported_dataset.path, \"r\") as f:\n",
+        "        data = f.read()\n",
+        "    print(\"Imported Dataset:\", data)\n",
+        "\n",
+        "    # Use model.get() to get a Model artifact, which has a .metadata dictionary\n",
+        "    # to store arbitrary metadata for the output artifact. This metadata will be\n",
+        "    # recorded in Managed Metadata and can be queried later. It will also show up\n",
+        "    # in the UI.\n",
+        "    model.metadata[\"accuracy\"] = 0.9\n",
+        "    model.metadata[\"framework\"] = \"Tensorflow\"\n",
+        "    model.metadata[\"time_to_train_in_seconds\"] = 257\n",
+        "\n",
+        "    artifact_contents = \"{}\\n{}\".format(dataset_one_contents, dataset_two_contents)\n",
+        "    output_message = \" \".join([message for _ in range(num_steps)])\n",
+        "    return (output_message, artifact_contents)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "OOIzr3g5MK8q"
+      },
+      "source": [
+        "Finally, define a small component that takes as input the `generic_artifact` returned by the `train` component function, and reads and prints the Artifact's contents."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Ta_n1Ww_C5tQ"
+      },
+      "outputs": [],
+      "source": [
+        "@component\n",
+        "def read_artifact_input(\n",
+        "    generic: Input[Artifact],\n",
+        "):\n",
+        "    with open(generic.path, \"r\") as input_file:\n",
+        "        generic_contents = input_file.read()\n",
+        "        print(f\"generic contents: {generic_contents}\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "fmKIkUCoZeAu"
+      },
+      "source": [
+        "### Define a pipeline that uses your components and the Importer\n",
+        "\n",
+        "Next, define a pipeline that uses the components that were built in the previous section, and also shows the use of the `kfp.dsl.importer`.  \n",
+        "\n",
+        "This example uses the `importer` to create, in this case, a `Dataset` artifact from an existing URI.\n",
+        "\n",
+        "Note that the `train_task` step takes as inputs three of the outputs of the `preprocess_task` step, as well as the output of the `importer` step.\n",
+        "In the \"train\" inputs we refer to the `preprocess`  `output_parameter`, which gives us the output string directly.\n",
+        "\n",
+        "The `read_task` step takes as input the `train_task` `generic_artifact` output.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "289jqF_XUEUO"
+      },
+      "outputs": [],
+      "source": [
+        "@dsl.pipeline(\n",
+        "    # Default pipeline root. You can override it when submitting the pipeline.\n",
+        "    pipeline_root=PIPELINE_ROOT,\n",
+        "    # A name for the pipeline. Use to determine the pipeline Context.\n",
+        "    name=\"metadata-pipeline-v2\",\n",
+        ")\n",
+        "def pipeline(message: str):\n",
+        "    importer = kfp.dsl.importer(\n",
+        "        artifact_uri=\"gs://ml-pipeline-playground/shakespeare1.txt\",\n",
+        "        artifact_class=Dataset,\n",
+        "        reimport=False,\n",
+        "    )\n",
+        "    preprocess_task = preprocess(message=message)\n",
+        "    train_task = train(\n",
+        "        dataset_one=preprocess_task.outputs[\"output_dataset_one\"],\n",
+        "        dataset_two=preprocess_task.outputs[\"output_dataset_two\"],\n",
+        "        imported_dataset=importer.output,\n",
+        "        message=preprocess_task.outputs[\"output_parameter\"],\n",
+        "        num_steps=5,\n",
+        "    )\n",
+        "    read_task = read_artifact_input(  # noqa: F841\n",
+        "        train_task.outputs[\"generic_artifact\"]\n",
+        "    )"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2Hl1iYEKSzjP"
+      },
+      "source": [
+        "## Compile and run the pipeline\n",
+        "\n",
+        "Now, you're ready to compile the pipeline:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7PwUFV-MleGs"
+      },
+      "outputs": [],
+      "source": [
+        "from kfp.v2 import compiler\n",
+        "\n",
+        "compiler.Compiler().compile(\n",
+        "    pipeline_func=pipeline, package_path=\"component_io_job.json\"\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "qfNuzFswBB4g"
+      },
+      "source": [
+        "The pipeline compilation generates the `component_io_job.json` job spec file.\n",
+        "\n",
+        "Next, instantiate an API client object:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Hl5Q74_gkW2c"
+      },
+      "outputs": [],
+      "source": [
+        "from kfp.v2.google.client import AIPlatformClient  # noqa: F811\n",
+        "\n",
+        "api_client = AIPlatformClient(\n",
+        "    project_id=PROJECT_ID,\n",
+        "    region=REGION,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "_jrn6saiQsPh"
+      },
+      "source": [
+        "Then, you run the defined pipeline like this: "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "R4Ha4FoDQpkd"
+      },
+      "outputs": [],
+      "source": [
+        "response = api_client.create_run_from_job_spec(\n",
+        "    job_spec_path=\"component_io_job.json\",\n",
+        "    # pipeline_root=PIPELINE_ROOT,  # Override if needed.\n",
+        "    parameter_values={\"message\": \"Hello, World\"},\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "GvBTCP318RKs"
+      },
+      "source": [
+        "Click on the generated link to see your run in the Cloud Console.  It should look like this:\n",
+        "\n",
+        "<a href=\"https://storage.googleapis.com/amy-jo/images/mp/artifact_io2.png\" target=\"_blank\"><img src=\"https://storage.googleapis.com/amy-jo/images/mp/artifact_io2.png\" width=\"95%\"/></a>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0dsUfUGk2sOj"
+      },
+      "source": [
+        "## Cleaning up\n",
+        "\n",
+        "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
+        "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
+        "\n",
+        "Otherwise, you can delete the individual resources you created in this tutorial:\n",
+        "- delete Cloud Storage objects that were created.  Uncomment and run the command in the cell below **only if you are not using the `PIPELINE_ROOT` path for any other purpose**.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "yMsZxXBZ2sOo"
+      },
+      "outputs": [],
+      "source": [
+        "# Warning: this command will delete ALL Cloud Storage objects under the PIPELINE_ROOT path.\n",
+        "# ! gsutil -m rm -r $PIPELINE_ROOT"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "lightweight_functions_component_io_kfp.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}