fix: missing import

Co-authored-by: Andrew Ferlitsch <aferlitsch@google.com>

.cloud-build/cleanup/cleanup.py

@@ -5,6 +5,8 @@ from resource_cleanup_manager import (
     ModelResourceCleanupManager,
     EndpointResourceCleanupManager,
     ResourceCleanupManager,
+    MatchingEngineIndexEndpointResourceCleanupManager,
+    MatchingEngineIndexResourceCleanupManager,
 )
 
 rate_limit = RateLimit(max_count=25, per=60, greedy=False)
@@ -40,10 +42,12 @@ if is_dry_run:
     print("Starting cleanup in dry run mode...")
 
 # List of all cleanup managers
-managers = [
+managers: List[ResourceCleanupManager] = [
     DatasetResourceCleanupManager(),
     EndpointResourceCleanupManager(),
     ModelResourceCleanupManager(),  # ModelResourceCleanupManager must follow EndpointResourceCleanupManager due to deployed models blocking model deletion.
+    MatchingEngineIndexEndpointResourceCleanupManager(),
+    MatchingEngineIndexResourceCleanupManager(),
 ]
 
 run_cleanup_managers(managers=managers, is_dry_run=is_dry_run)

.cloud-build/cleanup/resource_cleanup_manager.py

@@ -109,3 +109,11 @@ class EndpointResourceCleanupManager(VertexAIResourceCleanupManager):
 
 class ModelResourceCleanupManager(VertexAIResourceCleanupManager):
     vertex_ai_resource = aiplatform.Model
+
+
+class MatchingEngineIndexResourceCleanupManager(VertexAIResourceCleanupManager):
+    vertex_ai_resource = aiplatform.MatchingEngineIndex
+
+
+class MatchingEngineIndexEndpointResourceCleanupManager(VertexAIResourceCleanupManager):
+    vertex_ai_resource = aiplatform.MatchingEngineIndexEndpoint

.cloud-build/execute_changed_notebooks_helper.py

@@ -38,7 +38,7 @@ from utils import NotebookProcessors, util
 
 # A buffer so that workers finish before the orchestrating job
 WORKER_TIMEOUT_BUFFER_IN_SECONDS: int = 60 * 60
-PYTHON_VERSION = "3.9" # Set default python version
+PYTHON_VERSION = "3.9"  # Set default python version
 
 
 def format_timedelta(delta: datetime.timedelta) -> str:
@@ -102,6 +102,7 @@ def _process_notebook(
             "VPC_NETWORK": variable_vpc_network,
         },
     )
+    unique_strings_preprocessor = NotebookProcessors.UniqueStringsPreprocessor()
 
     # Use no-execute preprocessor
     (
@@ -110,6 +111,7 @@ def _process_notebook(
     ) = remove_no_execute_cells_preprocessor.preprocess(nb)
 
     (nb, resources) = update_variables_preprocessor.preprocess(nb, resources)
+    (nb, resources) = unique_strings_preprocessor.preprocess(nb, resources)
 
     with open(notebook_path, mode="w", encoding="utf-8") as new_file:
         nbformat.write(nb, new_file)
@@ -127,13 +129,15 @@ def _get_notebook_python_version(notebook_path: str) -> str:
     src = file.read()
     nb_json = json.loads(src)
 
-    #Iterate over the cells in the ipynb
-    for cell in nb_json['cells']:
-        if cell['cell_type'] == 'markdown':
-            markdown = str.join('', cell['source'])
+    # Iterate over the cells in the ipynb
+    for cell in nb_json["cells"]:
+        if cell["cell_type"] == "markdown":
+            markdown = str.join("", cell["source"])
 
             # Look for the python version specification pattern
-            re_match = re.search('python version = (\d\.\d)', markdown, flags=re.IGNORECASE)
+            re_match = re.search(
+                "python version = (\d\.\d)", markdown, flags=re.IGNORECASE
+            )
             if re_match:
                 # get the version number
                 python_version = re_match.group(1)
@@ -201,7 +205,9 @@ def process_and_execute_notebook(
     operation = None
     try:
         # Get the python version for running the notebook if specified
-        notebook_exec_python_version = _get_notebook_python_version(notebook_path=notebook)
+        notebook_exec_python_version = _get_notebook_python_version(
+            notebook_path=notebook
+        )
         print(f"Running notebook with python {notebook_exec_python_version}")
 
         # Pre-process notebook by substituting variable names
@@ -230,7 +236,7 @@ def process_and_execute_notebook(
             private_pool_id=private_pool_id,
             private_pool_region=variable_region,
             timeout_in_seconds=timeout_in_seconds,
-            python_version=notebook_exec_python_version
+            python_version=notebook_exec_python_version,
         )
 
         operation_metadata = BuildOperationMetadata(mapping=operation.metadata)
@@ -443,7 +449,7 @@ def process_and_execute_notebooks(
                         result.log_url,
                         result.output_uri,
                         result.output_uri_web,
-                        result.logs_bucket
+                        result.logs_bucket,
                     ]
                     for result in results_sorted
                 ],
@@ -454,34 +460,34 @@ def process_and_execute_notebooks(
                     "log_url",
                     "output_uri",
                     "output_uri_web",
-                    "logs_bucket"
+                    "logs_bucket",
                 ],
             )
         )
 
         if len(notebooks) == 1:
-          print("="*100)
-          print("The notebook execution build log:\n")
-          print("="*100)
+            print("=" * 100)
+            print("The notebook execution build log:\n")
+            print("=" * 100)
 
-          build_id = results_sorted[0].build_id
-          logs_bucket_name = (results_sorted[0].logs_bucket).removeprefix("gs://")
-          log_file_name = f"log-{build_id}.txt"
+            build_id = results_sorted[0].build_id
+            logs_bucket_name = (results_sorted[0].logs_bucket).removeprefix("gs://")
+            log_file_name = f"log-{build_id}.txt"
 
-          log_contents = util.download_blob_into_memory(
-            bucket_name=logs_bucket_name,
-            blob_name=log_file_name,
-            download_as_text=True
+            log_contents = util.download_blob_into_memory(
+                bucket_name=logs_bucket_name,
+                blob_name=log_file_name,
+                download_as_text=True,
             )
 
-          # Remove extra steps from the log
-          match = re.search("starting Step #4", log_contents, flags=re.IGNORECASE)
+            # Remove extra steps from the log
+            match = re.search("starting Step #4", log_contents, flags=re.IGNORECASE)
 
-          if match is not None:
-            match_index = match.span()[0]
-            print(log_contents[match_index:])
-          else:
-            print(log_contents)
+            if match is not None:
+                match_index = match.span()[0]
+                print(log_contents[match_index:])
+            else:
+                print(log_contents)
 
         print("\n=== END RESULTS===\n")
 

.cloud-build/test_notebook_vm.txt

@@ -2,5 +2,4 @@ notebooks/official/vizier/gapic-vizier-multi-objective-optimization.ipynb
 notebooks/official/pipelines/lightweight_functions_component_io_kfp.ipynb
 notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-locally-trained-models.ipynb
 notebooks/official/custom/custom-tabular-bq-managed-dataset.ipynb
-notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb
 .cloud-build/tests/python_version_test.ipynb

.cloud-build/utils/NotebookProcessors.py

@@ -14,6 +14,8 @@
 # limitations under the License.
 
 from typing import Dict
+import random
+import string
 
 from nbconvert.preprocessors import Preprocessor
 
@@ -63,3 +65,36 @@ class UpdateVariablesPreprocessor(Preprocessor):
             executable_cells.append(cell)
         notebook.cells = executable_cells
         return notebook, resources
+
+
+# Generate a uuid of a specifed length
+def generate_uuid(length: int = 8) -> str:
+    return "".join(random.choices(string.ascii_lowercase + string.digits, k=length))
+
+
+class UniqueStringsPreprocessor(Preprocessor):
+    # A preprocessor that replaces strings that end with "-unique" or "_unique" with a uuid.
+
+    @staticmethod
+    def update_unique_strings(content: str):
+        # Replace strings that end with "-unique" or "_unique" with a uuid.
+
+        unique_id = generate_uuid()
+        return (
+            content.replace('-unique"', f'-{unique_id}"')
+            .replace("-unique'", f'-{unique_id}"')
+            .replace('_unique"', f'_{unique_id}"')
+            .replace("_unique'", f'_{unique_id}"')
+        )
+
+    def preprocess(self, notebook, resources=None):
+        executable_cells = []
+        for cell in notebook.cells:
+            if cell.cell_type == "code":
+                cell.source = self.update_unique_strings(
+                    content=cell.source,
+                )
+
+            executable_cells.append(cell)
+        notebook.cells = executable_cells
+        return notebook, resources

.cloud-build/utils/UpdateNotebookVariables.py

@@ -40,65 +40,3 @@ def get_updated_value(content: str, variable_name: str, variable_value: str) ->
         content,
         flags=re.M,
     )
-
-
-def test_update_value():
-    new_content = get_updated_value(
-        content='asdf\nPROJECT_ID = "[your-project-id]" #@param {type:"string"} \nasdf',
-        variable_name="PROJECT_ID",
-        variable_value="sample-project",
-    )
-    assert (
-        new_content
-        == 'asdf\nPROJECT_ID = "sample-project" #@param {type:"string"} \nasdf'
-    )
-
-
-def test_update_value_single_quotes():
-    new_content = get_updated_value(
-        content="PROJECT_ID = '[your-project-id]'",
-        variable_name="PROJECT_ID",
-        variable_value="sample-project",
-    )
-    assert new_content == "PROJECT_ID = 'sample-project'"
-
-
-def test_update_value_avoidance():
-    new_content = get_updated_value(
-        content="PROJECT_ID = shell_output[0] ",
-        variable_name="PROJECT_ID",
-        variable_value="sample-project",
-    )
-    assert new_content == "PROJECT_ID = shell_output[0] "
-
-
-def test_region():
-    new_content = get_updated_value(
-        content='REGION = "[your-region]"  # @param {type:"string"}',
-        variable_name="REGION",
-        variable_value="us-central1",
-    )
-    assert new_content == 'REGION = "us-central1"  # @param {type:"string"}'
-
-
-def test_region_equal_equals_ignore():
-    # Tests that == is ignored
-    new_content = get_updated_value(
-        content='REGION == "[your-region]"  # @param {type:"string"}',
-        variable_name="REGION",
-        variable_value="us-central1",
-    )
-    assert new_content == 'REGION == "[your-region]"  # @param {type:"string"}'
-
-
-def test_service_account():
-    # Tests that == is ignored
-    new_content = get_updated_value(
-        content='SERVICE_ACCOUNT = "[your-service-account]"  # @param {type:"string"}',
-        variable_name="SERVICE_ACCOUNT",
-        variable_value="12345-compute@developer.gserviceaccount.com",
-    )
-    assert (
-        new_content
-        == 'SERVICE_ACCOUNT = "12345-compute@developer.gserviceaccount.com"  # @param {type:"string"}'
-    )

.cloud-build/utils/tests/test_preprocessors.py

@@ -0,0 +1,14 @@
+from utils import NotebookProcessors
+
+
+def test_update_value():
+    # Test that the content was updated
+    preprocessor = NotebookProcessors.UniqueStringsPreprocessor()
+
+    content = 'PROJECT_ID = "your-project-id-unique"'
+
+    new_content = preprocessor.update_unique_strings(content)
+
+    assert new_content != content
+    assert new_content.startswith('PROJECT_ID = "your-project-id-')
+    assert new_content.endswith('"')

.cloud-build/utils/tests/test_update_notebook_variables.py

@@ -0,0 +1,63 @@
+from utils import UpdateNotebookVariables
+
+
+def test_update_value():
+    new_content = UpdateNotebookVariables.get_updated_value(
+        content='asdf\nPROJECT_ID = "[your-project-id]" #@param {type:"string"} \nasdf',
+        variable_name="PROJECT_ID",
+        variable_value="sample-project",
+    )
+    assert (
+        new_content
+        == 'asdf\nPROJECT_ID = "sample-project" #@param {type:"string"} \nasdf'
+    )
+
+
+def test_update_value_single_quotes():
+    new_content = UpdateNotebookVariables.get_updated_value(
+        content="PROJECT_ID = '[your-project-id]'",
+        variable_name="PROJECT_ID",
+        variable_value="sample-project",
+    )
+    assert new_content == "PROJECT_ID = 'sample-project'"
+
+
+def test_update_value_avoidance():
+    new_content = UpdateNotebookVariables.get_updated_value(
+        content="PROJECT_ID = shell_output[0] ",
+        variable_name="PROJECT_ID",
+        variable_value="sample-project",
+    )
+    assert new_content == "PROJECT_ID = shell_output[0] "
+
+
+def test_region():
+    new_content = UpdateNotebookVariables.get_updated_value(
+        content='REGION = "[your-region]"  # @param {type:"string"}',
+        variable_name="REGION",
+        variable_value="us-central1",
+    )
+    assert new_content == 'REGION = "us-central1"  # @param {type:"string"}'
+
+
+def test_region_equal_equals_ignore():
+    # Tests that == is ignored
+    new_content = UpdateNotebookVariables.get_updated_value(
+        content='REGION == "[your-region]"  # @param {type:"string"}',
+        variable_name="REGION",
+        variable_value="us-central1",
+    )
+    assert new_content == 'REGION == "[your-region]"  # @param {type:"string"}'
+
+
+def test_service_account():
+    # Tests that == is ignored
+    new_content = UpdateNotebookVariables.get_updated_value(
+        content='SERVICE_ACCOUNT = "[your-service-account]"  # @param {type:"string"}',
+        variable_name="SERVICE_ACCOUNT",
+        variable_value="12345-compute@developer.gserviceaccount.com",
+    )
+    assert (
+        new_content
+        == 'SERVICE_ACCOUNT = "12345-compute@developer.gserviceaccount.com"  # @param {type:"string"}'
+    )

.cloud-build/utils/util.py

@@ -61,9 +61,7 @@ def archive_code_and_upload(staging_bucket: str):
 
 
 def download_blob_into_memory(
-  bucket_name: str,
-  blob_name: str,
-  download_as_text: Optional[bool]=False
+    bucket_name: str, blob_name: str, download_as_text: Optional[bool] = False
 ) -> Union[bytes, str]:
     """
     Downloads a blob into memory as byte or as text if
@@ -79,13 +77,10 @@ def download_blob_into_memory(
 
     # Download the blob content
     if download_as_text:
-      contents = blob.download_as_text()
+        contents = blob.download_as_text()
     else:
-      contents = blob.download_as_bytes()
+        contents = blob.download_as_bytes()
 
-    print(
-        f"Downloaded storage object {blob_name} from bucket {bucket_name}."
-    )
+    print(f"Downloaded storage object {blob_name} from bucket {bucket_name}.")
 
     return contents
-

.github/workflows/linter/requirements.txt

@@ -2,9 +2,9 @@ git+https://github.com/tensorflow/docs
 ipython
 jupyter
 nbconvert
-black==22.6.0
-pyupgrade==2.34.0
+black==22.10.0
+pyupgrade==2.38.4
 isort==5.10.1
 flake8==4.0.1
-nbqa==1.4.0
+nbqa==1.5.3
 

community-content/CODEOWNERS

@@ -6,3 +6,4 @@
 /sklearn_text_classification_from_script_using_vertex_sdk       @maxhardt
 /pluto_on_workbench @wkharold
 /cpr-examples @samthrasher
+/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines @Ark-kun

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_logistic_regression_model_using_Scikit_learn_and_import_to_Vertex_AI/pipeline.component.yaml

@@ -0,0 +1,83 @@
+name: Train tabular classification logistic regression model using Scikit learn pipeline
+metadata:
+  annotations:
+    author: Alexey Volkov <alexey.volkov@ark-kun.com>
+    canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_classification_logistic_regression_model_using_Scikit_learn_and_import_to_Vertex_AI/pipeline.component.yaml
+    sdk: https://cloud-pipelines.net/pipeline-editor/
+implementation:
+  graph:
+    tasks:
+      Download from GCS:
+        componentRef:
+          digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
+        arguments:
+          GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
+        annotations:
+          editor.position: '{"x":40,"y":40,"width":180,"height":40}'
+      Select columns using Pandas on CSV data:
+        componentRef:
+          digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: Data
+              taskId: Download from GCS
+          column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
+        annotations:
+          editor.position: '{"x":40,"y":140,"width":180,"height":54}'
+      Fill all missing values using Pandas on CSV data:
+        componentRef:
+          digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: transformed_table
+              taskId: Select columns using Pandas on CSV data
+              type: CSV
+          replacement_value: '0'
+        annotations:
+          editor.position: '{"x":40,"y":250,"width":180,"height":54}'
+      Binarize column using Pandas on CSV data:
+        componentRef:
+          digest: d699afd4d7cae862708717cc160f4394ed0c04e536e9515923ef1e8865f01d44
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: transformed_table
+              taskId: Fill all missing values using Pandas on CSV data
+              type: CSV
+          column_name: tips
+          predicate: '> 0'
+          new_column_name: class
+        annotations:
+          editor.position: '{"x":40,"y":380,"width":180,"height":54}'
+      Train logistic regression model using scikit learn from CSV:
+        componentRef:
+          digest: a864625a822e4b1c8ef6fe4ae1454fd90f15438f70a6712bb4c30e0dda4d35b7
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/cb44b75c9c062fcc40c2b905b2024b4493dbc62b/components/ML_frameworks/Scikit_learn/Train_logistic_regression_model/from_CSV/component.yaml
+        arguments:
+          dataset:
+            taskOutput:
+              outputName: transformed_table
+              taskId: Binarize column using Pandas on CSV data
+              type: CSV
+          label_column_name: class
+        annotations:
+          editor.position: '{"x":40,"y":510,"width":180,"height":70}'
+      Upload Scikit learn pickle model to Google Cloud Vertex AI:
+        componentRef:
+          digest: 81c91c8d7d21ec97e0872f669d68bd89edea87279d703685db54aa94743bebcd
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/workaround_for_buggy_KFPv2_compiler/component.yaml
+        arguments:
+          model:
+            taskOutput:
+              outputName: model
+              taskId: Train logistic regression model using scikit learn from CSV
+              type: ScikitLearnPickleModel
+        annotations:
+          editor.position: '{"x":40,"y":660,"width":180,"height":70}'
+    outputValues: {}

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_logistic_regression_model_using_Scikit_learn_and_import_to_Vertex_AI/pipeline.py

@@ -0,0 +1,73 @@
+# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
+from kfp import components
+
+# %% Loading components
+download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml")
+select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml")
+fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
+binarize_column_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml")
+train_logistic_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/cb44b75c9c062fcc40c2b905b2024b4493dbc62b/components/ML_frameworks/Scikit_learn/Train_logistic_regression_model/from_CSV/component.yaml")
+upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
+deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/workaround_for_buggy_KFPv2_compiler/component.yaml")
+
+# %% Pipeline definition
+def train_tabular_classification_logistic_regression_model_using_Scikit_learn_pipeline():
+    dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
+    feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]  # Excluded "trip_total"
+    label_column = "tips"
+    # Deploying the model might incur additional costs over time
+    deploy_model = False
+
+    classification_label_column = "class"
+    all_columns = [label_column] + feature_columns
+
+    training_data = download_from_gcs_op(
+        gcs_path=dataset_gcs_uri
+    ).outputs["Data"]
+
+    training_data = select_columns_using_Pandas_on_CSV_data_op(
+        table=training_data,
+        column_names=all_columns,
+    ).outputs["transformed_table"]
+
+    # Cleaning the NaN values.
+    training_data = fill_all_missing_values_using_Pandas_on_CSV_data_op(
+        table=training_data,
+        replacement_value="0",
+        #replacement_type_name="float",
+    ).outputs["transformed_table"]
+
+    classification_training_data = binarize_column_using_Pandas_on_CSV_data_op(
+        table=training_data,
+        column_name=label_column,
+        predicate="> 0",
+        new_column_name=classification_label_column,
+    ).outputs["transformed_table"]
+
+    model = train_logistic_regression_model_using_scikit_learn_from_CSV_op(
+        dataset=classification_training_data,
+        label_column_name=classification_label_column,
+        # Optional:
+        #penalty="l2",
+        #solver="lbfgs",
+        #max_iterations=100,
+        #multi_class_mode="auto",
+        #random_seed=0,
+    ).outputs["model"]
+
+    vertex_model_name = upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op(
+        model=model,
+    ).outputs["model_name"]
+
+    # Deploying the model might incur additional costs over time
+    if deploy_model:
+        sklearn_vertex_endpoint_name = deploy_model_to_endpoint_op(
+            model_name=vertex_model_name,
+        ).outputs["endpoint_name"]
+
+pipeline_func = train_tabular_classification_logistic_regression_model_using_Scikit_learn_pipeline
+
+# %% Pipeline submission
+if __name__ == '__main__':
+    from google.cloud import aiplatform
+    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_model_using_PyTorch_and_import_to_Vertex_AI/pipeline.component.yaml

@@ -0,0 +1,114 @@
+name: Train tabular classification model using PyTorch pipeline
+metadata:
+  annotations:
+    author: Alexey Volkov <alexey.volkov@ark-kun.com>
+    canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_classification_model_using_PyTorch_and_import_to_Vertex_AI/pipeline.component.yaml
+    sdk: https://cloud-pipelines.net/pipeline-editor/
+implementation:
+  graph:
+    tasks:
+      Download from GCS:
+        componentRef:
+          digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
+        arguments:
+          GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
+        annotations:
+          editor.position: '{"x":240,"y":40,"width":180,"height":40}'
+      Select columns using Pandas on CSV data:
+        componentRef:
+          digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: Data
+              taskId: Download from GCS
+          column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
+        annotations:
+          editor.position: '{"x":240,"y":140,"width":180,"height":54}'
+      Fill all missing values using Pandas on CSV data:
+        componentRef:
+          digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: transformed_table
+              taskId: Select columns using Pandas on CSV data
+              type: CSV
+          replacement_value: '0'
+        annotations:
+          editor.position: '{"x":240,"y":250,"width":180,"height":54}'
+      Create fully connected pytorch network:
+        componentRef:
+          digest: d03d8248fd358a0275ec33568ee7dd7dce576cc112b09dfafe2651e4d97e04a9
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1a2ef3eeb77bc278f33cad0dd29008ea2431e191/components/PyTorch/Create_fully_connected_network/component.yaml
+        arguments:
+          input_size: '7'
+          hidden_layer_sizes: '[10]'
+          activation_name: elu
+          output_activation_name: sigmoid
+        annotations:
+          editor.position: '{"x":40,"y":360,"width":180,"height":54}'
+      Binarize column using Pandas on CSV data:
+        componentRef:
+          digest: d699afd4d7cae862708717cc160f4394ed0c04e536e9515923ef1e8865f01d44
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: transformed_table
+              taskId: Fill all missing values using Pandas on CSV data
+              type: CSV
+          column_name: tips
+          predicate: ' > 0'
+          new_column_name: class
+        annotations:
+          editor.position: '{"x":240,"y":360,"width":180,"height":54}'
+      Train pytorch model from csv:
+        componentRef:
+          digest: 40f3185eb61e9727f41a4e0c05dd3d3b44bd802aa0f378cfc31756560033949a
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml
+        arguments:
+          model:
+            taskOutput:
+              outputName: model
+              taskId: Create fully connected pytorch network
+              type: PyTorchScriptModule
+          training_data:
+            taskOutput:
+              outputName: transformed_table
+              taskId: Binarize column using Pandas on CSV data
+              type: CSV
+          label_column_name: class
+          loss_function_name: binary_cross_entropy
+        annotations:
+          editor.position: '{"x":240,"y":490,"width":180,"height":40}'
+      Create PyTorch Model Archive with base handler:
+        componentRef:
+          digest: 8298b5ee1b0f0879f893add4cf352c8dec7cf9e21bb9db134c91a2d046cdb0ec
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/46d51383e6554b7f3ab4fd8cf614d8c2b422fb22/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml
+        arguments:
+          Model:
+            taskOutput:
+              outputName: trained_model
+              taskId: Train pytorch model from csv
+              type: PyTorchScriptModule
+          Model name: model
+          Model version: '1.0'
+        annotations:
+          editor.position: '{"x":240,"y":590,"width":180,"height":54}'
+      Upload PyTorch model archive to Google Cloud Vertex AI:
+        componentRef:
+          digest: 4450212fae7b9001482aca7eb78b28413c205506eccf08a04e7754a8dfa99004
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/workaround_for_buggy_KFPv2_compiler/component.yaml
+        arguments:
+          model_archive:
+            taskOutput:
+              outputName: Model archive
+              taskId: Create PyTorch Model Archive with base handler
+              type: PyTorchModelArchive
+        annotations:
+          editor.position: '{"x":240,"y":720,"width":180,"height":70}'
+    outputValues: {}

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_model_using_PyTorch_and_import_to_Vertex_AI/pipeline.py

@@ -0,0 +1,95 @@
+# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
+from kfp import components
+
+# %% Loading components
+download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml")
+select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml")
+fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
+binarize_column_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml")
+create_fully_connected_pytorch_network_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/1a2ef3eeb77bc278f33cad0dd29008ea2431e191/components/PyTorch/Create_fully_connected_network/component.yaml")
+train_pytorch_model_from_csv_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml")
+create_pytorch_model_archive_with_base_handler_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/46d51383e6554b7f3ab4fd8cf614d8c2b422fb22/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml")
+upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/workaround_for_buggy_KFPv2_compiler/component.yaml")
+deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/workaround_for_buggy_KFPv2_compiler/component.yaml")
+
+# %% Pipeline definition
+def train_tabular_classification_model_using_PyTorch_pipeline():
+    dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
+    feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]  # Excluded "trip_total"
+    label_column = "tips"
+    # Deploying the model might incur additional costs over time
+    deploy_model = False
+
+    classification_label_column = "class"
+    all_columns = [label_column] + feature_columns
+
+    training_data = download_from_gcs_op(
+        gcs_path=dataset_gcs_uri
+    ).outputs["Data"]
+
+    training_data = select_columns_using_Pandas_on_CSV_data_op(
+        table=training_data,
+        column_names=all_columns,
+    ).outputs["transformed_table"]
+
+    # Cleaning the NaN values.
+    training_data = fill_all_missing_values_using_Pandas_on_CSV_data_op(
+        table=training_data,
+        replacement_value="0",
+        #replacement_type_name="float",
+    ).outputs["transformed_table"]
+
+    classification_training_data = binarize_column_using_Pandas_on_CSV_data_op(
+        table=training_data,
+        column_name=label_column,
+        predicate=" > 0",
+        new_column_name=classification_label_column,
+    ).outputs["transformed_table"]
+
+    network = create_fully_connected_pytorch_network_op(
+        input_size=len(feature_columns),
+        # Optional:
+        hidden_layer_sizes=[10],
+        activation_name="elu",
+        output_activation_name="sigmoid",
+        # output_size=1,
+    ).outputs["model"]
+
+    model = train_pytorch_model_from_csv_op(
+        model=network,
+        training_data=classification_training_data,
+        label_column_name=classification_label_column,
+        loss_function_name="binary_cross_entropy",
+        # Optional:
+        #number_of_epochs=1,
+        #learning_rate=0.1,
+        #optimizer_name="Adadelta",
+        #optimizer_parameters={},
+        #batch_size=32,
+        #batch_log_interval=100,
+        #random_seed=0,
+    ).outputs["trained_model"]
+
+    model_archive = create_pytorch_model_archive_with_base_handler_op(
+        model=model,
+        # Optional:
+        # model_name="model",
+        # model_version="1.0",
+    ).outputs["Model archive"]
+
+    vertex_model_name = upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op(
+        model_archive=model_archive,
+    ).outputs["model_name"]
+
+    # Deploying the model might incur additional costs over time
+    if deploy_model:
+        vertex_endpoint_name = deploy_model_to_endpoint_op(
+            model_name=vertex_model_name,
+        ).outputs["endpoint_name"]
+
+pipeline_func=train_tabular_classification_model_using_PyTorch_pipeline
+
+# %% Pipeline submission
+if __name__ == '__main__':
+    from google.cloud import aiplatform
+    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_model_using_TensorFlow_and_import_to_Vertex_AI/pipeline.component.yaml

@@ -0,0 +1,132 @@
+name: Train tabular classification model using TensorFlow pipeline
+metadata:
+  annotations:
+    author: Alexey Volkov <alexey.volkov@ark-kun.com>
+    canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_classification_model_using_TensorFlow_and_import_to_Vertex_AI/pipeline.component.yaml
+    sdk: https://cloud-pipelines.net/pipeline-editor/
+implementation:
+  graph:
+    tasks:
+      Download from GCS:
+        componentRef:
+          digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
+        arguments:
+          GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
+        annotations:
+          editor.position: '{"x":40,"y":40,"width":180,"height":40}'
+      Select columns using Pandas on CSV data:
+        componentRef:
+          digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: Data
+              taskId: Download from GCS
+          column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
+        annotations:
+          editor.position: '{"x":40,"y":140,"width":180,"height":54}'
+      Fill all missing values using Pandas on CSV data:
+        componentRef:
+          digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: transformed_table
+              taskId: Select columns using Pandas on CSV data
+              type: CSV
+          replacement_value: '0'
+        annotations:
+          editor.position: '{"x":40,"y":250,"width":180,"height":54}'
+      Binarize column using Pandas on CSV data:
+        componentRef:
+          digest: d699afd4d7cae862708717cc160f4394ed0c04e536e9515923ef1e8865f01d44
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: transformed_table
+              taskId: Fill all missing values using Pandas on CSV data
+              type: CSV
+          column_name: tips
+          predicate: ' > 0'
+          new_column_name: class
+        annotations:
+          editor.position: '{"x":40,"y":370,"width":180,"height":54}'
+      Split rows into subsets:
+        componentRef:
+          digest: a609c3c9196484290f24a1174955f95b27f07a7b458aa5cb8cde28866cb2cb46
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: transformed_table
+              taskId: Binarize column using Pandas on CSV data
+              type: CSV
+          fraction_1: '0.8'
+        annotations:
+          editor.position: '{"x":170,"y":500,"width":180,"height":40}'
+      Create fully connected tensorflow network:
+        componentRef:
+          digest: bfcafbc5ce711b1f69cabf1338212d10d50136a73db9f9f7c984de7b80b4bfb0
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/9ca0f9eecf5f896f65b8538bbd809747052617d1/components/tensorflow/Create_fully_connected_network/component.yaml
+        arguments:
+          input_size: '7'
+          hidden_layer_sizes: '[10]'
+          activation_name: elu
+          output_activation_name: sigmoid
+        annotations:
+          editor.position: '{"x":370,"y":500,"width":180,"height":54}'
+      Train model using Keras on CSV:
+        componentRef:
+          digest: 42ae60c889034dbad74815653e95b4f7d576b5f47f803173e8679c7b54984609
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c504a4010348c50eaaf6d4337586ccc008f4dcef/components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml
+        arguments:
+          training_data:
+            taskOutput:
+              outputName: split_1
+              taskId: Split rows into subsets
+              type: CSV
+          model:
+            taskOutput:
+              outputName: model
+              taskId: Create fully connected tensorflow network
+              type: TensorflowSavedModel
+          label_column_name: class
+          loss_function_name: binary_crossentropy
+          number_of_epochs: '10'
+        annotations:
+          editor.position: '{"x":40,"y":620,"width":180,"height":54}'
+      Upload Tensorflow model to Google Cloud Vertex AI:
+        componentRef:
+          digest: 2e45263ff640b1a688e359b6936e27a81b2407749a84f340af2aa5547e0cb92c
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/workaround_for_buggy_KFPv2_compiler/component.yaml
+        arguments:
+          model:
+            taskOutput:
+              outputName: trained_model
+              taskId: Train model using Keras on CSV
+              type: TensorflowSavedModel
+        annotations:
+          editor.position: '{"x":40,"y":750,"width":180,"height":54}'
+      Predict with TensorFlow model on CSV data:
+        componentRef:
+          digest: 921bb1563e93a78233b8acceab87055b9154ccf5595d056028cf0396ca224cd4
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/59c759ce6f543184e30db6817d2a703879bc0f39/components/tensorflow/Predict/on_CSV/component.yaml
+        arguments:
+          dataset:
+            taskOutput:
+              outputName: split_2
+              taskId: Split rows into subsets
+              type: CSV
+          model:
+            taskOutput:
+              outputName: trained_model
+              taskId: Train model using Keras on CSV
+              type: TensorflowSavedModel
+          label_column_name: class
+        annotations:
+          editor.position: '{"x":240,"y":750,"width":180,"height":54}'
+    outputValues: {}

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_model_using_TensorFlow_and_import_to_Vertex_AI/pipeline.py

@@ -0,0 +1,106 @@
+# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
+from kfp import components
+
+# %% Loading components
+download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml")
+select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml")
+fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
+binarize_column_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml")
+split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
+create_fully_connected_tensorflow_network_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/9ca0f9eecf5f896f65b8538bbd809747052617d1/components/tensorflow/Create_fully_connected_network/component.yaml")
+train_model_using_Keras_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c504a4010348c50eaaf6d4337586ccc008f4dcef/components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml")
+predict_with_TensorFlow_model_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/59c759ce6f543184e30db6817d2a703879bc0f39/components/tensorflow/Predict/on_CSV/component.yaml")
+upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
+deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/workaround_for_buggy_KFPv2_compiler/component.yaml")
+
+# %% Pipeline definition
+def train_tabular_classification_model_using_TensorFlow_pipeline():
+    dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
+    feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]  # Excluded "trip_total"
+    label_column = "tips"
+    training_set_fraction = 0.8
+    # Deploying the model might incur additional costs over time
+    deploy_model = False
+
+    classification_label_column = "class"
+    all_columns = [label_column] + feature_columns
+
+    dataset = download_from_gcs_op(
+        gcs_path=dataset_gcs_uri
+    ).outputs["Data"]
+
+    dataset = select_columns_using_Pandas_on_CSV_data_op(
+        table=dataset,
+        column_names=all_columns,
+    ).outputs["transformed_table"]
+
+    dataset = fill_all_missing_values_using_Pandas_on_CSV_data_op(
+        table=dataset,
+        replacement_value="0",
+        # # Optional:
+        # column_names=None,  # =[...]
+    ).outputs["transformed_table"]
+
+    classification_dataset = binarize_column_using_Pandas_on_CSV_data_op(
+        table=dataset,
+        column_name=label_column,
+        predicate=" > 0",
+        new_column_name=classification_label_column,
+    ).outputs["transformed_table"]
+
+    split_task = split_rows_into_subsets_op(
+        table=classification_dataset,
+        fraction_1=training_set_fraction,
+    )
+    classification_training_data = split_task.outputs["split_1"]
+    classification_testing_data = split_task.outputs["split_2"]
+
+    network = create_fully_connected_tensorflow_network_op(
+        input_size=len(feature_columns),
+        # Optional:
+        hidden_layer_sizes=[10],
+        activation_name="elu",
+        output_activation_name="sigmoid",
+        # output_size=1,
+    ).outputs["model"]
+
+    model = train_model_using_Keras_on_CSV_op(
+        training_data=classification_training_data,
+        model=network,
+        label_column_name=classification_label_column,
+        # Optional:
+        loss_function_name="binary_crossentropy",
+        number_of_epochs=10,
+        #learning_rate=0.1,
+        #optimizer_name="Adadelta",
+        #optimizer_parameters={},
+        #batch_size=32,
+        #metric_names=["mean_absolute_error"],
+        #random_seed=0,
+    ).outputs["trained_model"]
+
+    predictions = predict_with_TensorFlow_model_on_CSV_data_op(
+        dataset=classification_testing_data,
+        model=model,
+        # label_column_name needs to be set when doing prediction on a dataset that has labels
+        label_column_name=classification_label_column,
+        # Optional:
+        # batch_size=1000,
+    ).outputs["predictions"]
+
+    vertex_model_name = upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op(
+        model=model,
+    ).outputs["model_name"]
+
+    # Deploying the model might incur additional costs over time
+    if deploy_model:
+        vertex_endpoint_name = deploy_model_to_endpoint_op(
+            model_name=vertex_model_name,
+        ).outputs["endpoint_name"]
+
+pipeline_func = train_tabular_classification_model_using_TensorFlow_pipeline
+
+# %% Pipeline submission
+if __name__ == '__main__':
+    from google.cloud import aiplatform
+    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_model_using_XGBoost_and_import_to_Vertex_AI/pipeline.component.yaml

@@ -0,0 +1,115 @@
+name: Train tabular classification model using XGBoost pipeline
+metadata:
+  annotations:
+    author: Alexey Volkov <alexey.volkov@ark-kun.com>
+    canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_classification_model_using_XGBoost_and_import_to_Vertex_AI/pipeline.component.yaml
+    sdk: https://cloud-pipelines.net/pipeline-editor/
+implementation:
+  graph:
+    tasks:
+      Download from GCS:
+        componentRef:
+          digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
+        arguments:
+          GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
+        annotations:
+          editor.position: '{"x":40,"y":40,"width":180,"height":40}'
+      Select columns using Pandas on CSV data:
+        componentRef:
+          digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: Data
+              taskId: Download from GCS
+          column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
+        annotations:
+          editor.position: '{"x":40,"y":140,"width":180,"height":54}'
+      Fill all missing values using Pandas on CSV data:
+        componentRef:
+          digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: transformed_table
+              taskId: Select columns using Pandas on CSV data
+              type: CSV
+          replacement_value: '0'
+        annotations:
+          editor.position: '{"x":40,"y":250,"width":180,"height":54}'
+      Binarize column using Pandas on CSV data:
+        componentRef:
+          digest: d699afd4d7cae862708717cc160f4394ed0c04e536e9515923ef1e8865f01d44
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: transformed_table
+              taskId: Fill all missing values using Pandas on CSV data
+              type: CSV
+          column_name: tips
+          predicate: '> 0'
+          new_column_name: class
+        annotations:
+          editor.position: '{"x":40,"y":380,"width":180,"height":54}'
+      Split rows into subsets:
+        componentRef:
+          digest: a609c3c9196484290f24a1174955f95b27f07a7b458aa5cb8cde28866cb2cb46
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: transformed_table
+              taskId: Binarize column using Pandas on CSV data
+              type: CSV
+          fraction_1: '0.8'
+        annotations:
+          editor.position: '{"x":170,"y":510,"width":180,"height":40}'
+      Train XGBoost model on CSV:
+        componentRef:
+          digest: 538c5a01eb38deaf532d619f0bbeaff4efc550fe1f0f776fc06791097b68ceac
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/58d3a47f904f32a64af8403330ba7e2134cae46d/components/XGBoost/Train/component.yaml
+        arguments:
+          training_data:
+            taskOutput:
+              outputName: split_1
+              taskId: Split rows into subsets
+              type: CSV
+          label_column_name: class
+          objective: binary:logistic
+        annotations:
+          editor.position: '{"x":40,"y":630,"width":180,"height":40}'
+      Upload XGBoost model to Google Cloud Vertex AI:
+        componentRef:
+          digest: 5a5a273c403670743820986c03a4175b7cb4595a556524fefcce403656286977
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/workaround_for_buggy_KFPv2_compiler/component.yaml
+        arguments:
+          model:
+            taskOutput:
+              outputName: model
+              taskId: Train XGBoost model on CSV
+              type: XGBoostModel
+        annotations:
+          editor.position: '{"x":40,"y":750,"width":180,"height":54}'
+      Xgboost predict on CSV:
+        componentRef:
+          digest: 0876233a0c7306fefec188bd70f059b46d1fb5aa57be231799570e3bbbdd0d95
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/4694ec97baccf59284c2a1db4aa2250c22291eab/components/XGBoost/Predict/component.yaml
+        arguments:
+          data:
+            taskOutput:
+              outputName: split_2
+              taskId: Split rows into subsets
+              type: CSV
+          model:
+            taskOutput:
+              outputName: model
+              taskId: Train XGBoost model on CSV
+              type: XGBoostModel
+          label_column_name: class
+        annotations:
+          editor.position: '{"x":240,"y":750,"width":180,"height":40}'
+    outputValues: {}

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_model_using_XGBoost_and_import_to_Vertex_AI/pipeline.py

@@ -0,0 +1,94 @@
+# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
+from kfp import components
+
+# %% Loading components
+download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml")
+select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml")
+fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
+binarize_column_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml")
+split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
+train_XGBoost_model_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/58d3a47f904f32a64af8403330ba7e2134cae46d/components/XGBoost/Train/component.yaml")
+xgboost_predict_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/4694ec97baccf59284c2a1db4aa2250c22291eab/components/XGBoost/Predict/component.yaml")
+upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
+deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/workaround_for_buggy_KFPv2_compiler/component.yaml")
+
+# %% Pipeline definition
+def train_tabular_classification_model_using_XGBoost_pipeline():
+    dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
+    feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]  # Excluded "trip_total"
+    label_column = "tips"
+    training_set_fraction = 0.8
+    # Deploying the model might incur additional costs over time
+    deploy_model = False
+
+    classification_label_column = "class"
+    all_columns = [label_column] + feature_columns
+
+    dataset = download_from_gcs_op(
+        gcs_path=dataset_gcs_uri
+    ).outputs["Data"]
+
+    dataset = select_columns_using_Pandas_on_CSV_data_op(
+        table=dataset,
+        column_names=all_columns,
+    ).outputs["transformed_table"]
+
+    dataset = fill_all_missing_values_using_Pandas_on_CSV_data_op(
+        table=dataset,
+        replacement_value="0",
+        # # Optional:
+        # column_names=None,  # =[...]
+    ).outputs["transformed_table"]
+
+    classification_dataset = binarize_column_using_Pandas_on_CSV_data_op(
+        table=dataset,
+        column_name=label_column,
+        predicate="> 0",
+        new_column_name=classification_label_column,
+    ).outputs["transformed_table"]
+
+    split_task = split_rows_into_subsets_op(
+        table=classification_dataset,
+        fraction_1=training_set_fraction,
+    )
+    classification_training_data = split_task.outputs["split_1"]
+    classification_testing_data = split_task.outputs["split_2"]
+
+    model = train_XGBoost_model_on_CSV_op(
+        training_data=classification_training_data,
+        label_column_name=classification_label_column,
+        objective="binary:logistic",
+        # Optional:
+        #starting_model=None,
+        #num_iterations=10,
+        #booster_params={},
+        #booster="gbtree",
+        #learning_rate=0.3,
+        #min_split_loss=0,
+        #max_depth=6,
+    ).outputs["model"]
+
+    # Predicting on the testing data
+    predictions = xgboost_predict_on_CSV_op(
+        data=classification_testing_data,
+        model=model,
+        # label_column needs to be set when doing prediction on a dataset that has labels
+        label_column_name=classification_label_column,
+    ).outputs["predictions"]
+
+    vertex_model_name = upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op(
+        model=model,
+    ).outputs["model_name"]
+
+    # Deploying the model might incur additional costs over time
+    if deploy_model:
+        vertex_endpoint_name = deploy_model_to_endpoint_op(
+            model_name=vertex_model_name,
+        ).outputs["endpoint_name"]
+
+pipeline_func = train_tabular_classification_model_using_XGBoost_pipeline
+
+# %% Pipeline submission
+if __name__ == '__main__':
+    from google.cloud import aiplatform
+    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_model_using_all_frameworks_and_import_to_Vertex_AI/pipeline.component.yaml

@@ -0,0 +1,257 @@
+name: Train tabular classification model using all frameworks pipeline
+metadata:
+  annotations:
+    author: Alexey Volkov <alexey.volkov@ark-kun.com>
+    canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_classification_model_using_all_frameworks_and_import_to_Vertex_AI/pipeline.component.yaml
+    sdk: https://cloud-pipelines.net/pipeline-editor/
+implementation:
+  graph:
+    tasks:
+      Download from GCS:
+        componentRef:
+          digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
+        arguments:
+          GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
+        annotations:
+          editor.position: '{"x":550,"y":40,"width":180,"height":40}'
+      Select columns using Pandas on CSV data:
+        componentRef:
+          digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: Data
+              taskId: Download from GCS
+          column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
+        annotations:
+          editor.position: '{"x":550,"y":140,"width":180,"height":54}'
+      Fill all missing values using Pandas on CSV data:
+        componentRef:
+          digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: transformed_table
+              taskId: Select columns using Pandas on CSV data
+              type: CSV
+          replacement_value: '0'
+        annotations:
+          editor.position: '{"x":550,"y":250,"width":180,"height":54}'
+      Binarize column using Pandas on CSV data:
+        componentRef:
+          digest: d699afd4d7cae862708717cc160f4394ed0c04e536e9515923ef1e8865f01d44
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: transformed_table
+              taskId: Fill all missing values using Pandas on CSV data
+              type: CSV
+          column_name: tips
+          predicate: ' > 0'
+          new_column_name: class
+        annotations:
+          editor.position: '{"x":550,"y":380,"width":180,"height":54}'
+      Split rows into subsets:
+        componentRef:
+          digest: a609c3c9196484290f24a1174955f95b27f07a7b458aa5cb8cde28866cb2cb46
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: transformed_table
+              taskId: Binarize column using Pandas on CSV data
+              type: CSV
+          fraction_1: '0.8'
+        annotations:
+          editor.position: '{"x":550,"y":490,"width":180,"height":40}'
+      Create fully connected pytorch network:
+        componentRef:
+          digest: d03d8248fd358a0275ec33568ee7dd7dce576cc112b09dfafe2651e4d97e04a9
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1a2ef3eeb77bc278f33cad0dd29008ea2431e191/components/PyTorch/Create_fully_connected_network/component.yaml
+        arguments:
+          input_size: '7'
+          hidden_layer_sizes: '[10]'
+          activation_name: elu
+          output_activation_name: sigmoid
+        annotations:
+          editor.position: '{"x":380,"y":620,"width":180,"height":54}'
+      Create fully connected tensorflow network:
+        componentRef:
+          digest: bfcafbc5ce711b1f69cabf1338212d10d50136a73db9f9f7c984de7b80b4bfb0
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/9ca0f9eecf5f896f65b8538bbd809747052617d1/components/tensorflow/Create_fully_connected_network/component.yaml
+        arguments:
+          input_size: '7'
+          hidden_layer_sizes: '[10]'
+          activation_name: elu
+          output_activation_name: sigmoid
+        annotations:
+          editor.position: '{"x":40,"y":630,"width":180,"height":54}'
+      Train model using Keras on CSV:
+        componentRef:
+          digest: 42ae60c889034dbad74815653e95b4f7d576b5f47f803173e8679c7b54984609
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c504a4010348c50eaaf6d4337586ccc008f4dcef/components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml
+        arguments:
+          training_data:
+            taskOutput:
+              outputName: split_1
+              taskId: Split rows into subsets
+              type: CSV
+          model:
+            taskOutput:
+              outputName: model
+              taskId: Create fully connected tensorflow network
+              type: TensorflowSavedModel
+          label_column_name: class
+          loss_function_name: binary_crossentropy
+          number_of_epochs: '10'
+        annotations:
+          editor.position: '{"x":40,"y":750,"width":180,"height":54}'
+      Train pytorch model from csv:
+        componentRef:
+          digest: 40f3185eb61e9727f41a4e0c05dd3d3b44bd802aa0f378cfc31756560033949a
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml
+        arguments:
+          model:
+            taskOutput:
+              outputName: model
+              taskId: Create fully connected pytorch network
+              type: PyTorchScriptModule
+          training_data:
+            taskOutput:
+              outputName: split_1
+              taskId: Split rows into subsets
+              type: CSV
+          label_column_name: class
+          loss_function_name: binary_cross_entropy
+        annotations:
+          editor.position: '{"x":380,"y":750,"width":180,"height":40}'
+      Train XGBoost model on CSV:
+        componentRef:
+          digest: 538c5a01eb38deaf532d619f0bbeaff4efc550fe1f0f776fc06791097b68ceac
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/58d3a47f904f32a64af8403330ba7e2134cae46d/components/XGBoost/Train/component.yaml
+        arguments:
+          training_data:
+            taskOutput:
+              outputName: split_1
+              taskId: Split rows into subsets
+              type: CSV
+          label_column_name: class
+          objective: binary:logistic
+        annotations:
+          editor.position: '{"x":720,"y":750,"width":180,"height":40}'
+      Train logistic regression model using scikit learn from CSV:
+        componentRef:
+          digest: a864625a822e4b1c8ef6fe4ae1454fd90f15438f70a6712bb4c30e0dda4d35b7
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/cb44b75c9c062fcc40c2b905b2024b4493dbc62b/components/ML_frameworks/Scikit_learn/Train_logistic_regression_model/from_CSV/component.yaml
+        arguments:
+          dataset:
+            taskOutput:
+              outputName: split_1
+              taskId: Split rows into subsets
+              type: CSV
+          label_column_name: class
+        annotations:
+          editor.position: '{"x":1030,"y":750,"width":180,"height":70}'
+      Predict with TensorFlow model on CSV data:
+        componentRef:
+          digest: 921bb1563e93a78233b8acceab87055b9154ccf5595d056028cf0396ca224cd4
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/59c759ce6f543184e30db6817d2a703879bc0f39/components/tensorflow/Predict/on_CSV/component.yaml
+        arguments:
+          dataset:
+            taskOutput:
+              outputName: split_2
+              taskId: Split rows into subsets
+              type: CSV
+          model:
+            taskOutput:
+              outputName: trained_model
+              taskId: Train model using Keras on CSV
+              type: TensorflowSavedModel
+          label_column_name: class
+        annotations:
+          editor.position: '{"x":160,"y":880,"width":180,"height":54}'
+      Create PyTorch Model Archive with base handler:
+        componentRef:
+          digest: 8298b5ee1b0f0879f893add4cf352c8dec7cf9e21bb9db134c91a2d046cdb0ec
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/46d51383e6554b7f3ab4fd8cf614d8c2b422fb22/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml
+        arguments:
+          Model:
+            taskOutput:
+              outputName: trained_model
+              taskId: Train pytorch model from csv
+              type: PyTorchScriptModule
+          Model name: model
+          Model version: '1.0'
+        annotations:
+          editor.position: '{"x":380,"y":880,"width":180,"height":54}'
+      Xgboost predict on CSV:
+        componentRef:
+          digest: 0876233a0c7306fefec188bd70f059b46d1fb5aa57be231799570e3bbbdd0d95
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/4694ec97baccf59284c2a1db4aa2250c22291eab/components/XGBoost/Predict/component.yaml
+        arguments:
+          data:
+            taskOutput:
+              outputName: split_2
+              taskId: Split rows into subsets
+              type: CSV
+          model:
+            taskOutput:
+              outputName: model
+              taskId: Train XGBoost model on CSV
+              type: XGBoostModel
+          label_column_name: class
+        annotations:
+          editor.position: '{"x":810,"y":880,"width":180,"height":40}'
+      Upload Scikit learn pickle model to Google Cloud Vertex AI:
+        componentRef:
+          digest: 81c91c8d7d21ec97e0872f669d68bd89edea87279d703685db54aa94743bebcd
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/workaround_for_buggy_KFPv2_compiler/component.yaml
+        arguments:
+          model:
+            taskOutput:
+              outputName: model
+              taskId: Train logistic regression model using scikit learn from CSV
+              type: ScikitLearnPickleModel
+        annotations:
+          editor.position: '{"x":1030,"y":880,"width":180,"height":70}'
+      Upload Tensorflow model to Google Cloud Vertex AI:
+        componentRef:
+          digest: 2e45263ff640b1a688e359b6936e27a81b2407749a84f340af2aa5547e0cb92c
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/workaround_for_buggy_KFPv2_compiler/component.yaml
+        arguments:
+          model:
+            taskOutput:
+              outputName: trained_model
+              taskId: Train model using Keras on CSV
+              type: TensorflowSavedModel
+        annotations:
+          editor.position: '{"x":40,"y":1010,"width":180,"height":54}'
+      Upload PyTorch model archive to Google Cloud Vertex AI:
+        componentRef:
+          digest: 4450212fae7b9001482aca7eb78b28413c205506eccf08a04e7754a8dfa99004
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/workaround_for_buggy_KFPv2_compiler/component.yaml
+        arguments:
+          model_archive:
+            taskOutput:
+              outputName: Model archive
+              taskId: Create PyTorch Model Archive with base handler
+              type: PyTorchModelArchive
+        annotations:
+          editor.position: '{"x":380,"y":1010,"width":180,"height":70}'
+      Upload XGBoost model to Google Cloud Vertex AI:
+        componentRef:
+          digest: 5a5a273c403670743820986c03a4175b7cb4595a556524fefcce403656286977
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/workaround_for_buggy_KFPv2_compiler/component.yaml
+        arguments:
+          model:
+            taskOutput:
+              outputName: model
+              taskId: Train XGBoost model on CSV
+              type: XGBoostModel
+        annotations:
+          editor.position: '{"x":720,"y":1010,"width":180,"height":54}'
+    outputValues: {}

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_model_using_all_frameworks_and_import_to_Vertex_AI/pipeline.py

@@ -0,0 +1,224 @@
+# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
+from kfp import components
+
+# %% Loading components
+download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml")
+select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml")
+fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
+binarize_column_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml")
+split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
+
+# TensorFlow
+create_fully_connected_tensorflow_network_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/9ca0f9eecf5f896f65b8538bbd809747052617d1/components/tensorflow/Create_fully_connected_network/component.yaml")
+train_model_using_Keras_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c504a4010348c50eaaf6d4337586ccc008f4dcef/components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml")
+predict_with_TensorFlow_model_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/59c759ce6f543184e30db6817d2a703879bc0f39/components/tensorflow/Predict/on_CSV/component.yaml")
+upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
+
+# PyTorch
+create_fully_connected_pytorch_network_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/1a2ef3eeb77bc278f33cad0dd29008ea2431e191/components/PyTorch/Create_fully_connected_network/component.yaml")
+train_pytorch_model_from_csv_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml")
+create_pytorch_model_archive_with_base_handler_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/46d51383e6554b7f3ab4fd8cf614d8c2b422fb22/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml")
+upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/workaround_for_buggy_KFPv2_compiler/component.yaml")
+
+# XGBoost
+train_XGBoost_model_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/58d3a47f904f32a64af8403330ba7e2134cae46d/components/XGBoost/Train/component.yaml")
+xgboost_predict_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/4694ec97baccf59284c2a1db4aa2250c22291eab/components/XGBoost/Predict/component.yaml")
+upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
+
+# Scikit-learn
+#train_linear_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/f807e02b54d4886c65a05f40848fd51c72407f40/components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml")
+train_logistic_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/cb44b75c9c062fcc40c2b905b2024b4493dbc62b/components/ML_frameworks/Scikit_learn/Train_logistic_regression_model/from_CSV/component.yaml")
+upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
+
+# Vertex AI
+deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/workaround_for_buggy_KFPv2_compiler/component.yaml")
+
+# %% Pipeline definition
+def train_tabular_classification_model_using_all_frameworks_pipeline():
+    dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
+    feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]  # Excluded "trip_total"
+    label_column = "tips"
+    training_set_fraction = 0.8
+    # Deploying the model might incur additional costs over time
+    deploy_model = False
+
+    classification_label_column = "class"
+    all_columns = [label_column] + feature_columns
+
+    dataset = download_from_gcs_op(
+        gcs_path=dataset_gcs_uri
+    ).outputs["Data"]
+
+    dataset = select_columns_using_Pandas_on_CSV_data_op(
+        table=dataset,
+        column_names=all_columns,
+    ).outputs["transformed_table"]
+
+    dataset = fill_all_missing_values_using_Pandas_on_CSV_data_op(
+        table=dataset,
+        replacement_value="0",
+        # # Optional:
+        # column_names=None,  # =[...]
+    ).outputs["transformed_table"]
+
+    classification_dataset = binarize_column_using_Pandas_on_CSV_data_op(
+        table=dataset,
+        column_name=label_column,
+        predicate=" > 0",
+        new_column_name=classification_label_column,
+    ).outputs["transformed_table"]
+
+    split_task = split_rows_into_subsets_op(
+        table=classification_dataset,
+        fraction_1=training_set_fraction,
+    )
+    classification_training_data = split_task.outputs["split_1"]
+    classification_testing_data = split_task.outputs["split_2"]
+
+    # TensorFlow
+    tensorflow_network = create_fully_connected_tensorflow_network_op(
+        input_size=len(feature_columns),
+        # Optional:
+        hidden_layer_sizes=[10],
+        activation_name="elu",
+        output_activation_name="sigmoid",
+        # output_size=1,
+    ).outputs["model"]
+
+    tensorflow_model = train_model_using_Keras_on_CSV_op(
+        training_data=classification_training_data,
+        model=tensorflow_network,
+        label_column_name=classification_label_column,
+        # Optional:
+        loss_function_name="binary_crossentropy",
+        number_of_epochs=10,
+        #learning_rate=0.1,
+        #optimizer_name="Adadelta",
+        #optimizer_parameters={},
+        #batch_size=32,
+        #metric_names=["mean_absolute_error"],
+        #random_seed=0,
+    ).outputs["trained_model"]
+
+    tensorflow_predictions = predict_with_TensorFlow_model_on_CSV_data_op(
+        dataset=classification_testing_data,
+        model=tensorflow_model,
+        # label_column_name needs to be set when doing prediction on a dataset that has labels
+        label_column_name=classification_label_column,
+        # Optional:
+        # batch_size=1000,
+    ).outputs["predictions"]
+
+    tensorflow_vertex_model_name = upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op(
+        model=tensorflow_model,
+    ).outputs["model_name"]
+
+    # Deploying the model might incur additional costs over time
+    if deploy_model:
+        tensorflow_vertex_endpoint_name = deploy_model_to_endpoint_op(
+            model_name=tensorflow_vertex_model_name,
+        ).outputs["endpoint_name"]
+
+    # PyTorch
+    pytorch_network = create_fully_connected_pytorch_network_op(
+        input_size=len(feature_columns),
+        # Optional:
+        hidden_layer_sizes=[10],
+        activation_name="elu",
+        output_activation_name="sigmoid",
+        # output_size=1,
+    ).outputs["model"]
+
+    pytorch_model = train_pytorch_model_from_csv_op(
+        model=pytorch_network,
+        training_data=classification_training_data,
+        label_column_name=classification_label_column,
+        loss_function_name="binary_cross_entropy",
+        # Optional:
+        #number_of_epochs=1,
+        #learning_rate=0.1,
+        #optimizer_name="Adadelta",
+        #optimizer_parameters={},
+        #batch_size=32,
+        #batch_log_interval=100,
+        #random_seed=0,
+    ).outputs["trained_model"]
+
+    pytorch_model_archive = create_pytorch_model_archive_with_base_handler_op(
+        model=pytorch_model,
+        # Optional:
+        # model_name="model",
+        # model_version="1.0",
+    ).outputs["Model archive"]
+
+    pytorch_vertex_model_name = upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op(
+        model_archive=pytorch_model_archive,
+    ).outputs["model_name"]
+
+    # Deploying the model might incur additional costs over time
+    if deploy_model:
+        pytorch_vertex_endpoint_name = deploy_model_to_endpoint_op(
+            model_name=pytorch_vertex_model_name,
+        ).outputs["endpoint_name"]
+
+    # XGBoost
+    xgboost_model = train_XGBoost_model_on_CSV_op(
+        training_data=classification_training_data,
+        label_column_name=classification_label_column,
+        objective="binary:logistic",
+        # Optional:
+        #starting_model=None,
+        #num_iterations=10,
+        #booster_params={},
+        #booster="gbtree",
+        #learning_rate=0.3,
+        #min_split_loss=0,
+        #max_depth=6,
+    ).outputs["model"]
+
+    # Predicting on the testing data
+    xgboost_predictions = xgboost_predict_on_CSV_op(
+        data=classification_testing_data,
+        model=xgboost_model,
+        # label_column needs to be set when doing prediction on a dataset that has labels
+        label_column_name=classification_label_column,
+    ).outputs["predictions"]
+
+    xgboost_vertex_model_name = upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op(
+        model=xgboost_model,
+    ).outputs["model_name"]
+
+    # Deploying the model might incur additional costs over time
+    if deploy_model:
+        xgboost_vertex_endpoint_name = deploy_model_to_endpoint_op(
+            model_name=xgboost_vertex_model_name,
+        ).outputs["endpoint_name"]
+
+    # Scikit-learn
+    sklearn_model = train_logistic_regression_model_using_scikit_learn_from_CSV_op(
+        dataset=classification_training_data,
+        label_column_name=classification_label_column,
+        # Optional:
+        #penalty="l2",
+        #solver="lbfgs",
+        #max_iterations=100,
+        #multi_class_mode="auto",
+        #random_seed=0,
+    ).outputs["model"]
+
+    sklearn_vertex_model_name = upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op(
+        model=sklearn_model,
+    ).outputs["model_name"]
+
+    # Deploying the model might incur additional costs over time
+    if deploy_model:
+        sklearn_vertex_endpoint_name = deploy_model_to_endpoint_op(
+            model_name=sklearn_vertex_model_name,
+        ).outputs["endpoint_name"]
+
+pipeline_func=train_tabular_classification_model_using_all_frameworks_pipeline
+
+# %% Pipeline submission
+if __name__ == '__main__':
+    from google.cloud import aiplatform
+    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_regression_linear_model_using_Scikit_learn_and_import_to_Vertex_AI/pipeline.component.yaml

@@ -0,0 +1,68 @@
+name: Train tabular regression linear model using Scikit learn pipeline
+metadata:
+  annotations:
+    author: Alexey Volkov <alexey.volkov@ark-kun.com>
+    canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_regression_linear_model_using_Scikit_learn_and_import_to_Vertex_AI/pipeline.component.yaml
+    sdk: https://cloud-pipelines.net/pipeline-editor/
+implementation:
+  graph:
+    tasks:
+      Download from GCS:
+        componentRef:
+          digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
+        arguments:
+          GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
+        annotations:
+          editor.position: '{"x":40,"y":40,"width":180,"height":40}'
+      Select columns using Pandas on CSV data:
+        componentRef:
+          digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: Data
+              taskId: Download from GCS
+          column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
+        annotations:
+          editor.position: '{"x":40,"y":140,"width":180,"height":54}'
+      Fill all missing values using Pandas on CSV data:
+        componentRef:
+          digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: transformed_table
+              taskId: Select columns using Pandas on CSV data
+              type: CSV
+          replacement_value: '0'
+        annotations:
+          editor.position: '{"x":40,"y":250,"width":180,"height":54}'
+      Train linear regression model using scikit learn from CSV:
+        componentRef:
+          digest: c7fe7912ab0d1fb45d201d452e9ce6be5544e7d8c6d229db7a4b931ff58560f3
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/f807e02b54d4886c65a05f40848fd51c72407f40/components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml
+        arguments:
+          dataset:
+            taskOutput:
+              outputName: transformed_table
+              taskId: Fill all missing values using Pandas on CSV data
+              type: CSV
+          label_column_name: tips
+        annotations:
+          editor.position: '{"x":40,"y":360,"width":180,"height":54}'
+      Upload Scikit learn pickle model to Google Cloud Vertex AI:
+        componentRef:
+          digest: 81c91c8d7d21ec97e0872f669d68bd89edea87279d703685db54aa94743bebcd
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/workaround_for_buggy_KFPv2_compiler/component.yaml
+        arguments:
+          model:
+            taskOutput:
+              outputName: model
+              taskId: Train linear regression model using scikit learn from CSV
+              type: ScikitLearnPickleModel
+        annotations:
+          editor.position: '{"x":40,"y":490,"width":180,"height":70}'
+    outputValues: {}

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_regression_linear_model_using_Scikit_learn_and_import_to_Vertex_AI/pipeline.py

@@ -0,0 +1,57 @@
+# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
+from kfp import components
+
+# %% Loading components
+download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml")
+select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml")
+fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
+train_linear_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/f807e02b54d4886c65a05f40848fd51c72407f40/components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml")
+upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
+deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/workaround_for_buggy_KFPv2_compiler/component.yaml")
+
+# %% Pipeline definition
+def train_tabular_regression_linear_model_using_Scikit_learn_pipeline():
+    dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
+    feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]  # Excluded "trip_total"
+    label_column = "tips"
+    all_columns = [label_column] + feature_columns
+    # Deploying the model might incur additional costs over time
+    deploy_model = False
+
+    training_data = download_from_gcs_op(
+        gcs_path=dataset_gcs_uri
+    ).outputs["Data"]
+
+    training_data = select_columns_using_Pandas_on_CSV_data_op(
+        table=training_data,
+        column_names=all_columns,
+    ).outputs["transformed_table"]
+
+    # Cleaning the NaN values.
+    training_data = fill_all_missing_values_using_Pandas_on_CSV_data_op(
+        table=training_data,
+        replacement_value="0",
+        #replacement_type_name="float",
+    ).outputs["transformed_table"]
+
+    model = train_linear_regression_model_using_scikit_learn_from_CSV_op(
+        dataset=training_data,
+        label_column_name=label_column,
+    ).outputs["model"]
+
+    vertex_model_name = upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op(
+        model=model,
+    ).outputs["model_name"]
+
+    # Deploying the model might incur additional costs over time
+    if deploy_model:
+        sklearn_vertex_endpoint_name = deploy_model_to_endpoint_op(
+            model_name=vertex_model_name,
+        ).outputs["endpoint_name"]
+
+pipeline_func = train_tabular_regression_linear_model_using_Scikit_learn_pipeline
+
+# %% Pipeline submission
+if __name__ == '__main__':
+    from google.cloud import aiplatform
+    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_regression_model_using_PyTorch_and_import_to_Vertex_AI/pipeline.component.yaml

@@ -0,0 +1,97 @@
+name: Train tabular regression model using PyTorch pipeline
+metadata:
+  annotations:
+    author: Alexey Volkov <alexey.volkov@ark-kun.com>
+    canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_regression_model_using_PyTorch_and_import_to_Vertex_AI/pipeline.component.yaml
+    sdk: https://cloud-pipelines.net/pipeline-editor/
+implementation:
+  graph:
+    tasks:
+      Download from GCS:
+        componentRef:
+          digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
+        arguments:
+          GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
+        annotations:
+          editor.position: '{"x":240,"y":40,"width":180,"height":40}'
+      Select columns using Pandas on CSV data:
+        componentRef:
+          digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: Data
+              taskId: Download from GCS
+          column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
+        annotations:
+          editor.position: '{"x":240,"y":130,"width":180,"height":54}'
+      Create fully connected pytorch network:
+        componentRef:
+          digest: d03d8248fd358a0275ec33568ee7dd7dce576cc112b09dfafe2651e4d97e04a9
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1a2ef3eeb77bc278f33cad0dd29008ea2431e191/components/PyTorch/Create_fully_connected_network/component.yaml
+        arguments:
+          input_size: '7'
+          hidden_layer_sizes: '[10]'
+          activation_name: elu
+        annotations:
+          editor.position: '{"x":40,"y":240,"width":180,"height":54}'
+      Fill all missing values using Pandas on CSV data:
+        componentRef:
+          digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: transformed_table
+              taskId: Select columns using Pandas on CSV data
+              type: CSV
+          replacement_value: '0'
+        annotations:
+          editor.position: '{"x":240,"y":240,"width":180,"height":54}'
+      Train pytorch model from csv:
+        componentRef:
+          digest: 40f3185eb61e9727f41a4e0c05dd3d3b44bd802aa0f378cfc31756560033949a
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml
+        arguments:
+          model:
+            taskOutput:
+              outputName: model
+              taskId: Create fully connected pytorch network
+              type: PyTorchScriptModule
+          training_data:
+            taskOutput:
+              outputName: transformed_table
+              taskId: Fill all missing values using Pandas on CSV data
+              type: CSV
+          label_column_name: tips
+        annotations:
+          editor.position: '{"x":240,"y":380,"width":180,"height":40}'
+      Create PyTorch Model Archive with base handler:
+        componentRef:
+          digest: 8298b5ee1b0f0879f893add4cf352c8dec7cf9e21bb9db134c91a2d046cdb0ec
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/46d51383e6554b7f3ab4fd8cf614d8c2b422fb22/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml
+        arguments:
+          Model:
+            taskOutput:
+              outputName: trained_model
+              taskId: Train pytorch model from csv
+              type: PyTorchScriptModule
+          Model name: model
+          Model version: '1.0'
+        annotations:
+          editor.position: '{"x":240,"y":500,"width":180,"height":54}'
+      Upload PyTorch model archive to Google Cloud Vertex AI:
+        componentRef:
+          digest: 4450212fae7b9001482aca7eb78b28413c205506eccf08a04e7754a8dfa99004
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/workaround_for_buggy_KFPv2_compiler/component.yaml
+        arguments:
+          model_archive:
+            taskOutput:
+              outputName: Model archive
+              taskId: Create PyTorch Model Archive with base handler
+              type: PyTorchModelArchive
+        annotations:
+          editor.position: '{"x":240,"y":630,"width":180,"height":70}'
+    outputValues: {}

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_regression_model_using_PyTorch_and_import_to_Vertex_AI/pipeline.py

@@ -0,0 +1,85 @@
+# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
+from kfp import components
+
+# %% Loading components
+download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml")
+select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml")
+fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
+create_fully_connected_pytorch_network_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/1a2ef3eeb77bc278f33cad0dd29008ea2431e191/components/PyTorch/Create_fully_connected_network/component.yaml")
+train_pytorch_model_from_csv_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml")
+create_pytorch_model_archive_with_base_handler_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/46d51383e6554b7f3ab4fd8cf614d8c2b422fb22/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml")
+upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/workaround_for_buggy_KFPv2_compiler/component.yaml")
+deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/workaround_for_buggy_KFPv2_compiler/component.yaml")
+
+# %% Pipeline definition
+def train_tabular_regression_model_using_PyTorch_pipeline():
+    dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
+    feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]  # Excluded "trip_total"
+    label_column = "tips"
+    all_columns = [label_column] + feature_columns
+    # Deploying the model might incur additional costs over time
+    deploy_model = False
+
+    training_data = download_from_gcs_op(
+        gcs_path=dataset_gcs_uri
+    ).outputs["Data"]
+
+    training_data = select_columns_using_Pandas_on_CSV_data_op(
+        table=training_data,
+        column_names=all_columns,
+    ).outputs["transformed_table"]
+
+    # Cleaning the NaN values.
+    training_data = fill_all_missing_values_using_Pandas_on_CSV_data_op(
+        table=training_data,
+        replacement_value="0",
+        #replacement_type_name="float",
+    ).outputs["transformed_table"]
+
+    network = create_fully_connected_pytorch_network_op(
+        input_size=len(feature_columns),
+        # Optional:
+        hidden_layer_sizes=[10],
+        activation_name="elu",
+        # output_activation_name=None,
+        # output_size=1,
+    ).outputs["model"]
+
+    model = train_pytorch_model_from_csv_op(
+        model=network,
+        training_data=training_data,
+        label_column_name=label_column,
+        # Optional:
+        #loss_function_name="mse_loss",
+        #number_of_epochs=1,
+        #learning_rate=0.1,
+        #optimizer_name="Adadelta",
+        #optimizer_parameters={},
+        #batch_size=32,
+        #batch_log_interval=100,
+        #random_seed=0,
+    ).outputs["trained_model"]
+
+    model_archive = create_pytorch_model_archive_with_base_handler_op(
+        model=model,
+        # Optional:
+        # model_name="model",
+        # model_version="1.0",
+    ).outputs["Model archive"]
+
+    vertex_model_name = upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op(
+        model_archive=model_archive,
+    ).outputs["model_name"]
+
+    # Deploying the model might incur additional costs over time
+    if deploy_model:
+        vertex_endpoint_name = deploy_model_to_endpoint_op(
+            model_name=vertex_model_name,
+        ).outputs["endpoint_name"]
+
+pipeline_func=train_tabular_regression_model_using_PyTorch_pipeline
+
+# %% Pipeline submission
+if __name__ == '__main__':
+    from google.cloud import aiplatform
+    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_regression_model_using_TensorFlow_and_import_to_Vertex_AI/pipeline.component.yaml

@@ -0,0 +1,116 @@
+name: Train tabular regression model using Tensorflow pipeline
+metadata:
+  annotations:
+    author: Alexey Volkov <alexey.volkov@ark-kun.com>
+    canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_regression_model_using_TensorFlow_and_import_to_Vertex_AI/pipeline.component.yaml
+    sdk: https://cloud-pipelines.net/pipeline-editor/
+implementation:
+  graph:
+    tasks:
+      Download from GCS:
+        componentRef:
+          digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
+        arguments:
+          GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
+        annotations:
+          editor.position: '{"x":40,"y":40,"width":180,"height":40}'
+      Select columns using Pandas on CSV data:
+        componentRef:
+          digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: Data
+              taskId: Download from GCS
+          column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
+        annotations:
+          editor.position: '{"x":40,"y":140,"width":180,"height":54}'
+      Fill all missing values using Pandas on CSV data:
+        componentRef:
+          digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: transformed_table
+              taskId: Select columns using Pandas on CSV data
+              type: CSV
+          replacement_value: '0'
+        annotations:
+          editor.position: '{"x":40,"y":250,"width":180,"height":54}'
+      Split rows into subsets:
+        componentRef:
+          digest: a609c3c9196484290f24a1174955f95b27f07a7b458aa5cb8cde28866cb2cb46
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: transformed_table
+              taskId: Fill all missing values using Pandas on CSV data
+              type: CSV
+          fraction_1: '0.8'
+        annotations:
+          editor.position: '{"x":170,"y":380,"width":180,"height":40}'
+      Create fully connected tensorflow network:
+        componentRef:
+          digest: bfcafbc5ce711b1f69cabf1338212d10d50136a73db9f9f7c984de7b80b4bfb0
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/9ca0f9eecf5f896f65b8538bbd809747052617d1/components/tensorflow/Create_fully_connected_network/component.yaml
+        arguments:
+          input_size: '7'
+          hidden_layer_sizes: '[10]'
+          activation_name: elu
+        annotations:
+          editor.position: '{"x":370,"y":380,"width":180,"height":54}'
+      Train model using Keras on CSV:
+        componentRef:
+          digest: 42ae60c889034dbad74815653e95b4f7d576b5f47f803173e8679c7b54984609
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c504a4010348c50eaaf6d4337586ccc008f4dcef/components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml
+        arguments:
+          training_data:
+            taskOutput:
+              outputName: split_1
+              taskId: Split rows into subsets
+              type: CSV
+          model:
+            taskOutput:
+              outputName: model
+              taskId: Create fully connected tensorflow network
+              type: TensorflowSavedModel
+          label_column_name: tips
+          number_of_epochs: '10'
+          metric_names: '["mean_absolute_error"]'
+        annotations:
+          editor.position: '{"x":40,"y":500,"width":180,"height":54}'
+      Upload Tensorflow model to Google Cloud Vertex AI:
+        componentRef:
+          digest: 2e45263ff640b1a688e359b6936e27a81b2407749a84f340af2aa5547e0cb92c
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/workaround_for_buggy_KFPv2_compiler/component.yaml
+        arguments:
+          model:
+            taskOutput:
+              outputName: trained_model
+              taskId: Train model using Keras on CSV
+              type: TensorflowSavedModel
+        annotations:
+          editor.position: '{"x":40,"y":630,"width":180,"height":54}'
+      Predict with TensorFlow model on CSV data:
+        componentRef:
+          digest: 921bb1563e93a78233b8acceab87055b9154ccf5595d056028cf0396ca224cd4
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/59c759ce6f543184e30db6817d2a703879bc0f39/components/tensorflow/Predict/on_CSV/component.yaml
+        arguments:
+          dataset:
+            taskOutput:
+              outputName: split_2
+              taskId: Split rows into subsets
+              type: CSV
+          model:
+            taskOutput:
+              outputName: trained_model
+              taskId: Train model using Keras on CSV
+              type: TensorflowSavedModel
+          label_column_name: tips
+        annotations:
+          editor.position: '{"x":240,"y":630,"width":180,"height":54}'
+    outputValues: {}

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_regression_model_using_TensorFlow_and_import_to_Vertex_AI/pipeline.py

@@ -0,0 +1,97 @@
+# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
+from kfp import components
+
+# %% Loading components
+download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml")
+select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml")
+fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
+split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
+create_fully_connected_tensorflow_network_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/9ca0f9eecf5f896f65b8538bbd809747052617d1/components/tensorflow/Create_fully_connected_network/component.yaml")
+train_model_using_Keras_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c504a4010348c50eaaf6d4337586ccc008f4dcef/components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml")
+predict_with_TensorFlow_model_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/59c759ce6f543184e30db6817d2a703879bc0f39/components/tensorflow/Predict/on_CSV/component.yaml")
+upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
+deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/workaround_for_buggy_KFPv2_compiler/component.yaml")
+
+# %% Pipeline definition
+def train_tabular_regression_model_using_Tensorflow_pipeline():
+    dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
+    feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]  # Excluded "trip_total"
+    label_column = "tips"
+    training_set_fraction = 0.8
+    # Deploying the model might incur additional costs over time
+    deploy_model = False
+
+    all_columns = [label_column] + feature_columns
+
+    dataset = download_from_gcs_op(
+        gcs_path=dataset_gcs_uri
+    ).outputs["Data"]
+
+    dataset = select_columns_using_Pandas_on_CSV_data_op(
+        table=dataset,
+        column_names=all_columns,
+    ).outputs["transformed_table"]
+
+    dataset = fill_all_missing_values_using_Pandas_on_CSV_data_op(
+        table=dataset,
+        replacement_value="0",
+        # # Optional:
+        # column_names=None,  # =[...]
+    ).outputs["transformed_table"]
+
+    split_task = split_rows_into_subsets_op(
+        table=dataset,
+        fraction_1=training_set_fraction,
+    )
+    training_data = split_task.outputs["split_1"]
+    testing_data = split_task.outputs["split_2"]
+
+    network = create_fully_connected_tensorflow_network_op(
+        input_size=len(feature_columns),
+        # Optional:
+        hidden_layer_sizes=[10],
+        activation_name="elu",
+        # output_activation_name=None,
+        # output_size=1,
+    ).outputs["model"]
+
+    model = train_model_using_Keras_on_CSV_op(
+        training_data=training_data,
+        model=network,
+        label_column_name=label_column,
+        # Optional:
+        #loss_function_name="mean_squared_error",
+        number_of_epochs=10,
+        #learning_rate=0.1,
+        #optimizer_name="Adadelta",
+        #optimizer_parameters={},
+        #batch_size=32,
+        metric_names=["mean_absolute_error"],
+        #random_seed=0,
+    ).outputs["trained_model"]
+
+    predictions = predict_with_TensorFlow_model_on_CSV_data_op(
+        dataset=testing_data,
+        model=model,
+        # label_column_name needs to be set when doing prediction on a dataset that has labels
+        label_column_name=label_column,
+        # Optional:
+        # batch_size=1000,
+    ).outputs["predictions"]
+
+    vertex_model_name = upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op(
+        model=model,
+    ).outputs["model_name"]
+
+    # Deploying the model might incur additional costs over time
+    if deploy_model:
+        vertex_endpoint_name = deploy_model_to_endpoint_op(
+            model_name=vertex_model_name,
+        ).outputs["endpoint_name"]
+
+pipeline_func=train_tabular_regression_model_using_Tensorflow_pipeline
+
+# %% Pipeline submission
+if __name__ == '__main__':
+    from google.cloud import aiplatform
+    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_regression_model_using_XGBoost_and_import_to_Vertex_AI/pipeline.component.yaml

@@ -0,0 +1,99 @@
+name: Train tabular regression model using XGBoost pipeline
+metadata:
+  annotations:
+    author: Alexey Volkov <alexey.volkov@ark-kun.com>
+    canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_regression_model_using_XGBoost_and_import_to_Vertex_AI/pipeline.component.yaml
+    sdk: https://cloud-pipelines.net/pipeline-editor/
+implementation:
+  graph:
+    tasks:
+      Download from GCS:
+        componentRef:
+          digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
+        arguments:
+          GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
+        annotations:
+          editor.position: '{"x":40,"y":40,"width":180,"height":40}'
+      Select columns using Pandas on CSV data:
+        componentRef:
+          digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: Data
+              taskId: Download from GCS
+          column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
+        annotations:
+          editor.position: '{"x":40,"y":140,"width":180,"height":54}'
+      Fill all missing values using Pandas on CSV data:
+        componentRef:
+          digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: transformed_table
+              taskId: Select columns using Pandas on CSV data
+              type: CSV
+          replacement_value: '0'
+        annotations:
+          editor.position: '{"x":40,"y":250,"width":180,"height":54}'
+      Split rows into subsets:
+        componentRef:
+          digest: a609c3c9196484290f24a1174955f95b27f07a7b458aa5cb8cde28866cb2cb46
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: transformed_table
+              taskId: Fill all missing values using Pandas on CSV data
+              type: CSV
+          fraction_1: '0.8'
+        annotations:
+          editor.position: '{"x":170,"y":360,"width":180,"height":40}'
+      Train XGBoost model on CSV:
+        componentRef:
+          digest: 538c5a01eb38deaf532d619f0bbeaff4efc550fe1f0f776fc06791097b68ceac
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/58d3a47f904f32a64af8403330ba7e2134cae46d/components/XGBoost/Train/component.yaml
+        arguments:
+          training_data:
+            taskOutput:
+              outputName: split_1
+              taskId: Split rows into subsets
+              type: CSV
+          label_column_name: tips
+        annotations:
+          editor.position: '{"x":40,"y":480,"width":180,"height":40}'
+      Upload XGBoost model to Google Cloud Vertex AI:
+        componentRef:
+          digest: 5a5a273c403670743820986c03a4175b7cb4595a556524fefcce403656286977
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/workaround_for_buggy_KFPv2_compiler/component.yaml
+        arguments:
+          model:
+            taskOutput:
+              outputName: model
+              taskId: Train XGBoost model on CSV
+              type: XGBoostModel
+        annotations:
+          editor.position: '{"x":40,"y":600,"width":180,"height":54}'
+      Xgboost predict on CSV:
+        componentRef:
+          digest: 0876233a0c7306fefec188bd70f059b46d1fb5aa57be231799570e3bbbdd0d95
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/4694ec97baccf59284c2a1db4aa2250c22291eab/components/XGBoost/Predict/component.yaml
+        arguments:
+          data:
+            taskOutput:
+              outputName: split_2
+              taskId: Split rows into subsets
+              type: CSV
+          model:
+            taskOutput:
+              outputName: model
+              taskId: Train XGBoost model on CSV
+              type: XGBoostModel
+          label_column_name: tips
+        annotations:
+          editor.position: '{"x":240,"y":600,"width":180,"height":40}'
+    outputValues: {}

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_regression_model_using_XGBoost_and_import_to_Vertex_AI/pipeline.py

@@ -0,0 +1,85 @@
+# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
+from kfp import components
+
+# %% Loading components
+download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml")
+select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml")
+fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
+split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
+train_XGBoost_model_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/58d3a47f904f32a64af8403330ba7e2134cae46d/components/XGBoost/Train/component.yaml")
+xgboost_predict_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/4694ec97baccf59284c2a1db4aa2250c22291eab/components/XGBoost/Predict/component.yaml")
+upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
+deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/workaround_for_buggy_KFPv2_compiler/component.yaml")
+
+# %% Pipeline definition
+def train_tabular_regression_model_using_XGBoost_pipeline():
+    dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
+    feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]  # Excluded "trip_total"
+    label_column = "tips"
+    training_set_fraction = 0.8
+    # Deploying the model might incur additional costs over time
+    deploy_model = False
+
+    all_columns = [label_column] + feature_columns
+
+    dataset = download_from_gcs_op(
+        gcs_path=dataset_gcs_uri
+    ).outputs["Data"]
+
+    dataset = select_columns_using_Pandas_on_CSV_data_op(
+        table=dataset,
+        column_names=all_columns,
+    ).outputs["transformed_table"]
+
+    dataset = fill_all_missing_values_using_Pandas_on_CSV_data_op(
+        table=dataset,
+        replacement_value="0",
+        # # Optional:
+        # column_names=None,  # =[...]
+    ).outputs["transformed_table"]
+
+    split_task = split_rows_into_subsets_op(
+        table=dataset,
+        fraction_1=training_set_fraction,
+    )
+    training_data = split_task.outputs["split_1"]
+    testing_data = split_task.outputs["split_2"]
+
+    model = train_XGBoost_model_on_CSV_op(
+        training_data=training_data,
+        label_column_name=label_column,
+        # Optional:
+        #starting_model=None,
+        #num_iterations=10,
+        #booster_params={},
+        #objective="reg:squarederror",
+        #booster="gbtree",
+        #learning_rate=0.3,
+        #min_split_loss=0,
+        #max_depth=6,
+    ).outputs["model"]
+
+    # Predicting on the testing data
+    predictions = xgboost_predict_on_CSV_op(
+        data=testing_data,
+        model=model,
+        # label_column needs to be set when doing prediction on a dataset that has labels
+        label_column_name=label_column,
+    ).outputs["predictions"]
+
+    vertex_model_name = upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op(
+        model=model,
+    ).outputs["model_name"]
+
+    # Deploying the model might incur additional costs over time
+    if deploy_model:
+        vertex_endpoint_name = deploy_model_to_endpoint_op(
+            model_name=vertex_model_name,
+        ).outputs["endpoint_name"]
+
+pipeline_func = train_tabular_regression_model_using_XGBoost_pipeline
+
+# %% Pipeline submission
+if __name__ == '__main__':
+    from google.cloud import aiplatform
+    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_regression_model_using_all_frameworks_and_import_to_Vertex_AI/pipeline.component.yaml

@@ -0,0 +1,238 @@
+name: Train tabular regression model using all frameworks pipeline
+metadata:
+  annotations:
+    author: Alexey Volkov <alexey.volkov@ark-kun.com>
+    canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_regression_model_using_all_frameworks_and_import_to_Vertex_AI/pipeline.component.yaml
+    sdk: https://cloud-pipelines.net/pipeline-editor/
+implementation:
+  graph:
+    tasks:
+      Download from GCS:
+        componentRef:
+          digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
+        arguments:
+          GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
+        annotations:
+          editor.position: '{"x":550,"y":40,"width":180,"height":40}'
+      Select columns using Pandas on CSV data:
+        componentRef:
+          digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: Data
+              taskId: Download from GCS
+          column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
+        annotations:
+          editor.position: '{"x":550,"y":140,"width":180,"height":54}'
+      Fill all missing values using Pandas on CSV data:
+        componentRef:
+          digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: transformed_table
+              taskId: Select columns using Pandas on CSV data
+              type: CSV
+          replacement_value: '0'
+        annotations:
+          editor.position: '{"x":550,"y":250,"width":180,"height":54}'
+      Split rows into subsets:
+        componentRef:
+          digest: a609c3c9196484290f24a1174955f95b27f07a7b458aa5cb8cde28866cb2cb46
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: transformed_table
+              taskId: Fill all missing values using Pandas on CSV data
+              type: CSV
+          fraction_1: '0.8'
+        annotations:
+          editor.position: '{"x":550,"y":360,"width":180,"height":40}'
+      Create fully connected pytorch network:
+        componentRef:
+          digest: d03d8248fd358a0275ec33568ee7dd7dce576cc112b09dfafe2651e4d97e04a9
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1a2ef3eeb77bc278f33cad0dd29008ea2431e191/components/PyTorch/Create_fully_connected_network/component.yaml
+        arguments:
+          input_size: '7'
+          hidden_layer_sizes: '[10]'
+          activation_name: elu
+        annotations:
+          editor.position: '{"x":380,"y":490,"width":180,"height":54}'
+      Create fully connected tensorflow network:
+        componentRef:
+          digest: bfcafbc5ce711b1f69cabf1338212d10d50136a73db9f9f7c984de7b80b4bfb0
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/9ca0f9eecf5f896f65b8538bbd809747052617d1/components/tensorflow/Create_fully_connected_network/component.yaml
+        arguments:
+          input_size: '7'
+          hidden_layer_sizes: '[10]'
+          activation_name: elu
+        annotations:
+          editor.position: '{"x":40,"y":500,"width":180,"height":54}'
+      Train model using Keras on CSV:
+        componentRef:
+          digest: 42ae60c889034dbad74815653e95b4f7d576b5f47f803173e8679c7b54984609
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c504a4010348c50eaaf6d4337586ccc008f4dcef/components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml
+        arguments:
+          training_data:
+            taskOutput:
+              outputName: split_1
+              taskId: Split rows into subsets
+              type: CSV
+          model:
+            taskOutput:
+              outputName: model
+              taskId: Create fully connected tensorflow network
+              type: TensorflowSavedModel
+          label_column_name: tips
+          number_of_epochs: '10'
+          metric_names: '["mean_absolute_error"]'
+        annotations:
+          editor.position: '{"x":40,"y":620,"width":180,"height":54}'
+      Train pytorch model from csv:
+        componentRef:
+          digest: 40f3185eb61e9727f41a4e0c05dd3d3b44bd802aa0f378cfc31756560033949a
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml
+        arguments:
+          model:
+            taskOutput:
+              outputName: model
+              taskId: Create fully connected pytorch network
+              type: PyTorchScriptModule
+          training_data:
+            taskOutput:
+              outputName: split_1
+              taskId: Split rows into subsets
+              type: CSV
+          label_column_name: tips
+        annotations:
+          editor.position: '{"x":380,"y":620,"width":180,"height":40}'
+      Train XGBoost model on CSV:
+        componentRef:
+          digest: 538c5a01eb38deaf532d619f0bbeaff4efc550fe1f0f776fc06791097b68ceac
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/58d3a47f904f32a64af8403330ba7e2134cae46d/components/XGBoost/Train/component.yaml
+        arguments:
+          training_data:
+            taskOutput:
+              outputName: split_1
+              taskId: Split rows into subsets
+              type: CSV
+          label_column_name: tips
+        annotations:
+          editor.position: '{"x":720,"y":620,"width":180,"height":40}'
+      Train linear regression model using scikit learn from CSV:
+        componentRef:
+          digest: c7fe7912ab0d1fb45d201d452e9ce6be5544e7d8c6d229db7a4b931ff58560f3
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/f807e02b54d4886c65a05f40848fd51c72407f40/components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml
+        arguments:
+          dataset:
+            taskOutput:
+              outputName: split_1
+              taskId: Split rows into subsets
+              type: CSV
+          label_column_name: tips
+        annotations:
+          editor.position: '{"x":1030,"y":620,"width":180,"height":54}'
+      Predict with TensorFlow model on CSV data:
+        componentRef:
+          digest: 921bb1563e93a78233b8acceab87055b9154ccf5595d056028cf0396ca224cd4
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/59c759ce6f543184e30db6817d2a703879bc0f39/components/tensorflow/Predict/on_CSV/component.yaml
+        arguments:
+          dataset:
+            taskOutput:
+              outputName: split_2
+              taskId: Split rows into subsets
+              type: CSV
+          model:
+            taskOutput:
+              outputName: trained_model
+              taskId: Train model using Keras on CSV
+              type: TensorflowSavedModel
+          label_column_name: tips
+        annotations:
+          editor.position: '{"x":160,"y":750,"width":180,"height":54}'
+      Create PyTorch Model Archive with base handler:
+        componentRef:
+          digest: 8298b5ee1b0f0879f893add4cf352c8dec7cf9e21bb9db134c91a2d046cdb0ec
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/46d51383e6554b7f3ab4fd8cf614d8c2b422fb22/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml
+        arguments:
+          Model:
+            taskOutput:
+              outputName: trained_model
+              taskId: Train pytorch model from csv
+              type: PyTorchScriptModule
+          Model name: model
+          Model version: '1.0'
+        annotations:
+          editor.position: '{"x":380,"y":750,"width":180,"height":54}'
+      Xgboost predict on CSV:
+        componentRef:
+          digest: 0876233a0c7306fefec188bd70f059b46d1fb5aa57be231799570e3bbbdd0d95
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/4694ec97baccf59284c2a1db4aa2250c22291eab/components/XGBoost/Predict/component.yaml
+        arguments:
+          data:
+            taskOutput:
+              outputName: split_2
+              taskId: Split rows into subsets
+              type: CSV
+          model:
+            taskOutput:
+              outputName: model
+              taskId: Train XGBoost model on CSV
+              type: XGBoostModel
+          label_column_name: tips
+        annotations:
+          editor.position: '{"x":810,"y":750,"width":180,"height":40}'
+      Upload Scikit learn pickle model to Google Cloud Vertex AI:
+        componentRef:
+          digest: 81c91c8d7d21ec97e0872f669d68bd89edea87279d703685db54aa94743bebcd
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/workaround_for_buggy_KFPv2_compiler/component.yaml
+        arguments:
+          model:
+            taskOutput:
+              outputName: model
+              taskId: Train linear regression model using scikit learn from CSV
+              type: ScikitLearnPickleModel
+        annotations:
+          editor.position: '{"x":1030,"y":750,"width":180,"height":70}'
+      Upload Tensorflow model to Google Cloud Vertex AI:
+        componentRef:
+          digest: 2e45263ff640b1a688e359b6936e27a81b2407749a84f340af2aa5547e0cb92c
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/workaround_for_buggy_KFPv2_compiler/component.yaml
+        arguments:
+          model:
+            taskOutput:
+              outputName: trained_model
+              taskId: Train model using Keras on CSV
+              type: TensorflowSavedModel
+        annotations:
+          editor.position: '{"x":40,"y":880,"width":180,"height":54}'
+      Upload PyTorch model archive to Google Cloud Vertex AI:
+        componentRef:
+          digest: 4450212fae7b9001482aca7eb78b28413c205506eccf08a04e7754a8dfa99004
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/workaround_for_buggy_KFPv2_compiler/component.yaml
+        arguments:
+          model_archive:
+            taskOutput:
+              outputName: Model archive
+              taskId: Create PyTorch Model Archive with base handler
+              type: PyTorchModelArchive
+        annotations:
+          editor.position: '{"x":380,"y":880,"width":180,"height":70}'
+      Upload XGBoost model to Google Cloud Vertex AI:
+        componentRef:
+          digest: 5a5a273c403670743820986c03a4175b7cb4595a556524fefcce403656286977
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/workaround_for_buggy_KFPv2_compiler/component.yaml
+        arguments:
+          model:
+            taskOutput:
+              outputName: model
+              taskId: Train XGBoost model on CSV
+              type: XGBoostModel
+        annotations:
+          editor.position: '{"x":720,"y":880,"width":180,"height":54}'
+    outputValues: {}

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_regression_model_using_all_frameworks_and_import_to_Vertex_AI/pipeline.py

@@ -0,0 +1,208 @@
+# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
+from kfp import components
+
+# %% Loading components
+download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml")
+select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml")
+fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
+split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
+
+# TensorFlow
+create_fully_connected_tensorflow_network_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/9ca0f9eecf5f896f65b8538bbd809747052617d1/components/tensorflow/Create_fully_connected_network/component.yaml")
+train_model_using_Keras_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c504a4010348c50eaaf6d4337586ccc008f4dcef/components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml")
+predict_with_TensorFlow_model_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/59c759ce6f543184e30db6817d2a703879bc0f39/components/tensorflow/Predict/on_CSV/component.yaml")
+upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
+
+# PyTorch
+create_fully_connected_pytorch_network_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/1a2ef3eeb77bc278f33cad0dd29008ea2431e191/components/PyTorch/Create_fully_connected_network/component.yaml")
+train_pytorch_model_from_csv_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml")
+create_pytorch_model_archive_with_base_handler_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/46d51383e6554b7f3ab4fd8cf614d8c2b422fb22/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml")
+upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/workaround_for_buggy_KFPv2_compiler/component.yaml")
+
+# XGBoost
+train_XGBoost_model_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/58d3a47f904f32a64af8403330ba7e2134cae46d/components/XGBoost/Train/component.yaml")
+xgboost_predict_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/4694ec97baccf59284c2a1db4aa2250c22291eab/components/XGBoost/Predict/component.yaml")
+upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
+
+# Scikit-learn
+train_linear_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/f807e02b54d4886c65a05f40848fd51c72407f40/components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml")
+upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
+
+# Vertex AI
+deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/workaround_for_buggy_KFPv2_compiler/component.yaml")
+
+# %% Pipeline definition
+def train_tabular_regression_model_using_all_frameworks_pipeline():
+    dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
+    feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]  # Excluded "trip_total"
+    label_column = "tips"
+    training_set_fraction = 0.8
+    # Deploying the model might incur additional costs over time
+    deploy_model = False
+
+    all_columns = [label_column] + feature_columns
+
+    dataset = download_from_gcs_op(
+        gcs_path=dataset_gcs_uri
+    ).outputs["Data"]
+
+    dataset = select_columns_using_Pandas_on_CSV_data_op(
+        table=dataset,
+        column_names=all_columns,
+    ).outputs["transformed_table"]
+
+    dataset = fill_all_missing_values_using_Pandas_on_CSV_data_op(
+        table=dataset,
+        replacement_value="0",
+        # # Optional:
+        # column_names=None,  # =[...]
+    ).outputs["transformed_table"]
+
+    split_task = split_rows_into_subsets_op(
+        table=dataset,
+        fraction_1=training_set_fraction,
+    )
+    training_data = split_task.outputs["split_1"]
+    testing_data = split_task.outputs["split_2"]
+
+    # TensorFlow
+    tensorflow_network = create_fully_connected_tensorflow_network_op(
+        input_size=len(feature_columns),
+        # Optional:
+        hidden_layer_sizes=[10],
+        activation_name="elu",
+        # output_activation_name=None,
+        # output_size=1,
+    ).outputs["model"]
+
+    tensorflow_model = train_model_using_Keras_on_CSV_op(
+        training_data=training_data,
+        model=tensorflow_network,
+        label_column_name=label_column,
+        # Optional:
+        #loss_function_name="mean_squared_error",
+        number_of_epochs=10,
+        #learning_rate=0.1,
+        #optimizer_name="Adadelta",
+        #optimizer_parameters={},
+        #batch_size=32,
+        metric_names=["mean_absolute_error"],
+        #random_seed=0,
+    ).outputs["trained_model"]
+
+    tensorflow_predictions = predict_with_TensorFlow_model_on_CSV_data_op(
+        dataset=testing_data,
+        model=tensorflow_model,
+        # label_column_name needs to be set when doing prediction on a dataset that has labels
+        label_column_name=label_column,
+        # Optional:
+        # batch_size=1000,
+    ).outputs["predictions"]
+
+    tensorflow_vertex_model_name = upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op(
+        model=tensorflow_model,
+    ).outputs["model_name"]
+
+    # Deploying the model might incur additional costs over time
+    if deploy_model:
+        tensorflow_vertex_endpoint_name = deploy_model_to_endpoint_op(
+            model_name=tensorflow_vertex_model_name,
+        ).outputs["endpoint_name"]
+
+    # PyTorch
+    pytorch_network = create_fully_connected_pytorch_network_op(
+        input_size=len(feature_columns),
+        # Optional:
+        hidden_layer_sizes=[10],
+        activation_name="elu",
+        # output_activation_name=None,
+        # output_size=1,
+    ).outputs["model"]
+
+    pytorch_model = train_pytorch_model_from_csv_op(
+        model=pytorch_network,
+        training_data=training_data,
+        label_column_name=label_column,
+        # Optional:
+        #loss_function_name="mse_loss",
+        #number_of_epochs=1,
+        #learning_rate=0.1,
+        #optimizer_name="Adadelta",
+        #optimizer_parameters={},
+        #batch_size=32,
+        #batch_log_interval=100,
+        #random_seed=0,
+    ).outputs["trained_model"]
+
+    pytorch_model_archive = create_pytorch_model_archive_with_base_handler_op(
+        model=pytorch_model,
+        # Optional:
+        # model_name="model",
+        # model_version="1.0",
+    ).outputs["Model archive"]
+
+    pytorch_vertex_model_name = upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op(
+        model_archive=pytorch_model_archive,
+    ).outputs["model_name"]
+
+    # Deploying the model might incur additional costs over time
+    if deploy_model:
+        pytorch_vertex_endpoint_name = deploy_model_to_endpoint_op(
+            model_name=pytorch_vertex_model_name,
+        ).outputs["endpoint_name"]
+
+    # XGBoost
+    xgboost_model = train_XGBoost_model_on_CSV_op(
+        training_data=training_data,
+        label_column_name=label_column,
+        # Optional:
+        #starting_model=None,
+        #num_iterations=10,
+        #booster_params={},
+        #objective="reg:squarederror",
+        #booster="gbtree",
+        #learning_rate=0.3,
+        #min_split_loss=0,
+        #max_depth=6,
+    ).outputs["model"]
+
+    # Predicting on the testing data
+    xgboost_predictions = xgboost_predict_on_CSV_op(
+        data=testing_data,
+        model=xgboost_model,
+        # label_column needs to be set when doing prediction on a dataset that has labels
+        label_column_name=label_column,
+    ).outputs["predictions"]
+
+    xgboost_vertex_model_name = upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op(
+        model=xgboost_model,
+    ).outputs["model_name"]
+
+    # Deploying the model might incur additional costs over time
+    if deploy_model:
+        xgboost_vertex_endpoint_name = deploy_model_to_endpoint_op(
+            model_name=xgboost_vertex_model_name,
+        ).outputs["endpoint_name"]
+
+    # Scikit-learn
+    sklearn_model = train_linear_regression_model_using_scikit_learn_from_CSV_op(
+        dataset=training_data,
+        label_column_name=label_column,
+    ).outputs["model"]
+
+    sklearn_vertex_model_name = upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op(
+        model=sklearn_model,
+    ).outputs["model_name"]
+
+    # Deploying the model might incur additional costs over time
+    if deploy_model:
+        sklearn_vertex_endpoint_name = deploy_model_to_endpoint_op(
+            model_name=sklearn_vertex_model_name,
+        ).outputs["endpoint_name"]
+
+pipeline_func=train_tabular_regression_model_using_all_frameworks_pipeline
+
+# %% Pipeline submission
+if __name__ == '__main__':
+    from google.cloud import aiplatform
+    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

community-content/pytorch_efficient_training/README.md

@@ -0,0 +1,29 @@
+# PyTorch Efficient Training Examples
+
+This folder provides PyTorch efficient training examples using ResNet-50 and ImageNet data.
+
+## Requirements
+
+```shell
+pip install --upgrade pip
+pip install -r requirements.txt
+```
+
+## Description
+
+* resnet.py - Train ResNet-50 on single GPU.
+* resnet_dp.py - Train ResNet-50 on single node multiple GPUs with `DataParallel` strategy.
+* resnet_ddp.py - Train ResNet-50 on single node multiple GPUs with `DistributedDataParallel` strategy.
+* resnet_ddp_wds.py - Train ResNet-50 on single node multiple GPUs with `DistributedDataParallel` strategy and `Webdataset`.
+* shard_imagenet.py - Shard ImagNet individual files into `tar` files.
+
+## Benchmark
+
+When run the benchmark on Nvidia T4 GPUs using ImageNet validation dataset, you can get the result like:
+Strategy              | Seconds/Epoch - Local Data | Seconds/Epoch - Cloud Data
+--------------------- | -------------------------- | --------------------------
+On 1 GPU              | 489                        | 804 (2x slower)
+On 4 GPUs (DP)        | 157                        | 738 (5x slower)
+On 4 GPUs (DDP)       | 134                        | 432 (3x slower)
+On 4 GPUs (DDP + WDS) | 131                        | 133 (same performance)
+

community-content/pytorch_efficient_training/requirements.txt

@@ -0,0 +1 @@
+webdataset == 0.2.26

community-content/pytorch_efficient_training/resnet.py

@@ -0,0 +1,197 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the \"License\");
+# you may not use this file except in compliance with the License.\n",
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an \"AS IS\" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Train resnet on single GPU."""
+
+import argparse
+import time
+
+from PIL import Image
+import torch
+from torch import nn
+import torchmetrics
+import torchvision
+from torchvision.models import resnet50
+
+
+class ImageFolder(torchvision.datasets.ImageFolder):
+  """Class for loading imagenet."""
+
+  def __init__(self, image_list_file, transform=None, target_transform=None):
+    self.samples = self._make_dataset(image_list_file)
+    self.loader = self._loader
+
+    self.imgs = self.samples
+    self.targets = [s[1] for s in self.samples]
+
+    self.transform = transform
+    self.target_transform = target_transform
+
+  def _make_dataset(self, image_list_file):
+    items = []
+    with open(image_list_file, 'r') as f:
+      for line in f:
+        item = line.strip().split(' ')
+        items.append((item[0], int(item[1])))
+    return items
+
+  def _loader(self, image_path):
+    with open(image_path, 'rb') as f:
+      img = Image.open(f)
+      img = img.convert('RGB')
+      return img
+
+
+def train(model, device, dataloader, optimizer):
+  model.train()
+  for image, target in dataloader:
+    image, target = image.to(device), target.to(device)
+    pred = model(image)
+    # pred.shape (N, C), target.shape (N)
+    loss = nn.functional.cross_entropy(pred, target)
+    optimizer.zero_grad()
+    loss.backward()
+    optimizer.step()
+  return loss
+
+
+def evaluate(model, device, dataloader, metric):
+  model.eval()
+  with torch.no_grad():
+    for image, target in dataloader:
+      image, target = image.to(device), target.to(device)
+      pred = model(image)
+      metric.update(pred, target)
+  accuracy = metric.compute()
+  metric.reset()
+  return accuracy
+
+
+def run_training(args):
+  """Run training and evaluation."""
+  # Create model.
+  model = resnet50(weights=None)
+  model = model.to(args.device)
+
+  # Create train dataloader.
+  train_dataset = ImageFolder(
+      image_list_file=args.train_data_path,
+      transform=torchvision.transforms.Compose([
+          torchvision.transforms.RandomResizedCrop(224),
+          torchvision.transforms.RandomHorizontalFlip(),
+          torchvision.transforms.ToTensor(),
+          torchvision.transforms.Normalize(
+              mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+      ]))
+  train_dataloader = torch.utils.data.DataLoader(
+      dataset=train_dataset,
+      batch_size=args.train_batch_size,
+      shuffle=True,
+      num_workers=args.dataloader_num_workers,
+      pin_memory=True)
+  print(f'Train dataloader | samples: {len(train_dataloader.dataset)}, '
+        f'num workers: {train_dataloader.num_workers}, '
+        f'batch size: {args.train_batch_size}, '
+        f'batches/epoch: {len(train_dataloader)}')
+
+  # Create eval dataloader.
+  eval_dataset = ImageFolder(
+      image_list_file=args.eval_data_path,
+      transform=torchvision.transforms.Compose([
+          torchvision.transforms.Resize(256),
+          torchvision.transforms.CenterCrop(224),
+          torchvision.transforms.ToTensor(),
+          torchvision.transforms.Normalize(
+              mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+      ]))
+  eval_dataloader = torch.utils.data.DataLoader(
+      dataset=eval_dataset,
+      batch_size=args.eval_batch_size,
+      shuffle=False,
+      num_workers=args.dataloader_num_workers,
+      pin_memory=True,
+      drop_last=True)
+  print(f'Eval dataloader | samples: {len(eval_dataloader.dataset)}, '
+        f'num workers: {eval_dataloader.num_workers}, '
+        f'batch size: {args.eval_batch_size}, '
+        f'batches/epoch: {len(eval_dataloader)}')
+
+  # Optimizer.
+  optimizer = torch.optim.SGD(model.parameters(), 0.1)
+
+  # Main loop.
+  metric = torchmetrics.classification.Accuracy(top_k=1).to(args.device)
+  for epoch in range(1, args.epochs + 1):
+    print(f'Running epoch {epoch}')
+
+    start = time.time()
+    train(model, args.device, train_dataloader, optimizer)
+    end = time.time()
+    print(f'Training finished in {(end - start):>0.3f} seconds')
+
+    start = time.time()
+    evaluate(model, args.device, eval_dataloader, metric)
+    end = time.time()
+    print(f'Evaluation finished in {(end - start):>0.3f} seconds')
+  print('Done')
+
+
+def create_args():
+  """Create main args."""
+  parser = argparse.ArgumentParser(
+      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+  parser.add_argument(
+      '--epochs',
+      default=1,
+      type=int,
+      help='number of total epochs to run')
+  parser.add_argument(
+      '--dataloader_num_workers',
+      default=2,
+      type=int,
+      help='number of workders for dataloader')
+  parser.add_argument(
+      '--train_data_path',
+      default='',
+      type=str,
+      help='path to training data')
+  parser.add_argument(
+      '--train_batch_size',
+      default=32,
+      type=int,
+      help='batch size for training')
+  parser.add_argument(
+      '--eval_data_path',
+      default='',
+      type=str,
+      help='path to evaluation data')
+  parser.add_argument(
+      '--eval_batch_size',
+      default=32,
+      type=int,
+      help='batch size for evaluation')
+  args = parser.parse_args()
+  return args
+
+
+def main():
+  args = create_args()
+  args.device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
+
+  print('Launch job on 1 GPU')
+  run_training(args)
+
+
+if __name__ == '__main__':
+  main()

community-content/pytorch_efficient_training/resnet_ddp.py

@@ -0,0 +1,234 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the \"License\");
+# you may not use this file except in compliance with the License.\n",
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an \"AS IS\" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Train resnet on multiple GPUs with DDP."""
+
+import argparse
+import os
+import time
+
+from PIL import Image
+import torch
+from torch import nn
+import torch.distributed as dist
+import torch.multiprocessing as mp
+import torchmetrics
+import torchvision
+from torchvision.models import resnet50
+
+
+class ImageFolder(torchvision.datasets.ImageFolder):
+  """Class for loading imagenet."""
+
+  def __init__(self, image_list_file, transform=None, target_transform=None):
+    self.samples = self._make_dataset(image_list_file)
+    self.loader = self._loader
+
+    self.imgs = self.samples
+    self.targets = [s[1] for s in self.samples]
+
+    self.transform = transform
+    self.target_transform = target_transform
+
+  def _make_dataset(self, image_list_file):
+    items = []
+    with open(image_list_file, 'r') as f:
+      for line in f:
+        item = line.strip().split(' ')
+        items.append((item[0], int(item[1])))
+    return items
+
+  def _loader(self, image_path):
+    with open(image_path, 'rb') as f:
+      img = Image.open(f)
+      img = img.convert('RGB')
+      return img
+
+
+def train(model, device, dataloader, optimizer):
+  model.train()
+  for image, target in dataloader:
+    image = image.to(device, non_blocking=True)
+    target = target.to(device, non_blocking=True)
+    pred = model(image)
+    # pred.shape (N, C), target.shape (N)
+    loss = nn.functional.cross_entropy(pred, target)
+    optimizer.zero_grad()
+    loss.backward()
+    optimizer.step()
+  return loss
+
+
+def evaluate(model, device, dataloader, metric):
+  model.eval()
+  with torch.no_grad():
+    for image, target in dataloader:
+      image = image.to(device, non_blocking=True)
+      target = target.to(device, non_blocking=True)
+      pred = model(image)
+      metric.update(pred, target)
+  accuracy = metric.compute()
+  metric.reset()
+  return accuracy
+
+
+def worker(gpu, args):
+  """Run training and evaluation."""
+  # Init process group.
+  print(f'Initiating process {gpu}')
+  dist.init_process_group(
+      backend='nccl',
+      init_method='env://',
+      world_size=args.gpus,
+      rank=gpu)
+
+  # Create model.
+  model = resnet50(weights=None)
+  torch.cuda.set_device(gpu)
+  model.to(args.device)
+  model = torch.nn.SyncBatchNorm.convert_sync_batchnorm(model)
+  model = nn.parallel.DistributedDataParallel(model, device_ids=[gpu])
+
+  # Create train dataloader.
+  train_dataset = ImageFolder(
+      image_list_file=args.train_data_path,
+      transform=torchvision.transforms.Compose([
+          torchvision.transforms.RandomResizedCrop(224),
+          torchvision.transforms.RandomHorizontalFlip(),
+          torchvision.transforms.ToTensor(),
+          torchvision.transforms.Normalize(
+              mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+      ]))
+  train_sampler = torch.utils.data.distributed.DistributedSampler(
+      train_dataset, num_replicas=args.gpus, rank=gpu)
+  train_dataloader = torch.utils.data.DataLoader(
+      dataset=train_dataset,
+      batch_size=args.train_batch_size,
+      shuffle=False,
+      num_workers=args.dataloader_num_workers,
+      pin_memory=True,
+      sampler=train_sampler)
+  if gpu == 0:
+    print(f'Train dataloader | samples: {len(train_dataloader.dataset)}, '
+          f'num workers: {train_dataloader.num_workers}, '
+          f'global batch size: {args.train_batch_size * args.gpus}, '
+          f'batches/epoch: {len(train_dataloader)}')
+
+  # Create eval dataloader.
+  eval_dataset = ImageFolder(
+      image_list_file=args.eval_data_path,
+      transform=torchvision.transforms.Compose([
+          torchvision.transforms.Resize(256),
+          torchvision.transforms.CenterCrop(224),
+          torchvision.transforms.ToTensor(),
+          torchvision.transforms.Normalize(
+              mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+      ]))
+  eval_sampler = torch.utils.data.distributed.DistributedSampler(
+      eval_dataset, num_replicas=args.gpus, rank=gpu)
+  eval_dataloader = torch.utils.data.DataLoader(
+      dataset=eval_dataset,
+      batch_size=args.eval_batch_size,
+      shuffle=False,
+      num_workers=args.dataloader_num_workers,
+      pin_memory=True,
+      drop_last=True,
+      sampler=eval_sampler)
+  if gpu == 0:
+    print(f'Eval dataloader | samples: {len(eval_dataloader.dataset)}, '
+          f'num workers: {eval_dataloader.num_workers}, '
+          f'batch size: {args.eval_batch_size}, '
+          f'batches/epoch: {len(eval_dataloader)}')
+
+  # Optimizer.
+  optimizer = torch.optim.SGD(model.parameters(), 0.1)
+
+  # Main loop.
+  metric = torchmetrics.classification.Accuracy(top_k=1).to(args.device)
+  for epoch in range(1, args.epochs + 1):
+    if gpu == 0:
+      print(f'Running epoch {epoch}')
+    train_sampler.set_epoch(epoch)
+
+    start = time.time()
+    train(model, args.device, train_dataloader, optimizer)
+    end = time.time()
+    if gpu == 0:
+      print(f'Training finished in {(end - start):>0.3f} seconds')
+
+    start = time.time()
+    evaluate(model, args.device, eval_dataloader, metric)
+    end = time.time()
+    if gpu == 0:
+      print(f'Evaluation finished in {(end - start):>0.3f} seconds')
+
+  if gpu == 0:
+    print('Done')
+
+
+def create_args():
+  """Create main args."""
+  parser = argparse.ArgumentParser(
+      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+  parser.add_argument(
+      '--gpus',
+      default=4,
+      type=int,
+      help='number of gpus to use')
+  parser.add_argument(
+      '--epochs',
+      default=1,
+      type=int,
+      help='number of total epochs to run')
+  parser.add_argument(
+      '--dataloader_num_workers',
+      default=2,
+      type=int,
+      help='number of workders for dataloader')
+  parser.add_argument(
+      '--train_data_path',
+      default='',
+      type=str,
+      help='path to training data')
+  parser.add_argument(
+      '--train_batch_size',
+      default=32,
+      type=int,
+      help='batch size for training per gpu')
+  parser.add_argument(
+      '--eval_data_path',
+      default='',
+      type=str,
+      help='path to evaluation data')
+  parser.add_argument(
+      '--eval_batch_size',
+      default=32,
+      type=int,
+      help='batch size for evaluation per gpu')
+  args = parser.parse_args()
+  return args
+
+
+def main():
+  args = create_args()
+  os.environ['MASTER_ADDR'] = 'localhost'
+  os.environ['MASTER_PORT'] = '8888'
+
+  args.device = 'cuda' if torch.cuda.is_available() else 'cpu'
+  print(f'Launch job on {args.gpus} GPUs with DDP')
+  mp.spawn(worker, nprocs=args.gpus, args=(args,))
+
+
+if __name__ == '__main__':
+  main()

community-content/pytorch_efficient_training/resnet_ddp_wds.py

@@ -0,0 +1,249 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the \"License\");
+# you may not use this file except in compliance with the License.\n",
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an \"AS IS\" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Train resnet on multiple GPUs with DDP."""
+
+import argparse
+import functools
+import itertools
+import math
+import os
+import time
+
+import torch
+from torch import nn
+import torch.distributed as dist
+import torch.multiprocessing as mp
+import torchmetrics
+from torchvision.models import resnet50
+from torchvision.transforms import transforms
+import webdataset as wds
+
+
+def wds_split(src, rank, world_size):
+  """Shards split function for webdataset."""
+  # The context of caller of this function is within multiple processes
+  # (by DDP world_size) and multiple workers (by dataloader_num_workers).
+  # So we totally have (world_size * num_workers) workers for processing data.
+  # NOTE: Raw data should be sharded to enough shards to make sure one process
+  # can handle at least one shard, otherwise the process may hang.
+  worker_id = 0
+  num_workers = 1
+  worker_info = torch.utils.data.get_worker_info()
+  if worker_info:
+    worker_id = worker_info.id
+    num_workers = worker_info.num_workers
+  for s in itertools.islice(src, rank * num_workers + worker_id, None,
+                            world_size * num_workers):
+    yield s
+
+
+def identity(x):
+  return x
+
+
+def create_wds_dataloader(rank, args, mode):
+  """Create webdataset dataset and dataloader."""
+  if mode == 'train':
+    transform = transforms.Compose([
+        transforms.RandomResizedCrop(224),
+        transforms.RandomHorizontalFlip(),
+        transforms.ToTensor(),
+        transforms.Normalize(
+            mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+    ])
+    data_path = args.train_data_path
+    data_size = args.train_data_size
+    batch_size_local = args.train_batch_size
+    batch_size_global = args.train_batch_size * args.gpus
+    # Since webdataset disallows partial batch, we pad the last batch for train.
+    batches = int(math.ceil(data_size / batch_size_global))
+  else:
+    transform = transforms.Compose([
+        transforms.Resize(256),
+        transforms.CenterCrop(224),
+        transforms.ToTensor(),
+        transforms.Normalize(
+            mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+    ])
+    data_path = args.eval_data_path
+    data_size = args.eval_data_size
+    batch_size_local = args.eval_batch_size
+    batch_size_global = args.eval_batch_size * args.gpus
+    # Since webdataset disallows partial batch, we drop the last batch for eval.
+    batches = int(data_size / batch_size_global)
+
+  dataset = wds.DataPipeline(
+      wds.SimpleShardList(data_path),
+      functools.partial(wds_split, rank=rank, world_size=args.gpus),
+      wds.tarfile_to_samples(),
+      wds.decode('pil'),
+      wds.to_tuple('jpg;png;jpeg cls'),
+      wds.map_tuple(transform, identity),
+      wds.batched(batch_size_local, partial=False),
+  )
+  num_workers = args.dataloader_num_workers
+  dataloader = wds.WebLoader(
+      dataset=dataset,
+      batch_size=None,
+      shuffle=False,
+      num_workers=num_workers,
+      persistent_workers=True if num_workers > 0 else False,
+      pin_memory=True).repeat(nbatches=batches)
+  print(f'{mode} dataloader | samples: {data_size}, '
+        f'num_workers: {num_workers}, '
+        f'local batch size: {batch_size_local}, '
+        f'global batch size: {batch_size_global}, '
+        f'batches: {batches}')
+  return dataloader
+
+
+def train(model, device, dataloader, optimizer):
+  model.train()
+  for image, target in dataloader:
+    image = image.to(device, non_blocking=True)
+    target = target.to(device, non_blocking=True)
+    pred = model(image)
+    # pred.shape (N, C), target.shape (N)
+    loss = nn.functional.cross_entropy(pred, target)
+    optimizer.zero_grad()
+    loss.backward()
+    optimizer.step()
+  return loss
+
+
+def evaluate(model, device, dataloader, metric):
+  model.eval()
+  with torch.no_grad():
+    for image, target in dataloader:
+      image = image.to(device, non_blocking=True)
+      target = target.to(device, non_blocking=True)
+      pred = model(image)
+      metric.update(pred, target)
+  accuracy = metric.compute()
+  metric.reset()
+  return accuracy
+
+
+def worker(gpu, args):
+  """Run training and evaluation."""
+  # Init process group.
+  print(f'Initiating process {gpu}')
+  dist.init_process_group(
+      backend='nccl',
+      init_method='env://',
+      world_size=args.gpus,
+      rank=gpu)
+
+  # Create model.
+  model = resnet50(weights=None)
+  torch.cuda.set_device(gpu)
+  model.to(args.device)
+  model = torch.nn.SyncBatchNorm.convert_sync_batchnorm(model)
+  model = nn.parallel.DistributedDataParallel(model, device_ids=[gpu])
+
+  # Create dataloader.
+  train_dataloader = create_wds_dataloader(gpu, args, 'train')
+  eval_dataloader = create_wds_dataloader(gpu, args, 'eval')
+
+  # Optimizer.
+  optimizer = torch.optim.SGD(model.parameters(), 0.1)
+
+  # Main loop.
+  metric = torchmetrics.classification.Accuracy(top_k=1).to(args.device)
+  for epoch in range(1, args.epochs + 1):
+    if gpu == 0:
+      print(f'Running epoch {epoch}')
+
+    start = time.time()
+    train(model, args.device, train_dataloader, optimizer)
+    end = time.time()
+    if gpu == 0:
+      print(f'Training finished in {(end - start):>0.3f} seconds')
+
+    start = time.time()
+    evaluate(model, args.device, eval_dataloader, metric)
+    end = time.time()
+    if gpu == 0:
+      print(f'Evaluation finished in {(end - start):>0.3f} seconds')
+
+  if gpu == 0:
+    print('Done')
+
+
+def create_args():
+  """Create main args."""
+  parser = argparse.ArgumentParser(
+      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+  parser.add_argument(
+      '--gpus',
+      default=4,
+      type=int,
+      help='number of gpus to use')
+  parser.add_argument(
+      '--epochs',
+      default=1,
+      type=int,
+      help='number of total epochs to run')
+  parser.add_argument(
+      '--dataloader_num_workers',
+      default=2,
+      type=int,
+      help='number of workders for dataloader')
+  parser.add_argument(
+      '--train_data_path',
+      default='',
+      type=str,
+      help='path to training data')
+  parser.add_argument(
+      '--train_batch_size',
+      default=32,
+      type=int,
+      help='batch size for training per gpu')
+  parser.add_argument(
+      '--train_data_size',
+      default=50000,
+      type=int,
+      help='data size for training')
+  parser.add_argument(
+      '--eval_data_path',
+      default='',
+      type=str,
+      help='path to evaluation data')
+  parser.add_argument(
+      '--eval_batch_size',
+      default=32,
+      type=int,
+      help='batch size for evaluation per gpu')
+  parser.add_argument(
+      '--eval_data_size',
+      default=50000,
+      type=int,
+      help='data size for evaluation')
+  args = parser.parse_args()
+  return args
+
+
+def main():
+  args = create_args()
+  os.environ['MASTER_ADDR'] = 'localhost'
+  os.environ['MASTER_PORT'] = '8888'
+
+  args.device = 'cuda' if torch.cuda.is_available() else 'cpu'
+  print(f'Launch job on {args.gpus} GPUs with DDP')
+  mp.spawn(worker, nprocs=args.gpus, args=(args,))
+
+
+if __name__ == '__main__':
+  main()

community-content/pytorch_efficient_training/resnet_dp.py

@@ -0,0 +1,207 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the \"License\");
+# you may not use this file except in compliance with the License.\n",
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an \"AS IS\" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Train resnet on multiple GPUs with DP."""
+
+import argparse
+import time
+
+from PIL import Image
+import torch
+from torch import nn
+import torchmetrics
+import torchvision
+from torchvision.models import resnet50
+
+
+class ImageFolder(torchvision.datasets.ImageFolder):
+  """Class for loading imagenet."""
+
+  def __init__(self, image_list_file, transform=None, target_transform=None):
+    self.samples = self._make_dataset(image_list_file)
+    self.loader = self._loader
+
+    self.imgs = self.samples
+    self.targets = [s[1] for s in self.samples]
+
+    self.transform = transform
+    self.target_transform = target_transform
+
+  def _make_dataset(self, image_list_file):
+    items = []
+    with open(image_list_file, 'r') as f:
+      for line in f:
+        item = line.strip().split(' ')
+        items.append((item[0], int(item[1])))
+    return items
+
+  def _loader(self, image_path):
+    with open(image_path, 'rb') as f:
+      img = Image.open(f)
+      img = img.convert('RGB')
+      return img
+
+
+def train(model, device, dataloader, optimizer):
+  model.train()
+  for image, target in dataloader:
+    image, target = image.to(device), target.to(device)
+    pred = model(image)
+    # pred.shape (N, C), target.shape (N)
+    loss = nn.functional.cross_entropy(pred, target)
+    optimizer.zero_grad()
+    loss.backward()
+    optimizer.step()
+  return loss
+
+
+def evaluate(model, device, dataloader, metric):
+  model.eval()
+  with torch.no_grad():
+    for image, target in dataloader:
+      image, target = image.to(device), target.to(device)
+      pred = model(image)
+      metric.update(pred, target)
+  accuracy = metric.compute()
+  metric.reset()
+  return accuracy
+
+
+def run_training(args):
+  """Run training and evaluation."""
+  # Create model.
+  model = resnet50(weights=None)
+  model = nn.DataParallel(model)
+  model = model.to(args.device)
+
+  # Create train dataloader.
+  train_dataset = ImageFolder(
+      image_list_file=args.train_data_path,
+      transform=torchvision.transforms.Compose([
+          torchvision.transforms.RandomResizedCrop(224),
+          torchvision.transforms.RandomHorizontalFlip(),
+          torchvision.transforms.ToTensor(),
+          torchvision.transforms.Normalize(
+              mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+      ]))
+  train_dataloader = torch.utils.data.DataLoader(
+      dataset=train_dataset,
+      batch_size=args.train_batch_size,
+      shuffle=True,
+      num_workers=args.dataloader_num_workers,
+      pin_memory=True)
+  print(f'Train dataloader | samples: {len(train_dataloader.dataset)}, '
+        f'num workers: {train_dataloader.num_workers}, '
+        f'global batch size: {args.train_batch_size}, '
+        f'batches/epoch: {len(train_dataloader)}')
+
+  # Create eval dataloader.
+  eval_dataset = ImageFolder(
+      image_list_file=args.eval_data_path,
+      transform=torchvision.transforms.Compose([
+          torchvision.transforms.Resize(256),
+          torchvision.transforms.CenterCrop(224),
+          torchvision.transforms.ToTensor(),
+          torchvision.transforms.Normalize(
+              mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+      ]))
+  eval_dataloader = torch.utils.data.DataLoader(
+      dataset=eval_dataset,
+      batch_size=args.eval_batch_size,
+      shuffle=False,
+      num_workers=args.dataloader_num_workers,
+      pin_memory=True,
+      drop_last=True)
+  print(f'Eval dataloader | samples: {len(eval_dataloader.dataset)}, '
+        f'num workers: {eval_dataloader.num_workers}, '
+        f'global batch size: {args.eval_batch_size}, '
+        f'batches/epoch: {len(eval_dataloader)}')
+
+  # Optimizer.
+  optimizer = torch.optim.SGD(model.parameters(), 0.1)
+
+  # Main loop.
+  metric = torchmetrics.classification.Accuracy(top_k=1).to(args.device)
+  for epoch in range(1, args.epochs + 1):
+    print(f'Running epoch {epoch}')
+
+    start = time.time()
+    train(model, args.device, train_dataloader, optimizer)
+    end = time.time()
+    print(f'Training finished in {(end - start):>0.3f} seconds')
+
+    start = time.time()
+    evaluate(model, args.device, eval_dataloader, metric)
+    end = time.time()
+    print(f'Evaluation finished in {(end - start):>0.3f} seconds')
+  print('Done')
+
+
+def create_args():
+  """Create main args."""
+  parser = argparse.ArgumentParser(
+      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+  parser.add_argument(
+      '--gpus',
+      default=4,
+      type=int,
+      help='number of gpus to use')
+  parser.add_argument(
+      '--epochs',
+      default=1,
+      type=int,
+      help='number of total epochs to run')
+  parser.add_argument(
+      '--dataloader_num_workers',
+      default=2,
+      type=int,
+      help='number of workders for dataloader')
+  parser.add_argument(
+      '--train_data_path',
+      default='',
+      type=str,
+      help='path to training data')
+  parser.add_argument(
+      '--train_batch_size',
+      default=32,
+      type=int,
+      help='batch size for training per gpu')
+  parser.add_argument(
+      '--eval_data_path',
+      default='',
+      type=str,
+      help='path to evaluation data')
+  parser.add_argument(
+      '--eval_batch_size',
+      default=32,
+      type=int,
+      help='batch size for evaluation per gpu')
+  args = parser.parse_args()
+  return args
+
+
+def main():
+  args = create_args()
+
+  args.device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
+  args.train_batch_size *= args.gpus
+  args.eval_batch_size *= args.gpus
+  args.dataloader_num_workers *= args.gpus
+
+  print(f'Launch job on {args.gpus} GPU with nn.DataParallel')
+  run_training(args)
+
+
+if __name__ == '__main__':
+  main()

community-content/pytorch_efficient_training/shard_imagenet.py

@@ -0,0 +1,98 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the \"License\");
+# you may not use this file except in compliance with the License.\n",
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an \"AS IS\" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+r"""Main function to shard ImageNet dataset.
+
+Example usage:
+python3 -u shard_imagenet.py \
+--image_list_file=/home/jupyter/data/imagenet/train_list.txt \
+--output_pattern=/home/jupyter/data/imagenet/validation-%06d.tar
+"""
+
+import argparse
+import os
+import random
+import webdataset as wds  # version: 0.2.26
+
+
+# NOTE: only supports writing to local path,
+# need gcsfuse mounting if want to write to gcs bucket.
+def write_shards(args):
+  """Shard individual data files."""
+  output_dir = os.path.dirname(args.output_pattern)
+  if not os.path.isdir(output_dir):
+    os.makedirs(output_dir)
+
+  items = []
+  # Image list file is a text file, each line is a pair (image_path, label).
+  with open(args.image_list_file, 'r') as f:
+    for line in f:
+      item = line.strip().split(' ')
+      items.append((item[0], int(item[1])))
+  # Shuffle items to avoid any large sequences of a single class
+  # in the dataset.
+  random.shuffle(items)
+
+  def _read_image(image_path):
+    with open(image_path, 'rb') as f:
+      return f.read()
+
+  with wds.ShardWriter(pattern=args.output_pattern,
+                       maxcount=args.max_images_per_shard,
+                       maxsize=args.max_bytes_per_shard) as sink:
+    for i, (image_path, target) in enumerate(items):
+      key = str(i)
+      image = _read_image(image_path)
+      sample = {'__key__': key, 'jpg': image, 'cls': target}
+      sink.write(sample)
+    if len(items) != sink.total:
+      raise ValueError('Items read {} != items written {}'.format(
+          len(items), sink.total))
+
+
+def create_args():
+  """Creates arg parser."""
+  parser = argparse.ArgumentParser(
+      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+  parser.add_argument(
+      '--image_list_file',
+      default='',
+      type=str,
+      help='path to image list file')
+  parser.add_argument(
+      '--output_pattern',
+      default='',
+      type=str,
+      help='the pattern for output shards, like /path/to/train-%06d.tar')
+  parser.add_argument(
+      '--max_images_per_shard',
+      default=10 * 1024,
+      type=int,
+      help='max number of images per shard')
+  parser.add_argument(
+      '--max_bytes_per_shard',
+      default=300 * 1024 * 1024,
+      type=int,
+      help='max bytes per shard')
+  args = parser.parse_args()
+  return args
+
+
+def main():
+  args = create_args()
+  write_shards(args)
+
+
+if __name__ == '__main__':
+  main()

notebooks/community/CODEOWNERS

@@ -12,7 +12,6 @@
 
 /managed_notebooks/
 /bigquery_ml/ @polong
-/sdk/SDK_FBProphet_Forecasting_Online.ipynb @brianchunkang
 /pipelines/google_cloud_pipeline_components_TPU_model_train_upload_deploy.ipynb @brianchunkang
 /explainable_ai/SDK_Custom_Container_XAI.ipynb @brianchunkang
 /matching_engine/sdk_matching_engine_for_indexing.ipynb @ivanmkc
@@ -23,6 +22,7 @@
 /feature_store @nayaknishant @morgandu
 /prediction @googleapis/vertex-prediction-team
 /vertex_endpoints/tf_hub_obj_detection/deploy_tfhub_object_detection_on_vertex_endpoints.ipynb @entrpn
+/vertex_endpoints/find_ideal_machine_type/find_ideal_machine_type/find_ideal_machine_type.ipynb @entrpn
 /vertex_endpoints/nvidia-triton/nvidia-triton-custom-container-prediction.ipynb @RajeshThallam
 /vertex_endpoints/optimized_tensorflow_runtime @vlasenkoalexey
 /notebooks/community/ml_ops/stage2/get_started_with_visionapi_and_automl.ipynb @mansari
@@ -34,3 +34,4 @@
 /notebooks/community/vertex-ai-samples/notebooks/community/model_registry/vertex_ai_model_registry_bqml_custom_model_versioning.ipynb @inardini
 /notebooks/community/vertex-ai-samples/notebooks/community/model_registry/vertex_ai_model_registry_automl_model_versioning.ipynb @inardini
 /notebooks/community/vizier/conversions_vertex_vizier_and_open_source_vizier.ipynb @halio-g
+/notebooks/community/experiments/vertex_ai_model_experimentation.ipynb @inardini @asobran

notebooks/community/ml_ops/stage2/get_started_vertex_hpt_r_kernel.ipynb

@@ -48,7 +48,7 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://github.com/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_hpt_r_kernel.ipynb\">\n",
+        "<a href=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_hpt_r_kernel.ipynb target='_blank'>",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",

notebooks/community/ml_ops/stage2/get_started_with_vertex_experiments_autologging_xgboost.ipynb

@@ -0,0 +1,929 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "copyright"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2022 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "title"
+      },
+      "source": [
+        "# E2E ML on GCP: MLOps stage 2 : Get started with autologging using Vertex AI Experiments for XGBoost models\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_with_vertex_experiments_autologging_xgboost.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_with_vertex_experiments_autologging_xgboost.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage2/get_started_with_vertex_experiments_autologging_xgboost.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "  </td>\n",
+        "</table>\n",
+        "<br/><br/><br/>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "overview:automl"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "\n",
+        "This tutorial demonstrates how to use the `Vertex AI Experiments` with DIY code to implement automatic logging of parameters and metrics for experiments."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "objective:automl,training,batch_prediction"
+      },
+      "source": [
+        "### Objective\n",
+        "\n",
+        "In this tutorial, you learn how to create an experiment for training an XGBoost model, and automatically log parameters and metrics using the enclosed do-it-yourself (DIY) code.\n",
+        "\n",
+        "This tutorial uses the following Google Cloud ML services and resources:\n",
+        "\n",
+        "- `Vertex AI Experiments`\n",
+        "\n",
+        "The steps performed include:\n",
+        "\n",
+        "- Construct the DIY autologging code.\n",
+        "- Construct training package with call to autologging.\n",
+        "- Train a model.\n",
+        "- View the experiment\n",
+        "- Delete the experiment."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dataset:custom,boston,lrg"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the [Iris dataset](https://www.tensorflow.org/datasets/catalog/iris) from [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/overview). This dataset does not require any feature engineering. The version of the dataset in this tutorial is stored in a public Cloud Storage bucket. The trained model predicts the type of Iris flower species from a class of three species: setosa, virginica, or versicolor."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "costs"
+      },
+      "source": [
+        "### Costs\n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI\n",
+        "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
+        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
+        "Calculator](https://cloud.google.com/products/calculator/)\n",
+        "to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "setup_local"
+      },
+      "source": [
+        "### Set up your local development environment\n",
+        "\n",
+        "If you are using Colab or Vertex Workbench AI Notebooks, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
+        "\n",
+        "Otherwise, make sure your environment meets this notebook's requirements. You need the following:\n",
+        "\n",
+        "- The Cloud Storage SDK\n",
+        "- Git\n",
+        "- Python 3\n",
+        "- virtualenv\n",
+        "- Jupyter notebook running in a virtual environment with Python 3\n",
+        "\n",
+        "The Cloud Storage guide to [Setting up a Python development environment](https://cloud.google.com/python/setup) and the [Jupyter installation guide](https://jupyter.org/install) provide detailed instructions for meeting these requirements. The following steps provide a condensed set of instructions:\n",
+        "\n",
+        "1. [Install and initialize the SDK](https://cloud.google.com/sdk/docs/).\n",
+        "\n",
+        "2. [Install Python 3](https://cloud.google.com/python/setup#installing_python).\n",
+        "\n",
+        "3. [Install virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv) and create a virtual environment that uses Python 3.  Activate the virtual environment.\n",
+        "\n",
+        "4. To install Jupyter, run `pip3 install jupyter` on the command-line in a terminal shell.\n",
+        "\n",
+        "5. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
+        "\n",
+        "6. Open this notebook in the Jupyter Notebook Dashboard.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "install_aip:mbsdk"
+      },
+      "source": [
+        "## Installation\n",
+        "\n",
+        "Install the following packages to execute this notebook."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "install_aip:mbsdk"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
+        "\n",
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "USER_FLAG = \"\"\n",
+        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\"\n",
+        "\n",
+        "! pip3 install --upgrade --quiet {USER_FLAG} google-cloud-aiplatform \\\n",
+        "                                             xgboost \\\n",
+        "                                             scikit-learn \\\n",
+        "                                             numpy"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "restart"
+      },
+      "source": [
+        "### Restart the kernel\n",
+        "\n",
+        "After you install the additional packages, you need to restart the notebook kernel so it can find the packages."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "restart"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "if not os.getenv(\"IS_TESTING\"):\n",
+        "    # Automatically restart kernel after installs\n",
+        "    import IPython\n",
+        "\n",
+        "    app = IPython.Application.instance()\n",
+        "    app.kernel.do_shutdown(True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "before_you_begin:nogpu"
+      },
+      "source": [
+        "## Before you begin\n",
+        "\n",
+        "### GPU runtime\n",
+        "\n",
+        "This tutorial does not require a GPU runtime.\n",
+        "\n",
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "2. [Make sure that billing is enabled for your project.](https://cloud.google.com/billing/docs/how-to/modify-project)\n",
+        "\n",
+        "3. [Enable the following APIs: Vertex AI APIs, Compute Engine APIs, and Cloud Storage.](https://console.cloud.google.com/flows/enableapi?apiid=ml.googleapis.com,compute_component,storage-component.googleapis.com)\n",
+        "\n",
+        "4. If you are running this notebook locally, you need to install the [Cloud SDK]((https://cloud.google.com/sdk)).\n",
+        "\n",
+        "5. Enter your project ID in the cell below. Then run the  cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$`."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "project_id"
+      },
+      "source": [
+        "#### Set your project ID\n",
+        "\n",
+        "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "set_project_id"
+      },
+      "outputs": [],
+      "source": [
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "autoset_project_id"
+      },
+      "outputs": [],
+      "source": [
+        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
+        "    # Get your GCP project id from gcloud\n",
+        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "    PROJECT_ID = shell_output[0]\n",
+        "    print(\"Project ID:\", PROJECT_ID)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "set_gcloud_project_id"
+      },
+      "outputs": [],
+      "source": [
+        "! gcloud config set project $PROJECT_ID"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "region"
+      },
+      "source": [
+        "#### Region\n",
+        "\n",
+        "You can also change the `REGION` variable, which is used for operations\n",
+        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
+        "\n",
+        "- Americas: `us-central1`\n",
+        "- Europe: `europe-west4`\n",
+        "- Asia Pacific: `asia-east1`\n",
+        "\n",
+        "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
+        "\n",
+        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "region"
+      },
+      "outputs": [],
+      "source": [
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "timestamp"
+      },
+      "source": [
+        "#### UUID\n",
+        "\n",
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "timestamp"
+      },
+      "outputs": [],
+      "source": [
+        "import random\n",
+        "import string\n",
+        "\n",
+        "\n",
+        "# Generate a uuid of a specifed length(default=8)\n",
+        "def generate_uuid(length: int = 8) -> str:\n",
+        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
+        "\n",
+        "\n",
+        "UUID = generate_uuid()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gcp_authenticate"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. \n",
+        "\n",
+        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
+        "\n",
+        "**Otherwise**, follow these steps:\n",
+        "\n",
+        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
+        "\n",
+        "**Click Create service account**.\n",
+        "\n",
+        "In the **Service account name** field, enter a name, and click **Create**.\n",
+        "\n",
+        "In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex\" into the filter box, and select **Vertex Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "\n",
+        "Click Create. A JSON file that contains your key downloads to your local environment.\n",
+        "\n",
+        "Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "gcp_authenticate"
+      },
+      "outputs": [],
+      "source": [
+        "# If you are running this notebook in Colab, run this cell and follow the\n",
+        "# instructions to authenticate your GCP account. This provides access to your\n",
+        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
+        "# requests.\n",
+        "\n",
+        "import os\n",
+        "import sys\n",
+        "\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = \"google.colab\" in sys.modules\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
+        "    if \"google.colab\" in sys.modules:\n",
+        "        from google.colab import auth as google_auth\n",
+        "\n",
+        "        google_auth.authenticate_user()\n",
+        "\n",
+        "    # If you are running this notebook locally, replace the string below with the\n",
+        "    # path to your service account key and run this cell to authenticate your GCP\n",
+        "    # account.\n",
+        "    elif not os.getenv(\"IS_TESTING\"):\n",
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "setup_vars"
+      },
+      "source": [
+        "### Import libraries and define constants"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "59963fb7178f"
+      },
+      "outputs": [],
+      "source": [
+        "import google.cloud.aiplatform as aiplatform\n",
+        "import numpy as np\n",
+        "import xgboost as xgb\n",
+        "from sklearn.metrics import accuracy_score, precision_score, recall_score\n",
+        "\n",
+        "# to suppress lint message (unused)\n",
+        "precision_score, recall_score"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "init_aip:mbsdk"
+      },
+      "source": [
+        "## Initialize Vertex AI SDK for Python\n",
+        "\n",
+        "Initialize the Vertex AI SDK for Python for your project and corresponding bucket."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "init_aip:mbsdk"
+      },
+      "outputs": [],
+      "source": [
+        "aiplatform.init(project=PROJECT_ID, location=REGION)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ae8f31c8c617"
+      },
+      "source": [
+        "## DIY code for autologging XGBoost models\n",
+        "\n",
+        "The code below implements autologging for XGBoost models.\n",
+        "\n",
+        "- `autologging()`: Initializes the experiment and uses heap injection to replace `xgboost.train()` symbols on the heap with the redirect wrapper function `VertexXGBtrain`.\n",
+        "\n",
+        "- `VertexXGBtrain`: A wrapper function for XGBoost train() function. Automatically logs hyperparameters and calls the underlyig function.\n",
+        "\n",
+        "- `VertexSKLaccuracy_score`: A wrapper function for scikit-learn accuracy_score() function. Automatically calls underlying function and logs the metrics results."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "8eb012e5d7ef"
+      },
+      "outputs": [],
+      "source": [
+        "def autolog(\n",
+        "    project: str = None,\n",
+        "    location: str = None,\n",
+        "    staging_bucket: str = None,\n",
+        "    experiment: str = None,\n",
+        "    run: str = None,\n",
+        "    framework: str = \"tf\",\n",
+        "):\n",
+        "    \"\"\"\n",
+        "    Enable automatic logging of parameters and metrics in Vertex AI Experiments,\n",
+        "    for corresponding framework.\n",
+        "\n",
+        "        project: The project ID\n",
+        "        location : The region\n",
+        "        staging_bucket: temporary bucket\n",
+        "        experiment: The name of the experiment\n",
+        "        run: The name of the run within the experiment\n",
+        "        framework: The ML framework for which a model is being trained.\n",
+        "    \"\"\"\n",
+        "    # autologging\n",
+        "    if framework == \"tf\":\n",
+        "        try:\n",
+        "            globals()[\"Sequential\"] = VertexTFSequential\n",
+        "            if \"tf\" in globals():\n",
+        "                tf.keras.Sequential = VertexTFSequential\n",
+        "            if \"tensorflow\" in globals():\n",
+        "                tensorflow.keras.Sequential = VertexTFSequential\n",
+        "        except:\n",
+        "            pass\n",
+        "\n",
+        "        try:\n",
+        "            globals()[\"Model\"] = VertexTFModel\n",
+        "            if \"tf\" in globals():\n",
+        "                tf.keras.Model = VertexTFModel\n",
+        "            if \"tensorflow\" in globals():\n",
+        "                tensorflow.keras.Model = VertexTFModel\n",
+        "        except:\n",
+        "            pass\n",
+        "    elif framework == \"xgb\":\n",
+        "        global real_xgb_train\n",
+        "        global real_accuracy_score, real_precision_score, real_recall_score\n",
+        "        import sklearn\n",
+        "\n",
+        "        try:\n",
+        "            if \"xgboost\" in globals():\n",
+        "                real_xgb_train = xgboost.train\n",
+        "                xgboost.train = VertexXGBtrain\n",
+        "        except:\n",
+        "            pass\n",
+        "\n",
+        "        try:\n",
+        "            if \"xgb\" in globals():\n",
+        "                real_xgb_train = xgb.train\n",
+        "                xgb.train = VertexXGBtrain\n",
+        "        except:\n",
+        "            pass\n",
+        "\n",
+        "        try:\n",
+        "            global accuracy_score, precision_score, recall_score\n",
+        "            if \"accuracy_score\" in globals():\n",
+        "                real_accuracy_score = sklearn.metrics.accuracy_score\n",
+        "                sklearn.metrics.accuracy_score = VertexSKLaccuracy_score\n",
+        "                accuracy_score = VertexSKLaccuracy_score\n",
+        "            if \"precision_score\" in globals():\n",
+        "                real_precision_score = sklearn.metrics.precision_score\n",
+        "                sklearn.metrics.precision_score = VertexSKLprecision_score\n",
+        "                precision_score = VertexSKLprecision_score\n",
+        "            if \"recall_score\" in globals():\n",
+        "                real_recall_score = sklearn.metrics.recall_score\n",
+        "                sklearn.metrics.recall_score = VertexSKLrecall_score\n",
+        "                recall_score = VertexSKLrecall_score\n",
+        "        except:\n",
+        "            pass\n",
+        "\n",
+        "    if project:\n",
+        "        aiplatform.init(\n",
+        "            project=project, location=location, staging_bucket=staging_bucket\n",
+        "        )\n",
+        "\n",
+        "    if experiment:\n",
+        "        aiplatform.init(experiment=experiment)\n",
+        "    if run:\n",
+        "        aiplatform.start_run(run)\n",
+        "\n",
+        "\n",
+        "def VertexXGBtrain(\n",
+        "    params,\n",
+        "    dtrain,\n",
+        "    num_boost_round=10,\n",
+        "    evals=None,\n",
+        "    obj=None,\n",
+        "    maximize=None,\n",
+        "    early_stopping_rounds=None,\n",
+        "    evals_result=None,\n",
+        "    verbose_eval=True,\n",
+        "    callbacks=None,\n",
+        "    custom_metric=None,\n",
+        "):\n",
+        "    \"\"\"\n",
+        "    Wrapper function for autologging training parameters with Vertex AI Experiments\n",
+        "    Args:\n",
+        "        same as underlying xgb.train() method\n",
+        "    \"\"\"\n",
+        "    global real_xgb_train\n",
+        "\n",
+        "    aiplatform.log_params({\"train.num_boost_round\": int(num_boost_round)})\n",
+        "\n",
+        "    if params:\n",
+        "        if \"booster\" in params:\n",
+        "            aiplatform.log_params({\"train.booster\": int(params[\"booster\"])})\n",
+        "\n",
+        "        # booster parameters\n",
+        "        if \"eta\" in params:\n",
+        "            aiplatform.log_params({\"train.eta\": int(params[\"eta\"])})\n",
+        "        if \"max_depth\" in params:\n",
+        "            aiplatform.log_params({\"train.max_depth\": int(params[\"max_depth\"])})\n",
+        "        if \"max_leaf_nodes\" in params:\n",
+        "            aiplatform.log_params(\n",
+        "                {\"train.max_leaf_nodes\": int(params[\"max_leaf_nodes\"])}\n",
+        "            )\n",
+        "        if \"gamma\" in params:\n",
+        "            aiplatform.log_params({\"train.gamma\": int(params[\"gamma\"])})\n",
+        "        if \"alpha\" in params:\n",
+        "            aiplatform.log_params({\"train.alpha\": int(params[\"alpha\"])})\n",
+        "\n",
+        "    return real_xgb_train(\n",
+        "        params=params,\n",
+        "        dtrain=dtrain,\n",
+        "        num_boost_round=num_boost_round,\n",
+        "        evals=evals,\n",
+        "        obj=obj,\n",
+        "        maximize=maximize,\n",
+        "        early_stopping_rounds=early_stopping_rounds,\n",
+        "        evals_result=evals_result,\n",
+        "        verbose_eval=verbose_eval,\n",
+        "        callbacks=callbacks,\n",
+        "        custom_metric=custom_metric,\n",
+        "    )\n",
+        "\n",
+        "\n",
+        "def VertexSKLaccuracy_score(labels, predictions):\n",
+        "    \"\"\"\n",
+        "    Wrapper function for autologging training metrics with Vertex AI Experiments\n",
+        "    Args:\n",
+        "        same as underlying accuracy_score function\n",
+        "    \"\"\"\n",
+        "    global real_accuracy_score\n",
+        "    accuracy = real_accuracy_score(labels, predictions)\n",
+        "    aiplatform.log_metrics({\"accuracy\": accuracy})\n",
+        "    return accuracy\n",
+        "\n",
+        "\n",
+        "def VertexSKLprecision_score(\n",
+        "    y_true,\n",
+        "    y_pred,\n",
+        "    *,\n",
+        "    labels=None,\n",
+        "    pos_label=1,\n",
+        "    average=\"binary\",\n",
+        "    sample_weight=None,\n",
+        "    zero_division=\"warn\",\n",
+        "):\n",
+        "    \"\"\"\n",
+        "    Wrapper function for autologging training metrics with Vertex AI Experiments\n",
+        "    Args:\n",
+        "        same as underlying precision_score function\n",
+        "    \"\"\"\n",
+        "    global real_precision_score\n",
+        "    precision = real_precision_score(\n",
+        "        y_true,\n",
+        "        y_pred,\n",
+        "        labels=labels,\n",
+        "        pos_label=pos_label,\n",
+        "        average=average,\n",
+        "        sample_weight=sample_weight,\n",
+        "        zero_division=zero_division,\n",
+        "    )\n",
+        "    aiplatform.log_metrics({\"precision\": precision})\n",
+        "    return precision\n",
+        "\n",
+        "\n",
+        "def VertexSKLrecall_score(\n",
+        "    y_true,\n",
+        "    y_pred,\n",
+        "    *,\n",
+        "    labels=None,\n",
+        "    pos_label=1,\n",
+        "    average=\"binary\",\n",
+        "    sample_weight=None,\n",
+        "    zero_division=\"warn\",\n",
+        "):\n",
+        "    \"\"\"\n",
+        "    Wrapper function for autologging training metrics with Vertex AI Experiments\n",
+        "    Args:\n",
+        "        same as underlying recall_score function\n",
+        "    \"\"\"\n",
+        "    global real_recall_score\n",
+        "    recall = real_recall_score(\n",
+        "        y_true,\n",
+        "        y_pred,\n",
+        "        labels=labels,\n",
+        "        pos_label=pos_label,\n",
+        "        average=average,\n",
+        "        sample_weight=sample_weight,\n",
+        "        zero_division=zero_division,\n",
+        "    )\n",
+        "    aiplatform.log_metrics({\"recall\": recall})\n",
+        "    return recall\n",
+        "\n",
+        "\n",
+        "class VertexXGBBooster(xgb.Booster):\n",
+        "    \"\"\"\n",
+        "    WIP\n",
+        "    \"\"\"\n",
+        "\n",
+        "    def __init__(self, params=None, cache=None, model_file=None):\n",
+        "        super().__init__(params, cache, model_file)\n",
+        "\n",
+        "    def boost(\n",
+        "        self, dtrain: xgb.core.DMatrix, grad: np.ndarray, hess: np.ndarray\n",
+        "    ) -> None:\n",
+        "        return super().boost(dtrain, grad, hess)\n",
+        "\n",
+        "    def eval(\n",
+        "        self, data: xgb.core.DMatrix, name: str = \"eval\", iteration: int = 0\n",
+        "    ) -> str:\n",
+        "        return super().eval(data, name, iteration)\n",
+        "\n",
+        "    def update(self, dtrain: xgb.core.DMatrix, iteration: int, fobj=None) -> None:\n",
+        "        return super().update(dtrain, iteration, fobj)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ce76826902c0"
+      },
+      "source": [
+        "### Train the XGBoost model with Vertex AI Experiments\n",
+        "\n",
+        "In the following code, you build, train and evaluate an XGBoost tabular model. The Python script includes the following calls to integrate `Vertex AI Experiments`:\n",
+        "\n",
+        "- command-line arguments: The arguments `experiment` and `run` are used to pass in the experiment and run names for the experiment.\n",
+        "- `autologging()`: Initializes the experiment and does the heap injection.\n",
+        "- `aiplatform.start_execution()`: Initializes a context for linking artifacts.\n",
+        "- `aiplatform.end_run()`: Ends the experiment.\n",
+        "\n",
+        "*Note:* The functions `xgb.train` and `accuracy_score` will be redirected to `VertexXGBtrain` and VertexSKLaccuracy_score, respectively, by heap injection. When subsequent calls are made to the `train()` and `accuracy()` function,s they will be executed as the corresponding `VertexXGBtrain` and `VertexSKLaccuracy_score` functions."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "WiSnFuDoox9W"
+      },
+      "outputs": [],
+      "source": [
+        "EXPERIMENT_NAME = f\"myexperiment{UUID}\"\n",
+        "RUN_NAME = \"run-1\"\n",
+        "\n",
+        "DATASET_DIR = \"gs://cloud-samples-data/ai-platform/iris\"\n",
+        "DATASET_DATA_URL = DATASET_DIR + \"/iris_data.csv\"\n",
+        "DATASET_LABELS_URL = DATASET_DIR + \"/iris_target.csv\"\n",
+        "\n",
+        "BOOSTED_ROUNDS = 20\n",
+        "\n",
+        "import logging\n",
+        "import os\n",
+        "import subprocess\n",
+        "import sys\n",
+        "\n",
+        "import hypertune\n",
+        "import numpy as np\n",
+        "import pandas as pd\n",
+        "import xgboost as xgb\n",
+        "from sklearn.model_selection import train_test_split\n",
+        "\n",
+        "\n",
+        "def get_data():\n",
+        "    # gsutil outputs everything to stderr so we need to divert it to stdout.\n",
+        "    subprocess.check_call(\n",
+        "        [\"gsutil\", \"cp\", DATASET_DATA_URL, \"data.csv\"], stderr=sys.stdout\n",
+        "    )\n",
+        "    # gsutil outputs everything to stderr so we need to divert it to stdout.\n",
+        "    subprocess.check_call(\n",
+        "        [\"gsutil\", \"cp\", DATASET_LABELS_URL, \"labels.csv\"], stderr=sys.stdout\n",
+        "    )\n",
+        "\n",
+        "    # Load data into pandas, then use `.values` to get NumPy arrays\n",
+        "    data = pd.read_csv(\"data.csv\").values\n",
+        "    labels = pd.read_csv(\"labels.csv\").values\n",
+        "\n",
+        "    # Convert one-column 2D array into 1D array for use with XGBoost\n",
+        "    labels = labels.reshape((labels.size,))\n",
+        "\n",
+        "    train_data, test_data, train_labels, test_labels = train_test_split(\n",
+        "        data, labels, test_size=0.2, random_state=7\n",
+        "    )\n",
+        "\n",
+        "    # Load data into DMatrix object\n",
+        "    dtrain = xgb.DMatrix(train_data, label=train_labels)\n",
+        "    return dtrain, test_data, test_labels\n",
+        "\n",
+        "\n",
+        "def train_model(dtrain):\n",
+        "    logging.info(\"Start training ...\")\n",
+        "    # Train XGBoost model\n",
+        "    params = {\"max_depth\": 3, \"objective\": \"multi:softmax\", \"num_class\": 3}\n",
+        "    model = xgb.train(params=params, dtrain=dtrain, num_boost_round=BOOSTED_ROUNDS)\n",
+        "    logging.info(\"Training completed\")\n",
+        "    return model\n",
+        "\n",
+        "\n",
+        "def evaluate_model(model, test_data, test_labels):\n",
+        "    dtest = xgb.DMatrix(test_data)\n",
+        "    pred = model.predict(dtest)\n",
+        "    predictions = [round(value) for value in pred]\n",
+        "    # evaluate predictions\n",
+        "    accuracy = accuracy_score(test_labels, predictions)\n",
+        "\n",
+        "    logging.info(f\"Evaluation completed with model accuracy: {accuracy}\")\n",
+        "\n",
+        "    # report metric for hyperparameter tuning\n",
+        "    hpt = hypertune.HyperTune()\n",
+        "    hpt.report_hyperparameter_tuning_metric(\n",
+        "        hyperparameter_metric_tag=\"accuracy\", metric_value=accuracy\n",
+        "    )\n",
+        "    return accuracy\n",
+        "\n",
+        "\n",
+        "# autologging\n",
+        "autolog(experiment=EXPERIMENT_NAME, run=RUN_NAME, framework=\"xgb\")\n",
+        "\n",
+        "with aiplatform.start_execution(\n",
+        "    schema_title=\"system.ContainerExecution\", display_name=\"example_training\"\n",
+        ") as execution:\n",
+        "    dtrain, test_data, test_labels = get_data()\n",
+        "    model = train_model(dtrain)\n",
+        "    accuracy = evaluate_model(model, test_data, test_labels)\n",
+        "\n",
+        "aiplatform.end_run()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5f40912e6500"
+      },
+      "source": [
+        "#### Get the experiment results\n",
+        "\n",
+        "Next, you use the experiment name as a parameter to the method `get_experiment_df()` to get the results of the experiment as a pandas dataframe."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7e9671712230"
+      },
+      "outputs": [],
+      "source": [
+        "experiment_df = aiplatform.get_experiment_df()\n",
+        "experiment_df = experiment_df[experiment_df.experiment_name == EXPERIMENT_NAME]\n",
+        "experiment_df.T"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e508c159d712"
+      },
+      "source": [
+        "#### Delete the experiment\n",
+        "\n",
+        "Since the experiment was created within a training script, to delete the experiment you use the `list()` method to obtain all the experiments for the project, and then filter on the experiment name."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "1a1b5fcbfde0"
+      },
+      "outputs": [],
+      "source": [
+        "experiments = aiplatform.Experiment.list()\n",
+        "for experiment in experiments:\n",
+        "    if experiment.name == EXPERIMENT_NAME:\n",
+        "        experiment.delete()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "cleanup:mbsdk"
+      },
+      "source": [
+        "# Cleaning up\n",
+        "\n",
+        "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
+        "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
+        "\n",
+        "Otherwise, you can delete the individual resources you created in this tutorial."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9eb897e0e728"
+      },
+      "outputs": [],
+      "source": [
+        "# There are no resources to cleanup"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "get_started_with_vertex_experiments_autologging_xgboost.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/community/ml_ops/stage3/get_started_with_dataproc_serverless_pipeline_components.ipynb

@@ -72,7 +72,7 @@
       "source": [
         "### Objective\n",
         "\n",
-        "In this tutorial, you learn how to use prebuilt `Google Cloud Pipeline Components` for `Dataproc Serverless` service. The documentation for the components can be found [here](https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-1.0.0/google_cloud_pipeline_components.experimental.dataproc.html).\n",
+        "In this tutorial, you learn how to use prebuilt `Google Cloud Pipeline Components` for `Dataproc Serverless` service. The documentation for the components can be found [here](https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-1.0.26/google_cloud_pipeline_components.v1.dataproc.html).\n",
         "\n",
         "This tutorial uses the following Google Cloud ML services:\n",
         "\n",
@@ -759,14 +759,14 @@
         "\n",
         "In this example, the `DataprocPySparkBatchOp` component takes the following parameters:\n",
         "\n",
-        "- `batch_id`: The batch ID to use for the Dataproc Batch workload.\n",
         "- `project_id`: The project ID.\n",
         "- `location`: The region.\n",
         "- `main_python_file_uri`: The URI of the main Python file.\n",
         "- `service_account`: The service account that runs the workload.\n",
         "- `args`: The arguments to pass to the PySpark program.\n",
+        "- `batch_id`: (Optional) The batch ID to use for the Dataproc Batch workload.\n",
         "\n",
-        "Learn more about the [Dataproc Serverless PySpark batch component](https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-1.0.0/google_cloud_pipeline_components.experimental.dataproc.html#google_cloud_pipeline_components.experimental.dataproc.DataprocPySparkBatchOp)."
+        "Learn more about the [Dataproc Serverless PySpark batch component](https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-1.0.26/google_cloud_pipeline_components.v1.dataproc.html#google_cloud_pipeline_components.v1.dataproc.DataprocPySparkBatchOp)."
       ]
     },
     {
@@ -799,16 +799,16 @@
         "    service_account: str = SERVICE_ACCOUNT,\n",
         "    args: list = ARGS,\n",
         "):\n",
-        "    from google_cloud_pipeline_components.experimental.dataproc import \\\n",
+        "    from google_cloud_pipeline_components.v1.dataproc import \\\n",
         "        DataprocPySparkBatchOp\n",
         "\n",
         "    _ = DataprocPySparkBatchOp(\n",
         "        project=project_id,\n",
         "        location=location,\n",
-        "        batch_id=batch_id,\n",
         "        main_python_file_uri=main_python_file_uri,\n",
         "        service_account=service_account,\n",
         "        args=args,\n",
+        "        batch_id=batch_id,  # `batch_id` is optional\n",
         "    )\n",
         "\n",
         "\n",
@@ -979,15 +979,15 @@
         "\n",
         "In this example, the `DataprocSparkBatchOp` component takes the following parameters:\n",
         "\n",
-        "- `batch_id`: The batch ID to use for the Dataproc Batch workload.\n",
         "- `project_id`: The project ID.\n",
         "- `location`: The region.\n",
         "- `main_class`: The main class.\n",
         "- `jar_file_uris`: The URIs of any required JARs to include in the executor and driver CLASSPATH.\n",
         "- `service_account`: The service account that runs the workload.\n",
         "- `args`: The arguments to pass to the Spark program.\n",
+        "- `batch_id`: (Optional) The batch ID to use for the Dataproc Batch workload.\n",
         "\n",
-        "Learn more about the [Dataproc Serverless Spark batch component](https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-1.0.0/google_cloud_pipeline_components.experimental.dataproc.html#google_cloud_pipeline_components.experimental.dataproc.DataprocSparkBatchOp)."
+        "Learn more about the [Dataproc Serverless Spark batch component](https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-1.0.26/google_cloud_pipeline_components.v1.dataproc.html#google_cloud_pipeline_components.v1.dataproc.DataprocSparkBatchOp)."
       ]
     },
     {
@@ -1019,17 +1019,17 @@
         "    service_account: str = SERVICE_ACCOUNT,\n",
         "    args: list = ARGS,\n",
         "):\n",
-        "    from google_cloud_pipeline_components.experimental.dataproc import \\\n",
+        "    from google_cloud_pipeline_components.v1.dataproc import \\\n",
         "        DataprocSparkBatchOp\n",
         "\n",
         "    _ = DataprocSparkBatchOp(\n",
         "        project=project_id,\n",
         "        location=location,\n",
-        "        batch_id=batch_id,\n",
         "        main_class=main_class,\n",
         "        jar_file_uris=jar_file_uris,\n",
         "        service_account=service_account,\n",
         "        args=args,\n",
+        "        batch_id=batch_id,  # `batch_id` is optional\n",
         "    )\n",
         "\n",
         "\n",
@@ -1281,14 +1281,14 @@
         "\n",
         "In this example, the `DataprocSparkSqlBatchOp` component takes the following parameters:\n",
         "\n",
-        "- `batch_id`: The batch ID to use for the Dataproc Batch workload.\n",
         "- `project_id`: The project ID.\n",
         "- `location`: The region.\n",
         "- `query_file_uri`: The URI of the file containing the SQL queries.\n",
         "- `query_variables`: The mapping of query variable names to values (equivalent to the Spark SQL command  `SET name=\"value\";`).\n",
         "- `service_account`: The service account that runs the workload.\n",
+        "- `batch_id`: (Optional) The batch ID to use for the Dataproc Batch workload.\n",
         "\n",
-        "Learn more about the [Dataproc Serverless Spark SQL batch component](https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-1.0.0/google_cloud_pipeline_components.experimental.dataproc.html#google_cloud_pipeline_components.experimental.dataproc.DataprocSparkSqlBatchOp)."
+        "Learn more about the [Dataproc Serverless Spark SQL batch component](https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-1.0.26/google_cloud_pipeline_components.v1.dataproc.html#google_cloud_pipeline_components.v1.dataproc.DataprocSparkSqlBatchOp)."
       ]
     },
     {
@@ -1326,16 +1326,16 @@
         "    query_variables: dict = QUERY_VARIABLES,\n",
         "    service_account: str = SERVICE_ACCOUNT,\n",
         "):\n",
-        "    from google_cloud_pipeline_components.experimental.dataproc import \\\n",
+        "    from google_cloud_pipeline_components.v1.dataproc import \\\n",
         "        DataprocSparkSqlBatchOp\n",
         "\n",
         "    _ = DataprocSparkSqlBatchOp(\n",
         "        project=project_id,\n",
         "        location=location,\n",
-        "        batch_id=batch_id,\n",
         "        query_file_uri=query_file_uri,\n",
         "        query_variables=query_variables,\n",
         "        service_account=service_account,\n",
+        "        batch_id=batch_id,  # `batch_id` is optional\n",
         "    )\n",
         "\n",
         "\n",
@@ -1502,14 +1502,14 @@
         "\n",
         "In this example, the `DataprocSparkRBatchOp` component takes the following parameters:\n",
         "\n",
-        "- `batch_id`: The batch ID to use for the Dataproc Batch workload.\n",
         "- `project_id`: The project ID.\n",
         "- `location`: The region.\n",
         "- `main_r_file_uri`: The URI of the main R file.\n",
         "- `service_account`: The service account that runs the workload.\n",
         "- `args`: The arguments to pass to the Spark program.\n",
+        "- `batch_id`: (Optional) The batch ID to use for the Dataproc Batch workload.\n",
         "\n",
-        "Learn more about the [Dataproc Serverless SparkR batch component](https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-1.0.0/google_cloud_pipeline_components.experimental.dataproc.html#google_cloud_pipeline_components.experimental.dataproc.DataprocSparkRBatchOp)."
+        "Learn more about the [Dataproc Serverless SparkR batch component](https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-1.0.26/google_cloud_pipeline_components.v1.dataproc.html#google_cloud_pipeline_components.v1.dataproc.DataprocSparkRBatchOp)."
       ]
     },
     {
@@ -1539,15 +1539,15 @@
         "    service_account: str = SERVICE_ACCOUNT,\n",
         "    args: list = ARGS,\n",
         "):\n",
-        "    from google_cloud_pipeline_components.experimental.dataproc import \\\n",
+        "    from google_cloud_pipeline_components.v1.dataproc import \\\n",
         "        DataprocSparkRBatchOp\n",
         "\n",
         "    _ = DataprocSparkRBatchOp(\n",
         "        project=project_id,\n",
         "        location=location,\n",
-        "        batch_id=batch_id,\n",
         "        main_r_file_uri=main_r_file_uri,\n",
         "        args=args,\n",
+        "        batch_id=batch_id,  # `batch_id` is optional\n",
         "    )\n",
         "\n",
         "\n",

notebooks/community/ml_ops/stage6/get_started_with_xgboost_model_online.ipynb

@@ -670,6 +670,56 @@
         "print(\"Train machine type\", DEPLOY_COMPUTE)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8ac30ffa72b5"
+      },
+      "source": [
+        "### Create the instance schema\n",
+        "\n",
+        "The instance.yaml file is used to validate the format of the input request before it goes to the model server. If you are sending correctly you don't need instance.yaml. If you get bad requests, the instance.yaml will give a more meaningful error msg than what you probably get from the model server.\n",
+        "\n",
+        "*The instance.yaml can also be reused as the input schema for Model Monitoring custom models.*\n",
+        "\n",
+        "\n",
+        "\n",
+        "Learn more about [Predict schemas](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/PredictSchemata)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "59becb56ad34"
+      },
+      "outputs": [],
+      "source": [
+        "yaml = \"\"\"type: array\n",
+        "properties:\n",
+        "  sepal_length:\n",
+        "    type: numeric\n",
+        "  sepal_width:\n",
+        "    type: numeric\n",
+        "  petal_length:\n",
+        "    type: numeric\n",
+        "  petal_width:\n",
+        "    type: numeric\n",
+        "required:\n",
+        "  - sepal_length\n",
+        "  - sepal_width\n",
+        "  - petal_length\n",
+        "  - petal_width\n",
+        "\"\"\"\n",
+        "\n",
+        "print(yaml)\n",
+        "\n",
+        "with open(\"instance.yaml\", \"w\") as f:\n",
+        "    f.write(yaml)\n",
+        "\n",
+        "! gsutil cp instance.yaml {BUCKET_URI}/instance.yaml"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -709,6 +759,7 @@
         "    display_name=f\"xgboost_iris_{UUID}\",\n",
         "    serving_container_image_uri=DEPLOY_IMAGE,\n",
         "    artifact_uri=MODEL_ARTIFACTS,\n",
+        "    instance_schema_uri=f\"{BUCKET_URI}/instance.yaml\",\n",
         "    is_default_version=True,\n",
         "    version_aliases=[\"v1\", \"version1\"],\n",
         "    version_description=\"This is the first version of the model\",\n",
@@ -749,7 +800,7 @@
       "outputs": [],
       "source": [
         "endpoint = aiplatform.Endpoint.create(\n",
-        "    display_name=\"flowers_\" + UUID,\n",
+        "    display_name=\"xgboost_iris_\" + UUID,\n",
         "    project=PROJECT_ID,\n",
         "    location=REGION,\n",
         "    labels={\"your_key\": \"your_value\"},\n",

notebooks/notebook_template.ipynb

@@ -31,6 +31,8 @@
       "source": [
         "# [TODO] Add your H1 title heading here\n",
         "\n",
+        "{TODO: Update the links below.} \n",
+        "\n",
         "<table align=\"left\">\n",
         "\n",
         "  <td>\n",
@@ -53,6 +55,17 @@
         "</table>"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "24743cf4a1e1"
+      },
+      "source": [
+        "**_NOTE_**: This notebook has been tested in the following environment:\n",
+        "\n",
+        "* Python version = 3.9"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -111,7 +124,7 @@
         "This tutorial uses billable components of Google Cloud:\n",
         "\n",
         "* Vertex AI\n",
-        "* {TODO: BigQyuery}\n",
+        "* {TODO: BigQuery}\n",
         "* Cloud Storage\n",
         "\n",
         "{TODO: Include links to pricing documentation for each product you listed above.\n",
@@ -125,62 +138,6 @@
         "to generate a cost estimate based on your projected usage."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "gCuSR8GkAgzl"
-      },
-      "source": [
-        "### Set up your local development environment\n",
-        "\n",
-        "**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets\n",
-        "all the requirements to run this notebook. You can skip this step."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "24743cf4a1e1"
-      },
-      "source": [
-        "**_NOTE_**: This notebook has been tested in the following environment:\n",
-        "\n",
-        "* Python version = 3.9"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "gCuSR8GkAgzl"
-      },
-      "source": [
-        "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
-        "You need the following:\n",
-        "\n",
-        "* The Google Cloud SDK\n",
-        "\n",
-        "The Google Cloud guide to [Setting up a Python development\n",
-        "environment](https://cloud.google.com/python/setup) and the [Jupyter\n",
-        "installation guide](https://jupyter.org/install) provide detailed instructions\n",
-        "for meeting these requirements. The following steps provide a condensed set of\n",
-        "instructions:\n",
-        "\n",
-        "1. [Install and initialize the Cloud SDK.](https://cloud.google.com/sdk/docs/)\n",
-        "\n",
-        "1. [Install Python 3.](https://cloud.google.com/python/setup#installing_python)\n",
-        "\n",
-        "1. [Install\n",
-        "   virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv)\n",
-        "   and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
-        "\n",
-        "1. To install Jupyter, run `pip3 install jupyter` on the\n",
-        "command-line in a terminal shell.\n",
-        "\n",
-        "1. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
-        "\n",
-        "1. Open this notebook in the Jupyter Notebook Dashboard."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -202,51 +159,32 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
-        "! pip3 install --upgrade google-cloud-aiplatform {USER_FLAG} -q\n",
-        "# TODO: Add remaining package installs here. All packages should be on a single pip install to resolve dependencies"
+        "# Install the packages\n",
+        "! pip3 install --user --upgrade google-cloud-aiplatform"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "hhq5zEbGg0XX"
+        "id": "58707a750154"
       },
       "source": [
-        "### Restart the kernel\n",
-        "\n",
-        "After you install the additional packages, you need to restart the notebook kernel so it can find the packages."
+        "### Colab only: Uncomment the following cell to restart the kernel."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "EzrelQZ22IZj"
+        "id": "f200f10a1da3"
       },
       "outputs": [],
       "source": [
-        "# Automatically restart kernel after installs\n",
-        "import os\n",
+        "# Automatically restart kernel after installs so that your environment can access the new packages\n",
+        "# import IPython\n",
         "\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    # Automatically restart kernel after installs\n",
-        "    import IPython\n",
-        "\n",
-        "    app = IPython.Application.instance()\n",
-        "    app.kernel.do_shutdown(True)"
+        "# app = IPython.Application.instance()\n",
+        "# app.kernel.do_shutdown(True)"
       ]
     },
     {
@@ -263,16 +201,11 @@
         "\n",
         "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
         "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "2. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
         "\n",
-        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com). {TODO: Update the APIs needed for your tutorial. Edit the API names, and update the link to append the API IDs, separating each one with a comma. For example, container.googleapis.com,cloudbuild.googleapis.com}\n",
+        "3. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com). {TODO: Update the APIs needed for your tutorial. Edit the API names, and update the link to append the API IDs, separating each one with a comma. For example, container.googleapis.com,cloudbuild.googleapis.com}\n",
         "\n",
-        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+        "4. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk)."
       ]
     },
     {
@@ -283,7 +216,10 @@
       "source": [
         "#### Set your project ID\n",
         "\n",
-        "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
+        "**If you don't know your project ID**, try the following:\n",
+        "* Run `gcloud config list`.\n",
+        "* Run `gcloud projects list`.\n",
+        "* See the support page: [Locate the project ID](https://support.google.com/googleapi/answer/7014113)"
       ]
     },
     {
@@ -294,57 +230,10 @@
       },
       "outputs": [],
       "source": [
-        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "riG_qUokg0XZ"
-      },
-      "outputs": [],
-      "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
-        "    # Get your GCP project id from gcloud\n",
-        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
-        "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID:\", PROJECT_ID)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "set_gcloud_project_id"
-      },
-      "outputs": [],
-      "source": [
-        "! gcloud config set project $PROJECT_ID"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "23988890fef6"
-      },
-      "source": [
-        "#### Get your project number {TODO: Include these cells if the notebook uses a project number}\n",
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}\n",
         "\n",
-        "Now that the project ID is set, you get your corresponding project number."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "2d6950574e1d"
-      },
-      "outputs": [],
-      "source": [
-        "shell_output = ! gcloud projects list --filter=\"PROJECT_ID:'{PROJECT_ID}'\" --format='value(PROJECT_NUMBER)'\n",
-        "PROJECT_NUMBER = shell_output[0]\n",
-        "print(\"Project Number:\", PROJECT_NUMBER)"
+        "# Set the project id\n",
+        "! gcloud config set project {PROJECT_ID}"
       ]
     },
     {
@@ -355,63 +244,18 @@
       "source": [
         "#### Region\n",
         "\n",
-        "You can also change the `REGION` variable, which is used for operations\n",
-        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. It is recommended that you choose the region closest to you.\n",
-        "\n",
-        "- Americas: `us-central1`\n",
-        "- Europe: `europe-west4`\n",
-        "- Asia Pacific: `asia-east1`\n",
-        "\n",
-        "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
-        "\n",
-        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
+        "You can also change the `REGION` variable used by Vertex AI. Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 1,
       "metadata": {
         "id": "region"
       },
       "outputs": [],
       "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "06571eb4063b"
-      },
-      "source": [
-        "#### UUID\n",
-        "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial.\n",
-        "\n",
-        "{TODO: replace the `TIMESTAMP` with `UUID` in official notebooks}"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "697568e92bd6"
-      },
-      "outputs": [],
-      "source": [
-        "import random\n",
-        "import string\n",
-        "\n",
-        "\n",
-        "# Generate a uuid of a specifed length(default=8)\n",
-        "def generate_uuid(length: int = 8) -> str:\n",
-        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
-        "\n",
-        "\n",
-        "UUID = generate_uuid()"
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
       ]
     },
     {
@@ -422,64 +266,68 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
-        "authenticated. \n",
-        "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions\n",
-        "when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "1. In the Cloud Console, go to the [**Create service account key**\n",
-        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
-        "\n",
-        "2. Click **Create service account**.\n",
-        "\n",
-        "3. In the **Service account name** field, enter a name, and\n",
-        "   click **Create**.\n",
-        "\n",
-        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
-        "into the filter box, and select\n",
-        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
-        "local environment.\n",
-        "\n",
-        "6. Enter the path to your service account key as the\n",
-        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
+        "Depending on your Jupyter environment, you may have to manually authenticate. Follow the relevant instructions below."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "74ccc9e52986"
+      },
+      "source": [
+        "**1. Vertex AI Workbench**\n",
+        "* Do nothing as you are already authenticated."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "de775a3773ba"
+      },
+      "source": [
+        "**2. Local JupyterLab instance, uncomment and run:**"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 2,
       "metadata": {
-        "id": "PyQmSRbKA8r-"
+        "id": "254614fa0c46"
       },
       "outputs": [],
       "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = \"google.colab\" in sys.modules\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS '[your-service-account-key-path]'"
+        "# ! gcloud auth login"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ef21552ccea8"
+      },
+      "source": [
+        "**3. Colab, uncomment and run:**"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 3,
+      "metadata": {
+        "id": "603adbbf0532"
+      },
+      "outputs": [],
+      "source": [
+        "# from google.colab import auth\n",
+        "# auth.authenticate_user()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "f6b2ccc891ed"
+      },
+      "source": [
+        "**4. Service account or other**\n",
+        "* See how to grant Cloud Storage permissions to your service account at https://cloud.google.com/storage/docs/gsutil/commands/iam#ch-examples."
       ]
     },
     {
@@ -490,20 +338,9 @@
       "source": [
         "### Create a Cloud Storage bucket\n",
         "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
+        "Create a storage bucket to store intermediate artifacts such as datasets.\n",
         "\n",
-        "\n",
-        "{TODO: Adjust wording in the first paragraph to fit your use case - explain how your tutorial uses the Cloud Storage bucket. The example below shows how Vertex AI uses the bucket for training.}\n",
-        "\n",
-        "When you submit a training job using the Vertex AI SDK, you upload a Python package\n",
-        "containing your training code to a Cloud Storage bucket. Vertex AI runs\n",
-        "the code from this package. In this tutorial, Vertex AI also saves the\n",
-        "trained model that results from your job in the same bucket. Using this model artifact, you can then\n",
-        "create Vertex AI model and endpoint resources in order to serve\n",
-        "online predictions.\n",
-        "\n",
-        "Set the name of your Cloud Storage bucket below. It must be unique across all\n",
-        "Cloud Storage buckets."
+        "- *{Note to notebook author: For any user-provided strings that need to be unique (like bucket names or model ID's), append \"-unique\" to the end so proper testing can occur}*"
       ]
     },
     {
@@ -514,21 +351,7 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "cf221059d072"
-      },
-      "outputs": [],
-      "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
-        "    BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+        "BUCKET_URI = \"gs://your-bucket-name-unique\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -554,99 +377,7 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "ucvCsknMCims"
-      },
-      "source": [
-        "Finally, validate access to your Cloud Storage bucket by examining its contents:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "vhOb7YnwClBb"
-      },
-      "outputs": [],
-      "source": [
-        "! gsutil ls -al $BUCKET_URI"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "set_service_account"
-      },
-      "source": [
-        "#### Service Account {TODO: Include these cells if the notebook specifies a service account}\n",
-        "\n",
-        "{TODO: What uses service account in the notebook; e.g., You use a service account to create Vertex AI Pipeline jobs.}. If you do not want to use your project's Compute Engine service account, set `SERVICE_ACCOUNT` to another service account ID."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "set_service_account"
-      },
-      "outputs": [],
-      "source": [
-        "SERVICE_ACCOUNT = \"[your-service-account]\"  # @param {type:\"string\"}"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "autoset_service_account"
-      },
-      "outputs": [],
-      "source": [
-        "if (\n",
-        "    SERVICE_ACCOUNT == \"\"\n",
-        "    or SERVICE_ACCOUNT is None\n",
-        "    or SERVICE_ACCOUNT == \"[your-service-account]\"\n",
-        "):\n",
-        "    # Get your service account from gcloud\n",
-        "    if not IS_COLAB:\n",
-        "        shell_output = !gcloud auth list 2>/dev/null\n",
-        "        SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
-        "\n",
-        "    else:  # IS_COLAB:\n",
-        "        shell_output = ! gcloud projects describe  $PROJECT_ID\n",
-        "        project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n",
-        "        SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n",
-        "\n",
-        "    print(\"Service Account:\", SERVICE_ACCOUNT)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "set_service_account:pipelines"
-      },
-      "source": [
-        "#### Set service account access for {TODO; e.g., Vertex AI Pipelines}\n",
-        "\n",
-        "Run the following commands to grant your service account access to {TODO; i.e., read and write pipeline artifacts} in the bucket that you created in the previous step. You only need to run this step once per service account."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "set_service_account:pipelines"
-      },
-      "outputs": [],
-      "source": [
-        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
-        "\n",
-        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "XoEqT2Y4DJmf"
+        "id": "960505627ddf"
       },
       "source": [
         "### Import libraries"
@@ -656,13 +387,11 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "pRUOFELefqf1"
+        "id": "PyQmSRbKA8r-"
       },
       "outputs": [],
       "source": [
-        "import google.cloud.aiplatform as aiplatform\n",
-        "\n",
-        "# TODO: import remaining libraries; e.g., tensorflow"
+        "from google.cloud import aiplatform"
       ]
     },
     {
@@ -673,7 +402,7 @@
       "source": [
         "### Initialize Vertex AI SDK for Python\n",
         "\n",
-        "Initialize the Vertex AI SDK for Python for your project and corresponding bucket."
+        "Initialize the Vertex AI SDK for Python for your project."
       ]
     },
     {
@@ -705,21 +434,21 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 1,
       "metadata": {
         "id": "sx_vKniMq9ZX"
       },
       "outputs": [],
       "source": [
+        "import os\n",
+        "\n",
         "# Delete endpoint resource\n",
-        "! gcloud ai endpoints delete $ENDPOINT_NAME --quiet --region $REGION\n",
+        "# e.g. `endpoint.delete()`\n",
         "\n",
         "# Delete model resource\n",
-        "! gcloud ai models delete $MODEL_NAME --quiet\n",
+        "# e.g. `model.delete()`\n",
         "\n",
         "# Delete Cloud Storage objects that were created\n",
-        "! gsutil -m rm -r $JOB_DIR\n",
-        "\n",
         "delete_bucket = False\n",
         "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
         "    ! gsutil -m rm -r $BUCKET_URI"

notebooks/official/CODEOWNERS

@@ -38,3 +38,6 @@
 /model_evaluation/automl_tabular_regression_model_evaluation.ipynb @soheilazangeneh
 /tabular_workflows/tabnet_on_vertex_pipelines.ipynb @sakagarwal
 /tabular_workflows/wide_and_deep_on_vertex_pipelines.ipynb @sakagarwal
+/model_evaluation/custom_tabular_classification_model_evaluation.ipynb @soheilazangeneh
+/sdk/SDK_FBProphet_Forecasting_Online.ipynb @brianchunkang
+/automl/sdk_automl_forecasting_hierarchical_batch.ipynb @ivanmkc

notebooks/official/automl/automl-text-classification.ipynb

@@ -2,7 +2,7 @@
   "cells": [
     {
       "cell_type": "code",
-      "execution_count": 54,
+      "execution_count": null,
       "metadata": {
         "id": "ur8xi4C7S06n"
       },
@@ -128,100 +128,29 @@
         "to generate a cost estimate based on your projected usage"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "lWEdiXsJg0XY"
-      },
-      "source": [
-        "## Before you begin\n",
-        "\n",
-        "**Note:** This notebook does not require a GPU runtime."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "a5cb73702a9b"
-      },
-      "source": [
-        "### Set up your local development environment\n",
-        "\n",
-        "**If you are using Colab or Workbench AI Notebooks**, your environment already meets\n",
-        "all the requirements to run this notebook. You can skip this step."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "gCuSR8GkAgzl"
-      },
-      "source": [
-        "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
-        "You need the following:\n",
-        "\n",
-        "* The Google Cloud SDK\n",
-        "* Git\n",
-        "* Python 3\n",
-        "* virtualenv\n",
-        "* Jupyter notebook running in a virtual environment with Python 3\n",
-        "\n",
-        "The Google Cloud guide to [Setting up a Python development\n",
-        "environment](https://cloud.google.com/python/setup) and the [Jupyter\n",
-        "installation guide](https://jupyter.org/install) provide detailed instructions\n",
-        "for meeting these requirements. The following steps provide a condensed set of\n",
-        "instructions:\n",
-        "\n",
-        "1. [Install and initialize the Cloud SDK.](https://cloud.google.com/sdk/docs/)\n",
-        "\n",
-        "1. [Install Python 3.](https://cloud.google.com/python/setup#installing_python)\n",
-        "\n",
-        "1. [Install\n",
-        "   virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv)\n",
-        "   and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
-        "\n",
-        "1. To install Jupyter, run `pip install jupyter` on the\n",
-        "command-line in a terminal shell.\n",
-        "\n",
-        "1. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
-        "\n",
-        "1. Open this notebook in the Jupyter Notebook Dashboard."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "db52a0a61fca"
       },
       "source": [
-        "### Install additional packages\n",
+        "### Installation\n",
         "\n",
         "Install the following packages for executing this notebook."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 55,
+      "execution_count": null,
       "metadata": {
         "id": "b75757581291"
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-        "\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
-        "! pip3 install {USER_FLAG} --upgrade google-cloud-aiplatform google-cloud-storage jsonlines -q"
+        "# install packages\n",
+        "! pip3 install --upgrade --quiet google-cloud-aiplatform \\\n",
+        "                                    google-cloud-storage \\\n",
+        "                                    jsonlines "
       ]
     },
     {
@@ -230,27 +159,22 @@
         "id": "e9255e3b156f"
       },
       "source": [
-        "### Restart the kernel\n",
-        "\n",
-        "Once you've installed the additional packages, you need to restart the notebook kernel so it can find the packages."
+        "### Colab Only: Uncomment the following cell to restart the kernel"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 56,
+      "execution_count": null,
       "metadata": {
         "id": "0c0b2427998a"
       },
       "outputs": [],
       "source": [
-        "import os\n",
+        "# Automatically restart kernel after installs so that your environment can access the new packages\n",
+        "# import IPython\n",
         "\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    # Automatically restart kernel after installs\n",
-        "    import IPython\n",
-        "\n",
-        "    app = IPython.Application.instance()\n",
-        "    app.kernel.do_shutdown(True)"
+        "# app = IPython.Application.instance()\n",
+        "# app.kernel.do_shutdown(True)"
       ]
     },
     {
@@ -259,62 +183,27 @@
         "id": "435b8e413535"
       },
       "source": [
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI, BigQuery, Compute Engine and Cloud Storage APIs](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,bigquery,compute_component,storage_component).\n",
-        "\n",
-        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands.\n",
+        "### Before you begin\n",
         "\n",
         "#### Set your project ID\n",
         "\n",
-        "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
+        "**If you don't know your project ID**, try the following:\n",
+        "-  Run `gcloud config list`\n",
+        "-  Run `gcloud projects list`\n",
+        "-  See the support page: [Locate the project ID](https://support.google.com/googleapi/answer/7014113)"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 1,
+      "execution_count": null,
       "metadata": {
         "id": "be175254a715"
       },
       "outputs": [],
       "source": [
-        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "db65832f7c1b"
-      },
-      "outputs": [],
-      "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
-        "    # Get your GCP project id from gcloud\n",
-        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
-        "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID:\", PROJECT_ID)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "ea86e5a1da1d"
-      },
-      "outputs": [],
-      "source": [
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# set the project id\n",
         "! gcloud config set project $PROJECT_ID"
       ]
     },
@@ -326,54 +215,19 @@
       "source": [
         "#### Region\n",
         "\n",
-        "You can also change the `REGION` variable, which is used for operations\n",
-        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
-        "\n",
-        "- Americas: `us-central1`\n",
-        "- Europe: `europe-west4`\n",
-        "- Asia Pacific: `asia-east1`\n",
-        "\n",
-        "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
-        "\n",
+        "You can also change the `REGION` variable used by Vertex AI. \n",
         "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 4,
+      "execution_count": null,
       "metadata": {
         "id": "ae43d96c4b1b"
       },
       "outputs": [],
       "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "5f4f5cccf897"
-      },
-      "source": [
-        "#### Timestamp\n",
-        "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 5,
-      "metadata": {
-        "id": "953fa6e5ddda"
-      },
-      "outputs": [],
-      "source": [
-        "from datetime import datetime\n",
-        "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}"
       ]
     },
     {
@@ -384,56 +238,54 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
+        "Depending on your Jupyter environment, you may have to manually authenticate. Follow the relevant instructions below.\n",
         "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
+        "**1. Vertex AI Workbench** \n",
+        "- Do nothing as you are already authenticated.\n",
         "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
-        "\n",
-        "1. **Click Create service account**.\n",
-        "\n",
-        "2. In the **Service account name** field, enter a name, and click **Create**.\n",
-        "\n",
-        "3. In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex AI\" into the filter box, and select **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "4. Click Create. A JSON file that contains your key downloads to your local environment.\n",
-        "\n",
-        "5. Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
+        "**2. Local JupyterLab Instance,** uncomment and run."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "oM1iC_MfAts1"
+        "id": "fbc9cd30cc4b"
       },
       "outputs": [],
       "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = \"google.colab\" in sys.modules\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+        "# ! gcloud auth login"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "cd0da2c26879"
+      },
+      "source": [
+        "**3. Colab,** uncomment and run:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "a336a05c6149"
+      },
+      "outputs": [],
+      "source": [
+        "# from google.colab import auth\n",
+        "# auth.authenticate_user()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0461097edfa5"
+      },
+      "source": [
+        "**4. Service Account or other**\n",
+        "- See all the authentication options here: [Google Cloud Platform Jupyter Notebook Authentication Guide](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/notebook_authentication_guide.ipynb)"
       ]
     },
     {
@@ -444,38 +296,21 @@
       "source": [
         "### Create a Cloud Storage bucket\n",
         "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "When you initialize the Vertex SDK for Python, you specify a Cloud Storage staging bucket. The staging bucket is where all the data associated with your dataset and model resources are retained across sessions.\n",
-        "\n",
-        "Set the name of your Cloud Storage bucket below. Bucket names must be globally unique across all Google Cloud projects, including those outside of your organization."
+        "Create a storage bucket to store intermediate artifacts such as datasets."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 6,
+      "execution_count": null,
       "metadata": {
         "id": "d2de92accb67"
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "BUCKET_NAME = \"your-bucket-name-unique\"  # @param {type:\"string\"}\n",
         "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": 7,
-      "metadata": {
-        "id": "5ba09496accc"
-      },
-      "outputs": [],
-      "source": [
-        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
-        "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -496,26 +331,6 @@
         "! gsutil mb -l $REGION $BUCKET_URI"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "c4cf2cdebb50"
-      },
-      "source": [
-        "Finally, validate access to your Cloud Storage bucket by examining its contents:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 9,
-      "metadata": {
-        "id": "96ad3d416327"
-      },
-      "outputs": [],
-      "source": [
-        "! gsutil ls -al $BUCKET_URI"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -527,14 +342,15 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 10,
+      "execution_count": null,
       "metadata": {
         "id": "152013538e59"
       },
       "outputs": [],
       "source": [
         "import jsonlines\n",
-        "from google.cloud import aiplatform, storage"
+        "from google.cloud import aiplatform, storage\n",
+        "from google.cloud.aiplatform import jobs"
       ]
     },
     {
@@ -550,7 +366,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 11,
+      "execution_count": null,
       "metadata": {
         "id": "740cd5c67c79"
       },
@@ -571,7 +387,7 @@
         "\n",
         "Using the Python SDK, you create a dataset and import the dataset in one call to `TextDataset.create()`, as shown in the following cell.\n",
         "\n",
-        "Creating and importing data is a long-running operation. This next step can take a while. The `create()` method waits for the operation to complete, outputting statements as the operation progresses. The statements contain the full name of the dataset that you will use in the following section.\n",
+        "Creating and importing data is a long-running operation. This next step can take a while. The `create()` method waits for the operation to complete, outputting statements as the operation progresses. The statements contain the full name of the dataset that you use in the following section.\n",
         "\n",
         "**Note**: You can close the noteboook while you wait for this operation to complete. "
       ]
@@ -586,7 +402,7 @@
       "source": [
         "# Use a timestamp to ensure unique resources\n",
         "src_uris = \"gs://cloud-ml-data/NL-classification/happiness.csv\"\n",
-        "display_name = f\"e2e-text-dataset-{TIMESTAMP}\"\n",
+        "display_name = \"e2e-text-dataset-unique\"\n",
         "\n",
         "text_dataset = aiplatform.TextDataset.create(\n",
         "    display_name=display_name,\n",
@@ -596,21 +412,14 @@
         ")"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "5b3cc427353a"
-      },
-      "source": [
-        "## Train your text classification model\n"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "68f10356cab9"
       },
       "source": [
+        "## Train your text classification model\n",
+        "\n",
         "Now you can begin training your model. Training the model is a two part process:\n",
         "\n",
         "1. **Define the training job.** You must provide a display name and the type of training you want when you define the training job.\n",
@@ -627,14 +436,14 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 16,
+      "execution_count": null,
       "metadata": {
         "id": "0aa0f01805ea"
       },
       "outputs": [],
       "source": [
         "# Define the training job\n",
-        "training_job_display_name = f\"e2e-text-training-job-{TIMESTAMP}\"\n",
+        "training_job_display_name = \"e2e-text-training-job-unique\"\n",
         "job = aiplatform.AutoMLTextTrainingJob(\n",
         "    display_name=training_job_display_name,\n",
         "    prediction_type=\"classification\",\n",
@@ -650,7 +459,7 @@
       },
       "outputs": [],
       "source": [
-        "model_display_name = f\"e2e-text-classification-model-{TIMESTAMP}\"\n",
+        "model_display_name = \"e2e-text-classification-model-unique\"\n",
         "\n",
         "# Run the training job\n",
         "model = job.run(\n",
@@ -711,7 +520,7 @@
       },
       "outputs": [],
       "source": [
-        "deployed_model_display_name = f\"e2e-deployed-text-classification-model-{TIMESTAMP}\"\n",
+        "deployed_model_display_name = \"e2e-deployed-text-classification-model-unique\"\n",
         "\n",
         "endpoint = model.deploy(\n",
         "    deployed_model_display_name=deployed_model_display_name, sync=True\n",
@@ -773,7 +582,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 23,
+      "execution_count": null,
       "metadata": {
         "id": "e4b838cbcd99"
       },
@@ -812,7 +621,7 @@
         "# Instantiate the Storage client and create the new bucket\n",
         "# from google.cloud import  storage\n",
         "storage_client = storage.Client()\n",
-        "bucket = storage_client.bucket(BUCKET_NAME)\n",
+        "bucket = storage_client.get_bucket(BUCKET_NAME)\n",
         "# Iterate over the prediction instances, creating a new TXT file\n",
         "# for each.\n",
         "input_file_data = []\n",
@@ -879,7 +688,7 @@
         "id": "cd014de40e2f"
       },
       "source": [
-        "## BatchPredictionJob"
+        "## Batch prediction job"
       ]
     },
     {
@@ -890,8 +699,6 @@
       },
       "outputs": [],
       "source": [
-        "from google.cloud.aiplatform import jobs\n",
-        "\n",
         "batch_job = jobs.BatchPredictionJob(batch_prediction_job_name)\n",
         "print(f\"Batch prediction job state: {str(batch_job.state)}\")"
       ]
@@ -972,7 +779,7 @@
         "id": "e375109b7e40"
       },
       "source": [
-        "## JsonLines"
+        "## Review results"
       ]
     },
     {
@@ -1048,16 +855,22 @@
         "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
         "    ! gsutil rm -r $BUCKET_URI\n",
         "\n",
+        "# Delete batch\n",
         "batch_job.delete()\n",
+        "\n",
+        "# Undeploy endpoint\n",
         "endpoint.undeploy_all()\n",
+        "\n",
         "# `force` parameter ensures that models are undeployed before deletion\n",
         "endpoint.delete()\n",
         "\n",
+        "# Delete model\n",
         "model.delete()\n",
         "\n",
+        "# Delete text dataset\n",
         "text_dataset.delete()\n",
         "\n",
-        "# Training job\n",
+        "# Delete training job\n",
         "job.delete()"
       ]
     },
@@ -1067,7 +880,7 @@
         "id": "fa6a8c434c79"
       },
       "source": [
-        "## Next Steps\n",
+        "## Next steps\n",
         "\n",
         "After completing this tutorial, see the following documentation pages to learn more about Vertex AI:\n",
         "\n",

notebooks/official/automl/sdk_automl_image_object_detection_batch.ipynb

@@ -8,7 +8,7 @@
       },
       "outputs": [],
       "source": [
-        "# Copyright 2021 Google LLC\n",
+        "# Copyright 2022 Google LLC\n",
         "#\n",
         "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
         "# you may not use this file except in compliance with the License.\n",
@@ -66,6 +66,17 @@
         "This tutorial demonstrates how to use the Vertex AI SDK to create image object detection models and do batch prediction using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dataset:salads,iod"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the Salads category of the [OpenImages dataset](https://www.tensorflow.org/datasets/catalog/open_images_v4) from [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/overview). This dataset does not require any feature engineering. The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket. The trained model predicts the bounding box locations and the corresponding type of salad items in an image from a class of five items: salad, seafood, tomato, baked goods, or cheese."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -74,7 +85,7 @@
       "source": [
         "### Objective\n",
         "\n",
-        "In this tutorial, you learn how to create an AutoML image object detection model from a Python script, and then do a batch prediction using the Vertex AI SDK. You can alternatively create and deploy models using the `gcloud` command-line tool or online using the Cloud Console.\n",
+        "In this tutorial, you create an AutoML image object detection model from a Python script, and then do a batch prediction using the Vertex AI SDK. You can alternatively create and deploy models using the `gcloud` command-line tool or online using the Cloud Console.\n",
         "\n",
         "The steps performed include:\n",
         "\n",
@@ -90,17 +101,6 @@
         "* Batch Prediction Service: Does a queued (batch) prediction for the entire set of instances in the background and stores the results in a Cloud Storage bucket when ready."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "dataset:salads,iod"
-      },
-      "source": [
-        "### Dataset\n",
-        "\n",
-        "The dataset used for this tutorial is the Salads category of the [OpenImages dataset](https://www.tensorflow.org/datasets/catalog/open_images_v4) from [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/overview). This dataset does not require any feature engineering. The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket. The trained model predicts the bounding box locations and the corresponding type of salad items in an image from a class of five items: salad, seafood, tomato, baked goods, or cheese."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -121,39 +121,6 @@
         "to generate a cost estimate based on your projected usage."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "setup_local"
-      },
-      "source": [
-        "### Set up your local development environment\n",
-        "\n",
-        "If you are using Colab or Google Cloud Notebooks, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
-        "\n",
-        "Otherwise, make sure your environment meets this notebook's requirements. You need the following:\n",
-        "\n",
-        "- The Cloud Storage SDK\n",
-        "- Git\n",
-        "- Python 3\n",
-        "- virtualenv\n",
-        "- Jupyter notebook running in a virtual environment with Python 3\n",
-        "\n",
-        "The Cloud Storage guide to [Setting up a Python development environment](https://cloud.google.com/python/setup) and the [Jupyter installation guide](https://jupyter.org/install) provide detailed instructions for meeting these requirements. The following steps provide a condensed set of instructions:\n",
-        "\n",
-        "1. [Install and initialize the SDK](https://cloud.google.com/sdk/docs/).\n",
-        "\n",
-        "2. [Install Python 3](https://cloud.google.com/python/setup#installing_python).\n",
-        "\n",
-        "3. [Install virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv) and create a virtual environment that uses Python 3.  Activate the virtual environment.\n",
-        "\n",
-        "4. To install Jupyter, run `pip3 install jupyter` on the command-line in a terminal shell.\n",
-        "\n",
-        "5. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
-        "\n",
-        "6. Open this notebook in the Jupyter Notebook Dashboard.\n"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -162,7 +129,7 @@
       "source": [
         "## Installation\n",
         "\n",
-        "Install the latest version of Vertex AI SDK for Python."
+        "Install the latest version of Cloud Storage, Bigquery and Vertex AI SDKs for Python."
       ]
     },
     {
@@ -173,44 +140,10 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
-        "# Google Cloud Notebook\n",
-        "if os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
-        "    USER_FLAG = \"--user\"\n",
-        "else:\n",
-        "    USER_FLAG = \"\"\n",
-        "\n",
-        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "install_storage"
-      },
-      "source": [
-        "Install the latest GA version of *google-cloud-storage* library."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "d6Pa6Sybv5mK"
-      },
-      "outputs": [],
-      "source": [
-        "! pip3 install -U --upgrade tensorflow google-cloud-storage $USER_FLAG"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "9_zWlX10v5mL"
-      },
-      "source": [
-        "Install the latest version of *tensorflow* library."
+        "# Install the packages.\n",
+        "! pip3 install --upgrade google-cloud-aiplatform \\\n",
+        "                         google-cloud-storage     \\\n",
+        "                         tensorflow -q"
       ]
     },
     {
@@ -219,9 +152,7 @@
         "id": "restart"
       },
       "source": [
-        "### Restart the kernel\n",
-        "\n",
-        "Once you've installed the additional packages, you need to restart the notebook kernel so it can find the packages."
+        "### Colab only: Uncomment the following cell to restart the kernel"
       ]
     },
     {
@@ -232,14 +163,20 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
+        "# Automatically restart kernel after installs so that your environment can access the new packages\n",
+        "# import IPython\n",
         "\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    # Automatically restart kernel after installs\n",
-        "    import IPython\n",
-        "\n",
-        "    app = IPython.Application.instance()\n",
-        "    app.kernel.do_shutdown(True)"
+        "# app = IPython.Application.instance()\n",
+        "# app.kernel.do_shutdown(True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "013daf3de88e"
+      },
+      "source": [
+        "## Before you begin"
       ]
     },
     {
@@ -248,28 +185,12 @@
         "id": "before_you_begin:nogpu"
       },
       "source": [
-        "## Before you begin\n",
+        "### Set your project ID\n",
         "\n",
-        "### GPU runtime\n",
-        "\n",
-        "This tutorial does not require a GPU runtime.\n",
-        "\n",
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "2. [Make sure that billing is enabled for your project.](https://cloud.google.com/billing/docs/how-to/modify-project)\n",
-        "\n",
-        "3. [Enable the following APIs: Vertex AI APIs, Compute Engine APIs, and Cloud Storage.](https://console.cloud.google.com/flows/enableapi?apiid=ml.googleapis.com,compute_component,storage-component.googleapis.com)\n",
-        "\n",
-        "4. If you are running this notebook locally, you will need to install the [Cloud SDK]((https://cloud.google.com/sdk)).\n",
-        "\n",
-        "5. Enter your project ID in the cell below. Then run the  cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$`."
+        "**If you don't know your project ID**, try the following:\n",
+        "* Run `gcloud config list`.\n",
+        "* Run `gcloud projects list`.\n",
+        "* See the support page: [Locate the project ID](https://support.google.com/googleapi/answer/7014113)"
       ]
     },
     {
@@ -280,56 +201,10 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}\n",
         "\n",
-        "PROJECT_ID = \"\""
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "autoset_project_id"
-      },
-      "outputs": [],
-      "source": [
-        "# Get your Google Cloud project ID from gcloud\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    shell_output = !gcloud config list --format 'value(core.project)' 2>/dev/null\n",
-        "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID: \", PROJECT_ID)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "W2F5WRyhv5mO"
-      },
-      "source": [
-        "Otherwise, set your project ID here."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "d7-MjQafv5mO"
-      },
-      "outputs": [],
-      "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
-        "    PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "set_gcloud_project_id"
-      },
-      "outputs": [],
-      "source": [
-        "! gcloud config set project $PROJECT_ID"
+        "# Set the project id\n",
+        "! gcloud config set project {PROJECT_ID}"
       ]
     },
     {
@@ -340,16 +215,7 @@
       "source": [
         "#### Region\n",
         "\n",
-        "You can also change the `REGION` variable, which is used for operations\n",
-        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
-        "\n",
-        "- Americas: `us-central1`\n",
-        "- Europe: `europe-west4`\n",
-        "- Asia Pacific: `asia-east1`\n",
-        "\n",
-        "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
-        "\n",
-        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)"
+        "You can also change the `REGION` variable used by Vertex AI. Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
       ]
     },
     {
@@ -360,41 +226,7 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"[your-region]\"  # @param {type:\"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "timestamp"
-      },
-      "source": [
-        "#### UUID\n",
-        "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "PzKW-zT_v5mR"
-      },
-      "outputs": [],
-      "source": [
-        "import random\n",
-        "import string\n",
-        "\n",
-        "\n",
-        "# Generate a uuid of a specifed length(default=8)\n",
-        "def generate_uuid(length: int = 8) -> str:\n",
-        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
-        "\n",
-        "\n",
-        "UUID = generate_uuid()"
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
       ]
     },
     {
@@ -405,53 +237,68 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Google Cloud Notebooks**, your environment is already authenticated.\n",
-        "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
-        "\n",
-        "**Click Create service account**.\n",
-        "\n",
-        "In the **Service account name** field, enter a name, and click **Create**.\n",
-        "\n",
-        "In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex\" into the filter box, and select **Vertex Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "Click Create. A JSON file that contains your key downloads to your local environment.\n",
-        "\n",
-        "Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
+        "Depending on your Jupyter environment, you may have to manually authenticate. Follow the relevant instructions below."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "FvQeFm3Gv5mR"
+      },
+      "source": [
+        "**1. Vertex AI Workbench**\n",
+        "* Do nothing as you are already authenticated."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ad1138a125ea"
+      },
+      "source": [
+        "**2. Local JupyterLab instance, uncomment and run:**"
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "FvQeFm3Gv5mR"
+        "id": "ce6043da7b33"
       },
       "outputs": [],
       "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Google Cloud Notebook, then don't execute this code\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+        "# ! gcloud auth login"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0367eac06a10"
+      },
+      "source": [
+        "**3. Colab, uncomment and run:**"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "21ad4dbb4a61"
+      },
+      "outputs": [],
+      "source": [
+        "# from google.colab import auth\n",
+        "# auth.authenticate_user()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "c13224697bfb"
+      },
+      "source": [
+        "**4. Service account or other**\n",
+        "* See how to grant Cloud Storage permissions to your service account at https://cloud.google.com/storage/docs/gsutil/commands/iam#ch-examples."
       ]
     },
     {
@@ -462,11 +309,7 @@
       "source": [
         "### Create a Cloud Storage bucket\n",
         "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "When you initialize the Vertex AI SDK for Python, you specify a Cloud Storage staging bucket. The staging bucket is where all the data associated with your dataset and model resources are retained across sessions.\n",
-        "\n",
-        "Set the name of your Cloud Storage bucket below. Bucket names must be globally unique across all Google Cloud projects, including those outside of your organization."
+        "Create a storage bucket to store intermediate artifacts such as datasets."
       ]
     },
     {
@@ -477,20 +320,7 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "autoset_bucket"
-      },
-      "outputs": [],
-      "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + UUID"
+        "BUCKET_URI = \"gs://your-bucket-name-unique\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -510,27 +340,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_URI"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "validate_bucket"
-      },
-      "source": [
-        "Finally, validate access to your Cloud Storage bucket by examining its contents:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "N9JY-esPv5mU"
-      },
-      "outputs": [],
-      "source": [
-        "! gsutil ls -al $BUCKET_URI"
+        "! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
       ]
     },
     {
@@ -539,10 +349,7 @@
         "id": "setup_vars"
       },
       "source": [
-        "### Set up variables\n",
-        "\n",
-        "Next, set up some variables used throughout the tutorial.\n",
-        "### Import libraries and define constants"
+        "### Import libraries"
       ]
     },
     {
@@ -575,7 +382,7 @@
       },
       "outputs": [],
       "source": [
-        "aiplatform.init(project=PROJECT_ID, staging_bucket=BUCKET_NAME, location=REGION)"
+        "aiplatform.init(project=PROJECT_ID, staging_bucket=BUCKET_URI, location=REGION)"
       ]
     },
     {
@@ -664,8 +471,9 @@
       },
       "outputs": [],
       "source": [
+        "DISPLAY_NAME = \"salads_unique\"\n",
         "dataset = aiplatform.ImageDataset.create(\n",
-        "    display_name=\"Salads\" + \"_\" + UUID,\n",
+        "    display_name=DISPLAY_NAME,\n",
         "    gcs_source=[IMPORT_FILE],\n",
         "    import_schema_uri=aiplatform.schema.dataset.ioformat.image.bounding_box,\n",
         ")\n",
@@ -713,7 +521,7 @@
       "outputs": [],
       "source": [
         "job = aiplatform.AutoMLImageTrainingJob(\n",
-        "    display_name=\"salads_\" + UUID,\n",
+        "    display_name=DISPLAY_NAME,\n",
         "    prediction_type=\"object_detection\",\n",
         "    multi_label=False,\n",
         "    model_type=\"CLOUD\",\n",
@@ -756,7 +564,7 @@
       "source": [
         "model = job.run(\n",
         "    dataset=dataset,\n",
-        "    model_display_name=\"salads_\" + UUID,\n",
+        "    model_display_name=DISPLAY_NAME,\n",
         "    training_fraction_split=0.8,\n",
         "    validation_fraction_split=0.1,\n",
         "    test_fraction_split=0.1,\n",
@@ -786,7 +594,8 @@
       "outputs": [],
       "source": [
         "# Get model resource ID\n",
-        "models = aiplatform.Model.list(filter=\"display_name=salads_\" + UUID)\n",
+        "filter_name = f\"display_name={DISPLAY_NAME}\"\n",
+        "models = aiplatform.Model.list(filter=filter_name)\n",
         "\n",
         "# Get a reference to the Model Service client\n",
         "client_options = {\"api_endpoint\": f\"{REGION}-aiplatform.googleapis.com\"}\n",
@@ -905,6 +714,7 @@
       "outputs": [],
       "source": [
         "import json\n",
+        "import os\n",
         "\n",
         "import tensorflow as tf\n",
         "\n",
@@ -957,7 +767,7 @@
       "outputs": [],
       "source": [
         "batch_predict_job = model.batch_predict(\n",
-        "    job_display_name=\"salads_\" + UUID,\n",
+        "    job_display_name=DISPLAY_NAME,\n",
         "    gcs_source=gcs_input_uri,\n",
         "    gcs_destination_prefix=BUCKET_URI,\n",
         "    machine_type=\"n1-standard-4\",\n",

notebooks/official/automl/sdk_automl_tabular_regression_batch_bq.ipynb

@@ -74,6 +74,14 @@
         "\n",
         "In this tutorial, you learn how to create an AutoML tabular regression model and deploy it for batch prediction using the Vertex AI SDK for Python. You can alternatively create and deploy models using the `gcloud` command-line tool or batch using the Cloud Console.\n",
         "\n",
+        "This tutorial uses the following Google Cloud ML services and resources:\n",
+        "\n",
+        "- Vertex AI Datasets (Tabular)\n",
+        "- Vertex AI Training (AutoML Tabular Training)\n",
+        "- Vertex AI Model Registry\n",
+        "- Vertex AI Endpoint\n",
+        "- Vertex AI Batch predictions\n",
+        "\n",
         "The steps performed include:\n",
         "\n",
         "- Create a Vertex AI `Dataset` resource.\n",
@@ -107,45 +115,11 @@
         "\n",
         "* Vertex AI\n",
         "* Cloud Storage\n",
+        "* BigQuery / BigQuery ML\n",
         "\n",
         "Learn about [Vertex AI\n",
-        "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
-        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
-        "Calculator](https://cloud.google.com/products/calculator/)\n",
-        "to generate a cost estimate based on your projected usage."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "setup_local"
-      },
-      "source": [
-        "### Set up your local development environment\n",
-        "\n",
-        "If you are using Colab or Vertex AI Workbench, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
-        "\n",
-        "Otherwise, make sure your environment meets this notebook's requirements. You need the following:\n",
-        "\n",
-        "- The Cloud Storage SDK\n",
-        "- Git\n",
-        "- Python 3\n",
-        "- virtualenv\n",
-        "- Jupyter notebook running in a virtual environment with Python 3\n",
-        "\n",
-        "The Cloud Storage guide to [Setting up a Python development environment](https://cloud.google.com/python/setup) and the [Jupyter installation guide](https://jupyter.org/install) provide detailed instructions for meeting these requirements. The following steps provide a condensed set of instructions:\n",
-        "\n",
-        "1. [Install and initialize the SDK](https://cloud.google.com/sdk/docs/).\n",
-        "\n",
-        "2. [Install Python 3](https://cloud.google.com/python/setup#installing_python).\n",
-        "\n",
-        "3. [Install virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv) and create a virtual environment that uses Python 3.  Activate the virtual environment.\n",
-        "\n",
-        "4. To install Jupyter, run `pip3 install jupyter` on the command-line in a terminal shell.\n",
-        "\n",
-        "5. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
-        "\n",
-        "6. Open this notebook in the Jupyter Notebook Dashboard.\n"
+        "pricing](https://cloud.google.com/vertex-ai/pricing), [Cloud Storage\n",
+        "pricing](https://cloud.google.com/storage/pricing) and [BigQuery pricing](https://cloud.google.com/bigquery/pricing) and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
       ]
     },
     {
@@ -156,66 +130,22 @@
       "source": [
         "## Installation\n",
         "\n",
-        "Install the latest version of the Vertex AI SDK for Python."
+        "Install the following packages required to execute this notebook."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "install_aip:mbsdk"
+        "id": "870f1b093d9c"
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
-        "# Google Cloud Notebook\n",
-        "if os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
-        "    USER_FLAG = \"--user\"\n",
-        "else:\n",
-        "    USER_FLAG = \"\"\n",
-        "\n",
-        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "install_storage"
-      },
-      "source": [
-        "Install the latest version of *google-cloud-storage*."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "install_storage"
-      },
-      "outputs": [],
-      "source": [
-        "! pip3 install -U google-cloud-storage $USER_FLAG"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "b2f8bf1a1c31"
-      },
-      "source": [
-        "Install the latest version of *google-cloud-bigquery*."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "fb18bb35a386"
-      },
-      "outputs": [],
-      "source": [
-        "! pip3 install -U \"google-cloud-bigquery[pandas]\" $USER_FLAG"
+        "# Install the packages\n",
+        "! pip3 install --upgrade --quiet google-cloud-aiplatform \\\n",
+        "                                    'google-cloud-bigquery[bqstorage,pandas]' \\\n",
+        "                                    google-cloud-storage \n",
+        "                                    "
       ]
     },
     {
@@ -224,9 +154,7 @@
         "id": "restart"
       },
       "source": [
-        "### Restart the kernel\n",
-        "\n",
-        "After installing the packages, restart the notebook kernel."
+        "### Colab only: Uncomment the following cell to restart the kernel."
       ]
     },
     {
@@ -237,14 +165,11 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
+        "# Automatically restart kernel after installs so that your environment can access the new packages\n",
+        "# import IPython\n",
         "\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    # Automatically restart kernel after installs\n",
-        "    import IPython\n",
-        "\n",
-        "    app = IPython.Application.instance()\n",
-        "    app.kernel.do_shutdown(True)"
+        "# app = IPython.Application.instance()\n",
+        "# app.kernel.do_shutdown(True)"
       ]
     },
     {
@@ -254,27 +179,12 @@
       },
       "source": [
         "## Before you begin\n",
+        "#### Set your project ID\n",
         "\n",
-        "### GPU runtime\n",
-        "\n",
-        "This tutorial does not require a GPU runtime.\n",
-        "\n",
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "2. [Make sure that billing is enabled for your project.](https://cloud.google.com/billing/docs/how-to/modify-project)\n",
-        "\n",
-        "3. [Enable the following APIs: Vertex AI APIs, Compute Engine APIs, and Cloud Storage.](https://console.cloud.google.com/flows/enableapi?apiid=ml.googleapis.com,compute_component,storage-component.googleapis.com)\n",
-        "\n",
-        "4. If you are running this notebook locally, install the [Cloud SDK]((https://cloud.google.com/sdk)).\n",
-        "\n",
-        "5. Enter your project ID in the cell below. Then run the  cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$`."
+        "**If you don't know your project ID**, try the following:\n",
+        "* Run `gcloud config list`.\n",
+        "* Run `gcloud projects list`.\n",
+        "* See the support page: [Locate the project ID](https://support.google.com/googleapi/answer/7014113)"
       ]
     },
     {
@@ -285,33 +195,10 @@
       },
       "outputs": [],
       "source": [
-        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "autoset_project_id"
-      },
-      "outputs": [],
-      "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
-        "    # Get your GCP project id from gcloud\n",
-        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
-        "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID:\", PROJECT_ID)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "set_gcloud_project_id"
-      },
-      "outputs": [],
-      "source": [
-        "! gcloud config set project $PROJECT_ID"
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# Set the project id\n",
+        "! gcloud config set project {PROJECT_ID}"
       ]
     },
     {
@@ -322,15 +209,7 @@
       "source": [
         "#### Region\n",
         "\n",
-        "You can also change the `REGION` variable, which is used for operations throughout the rest of this notebook. The following regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
-        "\n",
-        "- Americas: `us-central1`\n",
-        "- Europe: `europe-west4`\n",
-        "- Asia Pacific: `asia-east1`\n",
-        "\n",
-        "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
-        "\n",
-        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
+        "You can also change the `REGION` variable used by Vertex AI. Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
       ]
     },
     {
@@ -344,30 +223,6 @@
         "REGION = \"[your-region]\"  # @param {type: \"string\"}"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "timestamp"
-      },
-      "source": [
-        "#### Timestamp\n",
-        "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "timestamp"
-      },
-      "outputs": [],
-      "source": [
-        "from datetime import datetime\n",
-        "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -376,53 +231,53 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Google Cloud Notebooks**, your environment is already authenticated. Skip this step.\n",
+        "Depending on your Jupyter environment, you may have to manually authenticate. Follow the relevant instructions below.\n",
         "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
-        "\n",
-        "**Click Create service account**.\n",
-        "\n",
-        "In the **Service account name** field, enter a name, and click **Create**.\n",
-        "\n",
-        "In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex\" into the filter box, and select **Vertex Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "Click Create. A JSON file that contains your key downloads to your local environment.\n",
-        "\n",
-        "Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
+        "**1. Vertex AI Workbench**\n",
+        "* Do nothing as you are already authenticated.\n",
+        "**2. Local JupyterLab instance, uncomment and run:**\n"
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "gcp_authenticate"
+        "id": "457c78b08293"
       },
       "outputs": [],
       "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Google Cloud Notebook, then don't execute this code\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        ! gcloud auth login"
+        "# ! gcloud auth login"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d3e571ce6c56"
+      },
+      "source": [
+        "**3. Colab, uncomment and run:**"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "984a0526fb68"
+      },
+      "outputs": [],
+      "source": [
+        "# from google.colab import auth\n",
+        "# auth.authenticate_user()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2c549a59cca4"
+      },
+      "source": [
+        "**4. Service account or other**\n",
+        "* See how to grant Cloud Storage permissions to your service account at https://cloud.google.com/storage/docs/gsutil/commands/iam#ch-examples."
       ]
     },
     {
@@ -445,7 +300,8 @@
       },
       "outputs": [],
       "source": [
-        "import google.cloud.aiplatform as aiplatform"
+        "import google.cloud.aiplatform as aiplatform\n",
+        "from google.cloud import bigquery"
       ]
     },
     {
@@ -522,8 +378,6 @@
       },
       "outputs": [],
       "source": [
-        "from google.cloud import bigquery\n",
-        "\n",
         "# Create client in default region\n",
         "bq_client = bigquery.Client(\n",
         "    project=PROJECT_ID,\n",
@@ -540,13 +394,13 @@
       "outputs": [],
       "source": [
         "# Create training dataset in default region\n",
-        "TRAINING_INPUT_DATASET_ID = f\"gsod_training_{TIMESTAMP}\"\n",
+        "TRAINING_INPUT_DATASET_ID = \"gsod_training_unique\"\n",
         "bq_dataset = bigquery.Dataset(f\"{PROJECT_ID}.{TRAINING_INPUT_DATASET_ID}\")\n",
         "bq_dataset = bq_client.create_dataset(bq_dataset)\n",
         "print(f\"Created dataset {bq_client.project}.{bq_dataset.dataset_id}\")\n",
         "\n",
         "# Create test dataset in default region\n",
-        "PREDICTION_INPUT_DATASET_ID = f\"gsod_prediction_{TIMESTAMP}\"\n",
+        "PREDICTION_INPUT_DATASET_ID = \"gsod_prediction_unique\"\n",
         "bq_dataset = bigquery.Dataset(f\"{PROJECT_ID}.{PREDICTION_INPUT_DATASET_ID}\")\n",
         "bq_dataset = bq_client.create_dataset(bq_dataset)\n",
         "print(f\"Created dataset {bq_client.project}.{bq_dataset.dataset_id}\")"
@@ -625,7 +479,7 @@
       "outputs": [],
       "source": [
         "dataset = aiplatform.TabularDataset.create(\n",
-        "    display_name=\"NOAA historical weather data\" + \"_\" + TIMESTAMP,\n",
+        "    display_name=\"NOAA historical weather data_unique\",\n",
         "    bq_source=[f\"bq://{TRAINING_INPUT_TABLE_ID}\"],\n",
         ")\n",
         "\n",
@@ -696,7 +550,7 @@
       "outputs": [],
       "source": [
         "training_job = aiplatform.AutoMLTabularTrainingJob(\n",
-        "    display_name=\"gsod_\" + TIMESTAMP,\n",
+        "    display_name=\"job_unique\",\n",
         "    optimization_prediction_type=\"regression\",\n",
         "    optimization_objective=\"minimize-rmse\",\n",
         "    column_specs=COLUMN_SPECS,\n",
@@ -726,7 +580,7 @@
         "\n",
         "The `run` method when completed returns the `Model` resource.\n",
         "\n",
-        "The execution of the training pipeline will take upto 20 minutes."
+        "The execution of the training pipeline will take upto 3 hours."
       ]
     },
     {
@@ -739,7 +593,7 @@
       "source": [
         "model = training_job.run(\n",
         "    dataset=dataset,\n",
-        "    model_display_name=\"gsod_\" + TIMESTAMP,\n",
+        "    model_display_name=\"model_unique\",\n",
         "    training_fraction_split=0.6,\n",
         "    validation_fraction_split=0.2,\n",
         "    test_fraction_split=0.2,\n",
@@ -805,7 +659,7 @@
       "outputs": [],
       "source": [
         "# Create results dataset in default region\n",
-        "RESULTS_DATASET_ID = f\"gsod_results_{TIMESTAMP}\"\n",
+        "RESULTS_DATASET_ID = \"gsod_results_unique\"\n",
         "bq_dataset = bigquery.Dataset(f\"{PROJECT_ID}.{RESULTS_DATASET_ID}\")\n",
         "bq_dataset = bq_client.create_dataset(bq_dataset)\n",
         "print(f\"Created dataset {bq_client.project}.{bq_dataset.dataset_id}\")"
@@ -829,7 +683,9 @@
         "- `machine_type`: The type of machine to use for training.\n",
         "- `accelerator_type`: The hardware accelerator type.\n",
         "- `accelerator_count`: The number of accelerators to attach to a worker replica.\n",
-        "- `sync`: If set to True, the call will block while waiting for the asynchronous batch job to complete."
+        "- `sync`: If set to True, the call will block while waiting for the asynchronous batch job to complete.\n",
+        "\n",
+        "Batch prediction job takes roughly 1 hour to finish."
       ]
     },
     {
@@ -922,17 +778,8 @@
         "\n",
         "bq_client.delete_dataset(\n",
         "    f\"{PROJECT_ID}.{RESULTS_DATASET_ID}\", delete_contents=True, not_found_ok=True\n",
-        ")"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "cleanup:mbsdk"
-      },
-      "outputs": [],
-      "source": [
+        ")\n",
+        "\n",
         "# Delete Vertex AI resources\n",
         "dataset.delete()\n",
         "model.delete()\n",

notebooks/official/automl/sdk_automl_tabular_regression_online_bq.ipynb

@@ -116,39 +116,6 @@
         "to generate a cost estimate based on your projected usage."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "setup_local"
-      },
-      "source": [
-        "### Set up your local development environment\n",
-        "\n",
-        "If you are using Colab or Google Cloud Notebooks, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
-        "\n",
-        "Otherwise, make sure your environment meets this notebook's requirements. You need the following:\n",
-        "\n",
-        "- The Cloud Storage SDK\n",
-        "- Git\n",
-        "- Python 3\n",
-        "- virtualenv\n",
-        "- Jupyter notebook running in a virtual environment with Python 3\n",
-        "\n",
-        "The Cloud Storage guide to [Setting up a Python development environment](https://cloud.google.com/python/setup) and the [Jupyter installation guide](https://jupyter.org/install) provide detailed instructions for meeting these requirements. The following steps provide a condensed set of instructions:\n",
-        "\n",
-        "1. [Install and initialize the SDK](https://cloud.google.com/sdk/docs/).\n",
-        "\n",
-        "2. [Install Python 3](https://cloud.google.com/python/setup#installing_python).\n",
-        "\n",
-        "3. [Install virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv) and create a virtual environment that uses Python 3.  Activate the virtual environment.\n",
-        "\n",
-        "4. To install Jupyter, run `pip3 install jupyter` on the command-line in a terminal shell.\n",
-        "\n",
-        "5. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
-        "\n",
-        "6. Open this notebook in the Jupyter Notebook Dashboard.\n"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -168,35 +135,8 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
-        "# Google Cloud Notebook\n",
-        "if os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
-        "    USER_FLAG = \"--user\"\n",
-        "else:\n",
-        "    USER_FLAG = \"\"\n",
-        "\n",
-        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "install_storage"
-      },
-      "source": [
-        "Install the latest GA version of *google-cloud-storage* library as well."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "install_storage"
-      },
-      "outputs": [],
-      "source": [
-        "! pip3 install -U google-cloud-storage $USER_FLAG"
+        "! pip3 install --upgrade google-cloud-aiplatform \\\n",
+        "                            google-cloud-storage"
       ]
     },
     {
@@ -205,9 +145,7 @@
         "id": "restart"
       },
       "source": [
-        "### Restart the kernel\n",
-        "\n",
-        "Once you've installed the additional packages, you need to restart the notebook kernel so it can find the packages."
+        "### Colab only: Uncomment the following cell to restart the kernel"
       ]
     },
     {
@@ -218,14 +156,11 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
+        "# Automatically restart kernel after installs so that your environment can access the new packages\n",
+        "# import IPython\n",
         "\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    # Automatically restart kernel after installs\n",
-        "    import IPython\n",
-        "\n",
-        "    app = IPython.Application.instance()\n",
-        "    app.kernel.do_shutdown(True)"
+        "# app = IPython.Application.instance()\n",
+        "# app.kernel.do_shutdown(True)"
       ]
     },
     {
@@ -242,20 +177,11 @@
         "\n",
         "### Set up your Google Cloud project\n",
         "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
+        "**If you dont know your project ID,** try the following\n",
         "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "2. [Make sure that billing is enabled for your project.](https://cloud.google.com/billing/docs/how-to/modify-project)\n",
-        "\n",
-        "3. [Enable the following APIs: Vertex AI APIs, Compute Engine APIs, and Cloud Storage.](https://console.cloud.google.com/flows/enableapi?apiid=ml.googleapis.com,compute_component,storage-component.googleapis.com)\n",
-        "\n",
-        "4. If you are running this notebook locally, you will need to install the [Cloud SDK]((https://cloud.google.com/sdk)).\n",
-        "\n",
-        "5. Enter your project ID in the cell below. Then run the  cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$`."
+        "- Run `gcloud config list`\n",
+        "- Run `gcloud projects list`\n",
+        "- See the support page: [Locate the project ID](https://support.google.com/googleapi/answer/7014113)\n"
       ]
     },
     {
@@ -266,33 +192,10 @@
       },
       "outputs": [],
       "source": [
-        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "autoset_project_id"
-      },
-      "outputs": [],
-      "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
-        "    # Get your GCP project id from gcloud\n",
-        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
-        "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID:\", PROJECT_ID)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "set_gcloud_project_id"
-      },
-      "outputs": [],
-      "source": [
-        "! gcloud config set project $PROJECT_ID"
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# Set the project ID\n",
+        "! gcloud config set project {PROJECT_ID}"
       ]
     },
     {
@@ -303,16 +206,7 @@
       "source": [
         "#### Region\n",
         "\n",
-        "You can also change the `REGION` variable, which is used for operations\n",
-        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
-        "\n",
-        "- Americas: `us-central1`\n",
-        "- Europe: `europe-west4`\n",
-        "- Asia Pacific: `asia-east1`\n",
-        "\n",
-        "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
-        "\n",
-        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)"
+        "You can also change the `REGION` variable used by Vertex AI. Learn more about [Vertex AI Regions](https://cloud.google.com/vertex-ai/docs/general/locations)"
       ]
     },
     {
@@ -326,30 +220,6 @@
         "REGION = \"us-central1\"  # @param {type: \"string\"}"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "timestamp"
-      },
-      "source": [
-        "#### Timestamp\n",
-        "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "timestamp"
-      },
-      "outputs": [],
-      "source": [
-        "from datetime import datetime\n",
-        "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -358,53 +228,64 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Google Cloud Notebooks**, your environment is already authenticated. Skip this step.\n",
+        "Depending on your Jupyter environment, you may have to manually authenticate. Follow the relevant instructions below.\n",
         "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
+        "**1. Vertex AI workbench** \n",
+        "- Do nothing as you are already authenticated.\n",
         "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
-        "\n",
-        "**Click Create service account**.\n",
-        "\n",
-        "In the **Service account name** field, enter a name, and click **Create**.\n",
-        "\n",
-        "In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex\" into the filter box, and select **Vertex Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "Click Create. A JSON file that contains your key downloads to your local environment.\n",
-        "\n",
-        "Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
+        "**2. Local JupyterLab Instance, uncomment and run:**"
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "gcp_authenticate"
+        "id": "457c78b08293"
       },
       "outputs": [],
       "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Google Cloud Notebook, then don't execute this code\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+        "# ! gcloud auth login"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d3e571ce6c56"
+      },
+      "source": [
+        "**3. Colab, uncomment and run:**"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "984a0526fb68"
+      },
+      "outputs": [],
+      "source": [
+        "# from google.colab import auth\n",
+        "# auth.authenticate_user()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "764c0ac706e1"
+      },
+      "source": [
+        "**4. Service account or other**\n",
+        "- See all the authentication options here: [Google Cloud Platform Jupyter Notebook Authentication Guide](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/notebook_authentication_guide.ipynb)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "276cfd2c6167"
+      },
+      "source": [
+        "### Create a Cloud Storage bucket\n",
+        "Create a storage bucket to store intermediate artifacts such as datasets"
       ]
     },
     {
@@ -415,20 +296,7 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "autoset_bucket"
-      },
-      "outputs": [],
-      "source": [
-        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+        "BUCKET_URI = \"gs://test-bucket-unique\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -451,35 +319,12 @@
         "! gsutil mb -l $REGION $BUCKET_URI"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "validate_bucket"
-      },
-      "source": [
-        "Finally, validate access to your Cloud Storage bucket by examining its contents:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "validate_bucket"
-      },
-      "outputs": [],
-      "source": [
-        "! gsutil ls -al $BUCKET_URI"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "setup_vars"
       },
       "source": [
-        "### Set up variables\n",
-        "\n",
-        "Next, set up some variables used throughout the tutorial.\n",
         "### Import libraries and define constants"
       ]
     },
@@ -491,7 +336,11 @@
       },
       "outputs": [],
       "source": [
-        "import google.cloud.aiplatform as aiplatform"
+        "import os\n",
+        "\n",
+        "import google.cloud.aiplatform as aiplatform\n",
+        "\n",
+        "display_name = \"gsod_unique\""
       ]
     },
     {
@@ -574,7 +423,7 @@
       "outputs": [],
       "source": [
         "dataset = aiplatform.TabularDataset.create(\n",
-        "    display_name=\"NOAA historical weather data\" + \"_\" + TIMESTAMP,\n",
+        "    display_name=\"NOAA historical weather data_unique\",\n",
         "    bq_source=[IMPORT_FILE],\n",
         ")\n",
         "\n",
@@ -645,7 +494,7 @@
       "outputs": [],
       "source": [
         "job = aiplatform.AutoMLTabularTrainingJob(\n",
-        "    display_name=\"gsod_\" + TIMESTAMP,\n",
+        "    display_name=display_name,\n",
         "    optimization_prediction_type=\"regression\",\n",
         "    optimization_objective=\"minimize-rmse\",\n",
         "    column_transformations=TRANSFORMATIONS,\n",
@@ -688,7 +537,7 @@
       "source": [
         "model = job.run(\n",
         "    dataset=dataset,\n",
-        "    model_display_name=\"gsod_\" + TIMESTAMP,\n",
+        "    model_display_name=display_name,\n",
         "    training_fraction_split=0.6,\n",
         "    validation_fraction_split=0.2,\n",
         "    test_fraction_split=0.2,\n",
@@ -705,33 +554,22 @@
       },
       "source": [
         "## Review model evaluation scores\n",
-        "After your model has finished training, you can review the evaluation scores for it.\n",
-        "\n",
-        "First, you need to get a reference to the new model. As with datasets, you can either use the reference to the model variable you created when you deployed the model or you can list all of the models in your project."
+        "After your model training has finished, you can review the evaluation scores for it using the list_model_evaluations() method."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "evaluate_the_model:mbsdk"
+        "id": "4f674dcea72c"
       },
       "outputs": [],
       "source": [
-        "# Get model resource ID\n",
-        "models = aiplatform.Model.list(filter=\"display_name=gsod_\" + TIMESTAMP)\n",
-        "\n",
-        "# Get a reference to the Model Service client\n",
-        "client_options = {\"api_endpoint\": f\"{REGION}-aiplatform.googleapis.com\"}\n",
-        "model_service_client = aiplatform.gapic.ModelServiceClient(\n",
-        "    client_options=client_options\n",
-        ")\n",
-        "\n",
-        "model_evaluations = model_service_client.list_model_evaluations(\n",
-        "    parent=models[0].resource_name\n",
-        ")\n",
-        "model_evaluation = list(model_evaluations)[0]\n",
-        "print(model_evaluation)"
+        "model_evaluations = model.list_model_evaluations()\n",
+        "if len(model_evaluations) > 0:\n",
+        "    eval_res = model_evaluations[0].to_dict()\n",
+        "    evaluation_metrics = eval_res[\"metrics\"]\n",
+        "print(evaluation_metrics)"
       ]
     },
     {
@@ -881,23 +719,22 @@
       },
       "outputs": [],
       "source": [
-        "delete_all = True\n",
+        "# Delete the dataset using the Vertex dataset object\n",
+        "dataset.delete()\n",
         "\n",
-        "if delete_all:\n",
-        "    # Delete the dataset using the Vertex dataset object\n",
-        "    dataset.delete()\n",
+        "# Delete the model using the Vertex model object\n",
+        "model.delete()\n",
         "\n",
-        "    # Delete the model using the Vertex model object\n",
-        "    model.delete()\n",
+        "# Delete the endpoint using the Vertex endpoint object\n",
+        "endpoint.delete()\n",
         "\n",
-        "    # Delete the endpoint using the Vertex endpoint object\n",
-        "    endpoint.delete()\n",
+        "# Delete the AutoML trainig job\n",
+        "job.delete()\n",
         "\n",
-        "    # Delete the AutoML trainig job\n",
-        "    job.delete()\n",
+        "delete_bucket = False\n",
         "\n",
-        "    if os.getenv(\"IS_TESTING\"):\n",
-        "        ! gsutil rm -r $BUCKET_URI"
+        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "    ! gsutil rm -r $BUCKET_URI"
       ]
     }
   ],

notebooks/official/automl/sdk_automl_text_sentiment_analysis_online.ipynb

@@ -29,7 +29,7 @@
         "id": "title"
       },
       "source": [
-        "# Vertex SDK: AutoML training text sentiment analysis model for online prediction\n",
+        "# Vertex AI SDK: Training an AutoML text sentiment analysis model for online predictions\n",
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
@@ -45,6 +45,7 @@
         "  </td>\n",
         "  <td>\n",
         "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_text_sentiment_analysis_online.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
         "  </td>\n",
@@ -60,8 +61,7 @@
       "source": [
         "## Overview\n",
         "\n",
-        "\n",
-        "This tutorial demonstrates how to use the Vertex AI SDK to create text sentiment analysis models and do online prediction using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model."
+        "This tutorial demonstrates how to use the Vertex AI SDK to train and deploy an [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) text sentiment analysis model and get online predictions from it."
       ]
     },
     {
@@ -72,16 +72,23 @@
       "source": [
         "### Objective\n",
         "\n",
-        "In this tutorial, you learn how to create an AutoML text sentiment analysis model and deploy for online prediction from a Python script using the Vertex SDK. You can alternatively create and deploy models using the `gcloud` command-line tool or online using the Cloud Console.\n",
+        "In this tutorial, you learn how to create an AutoML text sentiment analysis model and deploy it for online predictions from a Python script using the Vertex AI SDK. You can alternatively create and deploy models using the `gcloud` command-line tool or online using the Cloud Console.\n",
+        "\n",
+        "This tutorial uses the following Google Cloud ML services and resources:\n",
+        "- Vertex AI Datasets\n",
+        "- Vertex AI Training (AutoML)\n",
+        "- Vertex AI Model Registry\n",
+        "- Vertex AI Endpoints\n",
         "\n",
         "The steps performed include:\n",
         "\n",
-        "- Create a Vertex `Dataset` resource.\n",
-        "- Create a training job for the model.\n",
-        "- View the model evaluation.\n",
-        "- Deploy the `Model` resource to a serving `Endpoint` resource.\n",
-        "- Make a prediction.\n",
-        "- Undeploy the `Model`."
+        "- Create a `Vertex AI Dataset` resource.\n",
+        "- Create a training job for the AutoML model on the dataset.\n",
+        "- View the model evaluation metrics.\n",
+        "- Deploy the `Vertex AI Model` resource to a serving `Vertex AI Endpoint`.\n",
+        "- Make a prediction request to the deployed model.\n",
+        "- Undeploy the model from endpoint.\n",
+        "- Perform clean up process."
       ]
     },
     {
@@ -92,7 +99,7 @@
       "source": [
         "### Dataset\n",
         "\n",
-        "The dataset used for this tutorial is the [Crowdflower Claritin-Twitter dataset](https://data.world/crowdflower/claritin-twitter) that consists of tweets tagged with sentiment, the author's gender, and whether or not they mention any of the top 10 adverse events reported to the FDA. The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket. In this tutorial, you will use the tweets' data to build an AutoML-text-sentiment-analysis model on Google Cloud platform."
+        "The dataset used for this tutorial is the [Crowdflower Claritin-Twitter dataset](https://data.world/crowdflower/claritin-twitter) that consists of tweets tagged with sentiment, the author's gender, and whether or not they mention any of the top 10 adverse events reported to the FDA. The version of the dataset you use in this tutorial is stored in a public Cloud Storage bucket. In this tutorial, you use the tweets data to build an AutoML text sentiment analysis model on Google Cloud platform."
       ]
     },
     {
@@ -123,29 +130,34 @@
       "source": [
         "### Set up your local development environment\n",
         "\n",
-        "If you are using Colab or Google Cloud Notebooks, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
+        "**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets\n",
+        "all the requirements to run this notebook. You can skip this step.\n",
         "\n",
-        "Otherwise, make sure your environment meets this notebook's requirements. You need the following:\n",
+        "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
+        "You need the following:\n",
         "\n",
-        "- The Cloud Storage SDK\n",
-        "- Git\n",
-        "- Python 3\n",
-        "- virtualenv\n",
-        "- Jupyter notebook running in a virtual environment with Python 3\n",
+        "* The Google Cloud SDK\n",
         "\n",
-        "The Cloud Storage guide to [Setting up a Python development environment](https://cloud.google.com/python/setup) and the [Jupyter installation guide](https://jupyter.org/install) provide detailed instructions for meeting these requirements. The following steps provide a condensed set of instructions:\n",
+        "The Google Cloud guide to [Setting up a Python development\n",
+        "environment](https://cloud.google.com/python/setup) and the [Jupyter\n",
+        "installation guide](https://jupyter.org/install) provide detailed instructions\n",
+        "for meeting these requirements. The following steps provide a condensed set of\n",
+        "instructions:\n",
         "\n",
-        "1. [Install and initialize the SDK](https://cloud.google.com/sdk/docs/).\n",
+        "1. [Install and initialize the Cloud SDK.](https://cloud.google.com/sdk/docs/)\n",
         "\n",
-        "2. [Install Python 3](https://cloud.google.com/python/setup#installing_python).\n",
+        "1. [Install Python 3.](https://cloud.google.com/python/setup#installing_python)\n",
         "\n",
-        "3. [Install virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv) and create a virtual environment that uses Python 3.  Activate the virtual environment.\n",
+        "1. [Install\n",
+        "   virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv)\n",
+        "   and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
         "\n",
-        "4. To install Jupyter, run `pip3 install jupyter` on the command-line in a terminal shell.\n",
+        "1. To install Jupyter, run `pip3 install jupyter` on the\n",
+        "command-line in a terminal shell.\n",
         "\n",
-        "5. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
+        "1. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
         "\n",
-        "6. Open this notebook in the Jupyter Notebook Dashboard.\n"
+        "1. Open this notebook in the Jupyter Notebook Dashboard."
       ]
     },
     {
@@ -156,7 +168,7 @@
       "source": [
         "## Installation\n",
         "\n",
-        "Install the latest version of Vertex SDK for Python."
+        "Install the latest version of Vertex AI SDK for Python."
       ]
     },
     {
@@ -169,35 +181,19 @@
       "source": [
         "import os\n",
         "\n",
-        "# Google Cloud Notebook\n",
-        "if os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
+        "\n",
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "USER_FLAG = \"\"\n",
+        "if IS_WORKBENCH_NOTEBOOK:\n",
         "    USER_FLAG = \"--user\"\n",
-        "else:\n",
-        "    USER_FLAG = \"\"\n",
         "\n",
-        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "install_storage"
-      },
-      "source": [
-        "Install the latest GA version of *google-cloud-storage* library as well.\n",
-        "\n",
-        "**Note**: You may encounter a PIP dependency error during the installation of the Google Cloud Storage package. This can be ignored as it will not affect the proper running of this script."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "install_storage"
-      },
-      "outputs": [],
-      "source": [
-        "! pip3 install -U google-cloud-storage $USER_FLAG"
+        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG -q\n",
+        "! pip3 install -U google-cloud-storage $USER_FLAG -q"
       ]
     },
     {
@@ -208,7 +204,7 @@
       "source": [
         "### Restart the kernel\n",
         "\n",
-        "Once you've installed the additional packages, you need to restart the notebook kernel so it can find the packages. The following cell will restart the kernel."
+        "After you install the additional packages, you need to restart the notebook kernel so it can find the packages."
       ]
     },
     {
@@ -219,6 +215,7 @@
       },
       "outputs": [],
       "source": [
+        "# Automatically restart kernel after installs\n",
         "import os\n",
         "\n",
         "if not os.getenv(\"IS_TESTING\"):\n",
@@ -237,26 +234,33 @@
       "source": [
         "## Before you begin\n",
         "\n",
-        "### GPU runtime\n",
-        "\n",
-        "This tutorial does not require a GPU runtime.\n",
-        "\n",
         "### Set up your Google Cloud project\n",
         "\n",
         "**The following steps are required, regardless of your notebook environment.**\n",
         "\n",
         "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
         "\n",
-        "2. [Make sure that billing is enabled for your project.](https://cloud.google.com/billing/docs/how-to/modify-project)\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
         "\n",
-        "3. [Enable the following APIs: Vertex AI APIs, Compute Engine APIs, and Cloud Storage.](https://console.cloud.google.com/flows/enableapi?apiid=ml.googleapis.com,compute_component,storage-component.googleapis.com)\n",
+        "1. [Enable the Vertex AI, Compute Engine, and Cloud Storage APIs.](https://console.cloud.google.com/flows/enableapi?apiid=ml.googleapis.com,compute_component,storage-component.googleapis.com)\n",
         "\n",
-        "4. If you are running this notebook locally, you will need to install the [Cloud SDK]((https://cloud.google.com/sdk)).\n",
+        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
         "\n",
-        "5. Enter your project ID in the cell below. Then run the  cell to make sure the\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
         "Cloud SDK uses the right project for all the commands in this notebook.\n",
         "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$`."
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "af794e75b7e3"
+      },
+      "source": [
+        "#### Set your project ID\n",
+        "\n",
+        "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
       ]
     },
     {
@@ -336,9 +340,9 @@
         "id": "timestamp"
       },
       "source": [
-        "#### Timestamp\n",
+        "#### UUID\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
       ]
     },
     {
@@ -349,9 +353,16 @@
       },
       "outputs": [],
       "source": [
-        "from datetime import datetime\n",
+        "import random\n",
+        "import string\n",
         "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+        "\n",
+        "# Generate a uuid of a specifed length(default=8)\n",
+        "def generate_uuid(length: int = 8) -> str:\n",
+        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
+        "\n",
+        "\n",
+        "UUID = generate_uuid()"
       ]
     },
     {
@@ -362,23 +373,31 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Google Cloud Notebooks**, your environment is already authenticated. Skip this step.\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
+        "authenticated. \n",
         "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
+        "**If you are using Colab**, run the cell below and follow the instructions\n",
+        "when prompted to authenticate your account via oAuth.\n",
         "\n",
         "**Otherwise**, follow these steps:\n",
         "\n",
-        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
+        "1. In the Cloud Console, go to the [**Create service account key**\n",
+        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
         "\n",
-        "**Click Create service account**.\n",
+        "2. Click **Create service account**.\n",
         "\n",
-        "In the **Service account name** field, enter a name, and click **Create**.\n",
+        "3. In the **Service account name** field, enter a name, and\n",
+        "   click **Create**.\n",
         "\n",
-        "In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex\" into the filter box, and select **Vertex Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
+        "into the filter box, and select\n",
+        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
         "\n",
-        "Click Create. A JSON file that contains your key downloads to your local environment.\n",
+        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
+        "local environment.\n",
         "\n",
-        "Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
+        "6. Enter the path to your service account key as the\n",
+        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
       ]
     },
     {
@@ -397,8 +416,11 @@
         "import os\n",
         "import sys\n",
         "\n",
-        "# If on Google Cloud Notebook, then don't execute this code\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = \"google.colab\" in sys.modules\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
         "    if \"google.colab\" in sys.modules:\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
@@ -408,7 +430,7 @@
         "    # path to your service account key and run this cell to authenticate your GCP\n",
         "    # account.\n",
         "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+        "        %env GOOGLE_APPLICATION_CREDENTIALS '[your-service-account-key-path]'"
       ]
     },
     {
@@ -421,9 +443,9 @@
         "\n",
         "**The following steps are required, regardless of your notebook environment.**\n",
         "\n",
-        "When you initialize the Vertex SDK for Python, you specify a Cloud Storage staging bucket. The staging bucket is where all the data associated with your dataset and model resources are retained across sessions.\n",
+        "When you initialize your Vertex AI SDK, you provide a Cloud Storage bucket to the SDK to serve as a staging bucket for the session. \n",
         "\n",
-        "Set the name of your Cloud Storage bucket below. Bucket names must be globally unique across all Google Cloud projects, including those outside of your organization."
+        "Set the name of your Cloud Storage bucket below. It must be unique across all Cloud Storage buckets."
       ]
     },
     {
@@ -434,7 +456,8 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
+        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
     {
@@ -445,8 +468,9 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
+        "    BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
     {
@@ -466,7 +490,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_NAME"
+        "! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
       ]
     },
     {
@@ -486,7 +510,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_NAME"
+        "! gsutil ls -al $BUCKET_URI"
       ]
     },
     {
@@ -495,10 +519,7 @@
         "id": "setup_vars"
       },
       "source": [
-        "### Set up variables\n",
-        "\n",
-        "Next, set up some variables used throughout the tutorial.\n",
-        "### Import libraries and define constants"
+        "### Import libraries"
       ]
     },
     {
@@ -518,9 +539,9 @@
         "id": "init_aip:mbsdk"
       },
       "source": [
-        "## Initialize Vertex SDK for Python\n",
+        "### Initialize Vertex AI SDK for Python\n",
         "\n",
-        "Initialize the Vertex SDK for Python for your project and corresponding bucket."
+        "Initialize the Vertex AI SDK for Python for your project and corresponding bucket."
       ]
     },
     {
@@ -531,18 +552,7 @@
       },
       "outputs": [],
       "source": [
-        "aiplatform.init(project=PROJECT_ID, staging_bucket=BUCKET_NAME)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "tutorial_start:automl"
-      },
-      "source": [
-        "# Tutorial\n",
-        "\n",
-        "Now you are ready to start creating your own AutoML text sentiment analysis model."
+        "aiplatform.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
       ]
     },
     {
@@ -551,9 +561,9 @@
         "id": "import_file:u_dataset,csv"
       },
       "source": [
-        "#### Location of Cloud Storage training data.\n",
+        "### Define the constants\n",
         "\n",
-        "Now set the variable `IMPORT_FILE` to the location of the CSV index file in Cloud Storage."
+        "Set the constants that you use in this tutorial."
       ]
     },
     {
@@ -564,7 +574,9 @@
       },
       "outputs": [],
       "source": [
+        "# Set the location of the CSV index file in Cloud Storage.\n",
         "IMPORT_FILE = \"gs://cloud-samples-data/language/claritin.csv\"\n",
+        "# Set the max. sentiment score\n",
         "SENTIMENT_MAX = 4"
       ]
     },
@@ -574,11 +586,11 @@
         "id": "quick_peek:csv"
       },
       "source": [
-        "#### Quick peek at your data\n",
+        "## Take a quick peek at your data\n",
         "\n",
-        "This tutorial uses a version of the Crowdflower Claritin-Twitter dataset that is stored in a public Cloud Storage bucket, using a CSV index file.\n",
+        "This tutorial uses a version of the `Crowdflower Claritin-Twitter` dataset which is stored in a public Cloud Storage bucket, using a CSV index file.\n",
         "\n",
-        "Start by doing a quick peek at the data. You count the number of examples by counting the number of rows in the CSV index file  (`wc -l`) and then peek at the first few rows."
+        "Start by taking a quick peek at the data. Further, count the number of examples by counting the number of rows in the CSV index file  (`wc -l`) and then print the first few rows."
       ]
     },
     {
@@ -604,12 +616,12 @@
         "id": "create_dataset:text,tst"
       },
       "source": [
-        "### Create the Dataset\n",
+        "## Create the Dataset\n",
         "\n",
-        "Next, create the `Dataset` resource using the `create` method for the `TextDataset` class, which takes the following parameters:\n",
+        "Now, create a `Vertex AI Dataset` resource using the `create` method of the `TextDataset` class, which takes the following parameters:\n",
         "\n",
-        "- `display_name`: The human readable name for the `Dataset` resource.\n",
-        "- `gcs_source`: A list of one or more dataset index files to import the data items into the `Dataset` resource.\n",
+        "- `display_name`: The human readable name for the dataset resource.\n",
+        "- `gcs_source`: A list of one or more dataset index files to import the data items into the dataset resource.\n",
         "- `import_schema_uri`: The data labeling schema for the data items.\n",
         "\n",
         "This operation may take several minutes."
@@ -624,7 +636,7 @@
       "outputs": [],
       "source": [
         "dataset = aiplatform.TextDataset.create(\n",
-        "    display_name=\"Crowdflower Claritin-Twitter\" + \"_\" + TIMESTAMP,\n",
+        "    display_name=\"Crowdflower Claritin-Twitter\" + \"_\" + UUID,\n",
         "    gcs_source=[IMPORT_FILE],\n",
         "    import_schema_uri=aiplatform.schema.dataset.ioformat.text.sentiment,\n",
         ")\n",
@@ -638,15 +650,18 @@
         "id": "create_automl_pipeline:text,tst"
       },
       "source": [
-        "### Create and run training pipeline\n",
+        "## Create and run training job\n",
         "\n",
-        "To train an AutoML model, you perform two steps: 1) create a training pipeline, and 2) run the pipeline.\n",
+        "In this section, to train an AutoML model, you perform these steps:\n",
         "\n",
-        "#### Create training pipeline\n",
+        "1) create a training job.\n",
+        "2) run the job.\n",
         "\n",
-        "An AutoML training pipeline is created with the `AutoMLTextTrainingJob` class, with the following parameters:\n",
+        "### Create a training job\n",
         "\n",
-        "- `display_name`: The human readable name for the `TrainingJob` resource.\n",
+        "An AutoML training job is created with the `AutoMLTextTrainingJob` class, with the following parameters:\n",
+        "\n",
+        "- `display_name`: The human readable name for the training job resource.\n",
         "- `prediction_type`: The type task to train the model for.\n",
         "  - `classification`: A text classification model.\n",
         "  - `sentiment`: A text sentiment analysis model.\n",
@@ -664,7 +679,7 @@
       "outputs": [],
       "source": [
         "job = aiplatform.AutoMLTextTrainingJob(\n",
-        "    display_name=\"claritin_\" + TIMESTAMP,\n",
+        "    display_name=\"claritin_\" + UUID,\n",
         "    prediction_type=\"sentiment\",\n",
         "    sentiment_max=SENTIMENT_MAX,\n",
         ")\n",
@@ -678,7 +693,7 @@
         "id": "run_automl_pipeline:text"
       },
       "source": [
-        "#### Run the training pipeline\n",
+        "### Run the training job\n",
         "\n",
         "Next, you run the training job by invoking the method `run`, with the following parameters:\n",
         "\n",
@@ -690,7 +705,7 @@
         "\n",
         "The `run` method when completed returns the `Model` resource.\n",
         "\n",
-        "The execution of the training pipeline will take upto 180 minutes."
+        "The execution of the training pipeline take upto 180 minutes."
       ]
     },
     {
@@ -703,7 +718,7 @@
       "source": [
         "model = job.run(\n",
         "    dataset=dataset,\n",
-        "    model_display_name=\"claritin_\" + TIMESTAMP,\n",
+        "    model_display_name=\"claritin_\" + UUID,\n",
         "    training_fraction_split=0.8,\n",
         "    validation_fraction_split=0.1,\n",
         "    test_fraction_split=0.1,\n",
@@ -717,9 +732,10 @@
       },
       "source": [
         "## Review model evaluation scores\n",
-        "After your model has finished training, you can review the evaluation scores for it.\n",
         "\n",
-        "First, you need to get a reference to the new model. As with datasets, you can either use the reference to the model variable you created when you deployed the model or you can list all of the models in your project."
+        "Once your model training has finished, you can review the evaluation scores.\n",
+        "\n",
+        "Firstly, you need to get a reference to the newly created model. As with datasets, you can either use the reference to the model variable you created when you deployed the model or you can list all of the models in your project and filter."
       ]
     },
     {
@@ -731,7 +747,7 @@
       "outputs": [],
       "source": [
         "# Get model resource ID\n",
-        "models = aiplatform.Model.list(filter=\"display_name=claritin_\" + TIMESTAMP)\n",
+        "models = aiplatform.Model.list(filter=\"display_name=claritin_\" + UUID)\n",
         "\n",
         "# Get a reference to the Model Service client\n",
         "client_options = {\"api_endpoint\": f\"{REGION}-aiplatform.googleapis.com\"}\n",
@@ -754,7 +770,9 @@
       "source": [
         "## Deploy the model\n",
         "\n",
-        "Next, deploy your model for online prediction. To deploy the model, you invoke the `deploy` method."
+        "Next, deploy your model to serve online predictions. To deploy the model, you invoke the `deploy` method of the model resource which in turn returns you the deployed endpoint.\n",
+        "\n",
+        "**Note:** Normally, an endpoint is created beforehand and is given as a reference while model deployment. By default, `deploy()` method creates an endpoint when an endpoint reference is not given."
       ]
     },
     {
@@ -774,9 +792,9 @@
         "id": "make_prediction"
       },
       "source": [
-        "## Send a online prediction request\n",
+        "## Send online prediction requests\n",
         "\n",
-        "Send a online prediction to your deployed model."
+        "In this step, you prepare some test instances from the dataset and send an online prediction request to your deployed model."
       ]
     },
     {
@@ -785,9 +803,9 @@
         "id": "get_test_item"
       },
       "source": [
-        "### Get test item\n",
+        "### Create test instances\n",
         "\n",
-        "You will use an arbitrary example out of the dataset as a test item. Don't be concerned that the example was likely used in training the model -- we just want to demonstrate how to make a prediction."
+        "You use an arbitrary example out of the dataset as a test item. Don't be concerned that the example was likely used in training the model. It is just to demonstrate how to make a prediction."
       ]
     },
     {
@@ -813,21 +831,21 @@
         "id": "predict_request:mbsdk,tst"
       },
       "source": [
-        "### Make the prediction\n",
+        "### Make the prediction request\n",
         "\n",
-        "Now that your `Model` resource is deployed to an `Endpoint` resource, you can do online predictions by sending prediction requests to the `Endpoint` resource.\n",
+        "Now that your model is deployed to an endpoint, you can send online prediction requests to the endpoint resource.\n",
         "\n",
-        "#### Request\n",
+        "#### Request format\n",
         "\n",
-        "The format of each instance is:\n",
+        "The format of each instance should be in JSON as below:\n",
         "\n",
         "     { 'content': text_string }\n",
         "\n",
-        "Since the predict() method can take multiple items (instances), send your single test item as a list of one test item.\n",
+        "Since the `predict()` method can take multiple instances, send your request as a list of one test instance.\n",
         "\n",
         "#### Response\n",
         "\n",
-        "The response from the predict() call is a Python dictionary with the following entries:\n",
+        "The response from the `predict()` call is a Python dictionary with the following entries:\n",
         "\n",
         "- `ids`: The internal assigned unique identifiers for each prediction request.\n",
         "- `sentiment`: The sentiment value.\n",
@@ -856,7 +874,7 @@
       "source": [
         "## Undeploy the model\n",
         "\n",
-        "When you are done doing predictions, you undeploy the model from the `Endpoint` resouce. This deprovisions all compute resources and ends billing for the deployed model."
+        "After you explore the predictions, you undeploy the model from the `Endpoint` resouce. This deprovisions all compute resources and ends billing for the deployed model."
       ]
     },
     {
@@ -883,11 +901,11 @@
         "\n",
         "Otherwise, you can delete the individual resources you created in this tutorial:\n",
         "\n",
-        "- Dataset\n",
-        "- Model\n",
-        "- Endpoint\n",
+        "- Vertex AI Dataset\n",
+        "- Vertex AI Model\n",
+        "- Vertex AI Endpoint\n",
         "- AutoML Training Job\n",
-        "- Cloud Storage Bucket"
+        "- Cloud Storage Bucket (set `delete_bucket` to **True** to delete the bucket)"
       ]
     },
     {
@@ -898,6 +916,8 @@
       },
       "outputs": [],
       "source": [
+        "delete_bucket = False\n",
+        "\n",
         "# Delete the dataset using the Vertex dataset object\n",
         "dataset.delete()\n",
         "\n",
@@ -911,8 +931,8 @@
         "job.delete()\n",
         "\n",
         "# Delete the Cloud storage bucket\n",
-        "if os.getenv(\"IS_TESTING\"):\n",
-        "    ! gsutil rm -r $BUCKET_NAME"
+        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "    ! gsutil rm -r $BUCKET_URI"
       ]
     }
   ],

notebooks/official/automl/sdk_automl_video_action_recognition_batch.ipynb

@@ -29,9 +29,10 @@
         "id": "title"
       },
       "source": [
-        "# Vertex SDK: AutoML training video action recognition model for batch prediction\n",
+        "# Vertex AI SDK: AutoML training video action recognition model for batch prediction\n",
         "\n",
         "<table align=\"left\">\n",
+        "\n",
         "  <td>\n",
         "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_video_action_recognition_batch.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
@@ -44,10 +45,11 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_video_action_recognition_batch.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/automl/sdk_automl_video_action_recognition_batch.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
-        "  </td>\n",
+        "  </td>                                                                                               \n",
         "</table>\n",
         "<br/><br/><br/>"
       ]
@@ -74,9 +76,16 @@
         "\n",
         "In this tutorial, you learn how to create an AutoML video action recognition model from a Python script, and then do a batch prediction using the Vertex AI SDK. You can alternatively create and deploy models using the `gcloud` command-line tool or online using the Cloud Console.\n",
         "\n",
+        "This tutorial uses the following Google Cloud ML services and resources:\n",
+        "\n",
+        "- Vertex AI Dataset\n",
+        "- Vertex AI Model\n",
+        "- Vertex AI Batch Prediction\n",
+        "\n",
+        "\n",
         "The steps performed include:\n",
         "\n",
-        "- Create a Vertex `Dataset` resource.\n",
+        "- Create a `Vertex AI Dataset` resource.\n",
         "- Train the model.\n",
         "- View the model evaluation.\n",
         "- Make a batch prediction.\n",
@@ -96,7 +105,7 @@
       "source": [
         "### Dataset\n",
         "\n",
-        "The dataset used for this tutorial is the golf swing recognition portion of the [Human Motion dataset from MIT](http://cbcl.mit.edu/publications/ps/Kuehne_etal_iccv11.pdf). The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket. The trained model will predict the start frame where an action of golf swing begins."
+        "The dataset used for this tutorial is the golf swing recognition portion of the [Human Motion dataset from MIT](http://cbcl.mit.edu/publications/ps/Kuehne_etal_iccv11.pdf). The version of the dataset you use in this tutorial is stored in a public Cloud Storage bucket. The trained model will predict the start frame where an action of golf swing begins."
       ]
     },
     {
@@ -127,29 +136,38 @@
       "source": [
         "### Set up your local development environment\n",
         "\n",
-        "If you are using Colab or Google Cloud Notebooks, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
+        "**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets\n",
+        "all the requirements to run this notebook.\n",
         "\n",
-        "Otherwise, make sure your environment meets this notebook's requirements. You need the following:\n",
+        "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
+        "You need the following:\n",
         "\n",
-        "- The Cloud Storage SDK\n",
-        "- Git\n",
-        "- Python 3\n",
-        "- virtualenv\n",
-        "- Jupyter notebook running in a virtual environment with Python 3\n",
+        "* The Google Cloud SDK\n",
+        "* Git\n",
+        "* Python 3\n",
+        "* virtualenv\n",
+        "* Jupyter notebook running in a virtual environment with Python 3\n",
         "\n",
-        "The Cloud Storage guide to [Setting up a Python development environment](https://cloud.google.com/python/setup) and the [Jupyter installation guide](https://jupyter.org/install) provide detailed instructions for meeting these requirements. The following steps provide a condensed set of instructions:\n",
+        "The Google Cloud guide to [Setting up a Python development\n",
+        "environment](https://cloud.google.com/python/setup) and the [Jupyter\n",
+        "installation guide](https://jupyter.org/install) provide detailed instructions\n",
+        "for meeting these requirements. The following steps provide a condensed set of\n",
+        "instructions:\n",
         "\n",
-        "1. [Install and initialize the SDK](https://cloud.google.com/sdk/docs/).\n",
+        "1. [Install and initialize the Cloud SDK.](https://cloud.google.com/sdk/docs/)\n",
         "\n",
-        "2. [Install Python 3](https://cloud.google.com/python/setup#installing_python).\n",
+        "1. [Install Python 3.](https://cloud.google.com/python/setup#installing_python)\n",
         "\n",
-        "3. [Install virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv) and create a virtual environment that uses Python 3.  Activate the virtual environment.\n",
+        "1. [Install\n",
+        "   virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv)\n",
+        "   and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
         "\n",
-        "4. To install Jupyter, run `pip3 install jupyter` on the command-line in a terminal shell.\n",
+        "1. To install Jupyter, run `pip3 install jupyter` on the\n",
+        "command-line in a terminal shell.\n",
         "\n",
-        "5. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
+        "1. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
         "\n",
-        "6. Open this notebook in the Jupyter Notebook Dashboard.\n"
+        "1. Open this notebook in the Jupyter Notebook Dashboard."
       ]
     },
     {
@@ -160,7 +178,7 @@
       "source": [
         "## Installation\n",
         "\n",
-        "Install the latest version of Vertex SDK for Python."
+        "Install the following packages required to execute this notebook. \n"
       ]
     },
     {
@@ -173,35 +191,17 @@
       "source": [
         "import os\n",
         "\n",
-        "# Google Cloud Notebook\n",
-        "if os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "USER_FLAG = \"\"\n",
+        "if IS_WORKBENCH_NOTEBOOK:\n",
         "    USER_FLAG = \"--user\"\n",
-        "else:\n",
-        "    USER_FLAG = \"\"\n",
         "\n",
-        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "install_storage"
-      },
-      "source": [
-        "Install the latest GA version of *google-cloud-storage* library as well.\n",
-        "\n",
-        "**Note**: You may encounter a PIP dependency error during the installation of the Google Cloud Storage package. This can be ignored as it will not affect the proper running of this script."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "install_storage"
-      },
-      "outputs": [],
-      "source": [
-        "! pip3 install -U google-cloud-storage $USER_FLAG"
+        "! pip3 install --upgrade --quiet {USER_FLAG} google-cloud-aiplatform google-cloud-storage"
       ]
     },
     {
@@ -212,7 +212,7 @@
       "source": [
         "### Restart the kernel\n",
         "\n",
-        "Once you've installed the additional packages, you need to restart the notebook kernel so it can find the packages. The following cell will restart the kernel."
+        "After you install the additional packages, you need to restart the notebook kernel so it can find the packages."
       ]
     },
     {
@@ -223,6 +223,7 @@
       },
       "outputs": [],
       "source": [
+        "# Automatically restart kernel after installs\n",
         "import os\n",
         "\n",
         "if not os.getenv(\"IS_TESTING\"):\n",
@@ -241,26 +242,33 @@
       "source": [
         "## Before you begin\n",
         "\n",
-        "### GPU runtime\n",
-        "\n",
-        "This tutorial does not require a GPU runtime.\n",
-        "\n",
         "### Set up your Google Cloud project\n",
         "\n",
         "**The following steps are required, regardless of your notebook environment.**\n",
         "\n",
         "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
         "\n",
-        "2. [Make sure that billing is enabled for your project.](https://cloud.google.com/billing/docs/how-to/modify-project)\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
         "\n",
-        "3. [Enable the following APIs: Vertex AI APIs, Compute Engine APIs, and Cloud Storage.](https://console.cloud.google.com/flows/enableapi?apiid=ml.googleapis.com,compute_component,storage-component.googleapis.com)\n",
+        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com). \n",
         "\n",
-        "4. If you are running this notebook locally, you will need to install the [Cloud SDK]((https://cloud.google.com/sdk)).\n",
+        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
         "\n",
-        "5. Enter your project ID in the cell below. Then run the  cell to make sure the\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
         "Cloud SDK uses the right project for all the commands in this notebook.\n",
         "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$`."
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5aee4379e8e5"
+      },
+      "source": [
+        "#### Set your project ID\n",
+        "\n",
+        "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
       ]
     },
     {
@@ -309,7 +317,7 @@
         "#### Region\n",
         "\n",
         "You can also change the `REGION` variable, which is used for operations\n",
-        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
+        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. It is recommended that you choose the region closest to you.\n",
         "\n",
         "- Americas: `us-central1`\n",
         "- Europe: `europe-west4`\n",
@@ -317,7 +325,7 @@
         "\n",
         "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
         "\n",
-        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)"
+        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
       ]
     },
     {
@@ -340,9 +348,9 @@
         "id": "timestamp"
       },
       "source": [
-        "#### Timestamp\n",
+        "#### UUID\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial.\n"
       ]
     },
     {
@@ -353,9 +361,16 @@
       },
       "outputs": [],
       "source": [
-        "from datetime import datetime\n",
+        "import random\n",
+        "import string\n",
         "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+        "\n",
+        "# Generate a uuid of a specifed length(default=8)\n",
+        "def generate_uuid(length: int = 8) -> str:\n",
+        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
+        "\n",
+        "\n",
+        "UUID = generate_uuid()"
       ]
     },
     {
@@ -366,23 +381,31 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Google Cloud Notebooks**, your environment is already authenticated. Skip this step.\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
+        "authenticated. \n",
         "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
+        "**If you are using Colab**, run the cell below and follow the instructions\n",
+        "when prompted to authenticate your account via oAuth.\n",
         "\n",
         "**Otherwise**, follow these steps:\n",
         "\n",
-        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
+        "1. In the Cloud Console, go to the [**Create service account key**\n",
+        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
         "\n",
-        "**Click Create service account**.\n",
+        "2. Click **Create service account**.\n",
         "\n",
-        "In the **Service account name** field, enter a name, and click **Create**.\n",
+        "3. In the **Service account name** field, enter a name, and\n",
+        "   click **Create**.\n",
         "\n",
-        "In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex\" into the filter box, and select **Vertex Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
+        "into the filter box, and select\n",
+        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
         "\n",
-        "Click Create. A JSON file that contains your key downloads to your local environment.\n",
+        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
+        "local environment.\n",
         "\n",
-        "Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
+        "6. Enter the path to your service account key as the\n",
+        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
       ]
     },
     {
@@ -401,8 +424,11 @@
         "import os\n",
         "import sys\n",
         "\n",
-        "# If on Google Cloud Notebook, then don't execute this code\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = \"google.colab\" in sys.modules\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
         "    if \"google.colab\" in sys.modules:\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
@@ -425,9 +451,9 @@
         "\n",
         "**The following steps are required, regardless of your notebook environment.**\n",
         "\n",
-        "When you initialize the Vertex SDK for Python, you specify a Cloud Storage staging bucket. The staging bucket is where all the data associated with your dataset and model resources are retained across sessions.\n",
+        "When you run a Vertex AI pipeline job using the Cloud SDK, your job stores the pipeline artifacts to a Cloud Storage bucket. In this tutorial, you create a Vertex AI Pipeline job that saves the artifacts like evaluation metrics and feature attributes to a Cloud Storage bucket.\n",
         "\n",
-        "Set the name of your Cloud Storage bucket below. Bucket names must be globally unique across all Google Cloud projects, including those outside of your organization."
+        "Set the name of your Cloud Storage bucket below. It must be unique across all Cloud Storage buckets."
       ]
     },
     {
@@ -438,7 +464,8 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
+        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
     {
@@ -449,8 +476,9 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
+        "    BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
     {
@@ -470,7 +498,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_NAME"
+        "! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
       ]
     },
     {
@@ -490,7 +518,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_NAME"
+        "! gsutil ls -al $BUCKET_URI"
       ]
     },
     {
@@ -499,9 +527,6 @@
         "id": "setup_vars"
       },
       "source": [
-        "### Set up variables\n",
-        "\n",
-        "Next, set up some variables used throughout the tutorial.\n",
         "### Import libraries and define constants"
       ]
     },
@@ -513,7 +538,11 @@
       },
       "outputs": [],
       "source": [
-        "import google.cloud.aiplatform as aiplatform"
+        "import json\n",
+        "import os\n",
+        "\n",
+        "import google.cloud.aiplatform as aiplatform\n",
+        "from google.cloud import storage"
       ]
     },
     {
@@ -522,9 +551,9 @@
         "id": "init_aip:mbsdk"
       },
       "source": [
-        "## Initialize Vertex SDK for Python\n",
+        "## Initialize Vertex AI SDK for Python\n",
         "\n",
-        "Initialize the Vertex SDK for Python for your project and corresponding bucket."
+        "Initialize the Vertex AI SDK for Python for your project and corresponding bucket."
       ]
     },
     {
@@ -535,7 +564,7 @@
       },
       "outputs": [],
       "source": [
-        "aiplatform.init(project=PROJECT_ID, staging_bucket=BUCKET_NAME)"
+        "aiplatform.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
       ]
     },
     {
@@ -546,15 +575,8 @@
       "source": [
         "# Tutorial\n",
         "\n",
-        "Now you are ready to start creating your own AutoML video action recognition model."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "import_file:u_dataset,csv"
-      },
-      "source": [
+        "Now you are ready to start creating your own AutoML video action recognition model.\n",
+        "\n",
         "#### Location of Cloud Storage training data.\n",
         "\n",
         "Now set the variable `IMPORT_FILES` to the location of the CSV index files in Cloud Storage."
@@ -629,7 +651,7 @@
       "outputs": [],
       "source": [
         "dataset = aiplatform.VideoDataset.create(\n",
-        "    display_name=\"Golf Swings\" + \"_\" + TIMESTAMP,\n",
+        "    display_name=\"Golf Swings\" + \"_\" + UUID,\n",
         "    gcs_source=IMPORT_FILES,\n",
         "    import_schema_uri=aiplatform.schema.dataset.ioformat.video.action_recognition,\n",
         ")\n",
@@ -645,17 +667,19 @@
       "source": [
         "### Create and run training pipeline\n",
         "\n",
-        "To train an AutoML model, you perform two steps: 1) create a training pipeline, and 2) run the pipeline.\n",
+        "To train an AutoML model, you perform two steps: \n",
+        "1. create a training pipeline.\n",
+        "2. run the pipeline.\n",
         "\n",
-        "#### Create training pipeline\n",
+        "#### Create the training pipeline\n",
         "\n",
         "An AutoML training pipeline is created with the `AutoMLVideoTrainingJob` class, with the following parameters:\n",
         "\n",
         "- `display_name`: The human readable name for the `TrainingJob` resource.\n",
         "- `prediction_type`: The type task to train the model for.\n",
-        "  - `classification`: A video classification model.\n",
-        "  - `object_tracking`: A video object tracking model.\n",
-        "  - `action_recognition`: A video action recognition model."
+        "    - `classification`: A video classification model.\n",
+        "    - `object_tracking`: A video object tracking model.\n",
+        "    - `action_recognition`: A video action recognition model."
       ]
     },
     {
@@ -667,7 +691,7 @@
       "outputs": [],
       "source": [
         "job = aiplatform.AutoMLVideoTrainingJob(\n",
-        "    display_name=\"golf_\" + TIMESTAMP,\n",
+        "    display_name=\"golf_\" + UUID,\n",
         "    prediction_type=\"action_recognition\",\n",
         ")\n",
         "\n",
@@ -704,7 +728,7 @@
       "source": [
         "model = job.run(\n",
         "    dataset=dataset,\n",
-        "    model_display_name=\"golf_\" + TIMESTAMP,\n",
+        "    model_display_name=\"golf_\" + UUID,\n",
         "    training_fraction_split=0.8,\n",
         "    test_fraction_split=0.2,\n",
         ")"
@@ -717,9 +741,7 @@
       },
       "source": [
         "## Review model evaluation scores\n",
-        "After your model has finished training, you can review the evaluation scores for it.\n",
-        "\n",
-        "First, you need to get a reference to the new model. As with datasets, you can either use the reference to the model variable you created when you deployed the model or you can list all of the models in your project."
+        "After your model has finished training, you can review the evaluation scores for it.\n"
       ]
     },
     {
@@ -730,18 +752,9 @@
       },
       "outputs": [],
       "source": [
-        "# Get model resource ID\n",
-        "models = aiplatform.Model.list(filter=\"display_name=golf_\" + TIMESTAMP)\n",
+        "# Get evaluations\n",
+        "model_evaluations = model.list_model_evaluations()\n",
         "\n",
-        "# Get a reference to the Model Service client\n",
-        "client_options = {\"api_endpoint\": f\"{REGION}-aiplatform.googleapis.com\"}\n",
-        "model_service_client = aiplatform.gapic.ModelServiceClient(\n",
-        "    client_options=client_options\n",
-        ")\n",
-        "\n",
-        "model_evaluations = model_service_client.list_model_evaluations(\n",
-        "    parent=models[0].resource_name\n",
-        ")\n",
         "model_evaluation = list(model_evaluations)[0]\n",
         "print(model_evaluation)"
       ]
@@ -754,18 +767,11 @@
       "source": [
         "## Send a batch prediction request\n",
         "\n",
-        "Send a batch prediction to your deployed model."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "get_test_items:batch_prediction"
-      },
-      "source": [
+        "Send a batch prediction request to your registered model.\n",
+        "\n",
         "### Get test item(s)\n",
         "\n",
-        "Now do a batch prediction to your Vertex model. You will use arbitrary examples out of the dataset as a test items. Don't be concerned that the examples were likely used in training the model as we just want to demonstrate how to make a prediction."
+        "Now send a batch prediction request to your Vertex AI model. You use arbitrary examples out of the dataset as a test items. Don't be concerned that the examples were likely used in training the model as we just want to demonstrate how to make a prediction."
       ]
     },
     {
@@ -799,7 +805,7 @@
       "source": [
         "### Make a batch input file\n",
         "\n",
-        "Now make a batch input file, which you store in your local Cloud Storage bucket. The batch input file can be either CSV or JSONL. You will use JSONL in this tutorial. For JSONL file, you make one dictionary entry per line for each video. The dictionary contains the key/value pairs:\n",
+        "Now make a batch input file, which you store in your local Cloud Storage bucket. The batch input file can be either CSV or JSONL. You use JSONL in this tutorial. For JSONL file, you make one dictionary entry per line for each video. The dictionary contains the key/value pairs:\n",
         "\n",
         "- `content`: The Cloud Storage path to the video.\n",
         "- `mimeType`: The content type. In our example, it is a `avi` file.\n",
@@ -815,12 +821,8 @@
       },
       "outputs": [],
       "source": [
-        "import json\n",
-        "\n",
-        "from google.cloud import storage\n",
-        "\n",
         "test_filename = \"test.jsonl\"\n",
-        "gcs_input_uri = BUCKET_NAME + \"/\" + test_filename\n",
+        "gcs_input_uri = BUCKET_URI + \"/\" + test_filename\n",
         "\n",
         "# Configure the test-data\n",
         "data_1 = {\n",
@@ -837,7 +839,7 @@
         "}\n",
         "\n",
         "# Upload the test-data to Cloud storage bucket\n",
-        "bucket = storage.Client(project=PROJECT_ID).bucket(BUCKET_NAME.replace(\"gs://\", \"\"))\n",
+        "bucket = storage.Client(project=PROJECT_ID).bucket(BUCKET_URI.replace(\"gs://\", \"\"))\n",
         "blob = bucket.blob(blob_name=test_filename)\n",
         "data = json.dumps(data_1) + \"\\n\" + json.dumps(data_2) + \"\\n\"\n",
         "blob.upload_from_string(data)\n",
@@ -855,7 +857,7 @@
       "source": [
         "### Make the batch prediction request\n",
         "\n",
-        "Now that your Model resource is trained, you can make a batch prediction by invoking the batch_predict() method, with the following parameters:\n",
+        "Now that your Vertex AI Model resource is trained, you can make a batch prediction by invoking the batch_predict() method, with the following parameters:\n",
         "\n",
         "- `job_display_name`: The human readable name for the batch prediction job.\n",
         "- `gcs_source`: A list of one or more batch request input files.\n",
@@ -872,9 +874,9 @@
       "outputs": [],
       "source": [
         "batch_predict_job = model.batch_predict(\n",
-        "    job_display_name=\"golf_\" + TIMESTAMP,\n",
+        "    job_display_name=\"golf_\" + UUID,\n",
         "    gcs_source=gcs_input_uri,\n",
-        "    gcs_destination_prefix=BUCKET_NAME,\n",
+        "    gcs_destination_prefix=BUCKET_URI,\n",
         "    sync=False,\n",
         ")\n",
         "\n",
@@ -941,7 +943,7 @@
         "\n",
         "for prediction_result in prediction_results:\n",
         "    gfile_name = f\"gs://{bp_iter_outputs.bucket.name}/{prediction_result}\".replace(\n",
-        "        BUCKET_NAME + \"/\", \"\"\n",
+        "        BUCKET_URI + \"/\", \"\"\n",
         "    )\n",
         "    data = bucket.get_blob(gfile_name).download_as_string()\n",
         "    data = json.loads(data)\n",
@@ -988,9 +990,10 @@
         "# Delete the batch prediction job using the Vertex batch prediction object\n",
         "batch_predict_job.delete()\n",
         "\n",
-        "# Delete the Cloud Storage bucket\n",
-        "if os.getenv(\"IS_TESTING\"):\n",
-        "    ! gsutil rm -r $BUCKET_NAME"
+        "# Delete Cloud Storage objects\n",
+        "delete_bucket = False\n",
+        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "    ! gsutil -m rm -r $BUCKET_URI"
       ]
     }
   ],

notebooks/official/automl/sdk_automl_video_object_tracking_batch.ipynb

@@ -18,7 +18,7 @@
         "#\n",
         "# Unless required by applicable law or agreed to in writing, software\n",
         "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
-        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,  either express or implied.\n",
         "# See the License for the specific language governing permissions and\n",
         "# limitations under the License."
       ]
@@ -73,7 +73,11 @@
       "source": [
         "### Objective\n",
         "\n",
-        "In this tutorial, you learn how to create an AutoML video object tracking model from a Python script, and then do a batch prediction using the Vertex SDK. You can alternatively create and deploy models using the `gcloud` command-line tool or online using the Cloud Console.\n",
+        "In this tutorial, you learn how to create an AutoML video object tracking model from a Python script, and then do a batch prediction using the Vertex AI SDK. You can alternatively create and deploy models using the `gcloud` command-line tool or online using the Cloud Console.\n",
+        "\n",
+        "This tutorial uses the following Google Cloud ML services and resources:\n",
+        "\n",
+        "- Vertex AI\n",
         "\n",
         "The steps performed include:\n",
         "\n",
@@ -97,7 +101,7 @@
       "source": [
         "### Dataset\n",
         "\n",
-        "The dataset used for this tutorial is the [Traffic](https://storage.googleapis.com/automl-video-demo-data/traffic_videos/traffic_videos_labels.csv) dataset. The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket."
+        "The dataset used for this tutorial is the [Traffic](https://storage.googleapis.com/automl-video-demo-data/traffic_videos/traffic_videos_labels.csv) dataset. The version of the dataset you use in this tutorial is stored in a public Cloud Storage bucket."
       ]
     },
     {
@@ -128,12 +132,11 @@
       "source": [
         "### Set up your local development environment\n",
         "\n",
-        "If you are using Colab or Google Cloud Notebooks, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
+        "**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
         "\n",
-        "Otherwise, make sure your environment meets this notebook's requirements. You need the following:\n",
+        "**Otherwise**, make sure your environment meets this notebook's requirements. You need the following:\n",
         "\n",
         "- The Cloud Storage SDK\n",
-        "- Git\n",
         "- Python 3\n",
         "- virtualenv\n",
         "- Jupyter notebook running in a virtual environment with Python 3\n",
@@ -161,7 +164,7 @@
       "source": [
         "## Installation\n",
         "\n",
-        "Install the latest version of Vertex SDK for Python."
+        "Install the following packages required to execute this notebook."
       ]
     },
     {
@@ -174,33 +177,19 @@
       "source": [
         "import os\n",
         "\n",
-        "# Google Cloud Notebook\n",
-        "if os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
+        "\n",
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "if IS_WORKBENCH_NOTEBOOK:\n",
         "    USER_FLAG = \"--user\"\n",
         "else:\n",
         "    USER_FLAG = \"\"\n",
         "\n",
-        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "install_storage"
-      },
-      "source": [
-        "Install the latest GA version of *google-cloud-storage* library as well."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "install_storage"
-      },
-      "outputs": [],
-      "source": [
-        "! pip3 install -U google-cloud-storage $USER_FLAG"
+        "! pip3 install --upgrade google-cloud-aiplatform google-cloud-storage $USER_FLAG -q"
       ]
     },
     {
@@ -211,7 +200,9 @@
       "source": [
         "### Restart the kernel\n",
         "\n",
-        "Once you've installed the additional packages, you need to restart the notebook kernel so it can find the packages."
+        "Once you've installed the additional packages, you need to restart the notebook kernel so it can find the packages.\n",
+        "\n",
+        "**Note: You may get a message saying \"Your session crashed for an unknown reason.\", this is expected. Once this cell has finished running, continue on. You do not need to re-run any of the cells above.**"
       ]
     },
     {
@@ -254,30 +245,43 @@
         "\n",
         "3. [Enable the following APIs: Vertex AI APIs, Compute Engine APIs, and Cloud Storage.](https://console.cloud.google.com/flows/enableapi?apiid=ml.googleapis.com,compute_component,storage-component.googleapis.com)\n",
         "\n",
-        "4. If you are running this notebook locally, you will need to install the [Cloud SDK]((https://cloud.google.com/sdk)).\n",
+        "4. If you are running this notebook locally, you need to install the [Cloud SDK]((https://cloud.google.com/sdk)).\n",
         "\n",
         "5. Enter your project ID in the cell below. Then run the  cell to make sure the\n",
         "Cloud SDK uses the right project for all the commands in this notebook.\n",
         "\n",
+        "6. (optional) You may also specify a service account to use to run Vertex AI Pipelines in the project.\n",
+        "\n",
         "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$`."
       ]
     },
     {
-      "cell_type": "code",
-      "execution_count": null,
+      "cell_type": "markdown",
       "metadata": {
-        "id": "set_project_id"
+        "id": "5aee4379e8e5"
       },
-      "outputs": [],
       "source": [
-        "PROJECT_ID = \"\"  # @param {type:\"string\"}"
+        "#### Set your project ID\n",
+        "\n",
+        "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "autoset_project_id"
+        "id": "dcdfccf50581"
+      },
+      "outputs": [],
+      "source": [
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "5bf9979b96ff"
       },
       "outputs": [],
       "source": [
@@ -292,7 +296,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "set_gcloud_project_id"
+        "id": "09021c90b34c"
       },
       "outputs": [],
       "source": [
@@ -338,9 +342,9 @@
         "id": "timestamp"
       },
       "source": [
-        "#### Timestamp\n",
+        "#### UUID\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
       ]
     },
     {
@@ -351,9 +355,16 @@
       },
       "outputs": [],
       "source": [
-        "from datetime import datetime\n",
+        "import random\n",
+        "import string\n",
         "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+        "\n",
+        "# Generate a uuid of a specifed length(default=8)\n",
+        "def generate_uuid(length: int = 8) -> str:\n",
+        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
+        "\n",
+        "\n",
+        "UUID = generate_uuid()"
       ]
     },
     {
@@ -364,23 +375,23 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Google Cloud Notebooks**, your environment is already authenticated. Skip this step.\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated.\n",
         "\n",
         "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
         "\n",
         "**Otherwise**, follow these steps:\n",
         "\n",
-        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
+        "- In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
         "\n",
-        "**Click Create service account**.\n",
+        "- **Click Create service account**.\n",
         "\n",
-        "In the **Service account name** field, enter a name, and click **Create**.\n",
+        "- In the **Service account name** field, enter a name, and click **Create**.\n",
         "\n",
-        "In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex\" into the filter box, and select **Vertex Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "- In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex\" into the filter box, and select **Vertex Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
         "\n",
-        "Click Create. A JSON file that contains your key downloads to your local environment.\n",
+        "- Click Create. A JSON file that contains your key downloads to your local environment.\n",
         "\n",
-        "Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
+        "- Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
       ]
     },
     {
@@ -399,8 +410,11 @@
         "import os\n",
         "import sys\n",
         "\n",
-        "# If on Google Cloud Notebook, then don't execute this code\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = \"google.colab\" in sys.modules\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
         "    if \"google.colab\" in sys.modules:\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
@@ -410,7 +424,7 @@
         "    # path to your service account key and run this cell to authenticate your GCP\n",
         "    # account.\n",
         "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+        "        %env GOOGLE_APPLICATION_CREDENTIALS '[your-service-account-key-path]'"
       ]
     },
     {
@@ -423,7 +437,7 @@
         "\n",
         "**The following steps are required, regardless of your notebook environment.**\n",
         "\n",
-        "When you initialize the Vertex SDK for Python, you specify a Cloud Storage staging bucket. The staging bucket is where all the data associated with your dataset and model resources are retained across sessions.\n",
+        "When you initialize the Vertex AI SDK for Python, you specify a Cloud Storage staging bucket. The staging bucket is where all the data associated with your dataset and model resources are retained across sessions.\n",
         "\n",
         "Set the name of your Cloud Storage bucket below. Bucket names must be globally unique across all Google Cloud projects, including those outside of your organization."
       ]
@@ -432,29 +446,31 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "bucket"
+        "id": "f78cf4290843"
       },
       "outputs": [],
       "source": [
-        "BUCKET_URI = \"\"  # @param {type:\"string\"}"
+        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "autoset_bucket"
+        "id": "219a24ea078b"
       },
       "outputs": [],
       "source": [
-        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
+        "    BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "create_bucket"
+        "id": "a8a62bec0259"
       },
       "source": [
         "**Only if your bucket doesn't already exist**: Run the following cell to create your Cloud Storage bucket."
@@ -464,7 +480,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "create_bucket"
+        "id": "91c46850b49b"
       },
       "outputs": [],
       "source": [
@@ -474,7 +490,7 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "validate_bucket"
+        "id": "4e69d430073b"
       },
       "source": [
         "Finally, validate access to your Cloud Storage bucket by examining its contents:"
@@ -484,7 +500,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "validate_bucket"
+        "id": "835eaacd691f"
       },
       "outputs": [],
       "source": [
@@ -497,9 +513,6 @@
         "id": "setup_vars"
       },
       "source": [
-        "### Set up variables\n",
-        "\n",
-        "Next, set up some variables used throughout the tutorial.\n",
         "### Import libraries and define constants"
       ]
     },
@@ -524,9 +537,9 @@
         "id": "init_aip:mbsdk"
       },
       "source": [
-        "## Initialize Vertex SDK for Python\n",
+        "## Initialize Vertex AI SDK for Python\n",
         "\n",
-        "Initialize the Vertex SDK for Python for your project and corresponding bucket."
+        "Initialize the Vertex AI SDK for Python for your project and corresponding bucket."
       ]
     },
     {
@@ -627,7 +640,7 @@
       "outputs": [],
       "source": [
         "dataset = aiplatform.VideoDataset.create(\n",
-        "    display_name=\"Traffic\" + \"_\" + TIMESTAMP,\n",
+        "    display_name=\"Traffic\" + \"_\" + UUID,\n",
         "    gcs_source=[IMPORT_FILE],\n",
         "    import_schema_uri=aiplatform.schema.dataset.ioformat.video.object_tracking,\n",
         ")\n",
@@ -665,7 +678,7 @@
       "outputs": [],
       "source": [
         "job = aiplatform.AutoMLVideoTrainingJob(\n",
-        "    display_name=\"traffic_\" + TIMESTAMP,\n",
+        "    display_name=\"traffic_\" + UUID,\n",
         "    prediction_type=\"object_tracking\",\n",
         ")\n",
         "\n",
@@ -689,7 +702,7 @@
         "\n",
         "The `run` method when completed returns the `Model` resource.\n",
         "\n",
-        "The execution of the training pipeline will take upto 5 hours."
+        "The execution of the training pipeline will take upto 4 hours."
       ]
     },
     {
@@ -702,7 +715,7 @@
       "source": [
         "model = job.run(\n",
         "    dataset=dataset,\n",
-        "    model_display_name=\"traffic_\" + TIMESTAMP,\n",
+        "    model_display_name=\"traffic_\" + UUID,\n",
         "    training_fraction_split=0.8,\n",
         "    test_fraction_split=0.2,\n",
         ")"
@@ -728,22 +741,28 @@
       },
       "outputs": [],
       "source": [
-        "# Get model resource ID\n",
-        "models = aiplatform.Model.list(filter=\"display_name=traffic_\" + TIMESTAMP)\n",
-        "\n",
-        "# Get a reference to the Model Service client\n",
-        "client_options = {\"api_endpoint\": f\"{REGION}-aiplatform.googleapis.com\"}\n",
-        "model_service_client = aiplatform.gapic.ModelServiceClient(\n",
-        "    client_options=client_options\n",
-        ")\n",
-        "\n",
-        "model_evaluations = model_service_client.list_model_evaluations(\n",
-        "    parent=models[0].resource_name\n",
-        ")\n",
+        "model_evaluations = model.list_model_evaluations()\n",
         "model_evaluation = list(model_evaluations)[0]\n",
         "print(model_evaluation)"
       ]
     },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b3377983702b"
+      },
+      "outputs": [],
+      "source": [
+        "# Print the evaluation metrics\n",
+        "for evaluation in model_evaluations:\n",
+        "    evaluation = evaluation.to_dict()\n",
+        "    print(\"Model's evaluation metrics from Training:\\n\")\n",
+        "    metrics = evaluation[\"metrics\"]\n",
+        "    for metric in metrics.keys():\n",
+        "        print(f\"metric: {metric}, value: {metrics[metric]}\\n\")"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -874,7 +893,7 @@
       "outputs": [],
       "source": [
         "batch_predict_job = model.batch_predict(\n",
-        "    job_display_name=\"traffic_\" + TIMESTAMP,\n",
+        "    job_display_name=\"traffic_\" + UUID,\n",
         "    gcs_source=gcs_input_uri,\n",
         "    gcs_destination_prefix=BUCKET_URI,\n",
         "    sync=False,\n",
@@ -966,13 +985,9 @@
         "Otherwise, you can delete the individual resources you created in this tutorial:\n",
         "\n",
         "- Dataset\n",
-        "- Pipeline\n",
         "- Model\n",
-        "- Endpoint\n",
         "- AutoML Training Job\n",
         "- Batch Job\n",
-        "- Custom Job\n",
-        "- Hyperparameter Tuning Job\n",
         "- Cloud Storage Bucket"
       ]
     },
@@ -996,7 +1011,8 @@
         "# Delete the batch prediction job using the Vertex batch prediction object\n",
         "batch_predict_job.delete()\n",
         "\n",
-        "if os.getenv(\"IS_TESTING\"):\n",
+        "delete_bucket = False\n",
+        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
         "    ! gsutil -m rm -r $BUCKET_URI"
       ]
     }

notebooks/official/bigquery_ml/bqml-online-prediction.ipynb

@@ -282,6 +282,17 @@
         "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
       ]
     },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3c8049930470"
+      },
+      "outputs": [],
+      "source": [
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -290,15 +301,11 @@
       },
       "outputs": [],
       "source": [
-        "PROJECT_ID = \"[YOUR-PROJECT-ID]\"\n",
-        "\n",
-        "# Get your Google Cloud project ID from gcloud\n",
-        "import os\n",
-        "\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    shell_output = !gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
+        "    # Get your GCP project id from gcloud\n",
+        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
         "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID: \", PROJECT_ID)"
+        "    print(\"Project ID:\", PROJECT_ID)"
       ]
     },
     {
@@ -318,8 +325,7 @@
       },
       "outputs": [],
       "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
-        "    PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
+        "! gcloud config set project $PROJECT_ID"
       ]
     },
     {

notebooks/official/custom/sdk-custom-image-classification-batch.ipynb

@@ -123,48 +123,6 @@
         "to generate a cost estimate based on your projected usage."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "a5cb73702a9b"
-      },
-      "source": [
-        "### Set up your local development environment\n",
-        "\n",
-        "**If you are using Colab or Workbench AI Notebooks**, your environment already meets\n",
-        "all the requirements to run this notebook. You can skip this step.\n",
-        "\n",
-        "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
-        "You need the following:\n",
-        "\n",
-        "* The Google Cloud SDK\n",
-        "* Git\n",
-        "* Python 3\n",
-        "* virtualenv\n",
-        "* Jupyter notebook running in a virtual environment with Python 3\n",
-        "\n",
-        "The Google Cloud guide to [Setting up a Python development\n",
-        "environment](https://cloud.google.com/python/setup) and the [Jupyter\n",
-        "installation guide](https://jupyter.org/install) provide detailed instructions\n",
-        "for meeting these requirements. The following steps provide a condensed set of\n",
-        "instructions:\n",
-        "\n",
-        "1. [Install and initialize the Cloud SDK.](https://cloud.google.com/sdk/docs/)\n",
-        "\n",
-        "1. [Install Python 3.](https://cloud.google.com/python/setup#installing_python)\n",
-        "\n",
-        "1. [Install\n",
-        "   virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv)\n",
-        "   and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
-        "\n",
-        "1. To install Jupyter, run `pip install jupyter` on the\n",
-        "command-line in a terminal shell.\n",
-        "\n",
-        "1. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
-        "\n",
-        "1. Open this notebook in the Jupyter Notebook Dashboard."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -184,306 +142,194 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
-        "! pip3 install {USER_FLAG} --upgrade google-cloud-aiplatform -q\n",
-        "! pip3 install {USER_FLAG} --upgrade google-cloud-storage -q\n",
-        "! pip3 install {USER_FLAG} --upgrade pillow -q\n",
-        "! pip3 install {USER_FLAG} --upgrade numpy -q"
+        "# Install the packages\n",
+        "! pip3 install --upgrade google-cloud-aiplatform \\\n",
+        "                        google-cloud-storage \\\n",
+        "                        pillow  \\\n",
+        "                        numpy    "
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "restart"
+        "id": "d98bc9fdd80d"
       },
       "source": [
-        "### Restart the kernel\n",
-        "\n",
-        "Once you've installed everything, you need to restart the notebook kernel so it can find the packages."
+        "### Colab only: Uncomment the following cell to restart the kernel."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "bzPxhxS5lugp"
+        "id": "0dbf29389c65"
       },
       "outputs": [],
       "source": [
-        "import os\n",
+        "# Automatically restart kernel after installs so that your environment can access the new packages\n",
+        "# import IPython\n",
         "\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    # Automatically restart kernel after installs\n",
-        "    import IPython\n",
-        "\n",
-        "    app = IPython.Application.instance()\n",
-        "    app.kernel.do_shutdown(True)"
+        "# app = IPython.Application.instance()\n",
+        "# app.kernel.do_shutdown(True)"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "before_you_begin"
+        "id": "013daf3de88e"
       },
       "source": [
-        "## Before you begin\n",
-        "\n",
-        "### Select a GPU runtime\n",
-        "\n",
-        "**Make sure you're running this notebook in a GPU runtime if you have that option. In Colab, select \"Runtime --> Change runtime type > GPU\"**\n",
-        "\n",
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "2. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "3. [Enable the Vertex AI API and Compute Engine API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component).\n",
-        "\n",
-        "4. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "5. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+        "## Before you begin"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "project_id"
+        "id": "8bc8a29f9001"
       },
       "source": [
         "#### Set your project ID\n",
         "\n",
-        "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
+        "**If you don't know your project ID**, try the following:\n",
+        "* Run `gcloud config list`.\n",
+        "* Run `gcloud projects list`.\n",
+        "* See the support page: [Locate the project ID](https://support.google.com/googleapi/answer/7014113)"
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "autoset_project_id"
+        "id": "e61aaa036444"
       },
       "outputs": [],
       "source": [
-        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "USd_pUT0lugr"
-      },
-      "outputs": [],
-      "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
-        "    # Get your GCP project id from gcloud\n",
-        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
-        "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID:\", PROJECT_ID)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "09021c90b34c"
-      },
-      "outputs": [],
-      "source": [
-        "! gcloud config set project $PROJECT_ID"
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# Set the project id\n",
+        "! gcloud config set project {PROJECT_ID}"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "88dd74c4c84e"
+        "id": "c4a624c8099d"
       },
       "source": [
         "#### Region\n",
         "\n",
-        "You can also change the `REGION` variable, which is used for operations\n",
-        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
-        "\n",
-        "- Americas: `us-central1`\n",
-        "- Europe: `europe-west4`\n",
-        "- Asia Pacific: `asia-east1`\n",
-        "\n",
-        "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
-        "\n",
-        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
+        "You can also change the `REGION` variable used by Vertex AI. Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "5c615e53149f"
+        "id": "f83bd6013894"
       },
       "outputs": [],
       "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "timestamp"
-      },
-      "source": [
-        "#### Timestamp\n",
-        "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append it onto the name of resources you create in this tutorial."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "c-pX32xalugs"
-      },
-      "outputs": [],
-      "source": [
-        "from datetime import datetime\n",
-        "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "gcp_authenticate"
+        "id": "08bfd1eb44ef"
       },
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Workbench AI Notebooks**, your environment is already\n",
-        "authenticated. Skip this step.\n",
-        "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions\n",
-        "when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "1. In the Cloud Console, go to the [**Create service account key**\n",
-        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
-        "\n",
-        "2. Click **Create service account**.\n",
-        "\n",
-        "3. In the **Service account name** field, enter a name, and\n",
-        "   click **Create**.\n",
-        "\n",
-        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
-        "into the filter box, and select\n",
-        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
-        "local environment.\n",
-        "\n",
-        "6. Enter the path to your service account key as the\n",
-        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
+        "Depending on your Jupyter environment, you may have to manually authenticate. Follow the relevant instructions below."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "af349043f23b"
+      },
+      "source": [
+        "**1. Vertex AI Workbench**\n",
+        "* Do nothing as you are already authenticated."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ad1138a125ea"
+      },
+      "source": [
+        "**2. Local JupyterLab instance, uncomment and run:**"
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "vF60K5v1lugs"
+        "id": "ce6043da7b33"
       },
       "outputs": [],
       "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = \"google.colab\" in sys.modules\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+        "# ! gcloud auth login"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "bucket:custom"
+        "id": "0367eac06a10"
+      },
+      "source": [
+        "**3. Colab, uncomment and run:**"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "21ad4dbb4a61"
+      },
+      "outputs": [],
+      "source": [
+        "# from google.colab import auth\n",
+        "# auth.authenticate_user()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "c13224697bfb"
+      },
+      "source": [
+        "**4. Service account or other**\n",
+        "* See how to grant Cloud Storage permissions to your service account at https://cloud.google.com/storage/docs/gsutil/commands/iam#ch-examples."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ddbea904fbe5"
       },
       "source": [
         "### Create a Cloud Storage bucket\n",
         "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "When you submit a training job using the Cloud SDK, you upload a Python package\n",
-        "containing your training code to a Cloud Storage bucket. Vertex AI runs\n",
-        "the code from this package. In this tutorial, Vertex AI also saves the\n",
-        "trained model that results from your job in the same bucket. Using this model artifact, you can then create Vertex AI model resources.\n",
-        "\n",
-        "Set the name of your Cloud Storage bucket below. It must be unique across all\n",
-        "Cloud Storage buckets."
+        "Create a storage bucket to store intermediate artifacts such as datasets."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "bucket"
+        "id": "751138cf3bd5"
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "autoset_bucket"
-      },
-      "outputs": [],
-      "source": [
-        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
-        "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
+        "BUCKET_URI = \"gs://your-bucket-name-unique\"  # @param {type:\"string\"}"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "create_bucket"
+        "id": "58cb4f5895f0"
       },
       "source": [
         "**Only if your bucket doesn't already exist**: Run the following cell to create your Cloud Storage bucket."
@@ -493,31 +339,11 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "Oz8J0vmSlugt"
+        "id": "5e1288505682"
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_URI"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "validate_bucket"
-      },
-      "source": [
-        "Finally, validate access to your Cloud Storage bucket by examining its contents:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "oadE10x2lugu"
-      },
-      "outputs": [],
-      "source": [
-        "! gsutil ls -al $BUCKET_URI"
+        "! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
       ]
     },
     {
@@ -538,7 +364,6 @@
       "outputs": [],
       "source": [
         "import os\n",
-        "import sys\n",
         "\n",
         "from google.cloud import aiplatform"
       ]
@@ -565,54 +390,17 @@
         "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=BUCKET_URI)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "accelerators:training,prediction"
-      },
-      "source": [
-        "#### Set hardware accelerators\n",
-        "\n",
-        "You can set hardware accelerators for training and prediction.\n",
-        "\n",
-        "Set the variables `TRAIN_GPU/TRAIN_NGPU` and `DEPLOY_GPU/DEPLOY_NGPU` to use a container image supporting a GPU and the number of GPUs allocated to the virtual machine (VM) instance. For example, to use a GPU container image with 4 Nvidia Telsa K80 GPUs allocated to each VM, you would specify:\n",
-        "\n",
-        "    (aiplatform.gapic.AcceleratorType.NVIDIA_TESLA_K80, 4)\n",
-        "\n",
-        "\n",
-        "Otherwise specify `(None, None)` to use a container image to run on a CPU.\n",
-        "\n",
-        "Learn more about [hardware accelerator support for your region](https://cloud.google.com/vertex-ai/docs/general/locations#accelerators).\n",
-        "\n",
-        "*Note*: TF releases before 2.3 for GPU support will fail to load the custom model in this tutorial. It is a known issue and fixed in TF 2.3. This is caused by static graph ops that are generated in the serving function. If you encounter this issue on your own custom models, use a container image for TF 2.3 with GPU support."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "xd5PLXDTlugv"
-      },
-      "outputs": [],
-      "source": [
-        "TRAIN_GPU, TRAIN_NGPU = (aiplatform.gapic.AcceleratorType.NVIDIA_TESLA_K80, 1)\n",
-        "\n",
-        "DEPLOY_GPU, DEPLOY_NGPU = (aiplatform.gapic.AcceleratorType.NVIDIA_TESLA_K80, 1)"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "container:training,prediction"
       },
       "source": [
-        "#### Set pre-built containers\n",
+        "### Set pre-built containers\n",
         "\n",
-        "Set the pre-built Docker container image for training and prediction.\n",
+        "Vertex AI provides pre-built containers to run training and prediction.\n",
         "\n",
-        "For the latest list, see [Pre-built containers for training](https://cloud.google.com/ai-platform-unified/docs/training/pre-built-containers).\n",
-        "\n",
-        "For the latest list, see [Pre-built containers for prediction](https://cloud.google.com/ai-platform-unified/docs/predictions/pre-built-containers)."
+        "For the latest list, see [Pre-built containers for training](https://cloud.google.com/vertex-ai/docs/training/pre-built-containers) and [Pre-built containers for prediction](https://cloud.google.com/vertex-ai/docs/predictions/pre-built-containers)"
       ]
     },
     {
@@ -623,64 +411,11 @@
       },
       "outputs": [],
       "source": [
-        "TRAIN_VERSION = \"tf-gpu.2-1\"\n",
-        "DEPLOY_VERSION = \"tf2-gpu.2-1\"\n",
+        "TRAIN_VERSION = \"tf-cpu.2-9\"\n",
+        "DEPLOY_VERSION = \"tf2-cpu.2-9\"\n",
         "\n",
-        "TRAIN_IMAGE = \"{}-docker.pkg.dev/vertex-ai/training/{}:latest\".format(\n",
-        "    REGION.split(\"-\")[0], TRAIN_VERSION\n",
-        ")\n",
-        "DEPLOY_IMAGE = \"{}-docker.pkg.dev/vertex-ai/prediction/{}:latest\".format(\n",
-        "    REGION.split(\"-\")[0], DEPLOY_VERSION\n",
-        ")\n",
-        "\n",
-        "print(\"Training:\", TRAIN_IMAGE, TRAIN_GPU, TRAIN_NGPU)\n",
-        "print(\"Deployment:\", DEPLOY_IMAGE, DEPLOY_GPU, DEPLOY_NGPU)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "machine:training,prediction"
-      },
-      "source": [
-        "#### Set machine types\n",
-        "\n",
-        "Next, set the machine types to use for training and prediction.\n",
-        "\n",
-        "- Set the variables `TRAIN_COMPUTE` and `DEPLOY_COMPUTE` to configure your compute resources for training and prediction.\n",
-        " - `machine type`\n",
-        "     - `n1-standard`: 3.75GB of memory per vCPU\n",
-        "     - `n1-highmem`: 6.5GB of memory per vCPU\n",
-        "     - `n1-highcpu`: 0.9 GB of memory per vCPU\n",
-        " - `vCPUs`: number of \\[2, 4, 8, 16, 32, 64, 96 \\]\n",
-        "\n",
-        "*Note: The following is not supported for training:*\n",
-        "\n",
-        " - `standard`: 2 vCPUs\n",
-        " - `highcpu`: 2, 4 and 8 vCPUs\n",
-        "\n",
-        "*Note: You may also use n2 and e2 machine types for training and deployment, but they do not support GPUs*."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "YAXwbqKKlugv"
-      },
-      "outputs": [],
-      "source": [
-        "MACHINE_TYPE = \"n1-standard\"\n",
-        "\n",
-        "VCPU = \"4\"\n",
-        "TRAIN_COMPUTE = MACHINE_TYPE + \"-\" + VCPU\n",
-        "print(\"Train machine type\", TRAIN_COMPUTE)\n",
-        "\n",
-        "MACHINE_TYPE = \"n1-standard\"\n",
-        "\n",
-        "VCPU = \"4\"\n",
-        "DEPLOY_COMPUTE = MACHINE_TYPE + \"-\" + VCPU\n",
-        "print(\"Deploy machine type\", DEPLOY_COMPUTE)"
+        "TRAIN_IMAGE = \"us-docker.pkg.dev/vertex-ai/training/{}:latest\".format(TRAIN_VERSION)\n",
+        "DEPLOY_IMAGE = \"us-docker.pkg.dev/vertex-ai/prediction/{}:latest\".format(DEPLOY_VERSION)"
       ]
     },
     {
@@ -735,13 +470,11 @@
       },
       "outputs": [],
       "source": [
-        "JOB_NAME = \"custom_job_\" + TIMESTAMP\n",
+        "JOB_NAME = \"custom_job_unique\"\n",
         "MODEL_DIR = \"{}/{}\".format(BUCKET_URI, JOB_NAME)\n",
         "\n",
-        "if not TRAIN_NGPU or TRAIN_NGPU < 2:\n",
-        "    TRAIN_STRATEGY = \"single\"\n",
-        "else:\n",
-        "    TRAIN_STRATEGY = \"mirror\"\n",
+        "\n",
+        "TRAIN_STRATEGY = \"single\"\n",
         "\n",
         "EPOCHS = 20\n",
         "STEPS = 100\n",
@@ -923,26 +656,15 @@
         "    model_serving_container_image_uri=DEPLOY_IMAGE,\n",
         ")\n",
         "\n",
-        "MODEL_DISPLAY_NAME = \"cifar10-\" + TIMESTAMP\n",
+        "MODEL_DISPLAY_NAME = \"model_unique\"\n",
         "\n",
         "# Start the training\n",
-        "if TRAIN_GPU:\n",
-        "    model = job.run(\n",
-        "        model_display_name=MODEL_DISPLAY_NAME,\n",
-        "        args=CMDARGS,\n",
-        "        replica_count=1,\n",
-        "        machine_type=TRAIN_COMPUTE,\n",
-        "        accelerator_type=TRAIN_GPU.name,\n",
-        "        accelerator_count=TRAIN_NGPU,\n",
-        "    )\n",
-        "else:\n",
-        "    model = job.run(\n",
-        "        model_display_name=MODEL_DISPLAY_NAME,\n",
-        "        args=CMDARGS,\n",
-        "        replica_count=1,\n",
-        "        machine_type=TRAIN_COMPUTE,\n",
-        "        accelerator_count=0,\n",
-        "    )"
+        "\n",
+        "model = job.run(\n",
+        "    model_display_name=MODEL_DISPLAY_NAME,\n",
+        "    args=CMDARGS,\n",
+        "    replica_count=1,\n",
+        ")"
       ]
     },
     {
@@ -1111,7 +833,7 @@
         "MAX_NODES = 1\n",
         "\n",
         "# The name of the job\n",
-        "BATCH_PREDICTION_JOB_NAME = \"cifar10_batch-\" + TIMESTAMP\n",
+        "BATCH_PREDICTION_JOB_NAME = \"cifar10_batch_prediction_unique\"\n",
         "\n",
         "# Folder in the bucket to write results to\n",
         "DESTINATION_FOLDER = \"batch_prediction_results\"\n",
@@ -1127,11 +849,9 @@
         "    gcs_source=BATCH_PREDICTION_GCS_SOURCE,\n",
         "    gcs_destination_prefix=BATCH_PREDICTION_GCS_DEST_PREFIX,\n",
         "    model_parameters=None,\n",
-        "    machine_type=DEPLOY_COMPUTE,\n",
-        "    accelerator_type=DEPLOY_GPU,\n",
-        "    accelerator_count=DEPLOY_NGPU,\n",
         "    starting_replica_count=MIN_NODES,\n",
         "    max_replica_count=MAX_NODES,\n",
+        "    machine_type=\"n1-standard-4\",\n",
         "    sync=True,\n",
         ")"
       ]

notebooks/official/custom/sdk-custom-image-classification-online.ipynb

@@ -142,308 +142,176 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
-        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG -q\n",
-        "! pip3 install {USER_FLAG} --upgrade google-cloud-storage -q\n",
-        "! pip3 install {USER_FLAG} --upgrade pillow -q\n",
-        "! pip3 install {USER_FLAG} --upgrade numpy -q"
+        "! pip3 install --upgrade --quiet google-cloud-aiplatform google-cloud-storage pillow numpy\n"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "restart"
+        "id": "d98bc9fdd80d"
       },
       "source": [
-        "### Restart the kernel\n",
-        "\n",
-        "Once you've installed everything, you need to restart the notebook kernel so it can find the packages."
+        "### Colab only: Uncomment the following cell to restart the kernel."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "bzPxhxS5lugp"
+        "id": "0dbf29389c65"
       },
       "outputs": [],
       "source": [
-        "import os\n",
+        "# Automatically restart kernel after installs so that your environment can access the new packages\n",
+        "# import IPython\n",
         "\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    # Automatically restart kernel after installs\n",
-        "    import IPython\n",
-        "\n",
-        "    app = IPython.Application.instance()\n",
-        "    app.kernel.do_shutdown(True)"
+        "# app = IPython.Application.instance()\n",
+        "# app.kernel.do_shutdown(True)"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "before_you_begin"
+        "id": "eec0fc7a0963"
       },
       "source": [
         "## Before you begin\n",
         "\n",
-        "### Select a GPU runtime\n",
-        "\n",
-        "**Make sure you're running this notebook in a GPU runtime if you have that option. In Colab, select \"Runtime --> Change runtime type > GPU\"**\n",
-        "\n",
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "2. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "3. [Enable the Vertex AI API and Compute Engine API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component).\n",
-        "\n",
-        "4. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "5. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "project_id"
-      },
-      "source": [
         "#### Set your project ID\n",
         "\n",
-        "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
+        "**If you don't know your project ID**, try the following:\n",
+        "* Run `gcloud config list`.\n",
+        "* Run `gcloud projects list`.\n",
+        "* See the support page: [Locate the project ID](https://support.google.com/googleapi/answer/7014113)"
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "autoset_project_id"
+        "id": "e3ce64be5527"
       },
       "outputs": [],
       "source": [
-        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "USd_pUT0lugr"
-      },
-      "outputs": [],
-      "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
-        "    # Get your GCP project id from gcloud\n",
-        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
-        "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID:\", PROJECT_ID)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "250cb8c648d5"
-      },
-      "outputs": [],
-      "source": [
-        "! gcloud config set project $PROJECT_ID"
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# Set the project id\n",
+        "! gcloud config set project {PROJECT_ID}"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "2aa333eca058"
+        "id": "7f2e7f78a864"
       },
       "source": [
         "#### Region\n",
         "\n",
-        "You can also change the `REGION` variable, which is used for operations\n",
-        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
-        "\n",
-        "- Americas: `us-central1`\n",
-        "- Europe: `europe-west4`\n",
-        "- Asia Pacific: `asia-east1`\n",
-        "\n",
-        "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
-        "\n",
-        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
+        "You can also change the `REGION` variable used by Vertex AI. Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "d8b34ef9a3d0"
+        "id": "369a2258dd59"
       },
       "outputs": [],
       "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "timestamp"
-      },
-      "source": [
-        "#### Timestamp\n",
-        "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append it onto the name of resources you create in this tutorial."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "c-pX32xalugs"
-      },
-      "outputs": [],
-      "source": [
-        "from datetime import datetime\n",
-        "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "gcp_authenticate"
+        "id": "6a94297012d5"
       },
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Workbench AI Notebooks**, your environment is already\n",
-        "authenticated. Skip this step.\n",
+        "Depending on your Jupyter environment, you may have to manually authenticate. Follow the relevant instructions below.\n",
         "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions\n",
-        "when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "1. In the Cloud Console, go to the [**Create service account key**\n",
-        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
-        "\n",
-        "2. Click **Create service account**.\n",
-        "\n",
-        "3. In the **Service account name** field, enter a name, and\n",
-        "   click **Create**.\n",
-        "\n",
-        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
-        "into the filter box, and select\n",
-        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
-        "local environment.\n",
-        "\n",
-        "6. Enter the path to your service account key as the\n",
-        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
+        "**1. Vertex AI Workbench**\n",
+        "* Do nothing as you are already authenticated."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "28e3c4539627"
+      },
+      "source": [
+        "**2. Local JupyterLab instance, uncomment and run:**\n"
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "vF60K5v1lugs"
+        "id": "fbc9cd30cc4b"
       },
       "outputs": [],
       "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = \"google.colab\" in sys.modules\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+        "# ! gcloud auth login"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "bucket:custom"
+        "id": "79efab26ad02"
+      },
+      "source": [
+        "**3. Colab, uncomment and run:**"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "a336a05c6149"
+      },
+      "outputs": [],
+      "source": [
+        "# from google.colab import auth\n",
+        "# auth.authenticate_user()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e3c28b6b796b"
+      },
+      "source": [
+        "**4. Service account or other**\n",
+        "* See how to grant Cloud Storage permissions to your service account at https://cloud.google.com/storage/docs/gsutil/commands/iam#ch-examples."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "70a42f1033a3"
       },
       "source": [
         "### Create a Cloud Storage bucket\n",
         "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "When you submit a training job using the Cloud SDK, you upload a Python package\n",
-        "containing your training code to a Cloud Storage bucket. Vertex AI runs\n",
-        "the code from this package. In this tutorial, Vertex AI also saves the\n",
-        "trained model that results from your job in the same bucket. Using this model artifact, you can then\n",
-        "create Vertex AI model and endpoint resources in order to serve\n",
-        "online predictions.\n",
-        "\n",
-        "Set the name of your Cloud Storage bucket below. It must be unique across all\n",
-        "Cloud Storage buckets."
+        "Create a storage bucket to store intermediate artifacts such as datasets."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "bucket"
+        "id": "7c12c0866590"
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "autoset_bucket"
-      },
-      "outputs": [],
-      "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
-        "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
+        "BUCKET_URI = \"gs://your-bucket-name-unique\"  # @param {type:\"string\"}"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "create_bucket"
+        "id": "0cb016da6de3"
       },
       "source": [
         "**Only if your bucket doesn't already exist**: Run the following cell to create your Cloud Storage bucket."
@@ -453,31 +321,11 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "Oz8J0vmSlugt"
+        "id": "eddb0b63434c"
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_URI"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "validate_bucket"
-      },
-      "source": [
-        "Finally, validate access to your Cloud Storage bucket by examining its contents:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "oadE10x2lugu"
-      },
-      "outputs": [],
-      "source": [
-        "! gsutil ls -al $BUCKET_URI"
+        "! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
       ]
     },
     {
@@ -486,10 +334,6 @@
         "id": "setup_vars"
       },
       "source": [
-        "### Set up variables\n",
-        "\n",
-        "Next, set up some variables used throughout the tutorial.\n",
-        "\n",
         "### Import libraries and define constants"
       ]
     },
@@ -502,9 +346,10 @@
       "outputs": [],
       "source": [
         "import os\n",
-        "import sys\n",
         "\n",
-        "from google.cloud import aiplatform"
+        "import numpy as np\n",
+        "from google.cloud import aiplatform\n",
+        "from PIL import Image"
       ]
     },
     {
@@ -529,56 +374,17 @@
         "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=BUCKET_URI)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "accelerators:training,prediction"
-      },
-      "source": [
-        "#### Set hardware accelerators\n",
-        "\n",
-        "You can set hardware accelerators for both training and prediction.\n",
-        "\n",
-        "Set the variables `TRAIN_GPU/TRAIN_NGPU` and `DEPLOY_GPU/DEPLOY_NGPU` to use a container image supporting a GPU and the number of GPUs allocated to the virtual machine (VM) instance. For example, to use a GPU container image with 4 Nvidia Tesla K80 GPUs allocated to each VM, you would specify:\n",
-        "\n",
-        "    (aiplatform.gapic.AcceleratorType.NVIDIA_TESLA_K80, 4)\n",
-        "\n",
-        "See the [locations where accelerators are available](https://cloud.google.com/vertex-ai/docs/general/locations#accelerators).\n",
-        "\n",
-        "Otherwise specify `(None, None)` to use a container image to run on a CPU.\n",
-        "\n",
-        "*Note*: TensorFlow releases earlier than 2.3 for GPU support fail to load the custom model in this tutorial. This issue is caused by static graph operations that are generated in the serving function. This is a known issue, which is fixed in TensorFlow 2.3. If you encounter this issue with your own custom models, use a container image for TensorFlow 2.3 or later with GPU support."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "xd5PLXDTlugv"
-      },
-      "outputs": [],
-      "source": [
-        "TRAIN_GPU, TRAIN_NGPU = (aiplatform.gapic.AcceleratorType.NVIDIA_TESLA_K80, 1)\n",
-        "\n",
-        "DEPLOY_GPU, DEPLOY_NGPU = (aiplatform.gapic.AcceleratorType.NVIDIA_TESLA_K80, 1)"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "container:training,prediction"
       },
       "source": [
-        "#### Set pre-built containers\n",
+        "### Set pre-built containers\n",
         "\n",
+        "Vertex AI provides pre-built containers to run training and prediction.\n",
         "\n",
-        "Set the pre-built Docker container image for training and prediction.\n",
-        "\n",
-        "\n",
-        "For the latest list, see [Pre-built containers for training](https://cloud.google.com/ai-platform-unified/docs/training/pre-built-containers).\n",
-        "\n",
-        "\n",
-        "For the latest list, see [Pre-built containers for prediction](https://cloud.google.com/ai-platform-unified/docs/predictions/pre-built-containers)."
+        "For the latest list, see [Pre-built containers for training](https://cloud.google.com/vertex-ai/docs/training/pre-built-containers) and [Pre-built containers for prediction](https://cloud.google.com/vertex-ai/docs/predictions/pre-built-containers)"
       ]
     },
     {
@@ -589,75 +395,11 @@
       },
       "outputs": [],
       "source": [
-        "TRAIN_VERSION = \"tf-gpu.2-1\"\n",
-        "DEPLOY_VERSION = \"tf2-gpu.2-1\"\n",
+        "TRAIN_VERSION = \"tf-cpu.2-9\"\n",
+        "DEPLOY_VERSION = \"tf2-cpu.2-9\"\n",
         "\n",
-        "TRAIN_IMAGE = \"{}-docker.pkg.dev/vertex-ai/training/{}:latest\".format(\n",
-        "    REGION.split(\"-\")[0], TRAIN_VERSION\n",
-        ")\n",
-        "DEPLOY_IMAGE = \"{}-docker.pkg.dev/vertex-ai/prediction/{}:latest\".format(\n",
-        "    REGION.split(\"-\")[0], DEPLOY_VERSION\n",
-        ")\n",
-        "\n",
-        "print(\"Training:\", TRAIN_IMAGE, TRAIN_GPU, TRAIN_NGPU)\n",
-        "print(\"Deployment:\", DEPLOY_IMAGE, DEPLOY_GPU, DEPLOY_NGPU)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "machine:training,prediction"
-      },
-      "source": [
-        "#### Set machine types\n",
-        "\n",
-        "Next, set the machine types to use for training and prediction.\n",
-        "\n",
-        "- Set the variables `TRAIN_COMPUTE` and `DEPLOY_COMPUTE` to configure your compute resources for training and prediction.\n",
-        " - `machine type`\n",
-        "     - `n1-standard`: 3.75GB of memory per vCPU\n",
-        "     - `n1-highmem`: 6.5GB of memory per vCPU\n",
-        "     - `n1-highcpu`: 0.9 GB of memory per vCPU\n",
-        " - `vCPUs`: number of \\[2, 4, 8, 16, 32, 64, 96 \\]\n",
-        "\n",
-        "*Note: The following is not supported for training:*\n",
-        "\n",
-        " - `standard`: 2 vCPUs\n",
-        " - `highcpu`: 2, 4 and 8 vCPUs\n",
-        "\n",
-        "*Note: You may also use n2 and e2 machine types for training and deployment, but they do not support GPUs*."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "YAXwbqKKlugv"
-      },
-      "outputs": [],
-      "source": [
-        "MACHINE_TYPE = \"n1-standard\"\n",
-        "\n",
-        "VCPU = \"4\"\n",
-        "TRAIN_COMPUTE = MACHINE_TYPE + \"-\" + VCPU\n",
-        "print(\"Train machine type\", TRAIN_COMPUTE)\n",
-        "\n",
-        "MACHINE_TYPE = \"n1-standard\"\n",
-        "\n",
-        "VCPU = \"4\"\n",
-        "DEPLOY_COMPUTE = MACHINE_TYPE + \"-\" + VCPU\n",
-        "print(\"Deploy machine type\", DEPLOY_COMPUTE)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "tutorial_start:custom"
-      },
-      "source": [
-        "# Tutorial\n",
-        "\n",
-        "Now you are ready to start creating your own custom-trained model with CIFAR10."
+        "TRAIN_IMAGE = \"us-docker.pkg.dev/vertex-ai/training/{}:latest\".format(TRAIN_VERSION)\n",
+        "DEPLOY_IMAGE = \"us-docker.pkg.dev/vertex-ai/prediction/{}:latest\".format(DEPLOY_VERSION)"
       ]
     },
     {
@@ -666,21 +408,17 @@
         "id": "train_custom_model"
       },
       "source": [
+        "# Tutorial\n",
+        "\n",
+        "Now you are ready to start creating your own custom-trained model with CIFAR10.\n",
         "## Train a model\n",
         "\n",
         "There are two ways you can train a custom model using a container image:\n",
         "\n",
         "- **Use a Google Cloud prebuilt container**. If you use a prebuilt container, you will additionally specify a Python package to install into the container image. This Python package contains your code for training a custom model.\n",
         "\n",
-        "- **Use your own custom container image**. If you use your own container, the container needs to contain your code for training a custom model."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "train_custom_job_args"
-      },
-      "source": [
+        "- **Use your own custom container image**. If you use your own container, the container needs to contain your code for training a custom model.\n",
+        "\n",
         "### Define the command args for the training script\n",
         "\n",
         "Prepare the command-line arguments to pass to your training script.\n",
@@ -701,13 +439,10 @@
       },
       "outputs": [],
       "source": [
-        "JOB_NAME = \"custom_job_\" + TIMESTAMP\n",
+        "JOB_NAME = \"custom_job_unique\"\n",
         "MODEL_DIR = \"{}/{}\".format(BUCKET_URI, JOB_NAME)\n",
         "\n",
-        "if not TRAIN_NGPU or TRAIN_NGPU < 2:\n",
-        "    TRAIN_STRATEGY = \"single\"\n",
-        "else:\n",
-        "    TRAIN_STRATEGY = \"mirror\"\n",
+        "TRAIN_STRATEGY = \"single\"\n",
         "\n",
         "EPOCHS = 20\n",
         "STEPS = 100\n",
@@ -885,30 +620,18 @@
         "    display_name=JOB_NAME,\n",
         "    script_path=\"task.py\",\n",
         "    container_uri=TRAIN_IMAGE,\n",
-        "    requirements=[\"tensorflow_datasets==1.3.0\"],\n",
+        "    requirements=[\"tensorflow_datasets\"],\n",
         "    model_serving_container_image_uri=DEPLOY_IMAGE,\n",
         ")\n",
         "\n",
-        "MODEL_DISPLAY_NAME = \"cifar10-\" + TIMESTAMP\n",
+        "MODEL_DISPLAY_NAME = \"cifar10_unique\"\n",
         "\n",
         "# Start the training\n",
-        "if TRAIN_GPU:\n",
-        "    model = job.run(\n",
-        "        model_display_name=MODEL_DISPLAY_NAME,\n",
-        "        args=CMDARGS,\n",
-        "        replica_count=1,\n",
-        "        machine_type=TRAIN_COMPUTE,\n",
-        "        accelerator_type=TRAIN_GPU.name,\n",
-        "        accelerator_count=TRAIN_NGPU,\n",
-        "    )\n",
-        "else:\n",
-        "    model = job.run(\n",
-        "        model_display_name=MODEL_DISPLAY_NAME,\n",
-        "        args=CMDARGS,\n",
-        "        replica_count=1,\n",
-        "        machine_type=TRAIN_COMPUTE,\n",
-        "        accelerator_count=0,\n",
-        "    )"
+        "model = job.run(\n",
+        "    model_display_name=MODEL_DISPLAY_NAME,\n",
+        "    args=CMDARGS,\n",
+        "    replica_count=1,\n",
+        ")"
       ]
     },
     {
@@ -962,44 +685,20 @@
       },
       "outputs": [],
       "source": [
-        "DEPLOYED_NAME = \"cifar10_deployed-\" + TIMESTAMP\n",
+        "DEPLOYED_NAME = \"cifar10_deployed_unique\"\n",
         "\n",
         "TRAFFIC_SPLIT = {\"0\": 100}\n",
         "\n",
         "MIN_NODES = 1\n",
         "MAX_NODES = 1\n",
         "\n",
-        "if DEPLOY_GPU:\n",
-        "    endpoint = model.deploy(\n",
-        "        deployed_model_display_name=DEPLOYED_NAME,\n",
-        "        traffic_split=TRAFFIC_SPLIT,\n",
-        "        machine_type=DEPLOY_COMPUTE,\n",
-        "        accelerator_type=DEPLOY_GPU.name,\n",
-        "        accelerator_count=DEPLOY_NGPU,\n",
-        "        min_replica_count=MIN_NODES,\n",
-        "        max_replica_count=MAX_NODES,\n",
-        "    )\n",
-        "else:\n",
-        "    endpoint = model.deploy(\n",
-        "        deployed_model_display_name=DEPLOYED_NAME,\n",
-        "        traffic_split=TRAFFIC_SPLIT,\n",
-        "        machine_type=DEPLOY_COMPUTE,\n",
-        "        accelerator_type=DEPLOY_COMPUTE.name,\n",
-        "        accelerator_count=0,\n",
-        "        min_replica_count=MIN_NODES,\n",
-        "        max_replica_count=MAX_NODES,\n",
-        "    )"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "make_prediction"
-      },
-      "source": [
-        "## Make an online prediction request\n",
         "\n",
-        "Send an online prediction request to your deployed model."
+        "endpoint = model.deploy(\n",
+        "    deployed_model_display_name=DEPLOYED_NAME,\n",
+        "    traffic_split=TRAFFIC_SPLIT,\n",
+        "    min_replica_count=MIN_NODES,\n",
+        "    max_replica_count=MAX_NODES,\n",
+        ")"
       ]
     },
     {
@@ -1008,6 +707,9 @@
         "id": "get_test_item:test"
       },
       "source": [
+        "## Make an online prediction request\n",
+        "\n",
+        "Send an online prediction request to your deployed model.\n",
         "### Get test data\n",
         "\n",
         "Download images from the CIFAR dataset and preprocess them.\n",
@@ -1053,9 +755,6 @@
       },
       "outputs": [],
       "source": [
-        "import numpy as np\n",
-        "from PIL import Image\n",
-        "\n",
         "# Load image data\n",
         "IMAGE_DIRECTORY = \"cifar_test_images\"\n",
         "\n",
@@ -1166,10 +865,6 @@
       },
       "outputs": [],
       "source": [
-        "delete_training_job = True\n",
-        "delete_model = True\n",
-        "delete_endpoint = True\n",
-        "\n",
         "# Warning: Setting this to true will delete everything in your bucket\n",
         "delete_bucket = False\n",
         "\n",

notebooks/official/explainable_ai/sdk_custom_tabular_regression_online_explain.ipynb

@@ -959,6 +959,7 @@
         "    script_path=\"custom/trainer/task.py\",\n",
         "    container_uri=TRAIN_IMAGE,\n",
         "    requirements=[\"gcsfs==0.7.1\", \"tensorflow-datasets==4.4\"],\n",
+        "    location=REGION,\n",
         ")\n",
         "\n",
         "print(job)"

notebooks/official/feature_store/sdk-feature-store-pandas.ipynb

@@ -29,7 +29,7 @@
         "id": "JAPoU8Sm5E6e"
       },
       "source": [
-        "# Using Vertex AI Feature Store with pandas DataFrame\n",
+        "# Using Vertex AI Feature Store with Pandas Dataframe\n",
         "\n",
         "<table align=\"left\">\n",
         "    <td>\n",
@@ -73,13 +73,17 @@
       "source": [
         "### Objective\n",
         "\n",
-        "In this notebook, you learn how to use `Vertex AI Feature Store` with pandas DataFrame.\n",
+        "In this notebook, you learn how to use `Vertex AI Feature Store` with pandas Dataframe.\n",
+        "\n",
+        "This tutorial uses the following Google Cloud ML services and resources:\n",
+        "\n",
+        "- Vertex AI Feature Store\n",
         "\n",
         "The steps performed include:\n",
         "\n",
         "- Ingest Feature values from Pandas DataFrame into Feature Store's Entity types.\n",
-        "- Read Entity Feature values from Online Feature Store into Pandas DataFrame.\n",
-        "- Batch serve Feature values from your Feature Store into Pandas DataFrame.\n",
+        "- Read Entity feature values from Online Feature Store into Pandas DataFrame.\n",
+        "- Batch serve feature values from your Feature Store into Pandas DataFrame.\n",
         "\n",
         "You also learn how Vertex AI Feature Store can be useful in the below scenarios:\n",
         "\n",
@@ -95,7 +99,7 @@
       "source": [
         "### Dataset\n",
         "\n",
-        "This tutorial uses a movie recommendation dataset as an example throughout all the notebooks including this one. The original task is to train a model to predict if a user is going to watch a movie and serve the model online."
+        "This tutorial is a part of the Feature Store tutorial notebooks. It uses a movie recommendation dataset as an example for demonstrating various functionalities of Feature Store. The original task is to train a model to predict if a user is going to watch a movie, and serve the model online."
       ]
     },
     {
@@ -146,12 +150,15 @@
       "source": [
         "import os\n",
         "\n",
-        "# The Google Cloud Notebook product has specific requirements\n",
-        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
         "\n",
-        "# Google Cloud Notebook requires dependencies to be installed with '--user'\n",
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
         "USER_FLAG = \"\"\n",
-        "if IS_GOOGLE_CLOUD_NOTEBOOK:\n",
+        "if IS_WORKBENCH_NOTEBOOK:\n",
         "    USER_FLAG = \"--user\"\n",
         "    \n",
         "! pip install -U {USER_FLAG} --upgrade google-cloud-aiplatform \\\n",
@@ -208,7 +215,7 @@
         "\n",
         "1. [Enable the Vertex AI API and Compute Engine API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component).\n",
         "\n",
-        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
         "\n",
         "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
         "Cloud SDK uses the right project for all the commands in this notebook.\n",
@@ -273,7 +280,7 @@
         "#### Region\n",
         "\n",
         "You can also change the `REGION` variable, which is used for operations\n",
-        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
+        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. It is recommended that you choose the region closest to you.\n",
         "\n",
         "- Americas: `us-central1`\n",
         "- Europe: `europe-west4`\n",
@@ -281,7 +288,7 @@
         "\n",
         "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
         "\n",
-        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)"
+        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
       ]
     },
     {
@@ -304,10 +311,29 @@
         "id": "dr--iN2kAylZ"
       },
       "source": [
-        "### Authenticate your Google Cloud account\n",
+        "#### UUID\n",
         "\n",
-        "**If you are using Google Cloud Notebooks**, your environment is already\n",
-        "authenticated. Skip this step."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "4e166d927e36"
+      },
+      "outputs": [],
+      "source": [
+        "import random\n",
+        "import string\n",
+        "\n",
+        "\n",
+        "# Generate a uuid of a specifed length(default=8)\n",
+        "def generate_uuid(length: int = 8) -> str:\n",
+        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
+        "\n",
+        "\n",
+        "UUID = generate_uuid()"
       ]
     },
     {
@@ -316,6 +342,11 @@
         "id": "sBCra4QMA2wR"
       },
       "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
+        "authenticated. \n",
+        "\n",
         "**If you are using Colab**, run the cell below and follow the instructions\n",
         "when prompted to authenticate your account via oAuth.\n",
         "\n",
@@ -333,7 +364,7 @@
         "into the filter box, and select\n",
         "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
         "\n",
-        "5. Click **Create**. A JSON file that contains your key downloads to your\n",
+        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
         "local environment.\n",
         "\n",
         "6. Enter the path to your service account key as the\n",
@@ -348,19 +379,19 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "import sys\n",
-        "\n",
         "# If you are running this notebook in Colab, run this cell and follow the\n",
         "# instructions to authenticate your GCP account. This provides access to your\n",
         "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
         "# requests.\n",
         "\n",
-        "# The Google Cloud Notebook product has specific requirements\n",
-        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
+        "import os\n",
+        "import sys\n",
         "\n",
-        "# If on Google Cloud Notebooks, then don't execute this code\n",
-        "if not IS_GOOGLE_CLOUD_NOTEBOOK:\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = \"google.colab\" in sys.modules\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
         "    if \"google.colab\" in sys.modules:\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
@@ -370,7 +401,7 @@
         "    # path to your service account key and run this cell to authenticate your GCP\n",
         "    # account.\n",
         "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+        "        %env GOOGLE_APPLICATION_CREDENTIALS '[your-service-account-key-path]'"
       ]
     },
     {
@@ -379,7 +410,7 @@
         "id": "XoEqT2Y4DJmf"
       },
       "source": [
-        "### Import libraries and define constants"
+        "### Import libraries"
       ]
     },
     {
@@ -393,18 +424,31 @@
         "import datetime\n",
         "\n",
         "import pandas as pd\n",
-        "from google.cloud import aiplatform\n",
-        "\n",
-        "aiplatform.init(project=PROJECT_ID, location=REGION)"
+        "from avro.datafile import DataFileReader\n",
+        "from avro.io import DatumReader\n",
+        "from google.cloud import aiplatform"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "9UvxYyGUimKw"
+        "id": "138407556b22"
       },
       "source": [
-        "## Create Feature Store Resources"
+        "### Initialize Vertex AI SDK for Python\n",
+        "\n",
+        "Initialize the Vertex AI SDK for Python for your project and region."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "8d2077ffee78"
+      },
+      "outputs": [],
+      "source": [
+        "aiplatform.init(project=PROJECT_ID, location=REGION)"
       ]
     },
     {
@@ -413,11 +457,12 @@
         "id": "buQBIv3ZL3A0"
       },
       "source": [
-        "### Create Feature Store\n",
+        "## Create a Feature Store\n",
         "\n",
-        "The method to create a Feature Store returns a\n",
-        "[long-running operation](https://google.aip.dev/151) (LRO). An LRO starts an asynchronous job. LROs are returned for other API\n",
-        "methods too, such as updating or deleting a featurestore. Running the code cell creates a featurestore and prints the process logs."
+        "The method to create a Feature Store in Vertex AI returns a\n",
+        "[long-running operation](https://google.aip.dev/151) (LRO). An LRO starts an asynchronous job. LROs are returned for other API methods too, such as updating or deleting a featurestore. \n",
+        "\n",
+        "Running the code cell below creates a featurestore and prints the process' logs."
       ]
     },
     {
@@ -428,9 +473,9 @@
       },
       "outputs": [],
       "source": [
+        "# Create featurestore\n",
         "movie_predictions_feature_store = aiplatform.Featurestore.create(\n",
-        "    featurestore_id=\"movie_predictions\",\n",
-        "    online_store_fixed_node_count=1,\n",
+        "    featurestore_id=f\"movie_predictions_{UUID}\", online_store_fixed_node_count=1\n",
         ")"
       ]
     },
@@ -440,7 +485,7 @@
         "id": "EpmJq75zXjmT"
       },
       "source": [
-        "### Create Entity Types\n",
+        "## Create Entity types\n",
         "\n",
         "Entity types can be created within the Featurestore class. Below, you create the `Users` entity type and `Movies` entity type. Process logs are printed in the output for each cell."
       ]
@@ -453,6 +498,7 @@
       },
       "outputs": [],
       "source": [
+        "# Create users entity type\n",
         "users_entity_type = movie_predictions_feature_store.create_entity_type(\n",
         "    entity_type_id=\"users\",\n",
         "    description=\"Users entity\",\n",
@@ -467,6 +513,7 @@
       },
       "outputs": [],
       "source": [
+        "# Create movies entity type\n",
         "movies_entity_type = movie_predictions_feature_store.create_entity_type(\n",
         "    entity_type_id=\"movies\",\n",
         "    description=\"Movies entity\",\n",
@@ -479,8 +526,11 @@
         "id": "FJW4q-0jO2Xf"
       },
       "source": [
-        "### Create Features\n",
-        "Features can be created within each entity type. Add defining features to the `Users` entity type and `Movies` entity type by using the following methods."
+        "## Create Features\n",
+        "Features can be created within each entity type. Add defined features to the `Users` entity type and `Movies` entity type by using the following methods.\n",
+        "\n",
+        "### Add features using *create_feature* method\n",
+        "Provide the feature information like id, type and description to the `create_feature` method of entity type."
       ]
     },
     {
@@ -491,18 +541,21 @@
       },
       "outputs": [],
       "source": [
+        "# Create age feature\n",
         "users_feature_age = users_entity_type.create_feature(\n",
         "    feature_id=\"age\",\n",
         "    value_type=\"INT64\",\n",
         "    description=\"User age\",\n",
         ")\n",
         "\n",
+        "# Create gender feature\n",
         "users_feature_gender = users_entity_type.create_feature(\n",
         "    feature_id=\"gender\",\n",
         "    value_type=\"STRING\",\n",
         "    description=\"User gender\",\n",
         ")\n",
         "\n",
+        "# Create liked_genres feature\n",
         "users_feature_liked_genres = users_entity_type.create_feature(\n",
         "    feature_id=\"liked_genres\",\n",
         "    value_type=\"STRING_ARRAY\",\n",
@@ -510,6 +563,18 @@
         ")"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ecb141839033"
+      },
+      "source": [
+        "### Add features using batch method\n",
+        "You can also create features using a config map in a dictionary format and the `batch_create_features` method. This way, you can add multiple features at once. \n",
+        "\n",
+        "Below, you define and create *title*, *genres* and *average_rating* features using the batch method."
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -531,17 +596,8 @@
         "        \"value_type\": \"DOUBLE\",\n",
         "        \"description\": \"The average rating for the movie, range is [1.0-5.0]\",\n",
         "    },\n",
-        "}"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "YhfOKJL_BvuM"
-      },
-      "outputs": [],
-      "source": [
+        "}\n",
+        "\n",
         "movie_features = movies_entity_type.batch_create_features(\n",
         "    feature_configs=movies_feature_configs,\n",
         ")"
@@ -553,18 +609,15 @@
         "id": "K3n5XdK8Xjmw"
       },
       "source": [
-        "## Ingest Feature Values into Entity Type from a Pandas DataFrame\n",
+        "## Ingest Feature values into Entity types from dataframes\n",
         "\n",
-        "You need to ingest feature values into your entity type containing the features, so you can later `read` (online) or `batch serve` (offline) the feature values from the entity type. In this step, you will learn how to ingest feature values from a Pandas DataFrame into an entity type. We can also import feature values from BigQuery or Google Cloud Storage.\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "BlqJ-QdTcs6W"
-      },
-      "source": [
-        "#### Get data from source files"
+        "You need to ingest feature values into your entity type containing the features. It is so that you can later `read` (online) or `batch serve` (offline) the feature values from the entity type. \n",
+        "\n",
+        "In this step, you learn how to ingest feature values from a Pandas dataframe into an entity type. You can also import feature values from BigQuery or Google Cloud Storage.\n",
+        "\n",
+        "### Get data from source\n",
+        "\n",
+        "Define the public data sources for users and movies and copy them locally into *avro* files."
       ]
     },
     {
@@ -583,17 +636,8 @@
         ")\n",
         "\n",
         "USERS_AVRO_FN = \"users.avro\"\n",
-        "MOVIES_AVRO_FN = \"movies.avro\""
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "KqIH_bS-5OW5"
-      },
-      "outputs": [],
-      "source": [
+        "MOVIES_AVRO_FN = \"movies.avro\"\n",
+        "\n",
         "! gsutil cp $GCS_USERS_AVRO_URI $USERS_AVRO_FN\n",
         "! gsutil cp $GCS_MOVIES_AVRO_URI $MOVIES_AVRO_FN"
       ]
@@ -604,7 +648,9 @@
         "id": "Fd6Z0jfR5OW5"
       },
       "source": [
-        "#### Load Avro Files into Pandas DataFrames"
+        "### Load data from avro files \n",
+        "\n",
+        "Load users and movies data from avro files into Pandas dataframes."
       ]
     },
     {
@@ -615,10 +661,7 @@
       },
       "outputs": [],
       "source": [
-        "from avro.datafile import DataFileReader\n",
-        "from avro.io import DatumReader\n",
-        "\n",
-        "\n",
+        "# Define a class for reading the avro data\n",
         "class AvroReader:\n",
         "    def __init__(self, data_file):\n",
         "        self.avro_reader = DataFileReader(open(data_file, \"rb\"), DatumReader())\n",
@@ -636,6 +679,7 @@
       },
       "outputs": [],
       "source": [
+        "# Load users data from avro file\n",
         "users_avro_reader = AvroReader(data_file=USERS_AVRO_FN)\n",
         "users_source_df = users_avro_reader.to_dataframe()\n",
         "print(users_source_df)"
@@ -649,6 +693,7 @@
       },
       "outputs": [],
       "source": [
+        "# Load movies data from avro file\n",
         "movies_avro_reader = AvroReader(data_file=MOVIES_AVRO_FN)\n",
         "movies_source_df = movies_avro_reader.to_dataframe()\n",
         "print(movies_source_df)"
@@ -660,7 +705,9 @@
         "id": "bgb0WGwX5OW6"
       },
       "source": [
-        "#### Ingest Feature Values into _Users_ Entity Type"
+        "### Ingest Feature values into Entity types\n",
+        "\n",
+        "Load the feature values into `users` entity type providing the id fields and time field."
       ]
     },
     {
@@ -685,7 +732,7 @@
         "id": "PCAdQ3cF5OW6"
       },
       "source": [
-        "#### Ingest Feature Values into _Movies_ Entity Type"
+        "Load the feature values into `movie` entity type providing the id fields and time field."
       ]
     },
     {
@@ -710,10 +757,12 @@
         "id": "pIYLZwao5OW6"
       },
       "source": [
-        "## Read/Online Serve Entity's Feature Values from Vertex AI Online Feature Store\n",
+        "## Read/serve Entity's feature values online from Feature Store\n",
         "\n",
         "Feature Store allows [online serving](https://cloud.google.com/vertex-ai/docs/featurestore/serving-online)\n",
-        "which lets you read feature values for small batches of entities. It works well when you want to read values of selected features from an entity or multiple entities in an entity type."
+        "which lets you read feature values for small batches of entities. It works well when you want to read values of selected features from an entity or multiple entities in an entity type.\n",
+        "\n",
+        "### Read feature values for users"
       ]
     },
     {
@@ -730,6 +779,15 @@
         "print(users_read_df)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "b2cfa09ef11d"
+      },
+      "source": [
+        "### Read feature values for movies"
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -751,18 +809,13 @@
         "id": "AK2Glzkq5OW7"
       },
       "source": [
-        "## Batch Serve Feature Values from Vertex AI Feature Store\n",
+        "## Batch serve feature values from Feature Store\n",
         "\n",
-        "Batch Serving is used to fetch a large batch of feature values for high-throughput, and is typically used for training a model or batch prediction. In this section, you learn how to prepare training examples by using the Feature Store's batch serve function."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "hxsotHUe5OW7"
-      },
-      "source": [
-        "#### Read instances from source file"
+        "Batch Serving is used to fetch a large batch of feature values for high-throughput, and is typically used for training a model or batch prediction. In this section, you learn how to prepare training examples by using the Feature Store's batch serve function.\n",
+        "\n",
+        "### Read instances from source file\n",
+        "\n",
+        "Define the source file and destination file. "
       ]
     },
     {
@@ -777,6 +830,15 @@
         "READ_INSTANCES_CSV_FN = \"data.csv\""
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3f2c558b649f"
+      },
+      "source": [
+        "Copy the instances from the source file to the destination file locally."
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -794,7 +856,9 @@
         "id": "T5DW1MFt5OW7"
       },
       "source": [
-        "#### Load CSV file into a Pandas DataFrame"
+        "### Load the instances\n",
+        "\n",
+        "Load the instances from CSV file into a Pandas dataframe."
       ]
     },
     {
@@ -815,7 +879,9 @@
         "id": "LsgNNH8G5OW8"
       },
       "source": [
-        "#### Change the Dtype of `Timestamp` to `Datetime64`"
+        "### Change the data type\n",
+        "\n",
+        "Change the data type of the timestamp field from `Timestamp` to `Datetime64`."
       ]
     },
     {
@@ -837,7 +903,9 @@
         "id": "ao1dC5Pc5OW8"
       },
       "source": [
-        "#### Batch Serve Feature Values from Movie Predictions Feature Store"
+        "### Batch serve feature values from Feature Store\n",
+        "\n",
+        "Serve the batch response to a dataframe and display the data."
       ]
     },
     {
@@ -858,43 +926,21 @@
         "movie_predictions_df"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "29gLNORP5OW8"
-      },
-      "source": [
-        "## Read the Updated Feature Values"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "XN84znoI5OW8"
       },
       "source": [
-        "#### Feature Values from last ingestion\n",
-        "Recall read from the Entity Type shows Feature Values from the last ingestion."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "wtmshq_n5OW9"
-      },
-      "outputs": [],
-      "source": [
-        "print(movies_read_df)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "feTUJjqG5OW9"
-      },
-      "source": [
-        "#### Ingest updated Feature Values"
+        "## Read the latest feature values\n",
+        "\n",
+        "In Feature Store, you access the latest or the last available feature values unless a specific time is provided. Now, you test this feature by ingesting new data to the entity types and reading it from the Feature Store.\n",
+        "\n",
+        "### Ingest updated feature values\n",
+        "\n",
+        "Now, you update the feature values by running the following cell. \n",
+        "\n",
+        "**Note:** For comparison, you can try printing the feature values read from the entity types earlier (those in `movies_read_df` variable). "
       ]
     },
     {
@@ -905,21 +951,13 @@
       },
       "outputs": [],
       "source": [
+        "# Create a dataframe for the new data\n",
         "update_movies_df = pd.DataFrame(\n",
         "    data=[[\"movie_03\", 4.3], [\"movie_04\", 4.8]],\n",
         "    columns=[\"movie_id\", \"average_rating\"],\n",
         ")\n",
-        "print(update_movies_df)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "aKKhSzUc5OW9"
-      },
-      "outputs": [],
-      "source": [
+        "\n",
+        "# Ingest the new data from the dataframe\n",
         "movies_entity_type.ingest_from_df(\n",
         "    feature_ids=[\"average_rating\"],\n",
         "    feature_time=datetime.datetime.now(),\n",
@@ -934,8 +972,9 @@
         "id": "s47WCIvL5OW9"
       },
       "source": [
-        "#### Latest Feature Values\n",
-        "Read from the Entity Type shows updated Feature values from the latest ingestion."
+        "### Fetch the latest feature values\n",
+        "\n",
+        "Reading from the entity type gives you the updated feature values from the latest ingestion."
       ]
     },
     {
@@ -953,23 +992,18 @@
         "print(update_movies_read_df)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "wsvCRzn_5OW9"
-      },
-      "source": [
-        "## Point-in-Time Correctness"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "R1YGRNsW5OW9"
       },
       "source": [
-        "#### Missing data\n",
-        "Recall Batch Serve from the last ingestion has some missing data in it."
+        "## Point-in-time correctness\n",
+        "\n",
+        "Vertex AI Feature Store captures feature values for a feature at a specific point in time. In case there are missing values in your past data, you can backfill them using batch serving.\n",
+        "\n",
+        "### Missing data\n",
+        "Recall that response from the batch serve from last ingestion has some missing data in it."
       ]
     },
     {
@@ -980,6 +1014,7 @@
       },
       "outputs": [],
       "source": [
+        "# Print the response\n",
         "print(movie_predictions_df)"
       ]
     },
@@ -989,7 +1024,9 @@
         "id": "abQRF6mx5OW-"
       },
       "source": [
-        "#### Backfill/Correct point-in-time data"
+        "### Backfill/correct point-in-time data\n",
+        "\n",
+        "Impute the missing data based on the timestamps."
       ]
     },
     {
@@ -1000,6 +1037,7 @@
       },
       "outputs": [],
       "source": [
+        "# Impute the users data\n",
         "backfill_users_df = pd.DataFrame(\n",
         "    data=[[\"bob\", 34, \"Male\", [\"Drama\"], \"2020-02-13 09:35:15\"]],\n",
         "    columns=[\"user_id\", \"age\", \"gender\", \"liked_genres\", \"update_time\"],\n",
@@ -1016,6 +1054,7 @@
       },
       "outputs": [],
       "source": [
+        "# Impute the movies data\n",
         "backfill_movies_df = pd.DataFrame(\n",
         "    data=[[\"movie_04\", 4.2, \"The Dark Knight\", \"Action\", \"2020-02-13 09:35:15\"]],\n",
         "    columns=[\"movie_id\", \"average_rating\", \"title\", \"genres\", \"update_time\"],\n",
@@ -1030,7 +1069,9 @@
         "id": "WXb4JUhu5OW-"
       },
       "source": [
-        "#### Ingest backfilled/corrected point-in-time data from dataframe"
+        "### Ingest the backfilled/corrected data\n",
+        "\n",
+        "Ingest the imputed point-in-time data from dataframe to the entity types in feature store."
       ]
     },
     {
@@ -1041,6 +1082,7 @@
       },
       "outputs": [],
       "source": [
+        "# Ingest the users data\n",
         "users_entity_type.ingest_from_df(\n",
         "    feature_ids=[\"age\", \"gender\", \"liked_genres\"],\n",
         "    feature_time=\"update_time\",\n",
@@ -1057,6 +1099,7 @@
       },
       "outputs": [],
       "source": [
+        "# Ingest the users data\n",
         "movies_entity_type.ingest_from_df(\n",
         "    feature_ids=[\"average_rating\", \"title\", \"genres\"],\n",
         "    feature_time=\"update_time\",\n",
@@ -1071,8 +1114,8 @@
         "id": "1e62Ku6W5OW_"
       },
       "source": [
-        "#### Latest ingestion with imputed missing data\n",
-        "Batch Serve from the latest ingestion with backfill/correction has reduced missing data."
+        "### Fetch the latest data\n",
+        "Batch serve the latest ingested data with backfill/correction to a dataframe to ensure the feature store is updated. "
       ]
     },
     {
@@ -1104,7 +1147,7 @@
         "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
         "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
         "\n",
-        "You can also keep the project but delete the featurestore by running the code below:"
+        "Otherwise, you can delete the individual resources you created in this tutorial:"
       ]
     },
     {
@@ -1115,6 +1158,7 @@
       },
       "outputs": [],
       "source": [
+        "# Delete the feature store\n",
         "movie_predictions_feature_store.delete(force=True)"
       ]
     }

notebooks/official/matching_engine/README.md

@@ -1,5 +1,6 @@
+## Vertex-AI: Matching Engine Notebook
 
-[Create Vertex AI Matching Engine index](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb)
+<a id="sdk_matching_engine_for_indexing"></a>[Create Vertex AI Matching Engine index](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb)
 
 Learn how to create Approximate Nearest Neighbor (ANN) Index, query against indexes, and validate the performance of the index.
 
@@ -11,29 +12,20 @@ The steps performed include:
 * Perform online query
 * Compute recall
 
-
-[Introduction to builtin Swivel embedding algorithm](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/matching_engine/intro-swivel.ipynb)
-
-Learn how to train custom embeddings using Vertex AI Pipelines and deploy the model for serving.
-
-The steps performed include:
-
-1. **Setup**: Importing the required libraries and setting your global variables.
-2. **Configure parameters**: Setting the appropriate parameter values for the pipeline job.
-3. **Train on Vertex AI Pipelines**: Create a Swivel job to Vertex Pipelines using pipeline template.
-4. **Deploy on Vertex AI Prediction**: Importing and deploying the trained model to a callable endpoint.
-5. **Predict**: Calling the deployed endpoint using online prediction.
-6. **Cleaning up**: Deleting resources created by this tutorial.
-
-[Introduction to builtin Two-towers embedding algorithm](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/matching_engine/two-tower-model-introduction.ipynb)
-
-Learn how to run the two-tower model.
-
-The steps performed include:
-1. **Setup**: Importing the required libraries and setting your global variables.
-2. **Configure parameters**: Setting the appropriate parameter values for the training job.
-3. **Train on Vertex AI Training**: Submitting a training job.
-4. **Deploy on Vertex AI Prediction**: Importing and deploying the trained model to a callable endpoint.
-5. **Predict**: Calling the deployed endpoint using online or batch prediction.
-6. **Hyperparameter tuning**: Running a hyperparameter tuning job.
-7. **Cleaning up**: Deleting resources created by this tutorial.
+    <details>
+    <summary>Example code snippet from the Notebook:</summary>
+    
+    * Create an IndexEndpoint with VPC Network
+        ```python
+        # [START aiplatform_sdk_matching_engine_for_indexing]
+        VPC_NETWORK = "[your-network-name]"
+        VPC_NETWORK_FULL = "projects/{}/global/networks/{}".format(PROJECT_NUMBER, VPC_NETWORK)
+        my_index_endpoint = aiplatform.MatchingEngineIndexEndpoint.create(
+            display_name="index_endpoint_for_demo",
+            description="index endpoint description",
+            network=VPC_NETWORK_FULL,
+        )
+        # [END aiplatform_sdk_matching_engine_for_indexing]
+        ```
+        [:notebook: sdk_matching_engine_for_indexing.ipynb](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb)
+    </details>

notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb

@@ -98,171 +98,172 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "5e2eba58ad71"
+        "id": "f0f1bea346db"
       },
       "source": [
-        "### Costs \n",
+        "## Installation\n",
         "\n",
-        "This tutorial uses billable components of Google Cloud:\n",
-        "\n",
-        "* Vertex AI\n",
-        "* Cloud Storage\n",
-        "\n",
-        "Learn about [Vertex AI\n",
-        "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
-        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
-        "Calculator](https://cloud.google.com/products/calculator/)\n",
-        "to generate a cost estimate based on your projected usage."
+        "Install the latest version of Cloud Storage, BigQuery and Vertex AI SDKs for Python."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "dfbccc635a17"
+      },
+      "outputs": [],
+      "source": [
+        "# Install the packages\n",
+        "! pip3 install --upgrade google-cloud-aiplatform \\\n",
+        "                        google-cloud-storage"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "d1e95a984673"
+        "id": "5b08ba354c6e"
+      },
+      "source": [
+        "### Colab only: Uncomment the following cell to restart the kernel."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "bea801acf6b5"
+      },
+      "outputs": [],
+      "source": [
+        "# Automatically restart kernel after installs so that your environment can access the new packages\n",
+        "# import IPython\n",
+        "\n",
+        "# app = IPython.Application.instance()\n",
+        "# app.kernel.do_shutdown(True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dd28c9e4f067"
       },
       "source": [
         "## Before you begin\n",
+        "#### Set your project ID\n",
         "\n",
-        "### Set up your Google Cloud project\n",
+        "**If you don't know your project ID**, try the following:\n",
+        "* Run `gcloud config list`.\n",
+        "* Run `gcloud projects list`.\n",
+        "* See the support page: [Locate the project ID](https://support.google.com/googleapi/answer/7014113)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "80c0215f05a0"
+      },
+      "outputs": [],
+      "source": [
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}\n",
         "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager).\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI API and Compute Engine API, and Service Networking API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component,servicenetworking.googleapis.com).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+        "# Set the project id\n",
+        "! gcloud config set project {PROJECT_ID}"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "2b9daa35336a"
+        "id": "4f4512bf63b3"
+      },
+      "source": [
+        "#### Region\n",
+        "\n",
+        "You can also change the `REGION` variable used by Vertex AI. Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "474be5183c27"
+      },
+      "outputs": [],
+      "source": [
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "949271bfebe3"
       },
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using a Vertex AI Workbench notebook**, your environment is already\n",
-        "authenticated. Skip this step."
+        "Depending on your Jupyter environment, you may have to manually authenticate. Follow the relevant instructions below."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "c6bed8c6a6b3"
+        "id": "b65b4ce80d9a"
       },
       "source": [
-        "**If you are using Colab**, run the cell below and follow the instructions\n",
-        "when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "1. In the Cloud Console, go to the [**Create service account key**\n",
-        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
-        "\n",
-        "2. Click **Create service account**.\n",
-        "\n",
-        "3. In the **Service account name** field, enter a name, and\n",
-        "   click **Create**.\n",
-        "\n",
-        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
-        "into the filter box, and select\n",
-        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
-        "local environment.\n",
-        "\n",
-        "6. Enter the path to your service account key as the\n",
-        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
+        "**1. Vertex AI Workbench**\n",
+        "* Do nothing as you are already authenticated."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "985cdbfe7372"
+      },
+      "source": [
+        "**2. Local JupyterLab instance, uncomment and run:**"
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "3e2b43c2d2bf"
+        "id": "fbc9cd30cc4b"
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "# The Vertex AI Workbench notebook product has specific requirements\n",
-        "IS_VERTEX_AI_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# If on a Vertex AI Workbench notebook, then don't execute this code\n",
-        "if not IS_VERTEX_AI_WORKBENCH_NOTEBOOK:\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, log in using gcloud\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        ! gcloud auth login"
+        "# ! gcloud auth login"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "WReHDGG5g0XY"
+        "id": "79efab26ad02"
       },
       "source": [
-        "#### Set your project ID\n",
-        "\n",
-        "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
+        "**3. Colab, uncomment and run:**"
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "beb72f394541"
+        "id": "a336a05c6149"
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
-        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}\n",
-        "\n",
-        "# Get your Google Cloud project ID from gcloud\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    shell_output = !gcloud config list --format 'value(core.project)' 2>/dev/null\n",
-        "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID: \", PROJECT_ID)"
+        "# from google.colab import auth\n",
+        "# auth.authenticate_user()"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "f4c6d0a9e66c"
+        "id": "0c0a44fa330f"
       },
       "source": [
-        "Otherwise, set your project ID here."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "1dc3fa9ac4f7"
-      },
-      "outputs": [],
-      "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
-        "    PROJECT_ID = \"<your_project_id>\"  # @param {type:\"string\"}"
+        "**4. Service account or other**\n",
+        "* See how to grant Cloud Storage permissions to your service account at https://cloud.google.com/storage/docs/gsutil/commands/iam#ch-examples."
       ]
     },
     {
@@ -271,7 +272,8 @@
         "id": "4962667eec8e"
       },
       "source": [
-        "* **Prepare a VPC network**.  To reduce any network overhead that might lead to unnecessary increase in overhead latency, it is best to call the ANN endpoints from your VPC via a direct [VPC Peering](https://cloud.google.com/vertex-ai/docs/general/vpc-peering) connection. \n",
+        "### Prepare a VPC network\n",
+        "To reduce any network overhead that might lead to unnecessary increase in overhead latency, it is best to call the ANN endpoints from your VPC via a direct [VPC Peering](https://cloud.google.com/vertex-ai/docs/general/vpc-peering) connection. \n",
         "  * The following section describes how to setup a VPC Peering connection if you don't have one. \n",
         "  * This is a one-time initial setup task. You can also reuse existing VPC network and skip this section."
       ]
@@ -284,7 +286,8 @@
       },
       "outputs": [],
       "source": [
-        "VPC_NETWORK = \"[your-vpc-network-name]\"  # @param {type:\"string\"}\n",
+        "# VPC_NETWORK = \"[your-vpc-network-name]\"  # @param {type:\"string\"}\n",
+        "VPC_NETWORK = \"matching-engine-test\"  # @param {type:\"string\"}\n",
         "\n",
         "PEERING_RANGE_NAME = \"ann-haystack-range\""
       ]
@@ -363,7 +366,7 @@
       },
       "outputs": [],
       "source": [
-        "! pip install -U google-cloud-aiplatform"
+        "! pip install --upgrade --quiet google-cloud-aiplatform grpcio-tools h5py"
       ]
     },
     {
@@ -375,27 +378,13 @@
         "Install the `h5py` to prepare sample dataset, and the `grpcio-tools` for querying against the index. "
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "-h5sqwOEZ5Yq"
-      },
-      "outputs": [],
-      "source": [
-        "! pip install -U grpcio-tools\n",
-        "! pip install -U h5py"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "hhq5zEbGg0XX"
       },
       "source": [
-        "### Restart the kernel\n",
-        "\n",
-        "After you install the additional packages, you need to restart the notebook kernel so it can find the packages."
+        "### Colab only: Uncomment the following cell to restart the kernel."
       ]
     },
     {
@@ -406,37 +395,11 @@
       },
       "outputs": [],
       "source": [
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    # Automatically restart kernel after installs\n",
-        "    import IPython\n",
+        "# Automatically restart kernel after installs so that your environment can access the new packages\n",
+        "# import IPython\n",
         "\n",
-        "    app = IPython.Application.instance()\n",
-        "    app.kernel.do_shutdown(True)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "q7tcBkCDI1_M"
-      },
-      "source": [
-        "### Random ID\n",
-        "\n",
-        "To avoid name collisions between users on resources created, create a random ID for each instance session, and append the id onto the name of resources you create in this tutorial."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "HpIK91y1IzDr"
-      },
-      "outputs": [],
-      "source": [
-        "import random\n",
-        "import string\n",
-        "\n",
-        "RANDOM_ID = \"\".join(random.choices(string.ascii_lowercase + string.digits, k=8))"
+        "# app = IPython.Application.instance()\n",
+        "# app.kernel.do_shutdown(True)"
       ]
     },
     {
@@ -447,15 +410,7 @@
       "source": [
         "### Create a Cloud Storage bucket\n",
         "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "Set the name of your Cloud Storage bucket below. It must be unique across all\n",
-        "Cloud Storage buckets.\n",
-        "\n",
-        "You may also change the `REGION` variable, which is used for operations\n",
-        "throughout the rest of this notebook. Make sure to [choose a region where Vertex AI services are\n",
-        "available](https://cloud.google.com/vertex-ai/docs/general/locations#available_regions). You may\n",
-        "not use a Multi-Regional Storage bucket for training with Vertex AI."
+        "Create a storage bucket to store intermediate artifacts such as datasets."
       ]
     },
     {
@@ -466,23 +421,7 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_URI = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "REGION = \"[your-region]\"  # @param {type:\"string\"}"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "cf221059d072"
-      },
-      "outputs": [],
-      "source": [
-        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + RANDOM_ID\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "BUCKET_URI = \"gs://your-bucket-name-unique\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -505,84 +444,6 @@
         "! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "ucvCsknMCims"
-      },
-      "source": [
-        "Finally, validate access to your Cloud Storage bucket by examining its contents:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "vhOb7YnwClBb"
-      },
-      "outputs": [],
-      "source": [
-        "! gsutil ls -al $BUCKET_URI"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "XoEqT2Y4DJmf"
-      },
-      "source": [
-        "### Import libraries and define constants"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "Y9Uo3tifg1kx"
-      },
-      "source": [
-        "Import the Vertex AI (unified) client library into your Python environment. \n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "f2d05ab4126a"
-      },
-      "outputs": [],
-      "source": [
-        "import h5py"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "76f7b9ffde0b"
-      },
-      "source": [
-        "Use gcloud to retrieve the project number."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "pRUOFELefqf1"
-      },
-      "outputs": [],
-      "source": [
-        "PROJECT_NUMBER = !gcloud projects list --filter=\"PROJECT_ID:'{PROJECT_ID}'\" --format='value(PROJECT_NUMBER)'\n",
-        "PROJECT_NUMBER = PROJECT_NUMBER[0]\n",
-        "\n",
-        "PARENT = \"projects/{}/locations/{}\".format(PROJECT_ID, REGION)\n",
-        "\n",
-        "print(\"PROJECT_ID: {}\".format(PROJECT_ID))\n",
-        "print(\"REGION: {}\".format(REGION))\n",
-        "\n",
-        "!gcloud config set project {PROJECT_ID} --quiet\n",
-        "!gcloud config set ai_platform/region {REGION} --quiet"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -627,6 +488,8 @@
       },
       "outputs": [],
       "source": [
+        "import h5py\n",
+        "\n",
         "# The number of nearest neighbors to be retrieved from database for each query.\n",
         "NUM_NEIGHBOURS = 10\n",
         "\n",
@@ -652,22 +515,42 @@
         "id": "aQIQSyF9GtSv"
       },
       "source": [
-        "Save the train split in JSONL format.\n"
+        "#### Save the train split in JSONL format.\n",
+        "\n",
+        "The data must be formatted in JSONL format, which means each embedding dictionary is written as a JSON string on its own line.\n",
+        "\n",
+        "Additionally, to demonstrate the filtering functionality, the `restricts` key is set such that each embedding has a different `class`, `even` or `odd`. These are used during the later matching step to filter for results.\n",
+        "See additional information of filtering here: https://cloud.google.com/vertex-ai/docs/matching-engine/filtering"
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "18wCiTwfG40P"
+        "id": "57fe2ce4b50f"
       },
       "outputs": [],
       "source": [
+        "import json\n",
+        "\n",
         "with open(\"glove100.json\", \"w\") as f:\n",
-        "    for i in range(len(train)):\n",
-        "        f.write('{\"id\":\"' + str(i) + '\",')\n",
-        "        f.write('\"embedding\":[' + \",\".join(str(x) for x in train[i]) + \"]}\")\n",
-        "        f.write(\"\\n\")"
+        "    embeddings_formatted = [\n",
+        "        json.dumps(\n",
+        "            {\n",
+        "                \"id\": str(index),\n",
+        "                \"embedding\": [str(value) for value in train[index]],\n",
+        "                \"restricts\": [\n",
+        "                    {\n",
+        "                        \"namespace\": \"class\",\n",
+        "                        \"allow_list\": [\"even\" if index % 2 == 0 else \"odd\"],\n",
+        "                    }\n",
+        "                ],\n",
+        "            }\n",
+        "        )\n",
+        "        + \"\\n\"\n",
+        "        for index, embedding in enumerate(train)\n",
+        "    ]\n",
+        "    f.writelines(embeddings_formatted)"
       ]
     },
     {
@@ -730,7 +613,7 @@
       "source": [
         "Create the ANN index configuration:\n",
         "\n",
-        "Please read the documentation to understand the various configuration parameters that can be used to tune the index\n"
+        "To learn more about configuring the index, see [Input data format and structure](https://cloud.google.com/vertex-ai/docs/matching-engine/match-eng-setup#input-data-format).\n"
       ]
     },
     {
@@ -742,7 +625,6 @@
       "outputs": [],
       "source": [
         "import os\n",
-        "import sys\n",
         "\n",
         "from google.cloud import aiplatform\n",
         "\n",
@@ -758,7 +640,7 @@
       "outputs": [],
       "source": [
         "tree_ah_index = aiplatform.MatchingEngineIndex.create_tree_ah_index(\n",
-        "    display_name=DISPLAY_NAME,\n",
+        "    display_name=DISPLAY_NAME_BRUTE_FORCE,\n",
         "    contents_delta_uri=EMBEDDINGS_INITIAL_URI,\n",
         "    dimensions=DIMENSIONS,\n",
         "    approximate_neighbors_count=150,\n",
@@ -878,8 +760,21 @@
       "outputs": [],
       "source": [
         "with open(\"glove100_incremental.json\", \"w\") as f:\n",
+        "    index = 0\n",
         "    f.write(\n",
-        "        '{\"id\":\"0\",\"embedding\":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]}\\n'\n",
+        "        json.dumps(\n",
+        "            {\n",
+        "                \"id\": str(index),\n",
+        "                \"embedding\": [str(0) for _ in train[index]],\n",
+        "                \"restricts\": [\n",
+        "                    {\n",
+        "                        \"namespace\": \"class\",\n",
+        "                        \"allow_list\": [\"even\" if index % 2 == 0 else \"odd\"],\n",
+        "                    }\n",
+        "                ],\n",
+        "            }\n",
+        "        )\n",
+        "        + \"\\n\"\n",
         "    )"
       ]
     },
@@ -965,6 +860,10 @@
       },
       "outputs": [],
       "source": [
+        "# Retrieve the project number\n",
+        "PROJECT_NUMBER = !gcloud projects list --filter=\"PROJECT_ID:'{PROJECT_ID}'\" --format='value(PROJECT_NUMBER)'\n",
+        "PROJECT_NUMBER = PROJECT_NUMBER[0]\n",
+        "\n",
         "VPC_NETWORK = \"[your-network-name]\"\n",
         "VPC_NETWORK_FULL = \"projects/{}/global/networks/{}\".format(PROJECT_NUMBER, VPC_NETWORK)\n",
         "VPC_NETWORK_FULL"
@@ -1023,7 +922,7 @@
       },
       "outputs": [],
       "source": [
-        "DEPLOYED_INDEX_ID = f\"tree_ah_glove_deployed_{RANDOM_ID}\""
+        "DEPLOYED_INDEX_ID = \"tree_ah_glove_deployed_unique\""
       ]
     },
     {
@@ -1058,7 +957,7 @@
       },
       "outputs": [],
       "source": [
-        "DEPLOYED_BRUTE_FORCE_INDEX_ID = f\"glove_brute_force_deployed_{RANDOM_ID}\""
+        "DEPLOYED_BRUTE_FORCE_INDEX_ID = \"glove_brute_force_deployed_unique\""
       ]
     },
     {
@@ -1084,16 +983,9 @@
       "source": [
         "## Create Online Queries\n",
         "\n",
-        "After you built your indexes, you may query against the deployed index through the online querying gRPC API (Match service) within the virtual machine instances from the same region (for example 'us-central1' in this tutorial).  "
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "IcXa9lSuB9AT"
-      },
-      "source": [
-        "Test your query:"
+        "After you built your indexes, you may query against the deployed index through the online querying gRPC API (Match service) within the virtual machine instances from the same region (for example 'us-central1' in this tutorial).\n",
+        "\n",
+        "The `filter` parameter is an optional way to filter for a subset of embeddings. In this case, only embeddings that have the `class` set as `even` are returned."
       ]
     },
     {
@@ -1104,9 +996,16 @@
       },
       "outputs": [],
       "source": [
+        "# Test query\n",
+        "from google.cloud.aiplatform.matching_engine.matching_engine_index_endpoint import \\\n",
+        "    Namespace\n",
+        "\n",
         "# Test query\n",
         "response = my_index_endpoint.match(\n",
-        "    deployed_index_id=DEPLOYED_INDEX_ID, queries=test[:1], num_neighbors=NUM_NEIGHBOURS\n",
+        "    deployed_index_id=DEPLOYED_INDEX_ID,\n",
+        "    queries=test[:1],\n",
+        "    num_neighbors=NUM_NEIGHBOURS,\n",
+        "    filter=[Namespace(\"class\", [\"even\"])],\n",
         ")\n",
         "\n",
         "response"

notebooks/official/migration/UJ10 Vertex SDK Custom Scikit-Learn with pre-built training container.ipynb

@@ -33,20 +33,67 @@
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/ai-platform-samples/blob/master/vertex-ai-samples/tree/master/notebooks/official/migration/UJ10%20Vertex%20SDK%20Custom%20Scikit-Learn%20with%20pre-built%20training%20container.ipynb\">\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ10%20Vertex%20SDK%20Custom%20Scikit-Learn%20with%20pre-built%20training%20container.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/ai-platform-samples/blob/master/vertex-ai-samples/tree/master/notebooks/official/migration/UJ10%20Vertex%20SDK%20Custom%20Scikit-Learn%20with%20pre-built%20training%20container.ipynb\">\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ10%20Vertex%20SDK%20Custom%20Scikit-Learn%20with%20pre-built%20training%20container.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
+        "     <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/migration/UJ10%20Vertex%20SDK%20Custom%20Scikit-Learn%20with%20pre-built%20training%20container.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "  </td>         \n",
         "</table>\n",
         "<br/><br/><br/>"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "7a8a13b86a8b"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "\n",
+        "This tutorial demonstrates how to use the Vertex AI SDK for Python to train and deploy a custom tabular classification scikit-learn model for batch prediction."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "618cfedf829a"
+      },
+      "source": [
+        "### Objective\n",
+        "\n",
+        "In this tutorial, you learn to use `Vertex AI Training` to create a custom trained model and use `Vertex AI Batch Prediction` to do a batch prediction on the trained model.\n",
+        "\n",
+        "\n",
+        "You learn how to create a custom-trained model from a Python script in a Docker container using the Vertex AI SDK for Python, and then do a prediction on the deployed model by sending data.\n",
+        "\n",
+        "This tutorial uses the following Google Cloud ML services:\n",
+        "\n",
+        "- `Vertex AI Training`\n",
+        "- `Vertex AI Batch Prediction`\n",
+        "- `Vertex AI Model` resource\n",
+        "- `Vertex AI Endpoint` resource\n",
+        "\n",
+        "The steps performed include:\n",
+        "\n",
+        "- Create a `Vertex AI` custom job for training a scikit-learn model.\n",
+        "- Upload the trained model artifacts as a `Model` resource.\n",
+        "- Make a batch prediction.\n",
+        "- Deploy model to a endpoint\n",
+        "- Make a online prediction"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -55,7 +102,7 @@
       "source": [
         "### Dataset\n",
         "\n",
-        "The dataset used for this tutorial is the UCI Machine Learning [US Census Data (1990) dataset](https://archive.ics.uci.edu/ml/datasets/US+Census+Data+(1990)).The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket.\n",
+        "The dataset used for this tutorial is the UCI Machine Learning [US Census Data (1990) dataset](https://archive.ics.uci.edu/ml/datasets/US+Census+Data+(1990)).The version of the dataset you use in this tutorial is stored in a public Cloud Storage bucket.\n",
         "\n",
         "The dataset predicts whether a persons income will be above $50K USD."
       ]
@@ -88,29 +135,37 @@
       "source": [
         "### Set up your local development environment\n",
         "\n",
-        "If you are using Colab or Google Cloud Notebooks, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
+        "**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
         "\n",
-        "Otherwise, make sure your environment meets this notebook's requirements. You need the following:\n",
+        "**_NOTE_**: This notebook has been tested in the following environment:\n",
         "\n",
-        "- The Cloud Storage SDK\n",
-        "- Git\n",
-        "- Python 3\n",
-        "- virtualenv\n",
-        "- Jupyter notebook running in a virtual environment with Python 3\n",
+        "* Python version = 3.9\n",
         "\n",
-        "The Cloud Storage guide to [Setting up a Python development environment](https://cloud.google.com/python/setup) and the [Jupyter installation guide](https://jupyter.org/install) provide detailed instructions for meeting these requirements. The following steps provide a condensed set of instructions:\n",
+        "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
+        "You need the following:\n",
         "\n",
-        "1. [Install and initialize the SDK](https://cloud.google.com/sdk/docs/).\n",
+        "* The Google Cloud SDK\n",
         "\n",
-        "2. [Install Python 3](https://cloud.google.com/python/setup#installing_python).\n",
+        "The Google Cloud guide to [Setting up a Python development\n",
+        "environment](https://cloud.google.com/python/setup) and the [Jupyter\n",
+        "installation guide](https://jupyter.org/install) provide detailed instructions\n",
+        "for meeting these requirements. The following steps provide a condensed set of\n",
+        "instructions:\n",
         "\n",
-        "3. [Install virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv) and create a virtual environment that uses Python 3.  Activate the virtual environment.\n",
+        "1. [Install and initialize the Cloud SDK.](https://cloud.google.com/sdk/docs/)\n",
         "\n",
-        "4. To install Jupyter, run `pip3 install jupyter` on the command-line in a terminal shell.\n",
+        "1. [Install Python 3.](https://cloud.google.com/python/setup#installing_python)\n",
         "\n",
-        "5. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
+        "1. [Install\n",
+        "   virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv)\n",
+        "   and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
         "\n",
-        "6. Open this notebook in the Jupyter Notebook Dashboard.\n"
+        "1. To install Jupyter, run `pip3 install jupyter` on the\n",
+        "command-line in a terminal shell.\n",
+        "\n",
+        "1. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
+        "\n",
+        "1. Open this notebook in the Jupyter Notebook Dashboard."
       ]
     },
     {
@@ -121,7 +176,7 @@
       "source": [
         "## Installation\n",
         "\n",
-        "Install the latest version of Vertex SDK for Python."
+        "Install the following packages required to execute this notebook. "
       ]
     },
     {
@@ -134,45 +189,18 @@
       "source": [
         "import os\n",
         "\n",
-        "# Google Cloud Notebook\n",
-        "if os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
-        "    USER_FLAG = \"--user\"\n",
-        "else:\n",
-        "    USER_FLAG = \"\"\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
         "\n",
-        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "install_storage"
-      },
-      "source": [
-        "Install the latest GA version of *google-cloud-storage* library as well."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "install_storage"
-      },
-      "outputs": [],
-      "source": [
-        "! pip3 install -U google-cloud-storage $USER_FLAG"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "install_tensorflow"
-      },
-      "outputs": [],
-      "source": [
-        "if os.getenv(\"IS_TESTING\"):\n",
-        "    ! pip3 install --upgrade tensorflow $USER_FLAG"
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "USER_FLAG = \"\"\n",
+        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\"\n",
+        "\n",
+        "! pip3 install --upgrade google-cloud-aiplatform google-cloud-storage tensorflow $USER_FLAG -q"
       ]
     },
     {
@@ -194,6 +222,7 @@
       },
       "outputs": [],
       "source": [
+        "# Automatically restart kernel after installs\n",
         "import os\n",
         "\n",
         "if not os.getenv(\"IS_TESTING\"):\n",
@@ -212,26 +241,33 @@
       "source": [
         "## Before you begin\n",
         "\n",
-        "### GPU runtime\n",
-        "\n",
-        "This tutorial does not require a GPU runtime.\n",
-        "\n",
         "### Set up your Google Cloud project\n",
         "\n",
         "**The following steps are required, regardless of your notebook environment.**\n",
         "\n",
         "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
         "\n",
-        "2. [Make sure that billing is enabled for your project.](https://cloud.google.com/billing/docs/how-to/modify-project)\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
         "\n",
-        "3. [Enable the following APIs: Vertex AI APIs, Compute Engine APIs, and Cloud Storage.](https://console.cloud.google.com/flows/enableapi?apiid=ml.googleapis.com,compute_component,storage-component.googleapis.com)\n",
+        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com). \n",
         "\n",
-        "4. If you are running this notebook locally, you will need to install the [Cloud SDK]((https://cloud.google.com/sdk)).\n",
+        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
         "\n",
-        "5. Enter your project ID in the cell below. Then run the  cell to make sure the\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
         "Cloud SDK uses the right project for all the commands in this notebook.\n",
         "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$`."
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5aee4379e8e5"
+      },
+      "source": [
+        "#### Set your project ID\n",
+        "\n",
+        "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
       ]
     },
     {
@@ -299,7 +335,10 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -308,9 +347,9 @@
         "id": "timestamp"
       },
       "source": [
-        "#### Timestamp\n",
+        "#### UUID\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
       ]
     },
     {
@@ -321,9 +360,16 @@
       },
       "outputs": [],
       "source": [
-        "from datetime import datetime\n",
+        "import random\n",
+        "import string\n",
         "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+        "\n",
+        "# Generate a uuid of a specifed length(default=8)\n",
+        "def generate_uuid(length: int = 8) -> str:\n",
+        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
+        "\n",
+        "\n",
+        "UUID = generate_uuid()"
       ]
     },
     {
@@ -334,23 +380,31 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Google Cloud Notebooks**, your environment is already authenticated. Skip this step.\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
+        "authenticated. \n",
         "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
+        "**If you are using Colab**, run the cell below and follow the instructions\n",
+        "when prompted to authenticate your account via oAuth.\n",
         "\n",
         "**Otherwise**, follow these steps:\n",
         "\n",
-        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
+        "1. In the Cloud Console, go to the [**Create service account key**\n",
+        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
         "\n",
-        "**Click Create service account**.\n",
+        "2. Click **Create service account**.\n",
         "\n",
-        "In the **Service account name** field, enter a name, and click **Create**.\n",
+        "3. In the **Service account name** field, enter a name, and\n",
+        "   click **Create**.\n",
         "\n",
-        "In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex\" into the filter box, and select **Vertex Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
+        "into the filter box, and select\n",
+        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
         "\n",
-        "Click Create. A JSON file that contains your key downloads to your local environment.\n",
+        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
+        "local environment.\n",
         "\n",
-        "Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
+        "6. Enter the path to your service account key as the\n",
+        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
       ]
     },
     {
@@ -369,8 +423,11 @@
         "import os\n",
         "import sys\n",
         "\n",
-        "# If on Google Cloud Notebook, then don't execute this code\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = \"google.colab\" in sys.modules\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
         "    if \"google.colab\" in sys.modules:\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
@@ -380,7 +437,7 @@
         "    # path to your service account key and run this cell to authenticate your GCP\n",
         "    # account.\n",
         "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+        "        %env GOOGLE_APPLICATION_CREDENTIALS '[your-service-account-key-path]'"
       ]
     },
     {
@@ -406,7 +463,8 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
+        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
     {
@@ -417,8 +475,9 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
+        "    BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
     {
@@ -438,7 +497,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_NAME"
+        "! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
       ]
     },
     {
@@ -458,7 +517,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_NAME"
+        "! gsutil ls -al $BUCKET_URI"
       ]
     },
     {
@@ -467,9 +526,6 @@
         "id": "setup_vars"
       },
       "source": [
-        "### Set up variables\n",
-        "\n",
-        "Next, set up some variables used throughout the tutorial.\n",
         "### Import libraries and define constants"
       ]
     },
@@ -481,7 +537,11 @@
       },
       "outputs": [],
       "source": [
-        "import google.cloud.aiplatform as aip"
+        "import json\n",
+        "import os\n",
+        "\n",
+        "import google.cloud.aiplatform as aip\n",
+        "import tensorflow as tf"
       ]
     },
     {
@@ -503,7 +563,7 @@
       },
       "outputs": [],
       "source": [
-        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_NAME)"
+        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
       ]
     },
     {
@@ -532,10 +592,13 @@
       "outputs": [],
       "source": [
         "TRAIN_VERSION = \"scikit-learn-cpu.0-23\"\n",
-        "DEPLOY_VERSION = \"sklearn-cpu.0-23\"\n",
+        "DEPLOY_VERSION = \"sklearn-cpu.1-0\"\n",
         "\n",
-        "TRAIN_IMAGE = \"gcr.io/cloud-aiplatform/training/{}:latest\".format(TRAIN_VERSION)\n",
-        "DEPLOY_IMAGE = \"gcr.io/cloud-aiplatform/prediction/{}:latest\".format(DEPLOY_VERSION)"
+        "TRAIN_IMAGE = \"us-docker.pkg.dev/vertex-ai/training/{}:latest\".format(TRAIN_VERSION)\n",
+        "DEPLOY_IMAGE = \"us-docker.pkg.dev/vertex-ai/prediction/{}:latest\".format(DEPLOY_VERSION)\n",
+        "\n",
+        "print(\"Training:\", TRAIN_IMAGE)\n",
+        "print(\"Deployment:\", DEPLOY_IMAGE)"
       ]
     },
     {
@@ -548,7 +611,7 @@
         "\n",
         "Next, set the machine type to use for training and prediction.\n",
         "\n",
-        "- Set the variables `TRAIN_COMPUTE` and `DEPLOY_COMPUTE` to configure  the compute resources for the VMs you will use for for training and prediction.\n",
+        "- Set the variables `TRAIN_COMPUTE` and `DEPLOY_COMPUTE` to configure  the compute resources for the VMs you use for for training and prediction.\n",
         " - `machine type`\n",
         "     - `n1-standard`: 3.75GB of memory per vCPU.\n",
         "     - `n1-highmem`: 6.5GB of memory per vCPU\n",
@@ -600,7 +663,7 @@
         "\n",
         "#### Package layout\n",
         "\n",
-        "Before you start the training, you will look at how a Python package is assembled for a custom training job. When unarchived, the package contains the following directory/file layout.\n",
+        "Before you start the training, you look at how a Python package is assembled for a custom training job. When unarchived, the package contains the following directory/file layout.\n",
         "\n",
         "- PKG-INFO\n",
         "- README.md\n",
@@ -616,7 +679,7 @@
         "\n",
         "#### Package Assembly\n",
         "\n",
-        "In the following cells, you will assemble the training package."
+        "In the following cells, you assemble the training package."
       ]
     },
     {
@@ -637,7 +700,7 @@
         "setup_cfg = \"[egg_info]\\n\\ntag_build =\\n\\ntag_date = 0\"\n",
         "! echo \"$setup_cfg\" > custom/setup.cfg\n",
         "\n",
-        "setup_py = \"import setuptools\\n\\nsetuptools.setup(\\n\\n    install_requires=[\\n\\n        'tensorflow_datasets==1.3.0',\\n\\n    ],\\n\\n    packages=setuptools.find_packages())\"\n",
+        "setup_py = \"import setuptools\\n\\nsetuptools.setup(\\n\\n    install_requires=[\\n\\n        'tensorflow_datasets',\\n\\n    ],\\n\\n    packages=setuptools.find_packages())\"\n",
         "! echo \"$setup_py\" > custom/setup.py\n",
         "\n",
         "pkg_info = \"Metadata-Version: 1.0\\n\\nName: US Census Data (1990) tabular binary classification\\n\\nVersion: 0.0.0\\n\\nSummary: Demostration training script\\n\\nHome-page: www.google.com\\n\\nAuthor: Google\\n\\nAuthor-email: aferlitsch@google.com\\n\\nLicense: Public\\n\\nDescription: Demo\\n\\nPlatform: Vertex\"\n",
@@ -679,7 +742,6 @@
         "parser.add_argument('--model-dir', dest='model_dir',\n",
         "                    default=os.getenv('AIP_MODEL_DIR'), type=str, help='Model dir.')\n",
         "args = parser.parse_args()\n",
-        "\n",
         "print('Python Version = {}'.format(sys.version))\n",
         "\n",
         "# Public bucket holding the census data\n",
@@ -795,6 +857,9 @@
         "subdirs = args.model_dir.split('/')[3:]\n",
         "subdir = subdirs[0]\n",
         "subdirs.pop(0)\n",
+        "\n",
+        "\n",
+        "\n",
         "for comp in subdirs:\n",
         "    subdir = os.path.join(subdir, comp)\n",
         "\n",
@@ -803,7 +868,7 @@
         "\n",
         "# Upload the model to GCS\n",
         "bucket = storage.Client().bucket(bucket)\n",
-        "blob = bucket.blob(subdir + '/model.joblib')\n",
+        "blob = bucket.blob(subdir + 'model.joblib')\n",
         "blob.upload_from_filename('model.joblib')"
       ]
     },
@@ -829,7 +894,7 @@
         "! rm -f custom.tar custom.tar.gz\n",
         "! tar cvf custom.tar custom\n",
         "! gzip custom.tar\n",
-        "! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_census.tar.gz"
+        "! gsutil cp custom.tar.gz $BUCKET_URI/trainer_census.tar.gz"
       ]
     },
     {
@@ -880,10 +945,10 @@
       "outputs": [],
       "source": [
         "job = aip.CustomTrainingJob(\n",
-        "    display_name=\"census_\" + TIMESTAMP,\n",
+        "    display_name=\"census_\" + UUID,\n",
         "    script_path=\"custom/trainer/task.py\",\n",
         "    container_uri=TRAIN_IMAGE,\n",
-        "    requirements=[\"gcsfs==0.7.1\", \"tensorflow-datasets==4.4\"],\n",
+        "    requirements=[\"gcsfs\", \"tensorflow-datasets\"],\n",
         ")\n",
         "\n",
         "print(job)"
@@ -924,7 +989,7 @@
       },
       "outputs": [],
       "source": [
-        "MODEL_DIR = \"{}/{}\".format(BUCKET_NAME, TIMESTAMP)\n",
+        "MODEL_DIR = \"{}/{}\".format(BUCKET_URI, UUID)\n",
         "\n",
         "\n",
         "job.run(\n",
@@ -971,7 +1036,7 @@
       "outputs": [],
       "source": [
         "model = aip.Model.upload(\n",
-        "    display_name=\"census_\" + TIMESTAMP,\n",
+        "    display_name=\"census_\" + UUID,\n",
         "    artifact_uri=MODEL_DIR,\n",
         "    serving_container_image_uri=DEPLOY_IMAGE,\n",
         "    sync=False,\n",
@@ -1021,7 +1086,7 @@
       "source": [
         "### Make test items\n",
         "\n",
-        "You will use synthetic data as a test data items. Don't be concerned that we are using synthetic data -- we just want to demonstrate how to make a prediction."
+        "You use synthetic data as test data items. Don't be concerned that we are using synthetic data -- we just want to demonstrate how to make a prediction."
       ]
     },
     {
@@ -1076,7 +1141,7 @@
       "source": [
         "### Make the batch input file\n",
         "\n",
-        "Now make a batch input file, which you will store in your local Cloud Storage bucket.  Each instance in the prediction request is a list of the form:\n",
+        "Now make a batch input file, which you store in your local Cloud Storage bucket.  Each instance in the prediction request is a list of the form:\n",
         "\n",
         "                        [ [ content_1], [content_2] ]\n",
         "\n",
@@ -1091,11 +1156,7 @@
       },
       "outputs": [],
       "source": [
-        "import json\n",
-        "\n",
-        "import tensorflow as tf\n",
-        "\n",
-        "gcs_input_uri = BUCKET_NAME + \"/\" + \"test.jsonl\"\n",
+        "gcs_input_uri = BUCKET_URI + \"/\" + \"test.jsonl\"\n",
         "with tf.io.gfile.GFile(gcs_input_uri, \"w\") as f:\n",
         "    for i in INSTANCES:\n",
         "        f.write(json.dumps(i) + \"\\n\")\n",
@@ -1134,9 +1195,9 @@
         "MAX_NODES = 1\n",
         "\n",
         "batch_predict_job = model.batch_predict(\n",
-        "    job_display_name=\"census_\" + TIMESTAMP,\n",
+        "    job_display_name=\"census_\" + UUID,\n",
         "    gcs_source=gcs_input_uri,\n",
-        "    gcs_destination_prefix=BUCKET_NAME,\n",
+        "    gcs_destination_prefix=BUCKET_URI,\n",
         "    instances_format=\"jsonl\",\n",
         "    predictions_format=\"jsonl\",\n",
         "    model_parameters=None,\n",
@@ -1248,8 +1309,6 @@
       },
       "outputs": [],
       "source": [
-        "import json\n",
-        "\n",
         "bp_iter_outputs = batch_predict_job.iter_outputs()\n",
         "\n",
         "prediction_results = list()\n",
@@ -1263,8 +1322,7 @@
         "    with tf.io.gfile.GFile(name=gfile_name, mode=\"r\") as gfile:\n",
         "        for line in gfile.readlines():\n",
         "            line = json.loads(line)\n",
-        "            print(line)\n",
-        "            break"
+        "            print(line)"
       ]
     },
     {
@@ -1323,7 +1381,7 @@
       },
       "outputs": [],
       "source": [
-        "DEPLOYED_NAME = \"census-\" + TIMESTAMP\n",
+        "DEPLOYED_NAME = \"census-\" + UUID\n",
         "\n",
         "TRAFFIC_SPLIT = {\"0\": 100}\n",
         "\n",
@@ -1357,15 +1415,6 @@
         "    INFO:google.cloud.aiplatform.models:Endpoint model deployed. Resource name: projects/759209241365/locations/us-central1/endpoints/4867177336350441472"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "endpoints_predict:migration,new,mbsdk"
-      },
-      "source": [
-        "### [predictions.online-prediction-automl](https://cloud.google.com/vertex-ai/docs/predictions/online-predictions-automl)"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -1374,7 +1423,7 @@
       "source": [
         "### Make test item\n",
         "\n",
-        "You will use synthetic data as a test data item. Don't be concerned that we are using synthetic data -- we just want to demonstrate how to make a prediction."
+        "You use synthetic data as a test data item. Don't be concerned that we are using synthetic data -- we just want to demonstrate how to make a prediction."
       ]
     },
     {
@@ -1491,13 +1540,10 @@
         "Otherwise, you can delete the individual resources you created in this tutorial:\n",
         "\n",
         "- Dataset\n",
-        "- Pipeline\n",
         "- Model\n",
         "- Endpoint\n",
         "- AutoML Training Job\n",
         "- Batch Job\n",
-        "- Custom Job\n",
-        "- Hyperparameter Tuning Job\n",
         "- Cloud Storage Bucket"
       ]
     },
@@ -1505,64 +1551,25 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "cleanup:mbsdk"
+        "id": "74ddbf65df4c"
       },
       "outputs": [],
       "source": [
-        "delete_all = True\n",
+        "# delete endpoint\n",
+        "endpoint.delete()\n",
         "\n",
-        "if delete_all:\n",
-        "    # Delete the dataset using the Vertex dataset object\n",
-        "    try:\n",
-        "        if \"dataset\" in globals():\n",
-        "            dataset.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
+        "# Delete the model using the Vertex model object\n",
+        "model.delete()\n",
         "\n",
-        "    # Delete the model using the Vertex model object\n",
-        "    try:\n",
-        "        if \"model\" in globals():\n",
-        "            model.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
+        "# Delete the AutoML or Pipeline training job\n",
+        "job.delete()\n",
         "\n",
-        "    # Delete the endpoint using the Vertex endpoint object\n",
-        "    try:\n",
-        "        if \"endpoint\" in globals():\n",
-        "            endpoint.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
+        "# Delete the batch prediction job using the Vertex batch prediction object\n",
+        "batch_predict_job.delete()\n",
         "\n",
-        "    # Delete the AutoML or Pipeline trainig job\n",
-        "    try:\n",
-        "        if \"dag\" in globals():\n",
-        "            dag.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the custom trainig job\n",
-        "    try:\n",
-        "        if \"job\" in globals():\n",
-        "            job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the batch prediction job using the Vertex batch prediction object\n",
-        "    try:\n",
-        "        if \"batch_predict_job\" in globals():\n",
-        "            batch_predict_job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the hyperparameter tuning job using the Vertex hyperparameter tuning object\n",
-        "    try:\n",
-        "        if \"hpt_job\" in globals():\n",
-        "            hpt_job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    if \"BUCKET_NAME\" in globals():\n",
-        "        ! gsutil rm -r $BUCKET_NAME"
+        "delete_bucket = False\n",
+        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "    ! gsutil -m rm -r $BUCKET_URI"
       ]
     }
   ],

notebooks/official/migration/UJ7 Vertex SDK AutoML Text Entity Extraction.ipynb

@@ -43,10 +43,58 @@
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
+        "         <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ7%20Vertex%20SDK%20AutoML%20Text%20Entity%20Extraction.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "  </td>   \n",
         "</table>\n",
         "<br/><br/><br/>"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2277f661a148"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "<a name=\"section-1\"></a>\n",
+        "\n",
+        "This notebook demonstrates how to create an AutoML Text Entity Extrasction Model, with a Vertex AI ncbi disease research dataset, and how to serve the model for batch prediction. It requires you provide a bucket where the dataset will be stored.\n",
+        "\n",
+        "Note: you may incur charges for training, prediction, storage or usage of other GCP products in connection with testing this SDK."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "f926ec7acab3"
+      },
+      "source": [
+        "### Objective\n",
+        "\n",
+        "The objective of this notebook is to build a AutoML Text Entity Extrasction Model. The following steps have been followed:\n",
+        "This tutorial uses the following Google Cloud ML services :\n",
+        "\n",
+        "* Vertex AI Dataset resource\n",
+        "* AutoML Training\n",
+        "* Vertex AI Model resource\n",
+        "* Vertex AI Batch Prediction\n",
+        "\n",
+        "The steps performed include the following:\n",
+        "\n",
+        "* Set your task name, and GCS prefix\n",
+        "* Copy AutoML video demo train data for creating managed dataset\n",
+        "* Create a dataset on Vertex AI.\n",
+        "* Configure a training job\n",
+        "* Launch a training job and create a model on Vertex AI\n",
+        "* Copy AutoML Video Demo Prediction Data for creating batch prediction job\n",
+        "* Perform batch prediction job on the model"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -119,7 +167,7 @@
       "source": [
         "## Installation\n",
         "\n",
-        "Install the latest version of Vertex SDK for Python."
+        "Install the packages required for executing this notebook."
       ]
     },
     {
@@ -138,7 +186,9 @@
         "else:\n",
         "    USER_FLAG = \"\"\n",
         "\n",
-        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG"
+        "! pip3 install --upgrade google-cloud-aiplatform \\\n",
+        "                                     cuda-python \\\n",
+        "                                      $USER_FLAG -q"
       ]
     },
     {
@@ -278,7 +328,7 @@
         "#### Region\n",
         "\n",
         "You can also change the `REGION` variable, which is used for operations\n",
-        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
+        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. It is recommended that you choose the region closest to you.\n",
         "\n",
         "- Americas: `us-central1`\n",
         "- Europe: `europe-west4`\n",
@@ -286,7 +336,7 @@
         "\n",
         "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
         "\n",
-        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)"
+        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
       ]
     },
     {
@@ -297,7 +347,10 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -306,9 +359,9 @@
         "id": "timestamp"
       },
       "source": [
-        "#### Timestamp\n",
+        "#### UUID\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
       ]
     },
     {
@@ -319,9 +372,16 @@
       },
       "outputs": [],
       "source": [
-        "from datetime import datetime\n",
+        "import random\n",
+        "import string\n",
         "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+        "\n",
+        "# Generate a uuid of a specifed length(default=8)\n",
+        "def generate_uuid(length: int = 8) -> str:\n",
+        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
+        "\n",
+        "\n",
+        "UUID = generate_uuid()"
       ]
     },
     {
@@ -332,23 +392,38 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Google Cloud Notebooks**, your environment is already authenticated. Skip this step.\n",
-        "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
+        "authenticated. Skip this step."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "f1d7a972141f"
+      },
+      "source": [
+        "**If you are using Colab**, run the cell below and follow the instructions\n",
+        "when prompted to authenticate your account via oAuth.\n",
         "\n",
         "**Otherwise**, follow these steps:\n",
         "\n",
-        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
+        "1. In the Cloud Console, go to the [**Create service account key**\n",
+        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
         "\n",
-        "**Click Create service account**.\n",
+        "2. Click **Create service account**.\n",
         "\n",
-        "In the **Service account name** field, enter a name, and click **Create**.\n",
+        "3. In the **Service account name** field, enter a name, and\n",
+        "   click **Create**.\n",
         "\n",
-        "In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex\" into the filter box, and select **Vertex Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
+        "into the filter box, and select\n",
+        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
         "\n",
-        "Click Create. A JSON file that contains your key downloads to your local environment.\n",
+        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
+        "local environment.\n",
         "\n",
-        "Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
+        "6. Enter the path to your service account key as the\n",
+        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
       ]
     },
     {
@@ -367,8 +442,11 @@
         "import os\n",
         "import sys\n",
         "\n",
-        "# If on Google Cloud Notebook, then don't execute this code\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = \"google.colab\" in sys.modules\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
         "    if \"google.colab\" in sys.modules:\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
@@ -391,9 +469,16 @@
         "\n",
         "**The following steps are required, regardless of your notebook environment.**\n",
         "\n",
-        "When you initialize the Vertex SDK for Python, you specify a Cloud Storage staging bucket. The staging bucket is where all the data associated with your dataset and model resources are retained across sessions.\n",
         "\n",
-        "Set the name of your Cloud Storage bucket below. Bucket names must be globally unique across all Google Cloud projects, including those outside of your organization."
+        "When you submit a training job using the Cloud SDK, you upload a Python package\n",
+        "containing your training code to a Cloud Storage bucket. Vertex AI runs\n",
+        "the code from this package. In this tutorial, Vertex AI also saves the\n",
+        "trained model that results from your job in the same bucket. Using this model artifact, you can then\n",
+        "create Vertex AI model and endpoint resources in order to serve\n",
+        "online predictions.\n",
+        "\n",
+        "Set the name of your Cloud Storage bucket below. It must be unique across all\n",
+        "Cloud Storage buckets."
       ]
     },
     {
@@ -404,7 +489,8 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
+        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
     {
@@ -415,8 +501,9 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
+        "    BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
     {
@@ -436,7 +523,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_NAME"
+        "! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
       ]
     },
     {
@@ -456,7 +543,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_NAME"
+        "! gsutil ls -al $BUCKET_URI"
       ]
     },
     {
@@ -488,7 +575,7 @@
         "id": "init_aip:mbsdk"
       },
       "source": [
-        "## Initialize Vertex SDK for Python\n",
+        "## Initialize Vertex AI SDK for Python\n",
         "\n",
         "Initialize the Vertex SDK for Python for your project and corresponding bucket."
       ]
@@ -603,7 +690,7 @@
       "outputs": [],
       "source": [
         "dataset = aip.TextDataset.create(\n",
-        "    display_name=\"NCBI Biomedical\" + \"_\" + TIMESTAMP,\n",
+        "    display_name=\"NCBI Biomedical\" + \"_\" + UUID,\n",
         "    gcs_source=[IMPORT_FILE],\n",
         "    import_schema_uri=aip.schema.dataset.ioformat.text.extraction,\n",
         ")\n",
@@ -682,7 +769,7 @@
       "outputs": [],
       "source": [
         "dag = aip.AutoMLTextTrainingJob(\n",
-        "    display_name=\"biomedical_\" + TIMESTAMP, prediction_type=\"extraction\"\n",
+        "    display_name=\"biomedical_\" + UUID, prediction_type=\"extraction\"\n",
         ")\n",
         "\n",
         "print(dag)"
@@ -730,7 +817,7 @@
       "source": [
         "model = dag.run(\n",
         "    dataset=dataset,\n",
-        "    model_display_name=\"biomedical_\" + TIMESTAMP,\n",
+        "    model_display_name=\"biomedical_\" + UUID,\n",
         "    training_fraction_split=0.8,\n",
         "    validation_fraction_split=0.1,\n",
         "    test_fraction_split=0.1,\n",
@@ -801,7 +888,7 @@
       "outputs": [],
       "source": [
         "# Get model resource ID\n",
-        "models = aip.Model.list(filter=\"display_name=biomedical_\" + TIMESTAMP)\n",
+        "models = aip.Model.list(filter=\"display_name=biomedical_\" + UUID)\n",
         "\n",
         "# Get a reference to the Model Service client\n",
         "client_options = {\"api_endpoint\": f\"{REGION}-aiplatform.googleapis.com\"}\n",
@@ -925,14 +1012,14 @@
         "\n",
         "import tensorflow as tf\n",
         "\n",
-        "gcs_test_item_1 = BUCKET_NAME + \"/test1.txt\"\n",
+        "gcs_test_item_1 = BUCKET_URI + \"/test1.txt\"\n",
         "with tf.io.gfile.GFile(gcs_test_item_1, \"w\") as f:\n",
         "    f.write(test_item_1 + \"\\n\")\n",
-        "gcs_test_item_2 = BUCKET_NAME + \"/test2.txt\"\n",
+        "gcs_test_item_2 = BUCKET_URI + \"/test2.txt\"\n",
         "with tf.io.gfile.GFile(gcs_test_item_2, \"w\") as f:\n",
         "    f.write(test_item_2 + \"\\n\")\n",
         "\n",
-        "gcs_input_uri = BUCKET_NAME + \"/test.jsonl\"\n",
+        "gcs_input_uri = BUCKET_URI + \"/test.jsonl\"\n",
         "with tf.io.gfile.GFile(gcs_input_uri, \"w\") as f:\n",
         "    data = {\"content\": gcs_test_item_1, \"mime_type\": \"text/plain\"}\n",
         "    f.write(json.dumps(data) + \"\\n\")\n",
@@ -968,9 +1055,9 @@
       "outputs": [],
       "source": [
         "batch_predict_job = model.batch_predict(\n",
-        "    job_display_name=\"biomedical_\" + TIMESTAMP,\n",
+        "    job_display_name=\"biomedical_\" + UUID,\n",
         "    gcs_source=gcs_input_uri,\n",
-        "    gcs_destination_prefix=BUCKET_NAME,\n",
+        "    gcs_destination_prefix=BUCKET_URI,\n",
         "    sync=False,\n",
         ")\n",
         "\n",
@@ -1311,60 +1398,31 @@
       },
       "outputs": [],
       "source": [
-        "delete_all = True\n",
+        "# Delete the dataset using the Vertex dataset object\n",
         "\n",
-        "if delete_all:\n",
-        "    # Delete the dataset using the Vertex dataset object\n",
-        "    try:\n",
-        "        if \"dataset\" in globals():\n",
-        "            dataset.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
+        "dataset.delete()\n",
         "\n",
-        "    # Delete the model using the Vertex model object\n",
-        "    try:\n",
-        "        if \"model\" in globals():\n",
-        "            model.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
+        "# Delete the model using the Vertex model object\n",
         "\n",
-        "    # Delete the endpoint using the Vertex endpoint object\n",
-        "    try:\n",
-        "        if \"endpoint\" in globals():\n",
-        "            endpoint.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
+        "model.delete()\n",
         "\n",
-        "    # Delete the AutoML or Pipeline trainig job\n",
-        "    try:\n",
-        "        if \"dag\" in globals():\n",
-        "            dag.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
+        "# Delete the endpoint using the Vertex endpoint object\n",
         "\n",
-        "    # Delete the custom trainig job\n",
-        "    try:\n",
-        "        if \"job\" in globals():\n",
-        "            job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
+        "endpoint.delete()\n",
+        "# Delete the AutoML or Pipeline trainig job\n",
         "\n",
-        "    # Delete the batch prediction job using the Vertex batch prediction object\n",
-        "    try:\n",
-        "        if \"batch_predict_job\" in globals():\n",
-        "            batch_predict_job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
+        "dag.delete()\n",
         "\n",
-        "    # Delete the hyperparameter tuning job using the Vertex hyperparameter tuning object\n",
-        "    try:\n",
-        "        if \"hpt_job\" in globals():\n",
-        "            hpt_job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
+        "# Delete the batch prediction job using the Vertex batch prediction object\n",
         "\n",
-        "    if \"BUCKET_NAME\" in globals():\n",
-        "        ! gsutil rm -r $BUCKET_NAME"
+        "batch_predict_job.delete()\n",
+        "\n",
+        "# Delete the hyperparameter tuning job using the Vertex hyperparameter tuning object\n",
+        "\n",
+        "# Delete GCS bucket.\n",
+        "delete_bucket = False\n",
+        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "    ! gsutil -m rm -r $BUCKET_URI"
       ]
     }
   ],

notebooks/official/migration/UJ8 Vertex SDK AutoML Text Sentiment Analysis.ipynb

@@ -3,6 +3,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "bdccc50b",
       "metadata": {
         "id": "copyright"
       },
@@ -25,6 +26,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "c6c22009",
       "metadata": {
         "id": "title:migration,new"
       },
@@ -56,17 +58,19 @@
     },
     {
       "cell_type": "markdown",
+      "id": "b3558cd7",
       "metadata": {
         "id": "dataset:claritin,tst"
       },
       "source": [
         "### Dataset\n",
         "\n",
-        "The dataset used for this tutorial is the [Crowdflower Claritin-Twitter dataset](https://data.world/crowdflower/claritin-twitter) from [data.world Datasets](https://data.world). The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket."
+        "The dataset used for this tutorial is the [Crowdflower Claritin-Twitter dataset](https://data.world/crowdflower/claritin-twitter) from [data.world Datasets](https://data.world). The version of the dataset you use in this tutorial is stored in a public Cloud Storage bucket."
       ]
     },
     {
       "cell_type": "markdown",
+      "id": "9b9362da",
       "metadata": {
         "id": "costs"
       },
@@ -87,6 +91,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "05425dbe",
       "metadata": {
         "id": "setup_local"
       },
@@ -120,6 +125,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "070c64e0",
       "metadata": {
         "id": "install_aip:mbsdk"
       },
@@ -132,6 +138,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "f6b14b99",
       "metadata": {
         "id": "install_aip:mbsdk"
       },
@@ -145,42 +152,12 @@
         "else:\n",
         "    USER_FLAG = \"\"\n",
         "\n",
-        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "install_storage"
-      },
-      "source": [
-        "Install the latest GA version of *google-cloud-storage* library as well."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "install_storage"
-      },
-      "outputs": [],
-      "source": [
-        "! pip3 install -U google-cloud-storage $USER_FLAG"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "install_tensorflow"
-      },
-      "outputs": [],
-      "source": [
-        "! pip3 install --upgrade tensorflow $USER_FLAG"
+        "! pip3 install --upgrade google-cloud-aiplatform google-cloud-storage tensorflow $USER_FLAG -q"
       ]
     },
     {
       "cell_type": "markdown",
+      "id": "81f60b84",
       "metadata": {
         "id": "restart"
       },
@@ -193,6 +170,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "a95627f0",
       "metadata": {
         "id": "restart"
       },
@@ -210,6 +188,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "b7f6b038",
       "metadata": {
         "id": "before_you_begin:nogpu"
       },
@@ -230,7 +209,7 @@
         "\n",
         "3. [Enable the following APIs: Vertex AI APIs, Compute Engine APIs, and Cloud Storage.](https://console.cloud.google.com/flows/enableapi?apiid=ml.googleapis.com,compute_component,storage-component.googleapis.com)\n",
         "\n",
-        "4. If you are running this notebook locally, you will need to install the [Cloud SDK]((https://cloud.google.com/sdk)).\n",
+        "4. If you are running this notebook locally, you need to install the [Cloud SDK]((https://cloud.google.com/sdk)).\n",
         "\n",
         "5. Enter your project ID in the cell below. Then run the  cell to make sure the\n",
         "Cloud SDK uses the right project for all the commands in this notebook.\n",
@@ -241,6 +220,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "f55cca7c",
       "metadata": {
         "id": "set_project_id"
       },
@@ -252,6 +232,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "6917314c",
       "metadata": {
         "id": "autoset_project_id"
       },
@@ -267,6 +248,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "53236aa8",
       "metadata": {
         "id": "set_gcloud_project_id"
       },
@@ -277,6 +259,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "c009cc18",
       "metadata": {
         "id": "region"
       },
@@ -298,47 +281,61 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "071a11c0",
       "metadata": {
         "id": "region"
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+        "REGION = \"us-central1\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
       "cell_type": "markdown",
+      "id": "ae48374d",
       "metadata": {
         "id": "timestamp"
       },
       "source": [
-        "#### Timestamp\n",
+        "#### UUID\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "41ba0990",
       "metadata": {
         "id": "timestamp"
       },
       "outputs": [],
       "source": [
-        "from datetime import datetime\n",
+        "import random\n",
+        "import string\n",
         "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+        "\n",
+        "# Generate a uuid of a specifed length(default=8)\n",
+        "def generate_uuid(length: int = 8) -> str:\n",
+        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
+        "\n",
+        "\n",
+        "UUID = generate_uuid()"
       ]
     },
     {
       "cell_type": "markdown",
+      "id": "2128e871",
       "metadata": {
         "id": "gcp_authenticate"
       },
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Google Cloud Notebooks**, your environment is already authenticated. Skip this step.\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated.\n",
         "\n",
         "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
         "\n",
@@ -360,6 +357,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "433e860c",
       "metadata": {
         "id": "gcp_authenticate"
       },
@@ -373,8 +371,11 @@
         "import os\n",
         "import sys\n",
         "\n",
-        "# If on Google Cloud Notebook, then don't execute this code\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = \"google.colab\" in sys.modules\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
         "    if \"google.colab\" in sys.modules:\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
@@ -389,6 +390,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "b57cb5f6",
       "metadata": {
         "id": "bucket:mbsdk"
       },
@@ -405,28 +407,33 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "61b082b1",
       "metadata": {
         "id": "bucket"
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
+        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "ff81b3cc",
       "metadata": {
         "id": "autoset_bucket"
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
+        "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
       ]
     },
     {
       "cell_type": "markdown",
+      "id": "f8c009cd",
       "metadata": {
         "id": "create_bucket"
       },
@@ -437,16 +444,18 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "2f881cb5",
       "metadata": {
         "id": "create_bucket"
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_NAME"
+        "! gsutil mb -l $REGION $BUCKET_URI"
       ]
     },
     {
       "cell_type": "markdown",
+      "id": "d746d0f0",
       "metadata": {
         "id": "validate_bucket"
       },
@@ -457,16 +466,18 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "8c435668",
       "metadata": {
         "id": "validate_bucket"
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_NAME"
+        "! gsutil ls -al $BUCKET_URI"
       ]
     },
     {
       "cell_type": "markdown",
+      "id": "f578b01b",
       "metadata": {
         "id": "setup_vars"
       },
@@ -480,6 +491,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "f41ecf1e",
       "metadata": {
         "id": "import_aip:mbsdk"
       },
@@ -490,28 +502,31 @@
     },
     {
       "cell_type": "markdown",
+      "id": "292245fd",
       "metadata": {
         "id": "init_aip:mbsdk"
       },
       "source": [
-        "## Initialize Vertex SDK for Python\n",
+        "## Initialize Vertex AI SDK for Python\n",
         "\n",
-        "Initialize the Vertex SDK for Python for your project and corresponding bucket."
+        "Initialize the Vertex AI SDK for Python for your project and corresponding bucket."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "56dc88d8",
       "metadata": {
         "id": "init_aip:mbsdk"
       },
       "outputs": [],
       "source": [
-        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_NAME)"
+        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
       ]
     },
     {
       "cell_type": "markdown",
+      "id": "87e20f86",
       "metadata": {
         "id": "import_file:u_dataset,csv"
       },
@@ -524,6 +539,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "fffdec5d",
       "metadata": {
         "id": "import_file:claritin,csv,tst"
       },
@@ -535,6 +551,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "9c8d950b",
       "metadata": {
         "id": "quick_peek:csv"
       },
@@ -549,6 +566,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "da6d0980",
       "metadata": {
         "id": "quick_peek:csv"
       },
@@ -568,6 +586,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "00e9f81e",
       "metadata": {
         "id": "create_a_dataset:migration"
       },
@@ -577,6 +596,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "14b72768",
       "metadata": {
         "id": "datasets_create:migration,new,mbsdk"
       },
@@ -586,6 +606,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "00d777bd",
       "metadata": {
         "id": "create_dataset:text,tst"
       },
@@ -604,13 +625,14 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "ca8a6f66",
       "metadata": {
         "id": "create_dataset:text,tst"
       },
       "outputs": [],
       "source": [
         "dataset = aip.TextDataset.create(\n",
-        "    display_name=\"Crowdflower Claritin-Twitter\" + \"_\" + TIMESTAMP,\n",
+        "    display_name=\"Crowdflower Claritin-Twitter\" + \"_\" + UUID,\n",
         "    gcs_source=[IMPORT_FILE],\n",
         "    import_schema_uri=aip.schema.dataset.ioformat.text.sentiment,\n",
         ")\n",
@@ -620,6 +642,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "068df169",
       "metadata": {
         "id": "create_dataset:text,tst"
       },
@@ -639,6 +662,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "fb50a4ce",
       "metadata": {
         "id": "train_a_model:migration"
       },
@@ -648,6 +672,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "293160ba",
       "metadata": {
         "id": "trainingpipelines_create:migration,new,mbsdk"
       },
@@ -657,6 +682,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "84801634",
       "metadata": {
         "id": "create_automl_pipeline:text,tst"
       },
@@ -683,13 +709,14 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "69eaae0e",
       "metadata": {
         "id": "create_automl_pipeline:text,tst"
       },
       "outputs": [],
       "source": [
         "dag = aip.AutoMLTextTrainingJob(\n",
-        "    display_name=\"claritin_\" + TIMESTAMP,\n",
+        "    display_name=\"claritin_\" + UUID,\n",
         "    prediction_type=\"sentiment\",\n",
         "    sentiment_max=SENTIMENT_MAX,\n",
         ")\n",
@@ -699,6 +726,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "da9ecb4e",
       "metadata": {
         "id": "create_automl_pipeline:text,tst"
       },
@@ -710,6 +738,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "55f19997",
       "metadata": {
         "id": "run_automl_pipeline:text"
       },
@@ -726,12 +755,13 @@
         "\n",
         "The `run` method when completed returns the `Model` resource.\n",
         "\n",
-        "The execution of the training pipeline will take upto 20 minutes."
+        "The execution of the training pipeline take upto 20 minutes."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "6149074c",
       "metadata": {
         "id": "run_automl_pipeline:text"
       },
@@ -739,7 +769,7 @@
       "source": [
         "model = dag.run(\n",
         "    dataset=dataset,\n",
-        "    model_display_name=\"claritin_\" + TIMESTAMP,\n",
+        "    model_display_name=\"claritin_\" + UUID,\n",
         "    training_fraction_split=0.8,\n",
         "    validation_fraction_split=0.1,\n",
         "    test_fraction_split=0.1,\n",
@@ -748,6 +778,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "6e8fe148",
       "metadata": {
         "id": "run_automl_pipeline:text"
       },
@@ -773,6 +804,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "c25dee28",
       "metadata": {
         "id": "evaluate_the_model:migration"
       },
@@ -782,6 +814,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "903e8226",
       "metadata": {
         "id": "models_evaluations_list:migration,new"
       },
@@ -791,6 +824,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "cb2d95f3",
       "metadata": {
         "id": "evaluate_the_model:mbsdk"
       },
@@ -804,13 +838,14 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "9b1ec312",
       "metadata": {
         "id": "evaluate_the_model:mbsdk"
       },
       "outputs": [],
       "source": [
         "# Get model resource ID\n",
-        "models = aip.Model.list(filter=\"display_name=claritin_\" + TIMESTAMP)\n",
+        "models = aip.Model.list(filter=\"display_name=claritin_\" + UUID)\n",
         "\n",
         "# Get a reference to the Model Service client\n",
         "client_options = {\"api_endpoint\": f\"{REGION}-aiplatform.googleapis.com\"}\n",
@@ -825,6 +860,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "9eab460e",
       "metadata": {
         "id": "evaluate_the_model:mbsdk"
       },
@@ -865,6 +901,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "d4111c50",
       "metadata": {
         "id": "make_batch_predictions:migration"
       },
@@ -874,6 +911,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "f73fad68",
       "metadata": {
         "id": "batchpredictionjobs_create:migration,new,mbsdk"
       },
@@ -883,18 +921,20 @@
     },
     {
       "cell_type": "markdown",
+      "id": "ba77f1c7",
       "metadata": {
         "id": "get_test_items:batch_prediction"
       },
       "source": [
         "### Get test item(s)\n",
         "\n",
-        "Now do a batch prediction to your Vertex model. You will use arbitrary examples out of the dataset as a test items. Don't be concerned that the examples were likely used in training the model -- we just want to demonstrate how to make a prediction."
+        "Now do a batch prediction to your Vertex model. You use arbitrary examples out of the dataset as a test items. Don't be concerned that the examples were likely used in training the model -- we just want to demonstrate how to make a prediction."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "1c9fc91f",
       "metadata": {
         "id": "get_test_items:automl,tst,csv"
       },
@@ -916,13 +956,14 @@
     },
     {
       "cell_type": "markdown",
+      "id": "2a18c8e2",
       "metadata": {
         "id": "make_batch_file:automl,text"
       },
       "source": [
         "### Make the batch input file\n",
         "\n",
-        "Now make a batch input file, which you will store in your local Cloud Storage bucket. The batch input file can only be in JSONL format. For JSONL file, you make one dictionary entry per line for each data item (instance). The dictionary contains the key/value pairs:\n",
+        "Now make a batch input file, which you store in your local Cloud Storage bucket. The batch input file can only be in JSONL format. For JSONL file, you make one dictionary entry per line for each data item (instance). The dictionary contains the key/value pairs:\n",
         "\n",
         "- `content`: The Cloud Storage path to the file with the text item.\n",
         "- `mime_type`: The content type. In our example, it is a `text` file.\n",
@@ -935,6 +976,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "70461c41",
       "metadata": {
         "id": "make_batch_file:automl,text"
       },
@@ -944,14 +986,14 @@
         "\n",
         "import tensorflow as tf\n",
         "\n",
-        "gcs_test_item_1 = BUCKET_NAME + \"/test1.txt\"\n",
+        "gcs_test_item_1 = BUCKET_URI + \"/test1.txt\"\n",
         "with tf.io.gfile.GFile(gcs_test_item_1, \"w\") as f:\n",
         "    f.write(test_item_1 + \"\\n\")\n",
-        "gcs_test_item_2 = BUCKET_NAME + \"/test2.txt\"\n",
+        "gcs_test_item_2 = BUCKET_URI + \"/test2.txt\"\n",
         "with tf.io.gfile.GFile(gcs_test_item_2, \"w\") as f:\n",
         "    f.write(test_item_2 + \"\\n\")\n",
         "\n",
-        "gcs_input_uri = BUCKET_NAME + \"/test.jsonl\"\n",
+        "gcs_input_uri = BUCKET_URI + \"/test.jsonl\"\n",
         "with tf.io.gfile.GFile(gcs_input_uri, \"w\") as f:\n",
         "    data = {\"content\": gcs_test_item_1, \"mime_type\": \"text/plain\"}\n",
         "    f.write(json.dumps(data) + \"\\n\")\n",
@@ -964,6 +1006,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "254cbdbb",
       "metadata": {
         "id": "batch_request:mbsdk"
       },
@@ -975,21 +1018,22 @@
         "- `job_display_name`: The human readable name for the batch prediction job.\n",
         "- `gcs_source`: A list of one or more batch request input files.\n",
         "- `gcs_destination_prefix`: The Cloud Storage location for storing the batch prediction resuls.\n",
-        "- `sync`: If set to True, the call will block while waiting for the asynchronous batch job to complete."
+        "- `sync`: If set to True, the call block while waiting for the asynchronous batch job to complete."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "8f3cf8b6",
       "metadata": {
         "id": "batch_request:mbsdk"
       },
       "outputs": [],
       "source": [
         "batch_predict_job = model.batch_predict(\n",
-        "    job_display_name=\"claritin_\" + TIMESTAMP,\n",
+        "    job_display_name=\"claritin_\" + UUID,\n",
         "    gcs_source=gcs_input_uri,\n",
-        "    gcs_destination_prefix=BUCKET_NAME,\n",
+        "    gcs_destination_prefix=BUCKET_URI,\n",
         "    sync=False,\n",
         ")\n",
         "\n",
@@ -998,6 +1042,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "530dbf5b",
       "metadata": {
         "id": "batch_request:mbsdk"
       },
@@ -1017,6 +1062,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "89414481",
       "metadata": {
         "id": "batch_request_wait:mbsdk"
       },
@@ -1029,6 +1075,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "a579bd4a",
       "metadata": {
         "id": "batch_request_wait:mbsdk"
       },
@@ -1039,6 +1086,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "2cba4cc6",
       "metadata": {
         "id": "batch_request_wait:mbsdk"
       },
@@ -1073,6 +1121,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "c46e3e76",
       "metadata": {
         "id": "get_batch_prediction:mbsdk,tst"
       },
@@ -1091,6 +1140,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "d2af5ea8",
       "metadata": {
         "id": "get_batch_prediction:mbsdk,tst"
       },
@@ -1119,6 +1169,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "9fc83253",
       "metadata": {
         "id": "get_batch_prediction:mbsdk,tst"
       },
@@ -1130,6 +1181,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "19466786",
       "metadata": {
         "id": "make_online_predictions:migration"
       },
@@ -1139,6 +1191,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "e97f1e55",
       "metadata": {
         "id": "deploy_model:migration,new,mbsdk"
       },
@@ -1148,6 +1201,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "d2745f77",
       "metadata": {
         "id": "deploy_model:mbsdk,automatic"
       },
@@ -1160,6 +1214,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "6d30aa15",
       "metadata": {
         "id": "deploy_model:mbsdk,automatic"
       },
@@ -1170,6 +1225,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "c2c876d0",
       "metadata": {
         "id": "deploy_model:mbsdk,automatic"
       },
@@ -1188,6 +1244,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "9bb982a8",
       "metadata": {
         "id": "endpoints_predict:migration,new,mbsdk"
       },
@@ -1197,18 +1254,20 @@
     },
     {
       "cell_type": "markdown",
+      "id": "246945bb",
       "metadata": {
         "id": "get_test_item"
       },
       "source": [
         "### Get test item\n",
         "\n",
-        "You will use an arbitrary example out of the dataset as a test item. Don't be concerned that the example was likely used in training the model -- we just want to demonstrate how to make a prediction."
+        "You use an arbitrary example out of the dataset as a test item. Don't be concerned that the example was likely used in training the model -- we just want to demonstrate how to make a prediction."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "e21c3f76",
       "metadata": {
         "id": "get_test_item:automl,tst,csv"
       },
@@ -1225,6 +1284,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "95ffe1ea",
       "metadata": {
         "id": "predict_request:mbsdk,tst"
       },
@@ -1253,6 +1313,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "16b7ab95",
       "metadata": {
         "id": "predict_request:mbsdk,tst"
       },
@@ -1266,6 +1327,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "f4c79c7f",
       "metadata": {
         "id": "predict_request:mbsdk,tst"
       },
@@ -1277,6 +1339,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "52717fb9",
       "metadata": {
         "id": "undeploy_model:mbsdk"
       },
@@ -1289,6 +1352,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "7c164d13",
       "metadata": {
         "id": "undeploy_model:mbsdk"
       },
@@ -1299,6 +1363,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "4b844c87",
       "metadata": {
         "id": "cleanup:mbsdk"
       },
@@ -1324,6 +1389,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "2ea906d0",
       "metadata": {
         "id": "cleanup:mbsdk"
       },
@@ -1382,7 +1448,7 @@
         "        print(e)\n",
         "\n",
         "    if \"BUCKET_NAME\" in globals():\n",
-        "        ! gsutil rm -r $BUCKET_NAME"
+        "        ! gsutil rm -r $BUCKET_URI"
       ]
     }
   ],

notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb

@@ -117,64 +117,15 @@
         "to generate a cost estimate based on your projected usage."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "ze4-nDLfK4pw"
-      },
-      "source": [
-        "### Set up your local development environment\n",
-        "\n",
-        "**If you are using Colab or Vertex AI Workbench**, your environment already meets\n",
-        "all the requirements to run this notebook. You can skip this step."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "gCuSR8GkAgzl"
-      },
-      "source": [
-        "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
-        "You need the following:\n",
-        "\n",
-        "* The Google Cloud SDK\n",
-        "* Git\n",
-        "* Python 3\n",
-        "* virtualenv\n",
-        "* Jupyter notebook running in a virtual environment with Python 3\n",
-        "\n",
-        "The Google Cloud guide to [Setting up a Python development\n",
-        "environment](https://cloud.google.com/python/setup) and the [Jupyter\n",
-        "installation guide](https://jupyter.org/install) provide detailed instructions\n",
-        "for meeting these requirements. The following steps provide a condensed set of\n",
-        "instructions:\n",
-        "\n",
-        "1. [Install and initialize the Cloud SDK.](https://cloud.google.com/sdk/docs/)\n",
-        "\n",
-        "1. [Install Python 3.](https://cloud.google.com/python/setup#installing_python)\n",
-        "\n",
-        "1. [Install\n",
-        "   virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv)\n",
-        "   and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
-        "\n",
-        "1. To install Jupyter, run `pip install jupyter` on the\n",
-        "command-line in a terminal shell.\n",
-        "\n",
-        "1. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
-        "\n",
-        "1. Open this notebook in the Jupyter Notebook Dashboard."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "i7EUnXsZhAGF"
       },
       "source": [
-        "### Install additional packages\n",
+        "### Installation\n",
         "\n",
-        "Install additional package dependencies not installed in your notebook environment."
+        "Install the packages required for executing this notebook."
       ]
     },
     {
@@ -185,23 +136,9 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
-        "\n",
-        "! pip3 install -U tensorflow $USER_FLAG\n",
-        "! python3 -m pip3 install {USER_FLAG} google-cloud-aiplatform --upgrade\n",
-        "! pip3 install scikit-learn {USER_FLAG}"
+        "! pip3 install --upgrade tensorflow \\\n",
+        "                         google-cloud-aiplatform \\\n",
+        "                         scikit-learn -q"
       ]
     },
     {
@@ -210,9 +147,7 @@
         "id": "hhq5zEbGg0XX"
       },
       "source": [
-        "### Restart the kernel\n",
-        "\n",
-        "After you install the additional packages, you need to restart the notebook kernel so it can find the packages."
+        "### Colab only: Uncomment the following cell to restart the kernel."
       ]
     },
     {
@@ -223,15 +158,11 @@
       },
       "outputs": [],
       "source": [
-        "# Automatically restart kernel after installs\n",
-        "import os\n",
+        "# Automatically restart kernel after installs so that your environment can access the new packages\n",
+        "# import IPython\n",
         "\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    # Automatically restart kernel after installs\n",
-        "    import IPython\n",
-        "\n",
-        "    app = IPython.Application.instance()\n",
-        "    app.kernel.do_shutdown(True)"
+        "# app = IPython.Application.instance()\n",
+        "# app.kernel.do_shutdown(True)"
       ]
     },
     {
@@ -240,46 +171,21 @@
         "id": "lWEdiXsJg0XY"
       },
       "source": [
-        "## Before you begin\n",
-        "\n",
-        "### Select a GPU runtime\n",
-        "\n",
-        "**Make sure you're running this notebook in a GPU runtime if you have that option. In Colab, select \"Runtime --> Change runtime type > GPU\"**"
+        "## Before you begin"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "BF1j6f9HApxa"
-      },
-      "source": [
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI API and Compute Engine API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component).\n",
-        "\n",
-        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "WReHDGG5g0XY"
+        "id": "8bc8a29f9001"
       },
       "source": [
         "#### Set your project ID\n",
         "\n",
-        "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
+        "**If you don't know your project ID**, try the following:\n",
+        "* Run `gcloud config list`.\n",
+        "* Run `gcloud projects list`.\n",
+        "* See the support page: [Locate the project ID](https://support.google.com/googleapi/answer/7014113)"
       ]
     },
     {
@@ -290,33 +196,10 @@
       },
       "outputs": [],
       "source": [
-        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "oM1iC_MfAts1"
-      },
-      "outputs": [],
-      "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
-        "    # Get your GCP project id from gcloud\n",
-        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
-        "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID:\", PROJECT_ID)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "TL9QIaVd9hvm"
-      },
-      "outputs": [],
-      "source": [
-        "!gcloud config set project $PROJECT_ID"
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# Set the project id\n",
+        "! gcloud config set project {PROJECT_ID}"
       ]
     },
     {
@@ -327,16 +210,7 @@
       "source": [
         "#### Region\n",
         "\n",
-        "You can also change the `REGION` variable, which is used for operations\n",
-        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
-        "\n",
-        "- Americas: `us-central1`\n",
-        "- Europe: `europe-west4`\n",
-        "- Asia Pacific: `asia-east1`\n",
-        "\n",
-        "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
-        "\n",
-        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
+        "You can also change the `REGION` variable used by Vertex AI. Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
       ]
     },
     {
@@ -347,41 +221,7 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "06571eb4063b"
-      },
-      "source": [
-        "#### UUID\n",
-        "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "697568e92bd6"
-      },
-      "outputs": [],
-      "source": [
-        "import random\n",
-        "import string\n",
-        "\n",
-        "\n",
-        "# Generate a uuid of length 8\n",
-        "def generate_uuid():\n",
-        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=8))\n",
-        "\n",
-        "\n",
-        "UUID = generate_uuid()"
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
       ]
     },
     {
@@ -392,8 +232,7 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Vertex AI Workbench**, your environment is already\n",
-        "authenticated. Skip this step."
+        "Depending on your Jupyter environment, you may have to manually authenticate. Follow the relevant instructions below."
       ]
     },
     {
@@ -402,28 +241,37 @@
         "id": "sBCra4QMA2wR"
       },
       "source": [
-        "**If you are using Colab**, run the cell below and follow the instructions\n",
-        "when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "1. In the Cloud Console, go to the [**Create service account key**\n",
-        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
-        "\n",
-        "2. Click **Create service account**.\n",
-        "\n",
-        "3. In the **Service account name** field, enter a name, and\n",
-        "   click **Create**.\n",
-        "\n",
-        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
-        "into the filter box, and select\n",
-        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
-        "local environment.\n",
-        "\n",
-        "6. Enter the path to your service account key as the\n",
-        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
+        "**1. Vertex AI Workbench**\n",
+        "* Do nothing as you are already authenticated."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ad1138a125ea"
+      },
+      "source": [
+        "**2. Local JupyterLab instance, uncomment and run:**"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ce6043da7b33"
+      },
+      "outputs": [],
+      "source": [
+        "# ! gcloud auth login"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0367eac06a10"
+      },
+      "source": [
+        "**3. Colab, uncomment and run:**"
       ]
     },
     {
@@ -434,26 +282,18 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "# If on Google Cloud Notebooks, then don't execute this code\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+        "# from google.colab import auth\n",
+        "# auth.authenticate_user()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "c13224697bfb"
+      },
+      "source": [
+        "**4. Service account or other**\n",
+        "* See how to grant Cloud Storage permissions to your service account at https://cloud.google.com/storage/docs/gsutil/commands/iam#ch-examples."
       ]
     },
     {
@@ -464,18 +304,7 @@
       "source": [
         "### Create a Cloud Storage bucket\n",
         "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "\n",
-        "When you submit a training job using the Vertex AI SDK, you upload a Python package\n",
-        "containing your training code to a Cloud Storage bucket. Vertex AI runs\n",
-        "the code from this package. In this tutorial, Vertex AI also saves the\n",
-        "trained model that results from your job in the same bucket. Using this model artifact, you can then\n",
-        "create Vertex AI model and endpoint resources in order to serve\n",
-        "online predictions.\n",
-        "\n",
-        "Set the name of your Cloud Storage bucket below. It must be unique across all\n",
-        "Cloud Storage buckets."
+        "Create a storage bucket to store intermediate artifacts such as datasets."
       ]
     },
     {
@@ -486,21 +315,7 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "cf221059d072"
-      },
-      "outputs": [],
-      "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
-        "    BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+        "BUCKET_URI = \"gs://your-bucket-name-unique\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -520,27 +335,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_URI"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "ucvCsknMCims"
-      },
-      "source": [
-        "Finally, validate access to your Cloud Storage bucket by examining its contents:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "vhOb7YnwClBb"
-      },
-      "outputs": [],
-      "source": [
-        "! gsutil ls -al $BUCKET_URI"
+        "! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
       ]
     },
     {
@@ -569,6 +364,8 @@
       },
       "outputs": [],
       "source": [
+        "import os\n",
+        "\n",
         "import pandas as pd\n",
         "from google.cloud import aiplatform\n",
         "from sklearn.metrics import mean_absolute_error, mean_squared_error\n",
@@ -601,28 +398,7 @@
       },
       "outputs": [],
       "source": [
-        "EXPERIMENT_NAME = \"\"  # @param {type:\"string\"}"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "jWQLXXNVN4Lv"
-      },
-      "source": [
-        "If EXEPERIMENT_NAME is not set, set a default one below:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "Q1QInYWOKsmo"
-      },
-      "outputs": [],
-      "source": [
-        "if EXPERIMENT_NAME == \"\" or EXPERIMENT_NAME is None:\n",
-        "    EXPERIMENT_NAME = \"my-experiment-\" + UUID"
+        "EXPERIMENT_NAME = \"my-experiment-unique\""
       ]
     },
     {
@@ -812,7 +588,9 @@
       },
       "outputs": [],
       "source": [
-        "aiplatform.start_run(\"custom-training-run-1\")  # Change this to your desired run name\n",
+        "aiplatform.start_run(\n",
+        "    \"custom-training-run-unique\"\n",
+        ")  # Change this to your desired run name\n",
         "parameters = {\"epochs\": 10, \"num_units\": 64}\n",
         "aiplatform.log_params(parameters)\n",
         "\n",
@@ -1049,6 +827,12 @@
         "# Delete dataset\n",
         "ds.delete()\n",
         "\n",
+        "# Delete experiment\n",
+        "experiment = aiplatform.Experiment(\n",
+        "    experiment_name=EXPERIMENT_NAME, project=PROJECT_ID, location=REGION\n",
+        ")\n",
+        "experiment.delete()\n",
+        "\n",
         "# Delete the training job\n",
         "job.delete()\n",
         "\n",

notebooks/official/model_evaluation/automl_tabular_regression_model_evaluation.ipynb

@@ -29,7 +29,7 @@
         "id": "JAPoU8Sm5E6e"
       },
       "source": [
-        "# Vertex AI Pipelines: Evaluating BatchPrediction results from AutoML Tabular regression model\n",
+        "# Vertex AI Pipelines: Evaluating batch prediction results from AutoML Tabular regression model\n",
         "\n",
         "<table align=\"left\">\n",
         "\n",
@@ -76,12 +76,11 @@
         "\n",
         "This tutorial uses the following Google Cloud ML services and resources:\n",
         "\n",
-        "- Vertex AI `AutoML`\n",
-        "- Vertex AI `TabularDataset` (AutoML)\n",
-        "- Vertex AI `AutoMLTabularTrainingJob`\n",
-        "- Vertex AI `BatchPrediction`\n",
-        "- Vertex AI `Pipeline`\n",
-        "- Vertex AI `Model Registry`\n",
+        "- Vertex AI Datasets (Tabular)\n",
+        "- Vertex AI Training (AutoML Tabular Training)\n",
+        "- Vertex AI Batch predictions\n",
+        "- Vertex AI Pipelines\n",
+        "- Vertex AI Model Registry\n",
         "\n",
         "\n",
         "The steps performed include:\n",
@@ -90,9 +89,9 @@
         "- Configure a `AutoMLTabularTrainingJob`\n",
         "- Run the `AutoMLTabularTrainingJob` which returns a model\n",
         "- Import a pre-trained `AutoML model resource` into the pipeline\n",
-        "- Run a `batch prediction` job\n",
+        "- Run a `batch prediction` job in the pipeline\n",
         "- Evaulate the AutoML model using the `regression evaluation component`\n",
-        "- Import the Classification Metrics to the AutoML model resource"
+        "- Import the Regression Metrics to the AutoML model resource"
       ]
     },
     {
@@ -210,16 +209,15 @@
         "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
         "    \"/opt/deeplearning/metadata/env_version\"\n",
         ")\n",
-        "\n",
         "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
         "USER_FLAG = \"\"\n",
         "if IS_WORKBENCH_NOTEBOOK:\n",
         "    USER_FLAG = \"--user\"\n",
         "\n",
-        "! pip3 install --upgrade google-cloud-aiplatform {USER_FLAG} -q\n",
-        "! pip3 install google-cloud-pipeline-components==1.0.20 {USER_FLAG} -q\n",
-        "! pip3 install --upgrade kfp {USER_FLAG} -q\n",
-        "! pip3 install --upgrade matplotlib {USER_FLAG} -q"
+        "! pip3 install --upgrade --quiet {USER_FLAG} google-cloud-aiplatform \\\n",
+        "                                         google-cloud-pipeline-components==1.0.26 \\\n",
+        "                                         kfp \\\n",
+        "                                         matplotlib "
       ]
     },
     {
@@ -708,7 +706,7 @@
         "\n",
         "Train a simple regression model using the created dataset using `Age` as the target column. \n",
         "\n",
-        "##### Set a display name and create the `AutoMLTabularTrainingJob` with appropriate data types specified for column transformations."
+        "**Set a display name and create the `AutoMLTabularTrainingJob` with appropriate data types specified for column transformations.**"
       ]
     },
     {
@@ -739,32 +737,25 @@
         "    TRAINING_JOB_DISPLAY_NAME = \"train-pet-agefinder-automl_\" + UUID"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "1009f66873f5"
-      },
-      "source": [
-        "#### Define AutoMLTabularTrainingJob"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "6cb41277f4f3"
       },
       "source": [
+        "### Define AutoML Tabular training job\n",
+        "\n",
         "An AutoML training job is created with the `AutoMLTabularTrainingJob` class, with the following parameters:\n",
         "\n",
         "- `display_name`: The human readable name for the `TrainingJob` resource.\n",
-        "- `optimization_prediction_type`: The type of prediction the AutoML Model is to produce. Ex: regression,classification\n",
-        "- `column_transformations`: Transformations to apply to the input columns (i.e. columns other than the targetColumn). Each transformation may produce multiple result values from the column's value, and all are used for training. When creating transformation for BigQuery Struct column, the column should be flattened using \".\" as the delimiter. Only columns with no child should have a transformation. If an input column has no transformations on it, such a column is ignored by the training, except for the targetColumn, which should have no transformations defined on. Only one of column_transformations or column_specs should be passed. Consider using column_specs as column_transformations will be deprecated eventually. If none of column_transformations or column_specs is passed, the local credentials being used will try setting column_transformations to \"auto\". To do this, the local credentials require read access to the Cloud Storage or BigQuery training data source.\n",
+        "- `optimization_prediction_type`: The type of prediction the AutoML Model is to produce. Ex: regression, classification.\n",
+        "- `column_transformations`: Transformations to apply to the input columns (i.e. columns other than the targetColumn). Each transformation may produce multiple result values from the column's value, and all are used for training. \n",
         "- `optimization_objective`: The optimization objective to minimize or maximize.\n",
         "    - `minimize-rmse`\n",
         "    - `minimize-mae`\n",
         "    - `minimize-rmsle`\n",
         "\n",
-        "To learn more about `AutoMLTabularTrainingJob` click [here](https://cloud.google.com/python/docs/reference/aiplatform/latest/google.cloud.aiplatform.AutoMLTabularTrainingJob)    "
+        "Learn about [AutoMLTabularTrainingJob](https://cloud.google.com/python/docs/reference/aiplatform/latest/google.cloud.aiplatform.AutoMLTabularTrainingJob)    "
       ]
     },
     {
@@ -805,7 +796,7 @@
         "id": "391c51c98647"
       },
       "source": [
-        "##### Set the display name for the model."
+        "#### Set the display name for the model."
       ]
     },
     {
@@ -836,21 +827,14 @@
         "    MODEL_DISPLAY_NAME = \"pet-agefinder-prediction-model_\" + UUID"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "97c335a53595"
-      },
-      "source": [
-        "#### Run the training job"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "de7e24205889"
       },
       "source": [
+        "#### Run the training job\n",
+        "\n",
         "Next, you start the training job by invoking the method `run`, with the following parameters:\n",
         "\n",
         "- `dataset`: The `Dataset` resource to train the model.\n",
@@ -859,10 +843,9 @@
         "- `validation_fraction_split`: The percentage of the dataset to use for validation.\n",
         "- `test_fraction_split`: The percentage of the dataset to use for test (holdout data).\n",
         "- `model_display_name`: The human readable name for the trained model.\n",
-        "- `disable_early_stopping`: If true, the entire budget is used.\n",
-        "- `budget_milli_node_hours`: (optional) Maximum training time specified in unit of millihours (1000 = hour).\n",
+        "- `budget_milli_node_hours`: The train budget of creating this Model, expressed in milli node hours i.e. 1,000 value in this field means 1 node hour. \n",
         "\n",
-        "The training job takes roughly 1.5-2 hours to finish."
+        "The training job takes roughly 3 hours to finish."
       ]
     },
     {
@@ -881,7 +864,6 @@
         "    validation_fraction_split=0.1,\n",
         "    test_fraction_split=0.1,\n",
         "    model_display_name=MODEL_DISPLAY_NAME,\n",
-        "    disable_early_stopping=False,\n",
         "    budget_milli_node_hours=1000,\n",
         ")"
       ]
@@ -932,14 +914,44 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "581a188f0453"
+        "id": "2241f3739e03"
       },
       "source": [
         "## Create Pipeline for evaluations\n",
         "\n",
-        "Now, you run a Vertex AI BatchPrediction job and generate evaluations and feature-attributions on its results. \n",
+        "Now, you run a Vertex AI BatchPrediction job and generate evaluations and feature-attributions on its results by creating a Vertex AI pipeline using the components available from the [google-cloud-pipeline-components](https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-1.0.17/index.html) python package.  \n",
         "\n",
-        "To do so, you create a Vertex AI pipeline using the components available from the [`google-cloud-pipeline-components`](https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-1.0.17/index.html) python package.\n"
+        "**Set a display name for your pipeline.**"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "d17a0268020f"
+      },
+      "outputs": [],
+      "source": [
+        "PIPELINE_DISPLAY_NAME = \"[your-pipeline-display-name]\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9e41d48d1f48"
+      },
+      "outputs": [],
+      "source": [
+        "# If no display name is set, use the default one\n",
+        "if (\n",
+        "    PIPELINE_DISPLAY_NAME == \"[your-pipeline-display-name]\"\n",
+        "    or PIPELINE_DISPLAY_NAME == \"\"\n",
+        "    or PIPELINE_DISPLAY_NAME is None\n",
+        "):\n",
+        "    PIPELINE_DISPLAY_NAME = (\n",
+        "        f\"vertex-evaluation-automl-tabular-regression-feature-attribution-{UUID}\"\n",
+        "    )"
       ]
     },
     {
@@ -954,21 +966,14 @@
         "\n",
         "The pipeline uses the following components:\n",
         "\n",
-        "- `GetVertexModelOp`: Gets a Vertex Model Artifact. For more details, please check [here](https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-1.0.17/google_cloud_pipeline_components.experimental.evaluation.html#google_cloud_pipeline_components.experimental.evaluation.GetVertexModelOp).\n",
-        "- `EvaluationDataSamplerOp`: Randomly downsamples an input dataset to a specified size for computing Vertex XAI feature attributions for AutoML Tables and custom models. Creates a Dataflow job with Apache Beam to downsample the dataset. For more details, please check [here](https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-1.0.17/google_cloud_pipeline_components.experimental.evaluation.html#google_cloud_pipeline_components.experimental.evaluation.EvaluationDataSamplerOp).\n",
-        "- `ModelBatchPredictOp`: Creates a Google Cloud Vertex BatchPredictionJob and waits for it to complete. For more details, please check [here](https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-1.0.17/google_cloud_pipeline_components.aiplatform.html#google_cloud_pipeline_components.aiplatform.ModelBatchPredictOp).\n",
-        "- `ModelEvaluationRegressionOp`: Compute evaluation metrics on a trained model’s batch prediction results. Creates a Dataflow job with Apache Beam and TFMA to compute evaluation metrics. Supports regression for tabular data.[here](https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-1.0.17/google_cloud_pipeline_components.experimental.evaluation.html#google_cloud_pipeline_components.experimental.evaluation.ModelEvaluationRegressionOp).\n",
-        "- `ModelEvaluationFeatureAttributionOp`: Compute feature attribution on a trained model’s batch explanation results. Creates a Dataflow job with Apache Beam and TFMA to compute feature attributions. For more details, please check [here](https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-1.0.17/google_cloud_pipeline_components.experimental.evaluation.html#google_cloud_pipeline_components.experimental.evaluation.ModelEvaluationFeatureAttributionOp).\n",
-        "- `ModelImportEvaluationOp`: Imports a model evaluation artifact to an existing Vertex model with ModelService.ImportModelEvaluation. For more details, please check [here](https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-1.0.17/google_cloud_pipeline_components.experimental.evaluation.html#google_cloud_pipeline_components.experimental.evaluation.ModelImportEvaluationOp)."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "ce6beLsXASnK"
-      },
-      "source": [
-        "## Model Evaluation"
+        "- `GetVertexModelOp`: Gets a Vertex AI Model Artifact. \n",
+        "- `EvaluationDataSamplerOp`: Randomly downsamples an input dataset to a specified size for computing Vertex XAI feature attributions for AutoML Tables and custom models. Creates a Dataflow job with Apache Beam to downsample the dataset.\n",
+        "- `ModelBatchPredictOp`: Creates a Google Cloud Vertex BatchPredictionJob and waits for it to complete.\n",
+        "- `ModelEvaluationRegressionOp`: Compute evaluation metrics on a trained model’s batch prediction results. Creates a Dataflow job with Apache Beam and TFMA to compute evaluation metrics. Supports regression for tabular data. \n",
+        "- `ModelEvaluationFeatureAttributionOp`: Compute feature attribution on a trained model’s batch explanation results. Creates a Dataflow job with Apache Beam and TFMA to compute feature attributions.\n",
+        "- `ModelImportEvaluationOp`: Imports a model evaluation artifact to an existing Vertex AI Model with ModelService.ImportModelEvaluation.\n",
+        "\n",
+        "Learn more about [Google Cloud Pipeline Evaluation Components](https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-1.0.26/google_cloud_pipeline_components.experimental.evaluation.html)"
       ]
     },
     {
@@ -979,9 +984,7 @@
       },
       "outputs": [],
       "source": [
-        "@kfp.dsl.pipeline(\n",
-        "    name=\"vertex-evaluation-automl-tabular-regression-feature-attribution\"\n",
-        ")\n",
+        "@kfp.dsl.pipeline(name=PIPELINE_DISPLAY_NAME)\n",
         "def evaluation_automl_tabular_feature_attribution_pipeline(\n",
         "    project: str,\n",
         "    location: str,\n",
@@ -990,14 +993,9 @@
         "    target_column_name: str,\n",
         "    batch_predict_gcs_source_uris: list,\n",
         "    batch_predict_instances_format: str,\n",
+        "    batch_predict_sample_size: int,\n",
         "    batch_predict_predictions_format: str = \"jsonl\",\n",
         "    batch_predict_machine_type: str = \"n1-standard-4\",\n",
-        "    batch_predict_explanation_metadata: dict = {},\n",
-        "    batch_predict_explanation_parameters: dict = {},\n",
-        "    batch_predict_explanation_data_sample_size: int = 10000,\n",
-        "    dataflow_max_num_workers: int = 5,\n",
-        "    dataflow_use_public_ips: bool = True,\n",
-        "    encryption_spec_key_name: str = \"\",\n",
         "):\n",
         "\n",
         "    from google_cloud_pipeline_components.aiplatform import ModelBatchPredictOp\n",
@@ -1016,7 +1014,7 @@
         "        root_dir=root_dir,\n",
         "        gcs_source_uris=batch_predict_gcs_source_uris,\n",
         "        instances_format=batch_predict_instances_format,\n",
-        "        sample_size=batch_predict_explanation_data_sample_size,\n",
+        "        sample_size=batch_predict_sample_size,\n",
         "    )\n",
         "\n",
         "    # Run Batch Explanations\n",
@@ -1030,25 +1028,20 @@
         "        predictions_format=batch_predict_predictions_format,\n",
         "        gcs_destination_output_uri_prefix=root_dir,\n",
         "        machine_type=batch_predict_machine_type,\n",
-        "        encryption_spec_key_name=encryption_spec_key_name,\n",
         "        # Set the explanation parameters\n",
         "        generate_explanation=True,\n",
-        "        explanation_parameters=batch_predict_explanation_parameters,\n",
-        "        explanation_metadata=batch_predict_explanation_metadata,\n",
         "    )\n",
         "\n",
         "    # Run evaluation based on prediction type and feature attribution component.\n",
-        "    # After, import the model evaluations to the Vertex model.\n",
+        "    # After, import the model evaluations to the Vertex AI model.\n",
         "    eval_task = ModelEvaluationRegressionOp(\n",
         "        project=project,\n",
         "        location=location,\n",
         "        root_dir=root_dir,\n",
-        "        ground_truth_column=target_column_name,\n",
+        "        target_field_name=target_column_name,\n",
         "        predictions_gcs_source=batch_explain_task.outputs[\"gcs_output_directory\"],\n",
         "        predictions_format=batch_predict_predictions_format,\n",
-        "        dataflow_max_workers_num=dataflow_max_num_workers,\n",
-        "        dataflow_use_public_ips=dataflow_use_public_ips,\n",
-        "        encryption_spec_key_name=encryption_spec_key_name,\n",
+        "        prediction_score_column=\"prediction.value\",\n",
         "    )\n",
         "\n",
         "    # Get Feature Attributions\n",
@@ -1058,9 +1051,6 @@
         "        root_dir=root_dir,\n",
         "        predictions_format=\"jsonl\",\n",
         "        predictions_gcs_source=batch_explain_task.outputs[\"gcs_output_directory\"],\n",
-        "        dataflow_max_workers_num=dataflow_max_num_workers,\n",
-        "        dataflow_use_public_ips=dataflow_use_public_ips,\n",
-        "        encryption_spec_key_name=encryption_spec_key_name,\n",
         "    )\n",
         "\n",
         "    ModelImportEvaluationOp(\n",
@@ -1104,37 +1094,7 @@
       "source": [
         "### Define the parameters to run the pipeline\n",
         "\n",
-        "Specify the required parameters to run the pipeline.\n",
-        "\n",
-        "Set a display name for your pipeline."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "8f17c5c7b3e3"
-      },
-      "outputs": [],
-      "source": [
-        "PIPELINE_DISPLAY_NAME = \"[your-pipeline-display-name]\"  # @param {type:\"string\"}"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "1aa7d7bbb1c9"
-      },
-      "outputs": [],
-      "source": [
-        "# If no display name is set, use the default one\n",
-        "if (\n",
-        "    PIPELINE_DISPLAY_NAME == \"[your-pipeline-display-name]\"\n",
-        "    or PIPELINE_DISPLAY_NAME == \"\"\n",
-        "    or PIPELINE_DISPLAY_NAME is None\n",
-        "):\n",
-        "    PIPELINE_DISPLAY_NAME = \"pet_agefinder_\" + UUID"
+        "Specify the required parameters to run the pipeline.\n"
       ]
     },
     {
@@ -1143,7 +1103,7 @@
         "id": "90f424d5dca0"
       },
       "source": [
-        "To pass the required arguments to the pipeline, you define the following paramters below:\n",
+        "To pass the required arguments to the pipeline, you define the following parameters below:\n",
         "\n",
         "- `project`: Project ID.\n",
         "- `location`: Region where the pipeline is run.\n",
@@ -1152,7 +1112,7 @@
         "- `target_column_name`: Name of the column to be used as the target for regression.\n",
         "- `batch_predict_gcs_source_uris`: List of the Cloud Storage bucket uris of input instances for batch prediction.\n",
         "- `batch_predict_instances_format`:  Format of the input instances for batch prediction. Can be \"jsonl\", \"csv\" or \"bigquery\".\n",
-        "- `batch_predict_explanation_data_sample_size`: Size of the samples to be considered for batch prediction and evaluation."
+        "- `batch_predict_sample_size`: Size of the samples to be considered for batch prediction and evaluation."
       ]
     },
     {
@@ -1164,6 +1124,7 @@
       "outputs": [],
       "source": [
         "PIPELINE_ROOT = f\"{BUCKET_URI}/pipeline_root/pet_agefinder_{UUID}\"\n",
+        "batch_predict_sample_size = 10000\n",
         "parameters = {\n",
         "    \"project\": PROJECT_ID,\n",
         "    \"location\": REGION,\n",
@@ -1172,7 +1133,7 @@
         "    \"target_column_name\": \"Age\",\n",
         "    \"batch_predict_gcs_source_uris\": [DATA_SOURCE],\n",
         "    \"batch_predict_instances_format\": \"csv\",\n",
-        "    \"batch_predict_explanation_data_sample_size\": 3000,\n",
+        "    \"batch_predict_sample_size\": batch_predict_sample_size,\n",
         "}"
       ]
     },
@@ -1185,7 +1146,7 @@
         "Next, you create the pipeline job, with the following parameters:\n",
         "\n",
         "- `display_name`: The user-defined name of this Pipeline.\n",
-        "- `template_path`: The path of PipelineJob or PipelineSpec JSON or YAML file. It can be a local path, a Google Cloud  Storage URI (e.g. \"gs://project.name\"), or an Artifact Registry URI (e.g. \"https://us-central1-kfp.pkg.dev/proj/repo/pack/latest\").\n",
+        "- `template_path`: The path of PipelineJob or PipelineSpec JSON or YAML file. It can be a local path, a Google Cloud  Storage URI, or an Artifact Registry URI.\n",
         "- `parameter_values`: The mapping from runtime parameter names to its values that control the pipeline run.\n",
         "- `enable_caching`: Whether to turn on caching for the run. If this is not set, defaults to the compile time settings, which are True for all tasks by default, while users may specify different caching options for individual tasks. If this is set, the setting applies to all tasks in the pipeline. Overrides the compile time settings.\n"
       ]
@@ -1196,7 +1157,9 @@
         "id": "e8dce0638349"
       },
       "source": [
-        "Run the pipeline using the configured `SERVICE_ACCOUNT`."
+        "Run the pipeline using the configured `SERVICE_ACCOUNT`.\n",
+        "\n",
+        "**The pipeline takes about 2 hours to complete.**\n"
       ]
     },
     {
@@ -1220,29 +1183,12 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "625960707c60"
-      },
-      "source": [
-        "## Model Evaluation"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "mKRTDi8ioXBY"
+        "id": "U2zocUvk2YVs"
       },
       "source": [
         "In the results from last step, click on the generated link to see your run in the Cloud Console.\n",
         "\n",
-        "In the UI, many of the pipeline DAG nodes will expand or collapse when you click on them. Here is a partially-expanded view of the DAG (click image to see larger version).\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "U2zocUvk2YVs"
-      },
-      "source": [
+        "In the UI, many of the pipeline DAG nodes will expand or collapse when you click on them. Here is a partially-expanded view of the DAG (click image to see larger version).\n",
         "<img src=\"images/automl_tabular_regression_evaluation_pipeline.PNG\" style=\"height:622px;width:726px\"></img>"
       ]
     },
@@ -1294,6 +1240,15 @@
         "### Visualize the metrics\n"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "14MBD57k0Fng"
+      },
+      "source": [
+        "After the evalution pipeline is finished, run the below cell to visualize the evaluation metrics."
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -1328,7 +1283,7 @@
         "\n",
         "Feature attributions indicate how much each feature in your model contributed to the predictions for each given instance.\n",
         "\n",
-        "To learn more about Feature Attributions click [here](https://cloud.google.com/vertex-ai/docs/explainable-ai/overview#feature_attributions)\n",
+        "Learn more about [Feature Attributions](https://cloud.google.com/vertex-ai/docs/explainable-ai/overview#feature_attributions)\n",
         "\n",
         "Run the below cell to get the feature attributions. "
       ]

notebooks/official/model_evaluation/automl_video_classification_model_evaluation.ipynb

@@ -807,6 +807,7 @@
         "    training_fraction_split=0.8,\n",
         "    test_fraction_split=0.2,\n",
         ")\n",
+        "\n",
         "print(model)"
       ]
     },
@@ -1037,7 +1038,7 @@
         "    model_name: str,\n",
         "    target_column_name: str,\n",
         "    ground_truth_gcs_uri: list,\n",
-        "    key_columns: list,\n",
+        "    class_labels: list = \"{}\",\n",
         "    batch_predict_instances_format: str = \"jsonl\",\n",
         "    batch_predict_predictions_format: str = \"jsonl\",\n",
         "    batch_predict_machine_type: str = \"n1-standard-16\",\n",
@@ -1048,8 +1049,8 @@
         "    from google_cloud_pipeline_components.aiplatform import ModelBatchPredictOp\n",
         "    from google_cloud_pipeline_components.experimental import evaluation\n",
         "    from google_cloud_pipeline_components.experimental.evaluation import (\n",
-        "        EvaluationDataSamplerOp, EvaluationDataSplitterOp,\n",
-        "        ModelEvaluationClassificationOp, ModelImportEvaluationOp)\n",
+        "        EvaluationDataSamplerOp, ModelEvaluationClassificationOp,\n",
+        "        ModelImportEvaluationOp, TargetFieldDataRemoverOp)\n",
         "\n",
         "    get_model_task = evaluation.GetVertexModelOp(model_resource_name=model_name)\n",
         "\n",
@@ -1063,13 +1064,13 @@
         "    )\n",
         "\n",
         "    # Run Data-splitter task\n",
-        "    data_splitter_task = EvaluationDataSplitterOp(\n",
+        "    data_splitter_task = TargetFieldDataRemoverOp(\n",
         "        project=project,\n",
         "        location=location,\n",
         "        root_dir=root_dir,\n",
         "        gcs_source_uris=data_sampler_task.outputs[\"gcs_output_directory\"],\n",
         "        instances_format=batch_predict_instances_format,\n",
-        "        ground_truth_column=target_column_name,\n",
+        "        target_field_name=target_column_name,\n",
         "    )\n",
         "\n",
         "    # Run Batch Prediction.\n",
@@ -1093,9 +1094,12 @@
         "        project=project,\n",
         "        location=location,\n",
         "        root_dir=root_dir,\n",
-        "        key_columns=key_columns,\n",
         "        ground_truth_gcs_source=data_sampler_task.outputs[\"gcs_output_directory\"],\n",
-        "        ground_truth_column=target_column_name,\n",
+        "        target_field_name=target_column_name,\n",
+        "        prediction_score_column=\"prediction.confidence\",\n",
+        "        prediction_label_column=\"prediction.displayName\",\n",
+        "        class_labels=[\"brush_hair\", \"cartwheel\"],\n",
+        "        ground_truth_format=batch_predict_instances_format,\n",
         "        predictions_format=batch_predict_predictions_format,\n",
         "        predictions_gcs_source=batch_predict_task.outputs[\"gcs_output_directory\"],\n",
         "    )\n",
@@ -1172,7 +1176,6 @@
         "    \"model_name\": MODEL_RSC_NAME,\n",
         "    \"target_column_name\": LABEL_COLUMN,\n",
         "    \"ground_truth_gcs_uri\": [gcs_ground_truth_uri],\n",
-        "    \"key_columns\": [\"content\", \"mimeType\", \"timeSegmentStart\", \"timeSegmentEnd\"],\n",
         "    \"batch_predict_instances_format\": \"jsonl\",\n",
         "    \"batch_predict_sample_size\": SAMPLE_SIZE,\n",
         "}"

notebooks/official/model_evaluation/custom_tabular_regression_model_evaluation.ipynb

@@ -29,7 +29,7 @@
         "id": "title"
       },
       "source": [
-        "# Vertex AI Pipelines: Evaluating batch prediction results from Custom Tabular regression model\n",
+        "# Vertex AI Pipelines: Evaluating batch prediction results from custom tabular regression model\n",
         "\n",
         "<table align=\"left\">\n",
         "\n",
@@ -194,7 +194,7 @@
         "    \n",
         "! pip3 install --upgrade --quiet {USER_FLAG} google-cloud-aiplatform \\\n",
         "                                    tensorflow \\\n",
-        "                                    google-cloud-pipeline-components \\\n",
+        "                                    google-cloud-pipeline-components==1.0.26 \\\n",
         "                                    kfp \\\n",
         "                                    matplotlib \\\n",
         "                                    google-cloud-storage "
@@ -652,7 +652,7 @@
         "\n",
         "Set the variables `TRAIN_GPU/TRAIN_NGPU` and `DEPLOY_GPU/DEPLOY_NGPU` to use a container image supporting a GPU and the number of GPUs allocated to the virtual machine (VM) instance. For example, to use a GPU container image with 4 Nvidia Telsa K80 GPUs allocated to each VM, you would specify:\n",
         "\n",
-        "    (aip.AcceleratorType.NVIDIA_TESLA_K80, 4)\n",
+        "    (aip.gapic.AcceleratorType.NVIDIA_TESLA_K80, 4)\n",
         "\n",
         "\n",
         "Otherwise specify `(None, None)` to use a container image to run on a CPU.\n",
@@ -805,7 +805,7 @@
         "\n",
         "Now you are ready to start creating your own custom model and training for Boston Housing. \n",
         "\n",
-        "[Learn more about custom model training on Vertex AI](https://cloud.google.com/vertex-ai/docs/training/custom-training)\n",
+        "Learn more about [custom model training on Vertex AI](https://cloud.google.com/vertex-ai/docs/training/custom-training)\n",
         "\n",
         "### Examine the training package\n",
         "\n",
@@ -1247,7 +1247,7 @@
         "\n",
         "\n",
         "\n",
-        "In the next code cell, define the parameters."
+        "In the next code cell, you define the parameters."
       ]
     },
     {
@@ -1276,7 +1276,7 @@
         "id": "781989a46a3b"
       },
       "source": [
-        "In the next code cell, define the metadata"
+        "**In the next code cell, you define the metadata**"
       ]
     },
     {
@@ -1333,11 +1333,11 @@
         "\n",
         "In the next cell, you write the contents of the instance_schema.yaml . You write the structure about the prediction instances you give to your batch prediction .\n",
         "\n",
-        "- Give the title and description.\n",
-        "- Give type of the input. In our case input to batch predictin is \n",
+        "- Specify the title and description.\n",
+        "- Specify type of the input. In your case input layer to batch prediction is \n",
         "**{\"dense_input\": [0.02715405449271202, 0.0, 0.027177177369594574, 0.0, 0.0010195195209234953, 0.009660660289227962, 0.1501501500606537, 0.0027548049110919237, 0.036036036908626556, 1.0, 0.03033033013343811, 0.04091591760516167, 0.043618619441986084]}**\n",
         "which is an object. Inside object, there are properties like dense_input.\n",
-        "- Give description about property\n",
+        "- Specify description about property\n",
         "- For each property, mention its type. \n",
         "- If type of the property is an array, mention the information about array items in `items` key.\n"
       ]
@@ -1365,28 +1365,18 @@
         "    description: 'Input values to model'\n"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "ef75c6f86088"
-      },
-      "source": [
-        "#### Make prediction schema yaml file"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "53f324aaf19a"
       },
       "source": [
+        "#### Make prediction schema yaml file\n",
+        "\n",
         "In the next cell, you write the contents of the prediction_schema.yaml . You write the structure about the prediction output you get from your batch prediction job.\n",
         "\n",
-        "In your case, output from batch prediction job is \n",
         "\n",
-        "**{\"instance\": {\"dense_input\": [0.02715405449271202, 0.0, 0.027177177369594574, 0.0, 0.0010195195209234953, 0.009660660289227962, 0.1501501500606537, 0.0027548049110919237, 0.036036036908626556, 1.0, 0.03033033013343811, 0.04091591760516167, 0.043618619441986084]}, \"prediction\": [0.522156954]}**\n",
-        "\n",
-        "Prediction output of batch prediction job is **\"prediction\": [0.522156954]**, which is of type array."
+        "Output of batch prediction job is \"prediction\": [value], which is of type array."
       ]
     },
     {
@@ -1438,15 +1428,15 @@
         "- `display_name`: The human readable name for the `Model` resource.\n",
         "- `artifact`: The Cloud Storage location of the trained model artifacts.\n",
         "- `serving_container_image_uri`: The serving container image.\n",
-        "- `instance_schema_uri`: Immutable. Points to a YAML file stored on Google Cloud Storage describing the format of a single instance.\n",
-        "- `prediction_schema_uri`: Immutable. Points to a YAML file stored on Google Cloud Storage describing the format of a single prediction produced by this model.\n",
+        "- `instance_schema_uri`: Points to a YAML file stored on Google Cloud Storage describing the format of a single instance.\n",
+        "- `prediction_schema_uri`: Points to a YAML file stored on Google Cloud Storage describing the format of a single prediction produced by this model.\n",
         "- `sync`: Whether to execute the upload asynchronously or synchronously.\n",
         "- `explanation_parameters`: Parameters to configure explaining for `Model`'s predictions.\n",
         "- `explanation_metadata`: Metadata describing the `Model`'s input and output for explanation.\n",
         "\n",
         "If the `upload()` method is run asynchronously, you can subsequently block until completion with the `wait()` method.\n",
         "\n",
-        "**Note:** If you want to configure explanations for the model, set `explanation_parameters`,  `explanation_metadata` parameters. Else do not set them."
+        "**Note:** If you want to configure explanations for the model, set `explanation_parameters`,  `explanation_metadata` parameters. Otherwise do not set them."
       ]
     },
     {
@@ -1559,7 +1549,7 @@
         "\n",
         "Now you create a pipeline for performing model evaluation.\n",
         "\n",
-        "### Create Pipeline for evaluations\n",
+        "### Create pipeline for evaluations\n",
         "\n",
         "Now, you run a Vertex AI BatchPrediction job and generate evaluations and feature-attributions on its results by creating a Vertex AI pipeline using the components available from the [google-cloud-pipeline-components](https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-1.0.17/index.html) python package. \n",
         "\n",
@@ -1610,7 +1600,7 @@
         "\n",
         "- `GetVertexModelOp`: Gets a Vertex AI Model resource Artifact. \n",
         "- `EvaluationDataSamplerOp`: Randomly downsamples an input dataset to a specified size for computing Vertex Explainable AI feature attributions for AutoML Tabular and custom models. Creates a Dataflow job with Apache Beam to downsample the dataset. \n",
-        "- `EvaluationDataSplitterOp`: Removes the Ground Truth columns from the input dataset for supporting unstructured AutoML models and custom models in Batch Prediction. Creates a Dataflow job with Apache Beam to remove the ground truth columns. \n",
+        "- `TargetFieldDataRemoverOp`: Removes the target field from the input dataset for supporting unstructured AutoML models and custom models for Vertex Batch Prediction. Creates a Dataflow job with Apache Beam to remove the target field.. \n",
         "- `ModelBatchPredictOp`: Creates a Google Cloud Vertex BatchPredictionJob and waits for it to complete. \n",
         "- `ModelEvaluationRegressionOp`: Compute evaluation metrics on a trained model’s batch prediction results. Creates a Dataflow job with Apache Beam and TFMA to compute evaluation metrics. Supports regression for tabular data.\n",
         "- `ModelEvaluationFeatureAttributionOp`: Compute feature attribution on a trained model’s batch explanation results. Creates a Dataflow job with Apache Beam and TFMA to compute feature attributions. \n",
@@ -1631,17 +1621,17 @@
         "\n",
         "{\"dense_input\": [0.004922522697597742, 0.0, 0.3608507513999939], \"MEDV\": 18.8}\n",
         "\n",
-        "2.If above output is give to data splitter with ground truth column as MEDV, then output is\n",
+        "2.If above output is give to target_field_data_remover with target_field_name as MEDV, then output is\n",
         "\n",
         "{\"dense_input\": [0.004922522697597742, 0.0, 0.3608507513999939]}\n",
         "\n",
-        "3.If output from data splitter is given as input to batch prediction, example output is\n",
+        "3.If output from target_field_data_remover is given as input to batch prediction, example output is\n",
         "\n",
         "{\"instance\": {\"dense_input\": [0.002855135127902031, 0.0, 0.1618601232767105]}, \"prediction\": [20.7158852], \"explanation\": {\"attributions\": [{\"outputName\": \"medv\", \"baselineOutputValue\": 6.1286516189575195, \"instanceOutputValue\": 20.715885162353516, \"outputIndex\": [0], \"featureAttributions\": {\"crim\": [-0.02773827149629295], \"zn\": [0.0], \"indus\": [-0.6544220782498746], \"chas\": [0.0], \"nox\": [1.5975404104438946], \"rm\": [12.334328035516096], \"age\": [1.9477325958880005], \"dis\": [-1.1212691238079533], \"rad\": [0.1904019388794652], \"tax\": [-1.0420063589728579], \"ptratio\": [0.6294915264341129], \"b\": [8.135296088546342], \"lstat\": [-7.403950636448881]}, \"approximationError\": 0.00012541217340836087}]}}\n",
         "\n",
-        "4.The output of the batch prediction is given as input for the `ModelEvaluationRegressionOp` component. For a custom model, the ground truth cannot be part of the batch prediction instance, so we provide the output of the data sampler with ground truths to `ModelEvaluationRegressionOp`'s `ground_truth_gcs_source` parameter.\n",
+        "4.The output of the batch prediction is given as input for the `ModelEvaluationRegressionOp` component. For a custom model, the ground truth cannot be part of the batch prediction instance, so you provide the output of the data sampler with ground truths to `ModelEvaluationRegressionOp`'s `ground_truth_gcs_source` parameter.\n",
         "\n",
-        "5.In `ModelImportEvaluationOp`, we import evaluation metrics and feature attributions to the model.\n"
+        "5.In `ModelImportEvaluationOp`, you import evaluation metrics and feature attributions to the model.\n"
       ]
     },
     {
@@ -1660,7 +1650,6 @@
         "    model_name: str,\n",
         "    target_column_name: str,\n",
         "    batch_predict_gcs_source_uris: list,\n",
-        "    key_columns: list,\n",
         "    batch_predict_instances_format: str,\n",
         "    batch_predict_sample_size: int,\n",
         "    batch_predict_predictions_format: str = \"jsonl\",\n",
@@ -1669,9 +1658,9 @@
         "\n",
         "    from google_cloud_pipeline_components.aiplatform import ModelBatchPredictOp\n",
         "    from google_cloud_pipeline_components.experimental.evaluation import (\n",
-        "        EvaluationDataSamplerOp, EvaluationDataSplitterOp, GetVertexModelOp,\n",
+        "        EvaluationDataSamplerOp, GetVertexModelOp,\n",
         "        ModelEvaluationFeatureAttributionOp, ModelEvaluationRegressionOp,\n",
-        "        ModelImportEvaluationOp)\n",
+        "        ModelImportEvaluationOp, TargetFieldDataRemoverOp)\n",
         "\n",
         "    # Get the Vertex AI model resource\n",
         "    get_model_task = GetVertexModelOp(model_resource_name=model_name)\n",
@@ -1687,13 +1676,13 @@
         "    )\n",
         "\n",
         "    # Run Data-splitter task\n",
-        "    data_splitter_task = EvaluationDataSplitterOp(\n",
+        "    data_splitter_task = TargetFieldDataRemoverOp(\n",
         "        project=project,\n",
         "        location=location,\n",
         "        root_dir=root_dir,\n",
         "        gcs_source_uris=data_sampler_task.outputs[\"gcs_output_directory\"],\n",
         "        instances_format=batch_predict_instances_format,\n",
-        "        ground_truth_column=target_column_name,\n",
+        "        target_field_name=target_column_name,\n",
         "    )\n",
         "\n",
         "    # Run Batch Explanations\n",
@@ -1720,10 +1709,9 @@
         "        predictions_gcs_source=batch_explain_task.outputs[\"gcs_output_directory\"],\n",
         "        ground_truth_format=\"jsonl\",\n",
         "        ground_truth_gcs_source=data_sampler_task.outputs[\"gcs_output_directory\"],\n",
-        "        key_columns=key_columns,\n",
         "        predictions_format=batch_predict_predictions_format,\n",
         "        prediction_score_column=\"prediction\",\n",
-        "        ground_truth_column=target_column_name,\n",
+        "        target_field_name=target_column_name,\n",
         "    )\n",
         "\n",
         "    # Get Feature Attributions\n",
@@ -1749,9 +1737,9 @@
         "id": "RqcRr7USbseH"
       },
       "source": [
-        "##### Compile the pipeline\n",
+        "#### Compile the pipeline\n",
         "\n",
-        "Next, compile the pipline to the `tabular_regression_pipline.json` file."
+        "Compile the pipeline and save the compiled pipeline in the file tabular_regression_pipline.json"
       ]
     },
     {
@@ -1774,7 +1762,7 @@
         "id": "zwrhHGm7bseH"
       },
       "source": [
-        "##### Define the parameters to run the pipeline\n",
+        "#### Define the parameters to run the pipeline\n",
         "\n",
         "Specify the required parameters to run the pipeline.\n",
         "\n",
@@ -1783,13 +1771,12 @@
         "\n",
         "- `project`: Project ID.\n",
         "- `location`: Region where the pipeline is run.\n",
-        "- `root_dir`: The GCS directory for keeping staging files and artifacts. A random subdirectory will be created under the directory to keep job info for resuming the job in case of failure.\n",
+        "- `root_dir`: The Cloud Storage directory for keeping staging files and artifacts. A random subdirectory will be created under the directory to keep job info for resuming the job in case of failure.\n",
         "- `model_name`: Resource name of the trained Custom Tabular Regression model.\n",
         "- `target_column_name`: Name of the column to be used as the target for regression.\n",
         "- `batch_predict_gcs_source_uris`: List of the Cloud Storage bucket uris of input instances for batch prediction.\n",
         "- `batch_predict_instances_format`:  Format of the input instances for batch prediction. Can be \"jsonl\", \"csv\" or \"bigquery\".\n",
-        "- `batch_predict_explanation_data_sample_size`: Size of the samples to be considered for batch prediction and evaluation.\n",
-        "- `key_columns`: The list of fields in the ground truth gcs source to format the joining key. Used to merge prediction instances with ground truth data."
+        "- `batch_predict_explanation_data_sample_size`: Size of the samples to be considered for batch prediction and evaluation.\n"
       ]
     },
     {
@@ -1813,7 +1800,6 @@
         "    ],\n",
         "    \"batch_predict_instances_format\": \"jsonl\",\n",
         "    \"batch_predict_sample_size\": batch_predict_sample_size,\n",
-        "    \"key_columns\": [\"dense_input\"],\n",
         "}"
       ]
     },
@@ -1823,7 +1809,7 @@
         "id": "zsd1Peh0bseI"
       },
       "source": [
-        "Next, you create the pipeline job, with the following parameters:\n",
+        "**Next, you create the pipeline job, with the following parameters:**\n",
         "\n",
         "- `display_name`: The user-defined name of this Pipeline.\n",
         "- `template_path`: The path of PipelineJob or PipelineSpec JSON or YAML file. It can be a local path, a Google Cloud  Storage URI (e.g. \"gs://project.name\"), or an Artifact Registry URI (e.g. \"https://us-central1-kfp.pkg.dev/proj/repo/pack/latest\").\n",
@@ -1874,7 +1860,7 @@
         "\n",
         "<img src=\"images/custom_tabular_regression_evaluation_pipeline.PNG\" style=\"height:622px;width:726px\"></img>\n",
         "\n",
-        "### Get the model evaluation results\n",
+        "## Get the model evaluation results\n",
         "\n",
         "After the evalution pipeline is finished, run the below cell to print the evaluation metrics."
       ]

notebooks/official/model_monitoring/batch_prediction_model_monitoring.ipynb

@@ -35,17 +35,17 @@
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_monitoring/batch_prediction_model_monitoring.ipynb\">\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_monitoring/batch_prediction_model_monitoring.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Open in Colab\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_monitoring/batch_prediction_model_monitoring.ipynb\">\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_monitoring/batch_prediction_model_monitoring.ipynb\">\n",
         "        <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td><td>\n",
-        "    <a href=\"https://console.cloud.google.com/ai-platform/notebooks/deploy-notebook?name=Model%20Monitoring&download_url=https%3A%2F%2Fraw.githubusercontent.com%2FGoogleCloudPlatform%2Fvertex-ai-samples%2Fmain%2Fnotebooks%2Fcommunity%2Fmodel_monitoring%2Fbatch_prediction_model_monitoring.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/ai-platform/notebooks/deploy-notebook?name=Model%20Monitoring&download_url=https%3A%2F%2Fraw.githubusercontent.com%2FGoogleCloudPlatform%2Fvertex-ai-samples%2Fmain%2Fnotebooks%2Fofficial%2Fmodel_monitoring%2Fbatch_prediction_model_monitoring.ipynb\">\n",
         "       <img src=\"https://www.gstatic.com/cloud/images/navigation/vertex-ai.svg\" alt=\"Google Cloud Notebooks\">Open in Workbench AI Notebook\n",
         "    </a>\n",
         "  </td> \n",
@@ -173,9 +173,11 @@
         "    USER_FLAG = \"--user\"\n",
         "\n",
         "# Install Python package dependencies.\n",
-        "! pip3 install -q {USER_FLAG} tensorflow-data-validation \\\n",
-        "                              google-api-core \\\n",
-        "                              google-cloud-aiplatform"
+        "! pip3 install -q {USER_FLAG} google-cloud-aiplatform \\\n",
+        "                              tensorflow-data-validation \\\n",
+        "                              protobuf==3.20.3\n",
+        "\n",
+        "! pip3 install -q {USER_FLAG} cachetools==5.2.0"
       ]
     },
     {
@@ -883,6 +885,8 @@
       },
       "outputs": [],
       "source": [
+        "import time\n",
+        "\n",
         "# If auto-testing, wait for request completion\n",
         "if os.getenv(\"IS_TESTING\"):\n",
         "    time.sleep(1800)"

notebooks/official/pipelines/google_cloud_pipeline_components_automl_images.ipynb

@@ -104,7 +104,7 @@
       "source": [
         "### Dataset\n",
         "\n",
-        "The dataset used for this tutorial is the [Flowers dataset](https://www.tensorflow.org/datasets/catalog/tf_flowers) from [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/overview). The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket. The trained model predicts the type of flower an image is from a class of five flowers: daisy, dandelion, rose, sunflower, or tulip."
+        "The dataset used for this tutorial is the [Flowers dataset](https://www.tensorflow.org/datasets/catalog/tf_flowers) from [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/overview). The version of the dataset you use in this tutorial is stored in a public Cloud Storage bucket. The trained model predicts the type of flower an image is from a class of five flowers: daisy, dandelion, rose, sunflower, or tulip."
       ]
     },
     {
@@ -182,19 +182,16 @@
         "import os\n",
         "\n",
         "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
         "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
         "    \"/opt/deeplearning/metadata/env_version\"\n",
         ")\n",
-        "\n",
         "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
         "USER_FLAG = \"\"\n",
         "if IS_WORKBENCH_NOTEBOOK:\n",
         "    USER_FLAG = \"--user\"\n",
         "\n",
-        "! pip3 install --upgrade google-cloud-aiplatform {USER_FLAG} -q\n",
-        "! pip3 install -U google-cloud-storage $USER_FLAG -q\n",
-        "! pip3 install $USER kfp google-cloud-pipeline-components --upgrade -q"
+        "! pip3 install --upgrade --quiet {USER_FLAG} google-cloud-aiplatform kfp google-cloud-pipeline-components google-cloud-storage"
       ]
     },
     {
@@ -226,27 +223,6 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "check_versions"
-      },
-      "source": [
-        "Check the versions of the packages you installed.  The KFP SDK version should be >=1.6."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "check_versions:kfp,gcpc"
-      },
-      "outputs": [],
-      "source": [
-        "! python3 -c \"import kfp; print('KFP SDK version: {}'.format(kfp.__version__))\"\n",
-        "! python3 -c \"import google_cloud_pipeline_components; print('google_cloud_pipeline_components version: {}'.format(google_cloud_pipeline_components.__version__))\""
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -277,6 +253,17 @@
         "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$`."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5aee4379e8e5"
+      },
+      "source": [
+        "#### Set your project ID\n",
+        "\n",
+        "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -342,7 +329,10 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -351,9 +341,9 @@
         "id": "timestamp"
       },
       "source": [
-        "#### Timestamp\n",
+        "#### UUID\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial.\n"
       ]
     },
     {
@@ -364,9 +354,16 @@
       },
       "outputs": [],
       "source": [
-        "from datetime import datetime\n",
+        "import random\n",
+        "import string\n",
         "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+        "\n",
+        "# Generate a uuid of a specifed length(default=8)\n",
+        "def generate_uuid(length: int = 8) -> str:\n",
+        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
+        "\n",
+        "\n",
+        "UUID = generate_uuid()"
       ]
     },
     {
@@ -472,7 +469,7 @@
       "outputs": [],
       "source": [
         "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
         "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
       ]
     },
@@ -493,7 +490,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_URI"
+        "! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
       ]
     },
     {
@@ -594,9 +591,6 @@
         "id": "setup_vars"
       },
       "source": [
-        "### Set up variables\n",
-        "\n",
-        "Next, set up some variables used throughout the tutorial.\n",
         "### Import libraries and define constants"
       ]
     },
@@ -608,8 +602,12 @@
       },
       "outputs": [],
       "source": [
+        "import os\n",
+        "from typing import Any, Dict, List\n",
+        "\n",
         "import google.cloud.aiplatform as aip\n",
-        "import kfp"
+        "import kfp\n",
+        "from kfp.v2 import compiler"
       ]
     },
     {
@@ -736,8 +734,6 @@
       },
       "outputs": [],
       "source": [
-        "from kfp.v2 import compiler  # noqa: F811\n",
-        "\n",
         "compiler.Compiler().compile(\n",
         "    pipeline_func=pipeline,\n",
         "    package_path=\"image classification_pipeline.json\".replace(\" \", \"_\"),\n",
@@ -763,7 +759,7 @@
       },
       "outputs": [],
       "source": [
-        "DISPLAY_NAME = \"flowers_\" + TIMESTAMP\n",
+        "DISPLAY_NAME = \"flowers_\" + UUID\n",
         "\n",
         "job = aip.PipelineJob(\n",
         "    display_name=DISPLAY_NAME,\n",
@@ -780,7 +776,7 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "view_pipeline_run:automl,image"
+        "id": "b6d7ccae0e3a"
       },
       "source": [
         "Click on the generated link to see your run in the Cloud Console.\n",
@@ -800,7 +796,7 @@
         "id": "cleanup:pipelines"
       },
       "source": [
-        "# Cleaning up\n",
+        "## Cleaning up\n",
         "\n",
         "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
         "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
@@ -808,6 +804,71 @@
         "Otherwise, you can delete the individual resources you created in this tutorial."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "463df8940f59"
+      },
+      "source": [
+        "### Get resources from the pipeline to clean up\n",
+        "#### Function to get details of a task"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "877740ce8d2f"
+      },
+      "outputs": [],
+      "source": [
+        "def get_task_detail(\n",
+        "    task_details: List[Dict[str, Any]], task_name: str\n",
+        ") -> List[Dict[str, Any]]:\n",
+        "    for task_detail in task_details:\n",
+        "        if task_detail.task_name == task_name:\n",
+        "            return task_detail"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3550a83b91e6"
+      },
+      "outputs": [],
+      "source": [
+        "pipeline_task_details = (\n",
+        "    job.gca_resource.job_detail.task_details\n",
+        ")  # fetch pipeline task details\n",
+        "\n",
+        "\n",
+        "# fetch endpoint from pipeline and delete the endpoint\n",
+        "endpoint_task = get_task_detail(pipeline_task_details, \"endpoint-create\")\n",
+        "endpoint_resourceName = (\n",
+        "    endpoint_task.outputs[\"endpoint\"].artifacts[0].metadata[\"resourceName\"]\n",
+        ")\n",
+        "endpoint = aip.Endpoint(endpoint_resourceName)\n",
+        "# undeploy model from endpoint\n",
+        "endpoint.undeploy_all()\n",
+        "endpoint.delete()\n",
+        "\n",
+        "# fetch model from pipeline and delete the model\n",
+        "model_task = get_task_detail(pipeline_task_details, \"automl-image-training-job\")\n",
+        "model_resourceName = model_task.outputs[\"model\"].artifacts[0].metadata[\"resourceName\"]\n",
+        "model = aip.Model(model_resourceName)\n",
+        "model.delete()\n",
+        "\n",
+        "\n",
+        "# fetch dataset from pipeline and delete the dataset\n",
+        "dataset_task = get_task_detail(pipeline_task_details, \"image-dataset-create\")\n",
+        "dataset_resourceName = (\n",
+        "    dataset_task.outputs[\"dataset\"].artifacts[0].metadata[\"resourceName\"]\n",
+        ")\n",
+        "dataset = aip.ImageDataset(dataset_resourceName)\n",
+        "dataset.delete()"
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,

notebooks/official/pipelines/google_cloud_pipeline_components_automl_tabular.ipynb

@@ -79,7 +79,7 @@
         "\n",
         "- `Vertex AI Pipelines`\n",
         "- `Google Cloud Pipeline Components`\n",
-        "- `Vertex AutoML`\n",
+        "- `Vertex AI AutoML`\n",
         "- `Vertex AI Model` resource\n",
         "- `Vertex AI Endpoint` resource\n",
         "\n",
@@ -93,7 +93,7 @@
         "- Compile the KFP pipeline.\n",
         "- Execute the KFP pipeline using `Vertex AI Pipelines`\n",
         "\n",
-        "The components are [documented here](https://google-cloud-pipeline-components.readthedocs.io/en/latest/google_cloud_pipeline_components.aiplatform.html#module-google_cloud_pipeline_components.aiplatform)."
+        "Learn more about [Google Cloud Pipeline Components](https://google-cloud-pipeline-components.readthedocs.io/en/latest/google_cloud_pipeline_components.aiplatform.html#module-google_cloud_pipeline_components.aiplatform)."
       ]
     },
     {
@@ -194,9 +194,9 @@
         "if IS_WORKBENCH_NOTEBOOK:\n",
         "    USER_FLAG = \"--user\"\n",
         "\n",
-        "! pip3 install --upgrade google-cloud-aiplatform {USER_FLAG} -q\n",
-        "! pip3 install -U google-cloud-storage $USER_FLAG -q\n",
-        "! pip3 install $USER kfp google-cloud-pipeline-components --upgrade -q"
+        "! pip3 install --upgrade google-cloud-aiplatform \\\n",
+        "                        google-cloud-storage \\\n",
+        "                        $USER kfp google-cloud-pipeline-components -q "
       ]
     },
     {
@@ -560,6 +560,11 @@
       },
       "outputs": [],
       "source": [
+        "import os\n",
+        "import sys\n",
+        "\n",
+        "IS_COLAB = \"google.colab\" in sys.modules\n",
+        "\n",
         "if (\n",
         "    SERVICE_ACCOUNT == \"\"\n",
         "    or SERVICE_ACCOUNT is None\n",
@@ -841,10 +846,6 @@
       },
       "outputs": [],
       "source": [
-        "delete_dataset = True\n",
-        "delete_pipeline = True\n",
-        "delete_model = True\n",
-        "delete_endpoint = True\n",
         "delete_bucket = True\n",
         "\n",
         "dataset_display_name = \"housing\"\n",
@@ -853,64 +854,33 @@
         "endpoint_display_name = \"train-automl-cal_housing_endpoint\"\n",
         "\n",
         "\n",
-        "if delete_endpoint:\n",
-        "    endpoints = aip.Endpoint.list(\n",
-        "        filter=f\"display_name={endpoint_display_name}\", order_by=\"create_time\"\n",
-        "    )\n",
-        "    if endpoints:\n",
-        "        endpoint = endpoints[0]\n",
-        "        endpoint.undeploy_all()\n",
-        "        endpoint.delete()\n",
-        "        print(\"Deleted endpoint:\", endpoint)\n",
+        "endpoints = aip.Endpoint.list(\n",
+        "    filter=f\"display_name={endpoint_display_name}\", order_by=\"create_time\"\n",
+        ")\n",
+        "if endpoints:\n",
+        "    endpoint = endpoints[0]\n",
+        "    endpoint.undeploy_all()\n",
+        "    endpoint.delete()\n",
+        "    print(\"Deleted endpoint:\", endpoint)\n",
         "\n",
-        "if delete_model:\n",
-        "    models = aip.Model.list(\n",
-        "        filter=f\"display_name={model_display_name}\", order_by=\"create_time\"\n",
-        "    )\n",
-        "    if models:\n",
-        "        model = models[0]\n",
-        "        model.delete()\n",
-        "        print(\"Deleted model:\", model)\n",
+        "models = aip.Model.list(\n",
+        "    filter=f\"display_name={model_display_name}\", order_by=\"create_time\"\n",
+        ")\n",
+        "if models:\n",
+        "    model = models[0]\n",
+        "    model.delete()\n",
+        "    print(\"Deleted model:\", model)\n",
         "\n",
-        "if delete_dataset:\n",
-        "    if \"tabular\" == \"tabular\":\n",
-        "        datasets = aip.TabularDataset.list(\n",
-        "            filter=f\"display_name={dataset_display_name}\", order_by=\"create_time\"\n",
-        "        )\n",
-        "        if datasets:\n",
-        "            dataset = datasets[0]\n",
-        "            dataset.delete()\n",
-        "            print(\"Deleted dataset:\", dataset)\n",
+        "datasets = aip.TabularDataset.list(\n",
+        "    filter=f\"display_name={dataset_display_name}\", order_by=\"create_time\"\n",
+        ")\n",
+        "if datasets:\n",
+        "    dataset = datasets[0]\n",
+        "    dataset.delete()\n",
+        "    print(\"Deleted dataset:\", dataset)\n",
         "\n",
-        "    if \"tabular\" == \"image\":\n",
-        "        datasets = aip.ImageDataset.list(\n",
-        "            filter=f\"display_name={dataset_display_name}\", order_by=\"create_time\"\n",
-        "        )\n",
-        "        if datasets:\n",
-        "            dataset = datasets[0]\n",
-        "            dataset.delete()\n",
-        "            print(\"Deleted dataset:\", dataset)\n",
-        "\n",
-        "    if \"tabular\" == \"text\":\n",
-        "        datasets = aip.TextDataset.list(\n",
-        "            filter=f\"display_name={dataset_display_name}\", order_by=\"create_time\"\n",
-        "        )\n",
-        "        if datasets:\n",
-        "            dataset = datasets[0]\n",
-        "            dataset.delete()\n",
-        "            print(\"Deleted dataset:\", dataset)\n",
-        "\n",
-        "    if \"tabular\" == \"video\":\n",
-        "        datasets = aip.VideoDataset.list(\n",
-        "            filter=f\"display_name={dataset_display_name}\", order_by=\"create_time\"\n",
-        "        )\n",
-        "        if datasets:\n",
-        "            dataset = datasets[0]\n",
-        "            dataset.delete()\n",
-        "            print(\"Deleted dataset:\", dataset)\n",
-        "\n",
-        "if delete_pipeline:\n",
-        "    job.delete()\n",
+        "   \n",
+        "job.delete()\n",
         "\n",
         "\n",
         "if delete_bucket and os.getenv(\"IS_TESTING\"):\n",

notebooks/official/pipelines/google_cloud_pipeline_components_model_train_upload_deploy.ipynb

@@ -229,27 +229,6 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "check_versions"
-      },
-      "source": [
-        "Check the versions of the packages you installed.  The KFP SDK version should be >=1.6."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "check_versions:kfp,gcpc"
-      },
-      "outputs": [],
-      "source": [
-        "! python3 -c \"import kfp; print('KFP SDK version: {}'.format(kfp.__version__))\"\n",
-        "! python3 -c \"import google_cloud_pipeline_components; print('google_cloud_pipeline_components version: {}'.format(google_cloud_pipeline_components.__version__))\""
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -280,6 +259,17 @@
         "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$`."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5aee4379e8e5"
+      },
+      "source": [
+        "#### Set your project ID\n",
+        "\n",
+        "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -621,8 +611,12 @@
       },
       "outputs": [],
       "source": [
+        "import os\n",
+        "from typing import Any, Dict, List\n",
+        "\n",
         "import google.cloud.aiplatform as aip\n",
-        "import kfp"
+        "import kfp\n",
+        "from kfp.v2 import compiler  # noqa: F811"
       ]
     },
     {
@@ -709,7 +703,7 @@
         "def pipeline(\n",
         "    project: str = PROJECT_ID,\n",
         "    model_display_name: str = MODEL_DISPLAY_NAME,\n",
-        "    serving_container_image_uri: str = \"us-docker.pkg.dev/cloud-aiplatform/prediction/tf2-cpu.2-3:latest\",\n",
+        "    serving_container_image_uri: str = \"us-docker.pkg.dev/vertex-ai/prediction/tf2-cpu.2-9:latest\",\n",
         "):\n",
         "    from google_cloud_pipeline_components.types import artifact_types\n",
         "    from google_cloud_pipeline_components.v1.custom_job import \\\n",
@@ -744,7 +738,7 @@
         "        artifact_class=artifact_types.UnmanagedContainerModel,\n",
         "        metadata={\n",
         "            \"containerSpec\": {\n",
-        "                \"imageUri\": \"us-docker.pkg.dev/cloud-aiplatform/prediction/tf2-cpu.2-3:latest\",\n",
+        "                \"imageUri\": \"us-docker.pkg.dev/vertex-ai/prediction/tf2-cpu.2-9:latest\",\n",
         "            },\n",
         "        },\n",
         "    ).after(custom_job_task)\n",
@@ -790,8 +784,6 @@
       },
       "outputs": [],
       "source": [
-        "from kfp.v2 import compiler  # noqa: F811\n",
-        "\n",
         "compiler.Compiler().compile(\n",
         "    pipeline_func=pipeline,\n",
         "    package_path=\"tabular_regression_pipeline.json\",\n",
@@ -860,15 +852,54 @@
         "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
         "\n",
         "Otherwise, you can delete the individual resources you created in this tutorial -- *Note:* this is auto-generated and not all resources may be applicable for this tutorial:\n",
+        "### Get resources from the pipline to clean up\n",
+        "Function to get details of a task"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "50f5f0d96294"
+      },
+      "outputs": [],
+      "source": [
+        "def get_task_detail(\n",
+        "    task_details: List[Dict[str, Any]], task_name: str\n",
+        ") -> List[Dict[str, Any]]:\n",
+        "    for task_detail in task_details:\n",
+        "        if task_detail.task_name == task_name:\n",
+        "            return task_detail"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "0966cd56fc3e"
+      },
+      "outputs": [],
+      "source": [
+        "pipeline_task_details = (\n",
+        "    job.gca_resource.job_detail.task_details\n",
+        ")  # fetch pipeline task details\n",
         "\n",
-        "- Dataset\n",
-        "- Pipeline\n",
-        "- Model\n",
-        "- Endpoint\n",
-        "- Batch Job\n",
-        "- Custom Job\n",
-        "- Hyperparameter Tuning Job\n",
-        "- Cloud Storage Bucket"
+        "\n",
+        "# fetch endpoint from pipeline and delete the endpoint\n",
+        "endpoint_task = get_task_detail(pipeline_task_details, \"endpoint-create\")\n",
+        "endpoint_resourceName = (\n",
+        "    endpoint_task.outputs[\"endpoint\"].artifacts[0].metadata[\"resourceName\"]\n",
+        ")\n",
+        "endpoint = aip.Endpoint(endpoint_resourceName)\n",
+        "# undeploy model from endpoint\n",
+        "endpoint.undeploy_all()\n",
+        "endpoint.delete()\n",
+        "\n",
+        "# fetch model from pipeline and delete the model\n",
+        "model_task = get_task_detail(pipeline_task_details, \"model-upload\")\n",
+        "model_resourceName = model_task.outputs[\"model\"].artifacts[0].metadata[\"resourceName\"]\n",
+        "model = aip.Model(model_resourceName)\n",
+        "model.delete()"
       ]
     },
     {
@@ -879,93 +910,6 @@
       },
       "outputs": [],
       "source": [
-        "delete_dataset = True\n",
-        "delete_pipeline = True\n",
-        "delete_model = True\n",
-        "delete_endpoint = True\n",
-        "delete_batchjob = True\n",
-        "delete_customjob = True\n",
-        "delete_hptjob = True\n",
-        "\n",
-        "try:\n",
-        "    if delete_model and \"DISPLAY_NAME\" in globals():\n",
-        "        models = aip.Model.list(\n",
-        "            filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
-        "        )\n",
-        "        model = models[0]\n",
-        "        aip.Model.delete(model)\n",
-        "        print(\"Deleted model:\", model)\n",
-        "except Exception as e:\n",
-        "    print(e)\n",
-        "\n",
-        "try:\n",
-        "    if delete_endpoint and \"DISPLAY_NAME\" in globals():\n",
-        "        endpoints = aip.Endpoint.list(\n",
-        "            filter=f\"display_name={DISPLAY_NAME}_endpoint\", order_by=\"create_time\"\n",
-        "        )\n",
-        "        endpoint = endpoints[0]\n",
-        "        endpoint.undeploy_all()\n",
-        "        aip.Endpoint.delete(endpoint.resource_name)\n",
-        "        print(\"Deleted endpoint:\", endpoint)\n",
-        "except Exception as e:\n",
-        "    print(e)\n",
-        "\n",
-        "if delete_dataset and \"DISPLAY_NAME\" in globals():\n",
-        "    if \"tabular\" == \"tabular\":\n",
-        "        try:\n",
-        "            datasets = aip.TabularDataset.list(\n",
-        "                filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
-        "            )\n",
-        "            dataset = datasets[0]\n",
-        "            aip.TabularDataset.delete(dataset.resource_name)\n",
-        "            print(\"Deleted dataset:\", dataset)\n",
-        "        except Exception as e:\n",
-        "            print(e)\n",
-        "\n",
-        "    if \"tabular\" == \"image\":\n",
-        "        try:\n",
-        "            datasets = aip.ImageDataset.list(\n",
-        "                filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
-        "            )\n",
-        "            dataset = datasets[0]\n",
-        "            aip.ImageDataset.delete(dataset.resource_name)\n",
-        "            print(\"Deleted dataset:\", dataset)\n",
-        "        except Exception as e:\n",
-        "            print(e)\n",
-        "\n",
-        "    if \"tabular\" == \"text\":\n",
-        "        try:\n",
-        "            datasets = aip.TextDataset.list(\n",
-        "                filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
-        "            )\n",
-        "            dataset = datasets[0]\n",
-        "            aip.TextDataset.delete(dataset.resource_name)\n",
-        "            print(\"Deleted dataset:\", dataset)\n",
-        "        except Exception as e:\n",
-        "            print(e)\n",
-        "\n",
-        "    if \"tabular\" == \"video\":\n",
-        "        try:\n",
-        "            datasets = aip.VideoDataset.list(\n",
-        "                filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
-        "            )\n",
-        "            dataset = datasets[0]\n",
-        "            aip.VideoDataset.delete(dataset.resource_name)\n",
-        "            print(\"Deleted dataset:\", dataset)\n",
-        "        except Exception as e:\n",
-        "            print(e)\n",
-        "\n",
-        "try:\n",
-        "    if delete_pipeline and \"DISPLAY_NAME\" in globals():\n",
-        "        pipelines = aip.PipelineJob.list(\n",
-        "            filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
-        "        )\n",
-        "        pipeline = pipelines[0]\n",
-        "        aip.PipelineJob.delete(pipeline.resource_name)\n",
-        "        print(\"Deleted pipeline:\", pipeline)\n",
-        "except Exception as e:\n",
-        "    print(e)\n",
-        "\n",
         "delete_bucket = False\n",
         "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
         "    ! gsutil rm -r $BUCKET_URI"

notebooks/official/pipelines/pipelines_intro_kfp.ipynb

@@ -104,39 +104,6 @@
         "to generate a cost estimate based on your projected usage."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "setup_local"
-      },
-      "source": [
-        "### Set up your local development environment\n",
-        "\n",
-        "If you are using Colab or Vertex AI Workbench Notebook, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
-        "\n",
-        "Otherwise, make sure your environment meets this notebook's requirements. You need the following:\n",
-        "\n",
-        "- The Cloud Storage SDK\n",
-        "- Git\n",
-        "- Python 3\n",
-        "- virtualenv\n",
-        "- Jupyter notebook running in a virtual environment with Python 3\n",
-        "\n",
-        "The Cloud Storage guide to [Setting up a Python development environment](https://cloud.google.com/python/setup) and the [Jupyter installation guide](https://jupyter.org/install) provide detailed instructions for meeting these requirements. The following steps provide a condensed set of instructions:\n",
-        "\n",
-        "1. [Install and initialize the SDK](https://cloud.google.com/sdk/docs/).\n",
-        "\n",
-        "2. [Install Python 3](https://cloud.google.com/python/setup#installing_python).\n",
-        "\n",
-        "3. [Install virtualenv](Ihttps://cloud.google.com/python/setup#installing_and_using_virtualenv) and create a virtual environment that uses Python 3.\n",
-        "\n",
-        "4. Activate that environment and run `pip3 install Jupyter` in a terminal shell to install Jupyter.\n",
-        "\n",
-        "5. Run `jupyter notebook` on the command line in a terminal shell to launch Jupyter.\n",
-        "\n",
-        "6. Open this notebook in the Jupyter Notebook Dashboard.\n"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -152,55 +119,39 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "F_jyqAW_2Ouu"
+        "id": "1fd00fa70a2a"
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
-        "! pip3 install --upgrade google-cloud-aiplatform {USER_FLAG} -q\n",
-        "! pip3 install -U google-cloud-storage {USER_FLAG} -q\n",
-        "! pip3 install {USER_FLAG} kfp google-cloud-pipeline-components --upgrade -q"
+        "# Install the packages\n",
+        "! pip3 install --upgrade google-cloud-aiplatform \\\n",
+        "                        google-cloud-storage \\\n",
+        "                        kfp \\\n",
+        "                        google-cloud-pipeline-components"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "restart"
+        "id": "blGlVGFYW9Pt"
       },
       "source": [
-        "### Restart the kernel\n",
-        "\n",
-        "Once you've installed the additional packages, you need to restart the notebook kernel so it can find the packages."
+        "### Colab only: Uncomment the following cell to restart the kernel."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "lr4jnWxh2Oux"
+        "id": "0JrvuK6LUYnQ"
       },
       "outputs": [],
       "source": [
-        "import os\n",
+        "# Automatically restart kernel after installs so that your environment can access the new packages\n",
+        "# import IPython\n",
         "\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    # Automatically restart kernel after installs\n",
-        "    import IPython\n",
-        "\n",
-        "    app = IPython.Application.instance()\n",
-        "    app.kernel.do_shutdown(True)"
+        "# app = IPython.Application.instance()\n",
+        "# app.kernel.do_shutdown(True)"
       ]
     },
     {
@@ -226,217 +177,143 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "before_you_begin:nogpu"
+        "id": "a47846030fef"
       },
       "source": [
-        "## Before you begin\n",
-        "\n",
-        "### GPU runtime\n",
-        "\n",
-        "This tutorial does not require a GPU runtime.\n",
-        "\n",
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "2. [Make sure that billing is enabled for your project.](https://cloud.google.com/billing/docs/how-to/modify-project)\n",
-        "\n",
-        "3. [Enable the Vertex AI APIs, Compute Engine APIs, and Cloud Storage.](https://console.cloud.google.com/flows/enableapi?apiid=ml.googleapis.com,compute_component,storage-component.googleapis.com)\n",
-        "\n",
-        "4. [The Google Cloud SDK](https://cloud.google.com/sdk) is already installed in Google Cloud Notebook.\n",
-        "\n",
-        "5. Enter your project ID in the cell below. Then run the  cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$`."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "set_project_id"
-      },
-      "outputs": [],
-      "source": [
-        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "autoset_project_id"
-      },
-      "outputs": [],
-      "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
-        "    # Get your GCP project id from gcloud\n",
-        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
-        "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID:\", PROJECT_ID)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "set_gcloud_project_id"
-      },
-      "outputs": [],
-      "source": [
-        "! gcloud config set project $PROJECT_ID"
+        "## Before you begin"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "region"
+        "id": "project_id"
+      },
+      "source": [
+        "#### Set your project ID\n",
+        "\n",
+        "**If you don't know your project ID**, try the following:\n",
+        "* Run `gcloud config list`.\n",
+        "* Run `gcloud projects list`.\n",
+        "* See the support page: [Locate the project ID](https://support.google.com/googleapi/answer/7014113)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3c8049930470"
+      },
+      "outputs": [],
+      "source": [
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# Set the project id\n",
+        "! gcloud config set project {PROJECT_ID}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "a54f9d7c1876"
       },
       "source": [
         "#### Region\n",
         "\n",
-        "You can also change the `REGION` variable, which is used for operations\n",
-        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
-        "\n",
-        "- Americas: `us-central1`\n",
-        "- Europe: `europe-west4`\n",
-        "- Asia Pacific: `asia-east1`\n",
-        "\n",
-        "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
-        "\n",
-        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)"
+        "You can also change the `REGION` variable used by Vertex AI. Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "04933ed28eef"
+        "id": "3aaadaaf9b30"
       },
       "outputs": [],
       "source": [
-        "REGION = \"[your-region]\"  # @param {type:\"string\"}"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "jvNx3KyF2Ou0"
-      },
-      "outputs": [],
-      "source": [
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "timestamp"
-      },
-      "source": [
-        "#### Timestamp\n",
-        "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "lFCteCD32Ou0"
-      },
-      "outputs": [],
-      "source": [
-        "from datetime import datetime\n",
-        "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "gcp_authenticate"
+        "id": "5c0404984792"
       },
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Vertex AI Workbench Notebook**, your environment is already authenticated. Skip this step.\n",
-        "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions\n",
-        "when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "1. In the Cloud Console, go to the [**Create service account key**\n",
-        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
-        "\n",
-        "2. Click **Create service account**.\n",
-        "\n",
-        "3. In the **Service account name** field, enter a name, and\n",
-        "   click **Create**.\n",
-        "\n",
-        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
-        "into the filter box, and select\n",
-        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
-        "local environment.\n",
-        "\n",
-        "6. Enter the path to your service account key as the\n",
-        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
+        "Depending on your Jupyter environment, you may have to manually authenticate. Follow the relevant instructions below."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "BaFKzJ_xXpvm"
+      },
+      "source": [
+        "**1. Vertex AI Workbench**\n",
+        "* Do nothing as you are already authenticated."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "_fV-KyGAX4Xl"
+      },
+      "source": [
+        "**2. Local JupyterLab instance, uncomment and run:**\n"
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "7Z2W27Sa2Ou1"
+        "id": "7uXB1HAPX6L_"
       },
       "outputs": [],
       "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = \"google.colab\" in sys.modules\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+        "# ! gcloud auth login"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "bucket:mbsdk"
+        "id": "Ab_TRMQIYCCX"
+      },
+      "source": [
+        "**3. Colab, uncomment and run:**"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "vx25htmYYExI"
+      },
+      "outputs": [],
+      "source": [
+        "# from google.colab import auth\n",
+        "# auth.authenticate_user()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "uZdA0-jBYGqt"
+      },
+      "source": [
+        "**4. Service account or other**\n",
+        "* See how to grant Cloud Storage permissions to your service account at https://cloud.google.com/storage/docs/gsutil/commands/iam#ch-examples."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "bucket:custom"
       },
       "source": [
         "### Create a Cloud Storage bucket\n",
         "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "When you initialize the Vertex AI SDK for Python, you specify a Cloud Storage staging bucket. The staging bucket is where all the data associated with your dataset and model resources are retained across sessions.\n",
-        "\n",
-        "Set the name of your Cloud Storage bucket below. Bucket names must be globally unique across all Google Cloud projects, including those outside of your organization."
+        "Create a storage bucket to store intermediate artifacts such as datasets."
       ]
     },
     {
@@ -447,21 +324,7 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "autoset_bucket"
-      },
-      "outputs": [],
-      "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
-        "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
+        "BUCKET_URI = \"gs://your-bucket-name-unique\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -477,79 +340,11 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "IXf7nYUc2Ou2"
+        "id": "Oz8J0vmSlugt"
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_URI"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "validate_bucket"
-      },
-      "source": [
-        "Finally, validate access to your Cloud Storage bucket by examining its contents:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "p23SI9T42Ou2"
-      },
-      "outputs": [],
-      "source": [
-        "! gsutil ls -al $BUCKET_URI"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "set_service_account"
-      },
-      "source": [
-        "#### Service Account\n",
-        "\n",
-        "**If you don't know your service account**, try to get your service account using `gcloud` command by executing the second cell below."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "NY-zkA0Y2Ou3"
-      },
-      "outputs": [],
-      "source": [
-        "SERVICE_ACCOUNT = \"[your-service-account]\"  # @param {type:\"string\"}"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "autoset_service_account"
-      },
-      "outputs": [],
-      "source": [
-        "if (\n",
-        "    SERVICE_ACCOUNT == \"\"\n",
-        "    or SERVICE_ACCOUNT is None\n",
-        "    or SERVICE_ACCOUNT == \"[your-service-account]\"\n",
-        "):\n",
-        "    # Get your service account from gcloud\n",
-        "    if not IS_COLAB:\n",
-        "        shell_output = !gcloud auth list 2>/dev/null\n",
-        "        SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
-        "\n",
-        "    if IS_COLAB:\n",
-        "        shell_output = ! gcloud projects describe  $PROJECT_ID\n",
-        "        project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n",
-        "        SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n",
-        "\n",
-        "    print(\"Service Account:\", SERVICE_ACCOUNT)"
+        "! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
       ]
     },
     {
@@ -563,6 +358,17 @@
         "Run the following commands to grant your service account access to read and write pipeline artifacts in the bucket that you created in the previous step -- you only need to run these once per service account."
       ]
     },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b556df542518"
+      },
+      "outputs": [],
+      "source": [
+        "SERVICE_ACCOUNT = \"[your-service-account@developer.gserviceaccount.com]\""
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -814,7 +620,7 @@
       "outputs": [],
       "source": [
         "@dsl.pipeline(\n",
-        "    name=\"hello-world-v2\",\n",
+        "    name=\"intro-pipeline-unique\",\n",
         "    description=\"A simple intro pipeline\",\n",
         "    pipeline_root=PIPELINE_ROOT,\n",
         ")\n",
@@ -871,7 +677,7 @@
       },
       "outputs": [],
       "source": [
-        "DISPLAY_NAME = \"intro_\" + TIMESTAMP\n",
+        "DISPLAY_NAME = \"intro_pipeline_job_unique\"\n",
         "\n",
         "job = aip.PipelineJob(\n",
         "    display_name=DISPLAY_NAME,\n",
@@ -949,6 +755,8 @@
       },
       "outputs": [],
       "source": [
+        "import os\n",
+        "\n",
         "if not os.getenv(\"IS_TESTING\"):\n",
         "    response = api_client.create_run_from_job_spec(\n",
         "        job_spec_path=\"intro_pipeline.json\",\n",
@@ -979,7 +787,7 @@
       "outputs": [],
       "source": [
         "job = aip.PipelineJob(\n",
-        "    display_name=\"intro_\" + TIMESTAMP,\n",
+        "    display_name=\"intro_pipeline_job_cached_unique\",\n",
         "    template_path=\"intro_pipeline.json\",\n",
         "    enable_caching=False,\n",
         ")\n",

notebooks/official/tabular_workflows/tabnet_on_vertex_pipelines.ipynb

@@ -77,7 +77,7 @@
         "This tutorial uses the following Google Cloud ML services and resources:\n",
         "\n",
         "- Vertex AI Training\n",
-        "- Vertex Pipelines\n",
+        "- Vertex AI Pipelines\n",
         "- Cloud Storage\n",
         "\n",
         "The steps performed include:\n",
@@ -97,7 +97,7 @@
         "### Dataset\n",
         "\n",
         "The dataset you will be using is [Bank Marketing](https://archive.ics.uci.edu/ml/datasets/bank+marketing).\n",
-        "The data is for direct marketing campaigns (phone calls) of a Portuguese banking institution. The binary classification goal is to predict if a client will subscribe a term deposit. For this notebook, we randomly selected 90% of the rows in the original dataset and saved them in a train.csv file hosted on Cloud Storage. To download the file, click [here](https://storage.googleapis.com/cloud-samples-data/vertex-ai/tabular-workflows/datasets/bank-marketing/train.csv)."
+        "The data is for direct marketing campaigns (phone calls) of a Portuguese banking institution. The binary classification goal is to predict if a client subscribe a term deposit. For this notebook, you randomly selected 90% of the rows in the original dataset and saved them in a train.csv file hosted on Cloud Storage. To download the file, click [here](https://storage.googleapis.com/cloud-samples-data/vertex-ai/tabular-workflows/datasets/bank-marketing/train.csv)."
       ]
     },
     {
@@ -128,7 +128,7 @@
       "source": [
         "### Set up your local development environment\n",
         "\n",
-        "**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets\n",
+        "**If you are using Colab or Vertex AI SDK Workbench Notebooks**, your environment already meets\n",
         "all the requirements to run this notebook. You can skip this step.\n",
         "\n",
         "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
@@ -251,7 +251,7 @@
         "\n",
         "3. [Enable the following APIs: Vertex AI APIs, Dataflow APIs, Compute Engine APIs, and Cloud Storage.](https://console.cloud.google.com/flows/enableapi?apiid=ml.googleapis.com,dataflow.googleapis.com,compute_component,storage-component.googleapis.com)\n",
         "\n",
-        "4. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "4. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
         "\n",
         "5. Enter your project ID in the cell below. Then run the  cell to make sure the\n",
         "Cloud SDK uses the right project for all the commands in this notebook.\n",
@@ -438,7 +438,7 @@
         "\n",
         "**The following steps are required, regardless of your notebook environment.**\n",
         "\n",
-        "All training related files (TF model checkpoint, TensorBoard file, etc) will be saved to the GCS bucket. The pipeline will not clean up the files since some of them might be useful for you, **please make sure to clean up the files**. For easy cleanup, you can set [GCS bucket level TTL](https://cloud.google.com/storage/docs/lifecycle).\n",
+        "All training related files (TF model checkpoint, TensorBoard file, etc) will be saved to the GCS bucket. The pipeline not clean up the files since some of them might be useful for you, **please make sure to clean up the files**. For easy cleanup, you can set [GCS bucket level TTL](https://cloud.google.com/storage/docs/lifecycle).\n",
         "\n",
         "Set the name of your Cloud Storage bucket below. Bucket names must be globally unique across all Google Cloud projects, including those outside of your organization.\n"
       ]
@@ -512,6 +512,77 @@
         "! gsutil ls -al $BUCKET_URI"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "44accda192d5"
+      },
+      "source": [
+        "#### Service Account\n",
+        "\n",
+        "You use a service account to create Vertex AI Pipeline jobs. If you do not want to use your project's Compute Engine service account, set `SERVICE_ACCOUNT` to another service account ID."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "c65d12a97f45"
+      },
+      "outputs": [],
+      "source": [
+        "SERVICE_ACCOUNT = \"[your-service-account]\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "604ae09ab6d3"
+      },
+      "outputs": [],
+      "source": [
+        "if (\n",
+        "    SERVICE_ACCOUNT == \"\"\n",
+        "    or SERVICE_ACCOUNT is None\n",
+        "    or SERVICE_ACCOUNT == \"[your-service-account]\"\n",
+        "):\n",
+        "    # Get your service account from gcloud\n",
+        "    if not IS_COLAB:\n",
+        "        shell_output = !gcloud auth list 2>/dev/null\n",
+        "        SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
+        "\n",
+        "    else:  # IS_COLAB:\n",
+        "        shell_output = ! gcloud projects describe  $PROJECT_ID\n",
+        "        project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n",
+        "        SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n",
+        "\n",
+        "    print(\"Service Account:\", SERVICE_ACCOUNT)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d1ecb60964d5"
+      },
+      "source": [
+        "#### Set service account access for Vertex AI Pipelines\n",
+        "Run the following commands to grant your service account access to read and write pipeline artifacts in the bucket that you created in the previous step. You only need to run this step once per service account."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "a592f0a380c2"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
+        "\n",
+        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -577,11 +648,13 @@
       },
       "outputs": [],
       "source": [
+        "# Get the mdoel artifacts path from task details.\n",
         "def get_model_artifacts_path(task_details: List[Dict[str, Any]], task_name: str) -> str:\n",
         "    task = get_task_detail(task_details, task_name)\n",
         "    return task.outputs[\"unmanaged_container_model\"].artifacts[0].uri\n",
         "\n",
         "\n",
+        "# Get the model uri from the task details.\n",
         "def get_model_uri(task_details: List[Dict[str, Any]]) -> str:\n",
         "    task = get_task_detail(task_details, \"model-upload\")\n",
         "    # in format https://<location>-aiplatform.googleapis.com/v1/projects/<project_number>/locations/<location>/models/<model_id>\n",
@@ -589,12 +662,14 @@
         "    return f\"https://console.cloud.google.com/vertex-ai/locations/{REGION}/models/{model_id}?project={PROJECT_ID}\"\n",
         "\n",
         "\n",
+        "# Get the bucket name and path.\n",
         "def get_bucket_name_and_path(uri: str) -> str:\n",
         "    no_prefix_uri = uri[len(\"gs://\") :]\n",
         "    splits = no_prefix_uri.split(\"/\")\n",
         "    return splits[0], \"/\".join(splits[1:])\n",
         "\n",
         "\n",
+        "# Get the content from the bucket.\n",
         "def download_from_gcs(uri: str) -> str:\n",
         "    bucket_name, path = get_bucket_name_and_path(uri)\n",
         "    storage_client = storage.Client(project=PROJECT_ID)\n",
@@ -603,6 +678,7 @@
         "    return blob.download_as_string()\n",
         "\n",
         "\n",
+        "# Upload content in to the bucket.\n",
         "def write_to_gcs(uri: str, content: str):\n",
         "    bucket_name, path = get_bucket_name_and_path(uri)\n",
         "    storage_client = storage.Client()\n",
@@ -611,6 +687,7 @@
         "    blob.upload_from_string(content)\n",
         "\n",
         "\n",
+        "# Get the task details by using task name.\n",
         "def get_task_detail(\n",
         "    task_details: List[Dict[str, Any]], task_name: str\n",
         ") -> List[Dict[str, Any]]:\n",
@@ -619,6 +696,7 @@
         "            return task_detail\n",
         "\n",
         "\n",
+        "# Get the model name from pipeline task details.\n",
         "def get_model_name(job_id: str) -> str:\n",
         "    pipeline_task_details = aiplatform.PipelineJob.get(\n",
         "        job_id\n",
@@ -627,6 +705,7 @@
         "    return upload_task_details.outputs[\"model\"].artifacts[0].metadata[\"resourceName\"]\n",
         "\n",
         "\n",
+        "# Get the evaluation metrics.\n",
         "def get_evaluation_metrics(\n",
         "    task_details: List[Dict[str, Any]],\n",
         ") -> str:\n",
@@ -681,7 +760,7 @@
       "source": [
         "### Configure feature transformation\n",
         "\n",
-        "Transformations can be specified using Feature Transform Engine (FTE) specific configurations. Below, we configure full auto transformations (i.e., `auto_transform_config`). FTE automatically configures a set of built-in transformations for each input column based on its data statistics. \n",
+        "Transformations can be specified using Feature Transform Engine (FTE) specific configurations. Below, you configure full auto transformations (i.e., `auto_transform_config`). FTE automatically configures a set of built-in transformations for each input column based on its data statistics. \n",
         "\n",
         "For a complete list of supported feature transformation configs and examples, please go [here](https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-1.0.15/google_cloud_pipeline_components.experimental.automl.tabular.html#google_cloud_pipeline_components.experimental.automl.tabular.FeatureTransformEngineOp)."
       ]
@@ -847,7 +926,7 @@
         "pipeline_job_root_dir = os.path.join(BUCKET_URI, \"tabnet_custom_job\")\n",
         "\n",
         "# max_steps and/or max_train_secs must be set. If both are\n",
-        "# specified, training will stop after either condition is met.\n",
+        "# specified, training stop after either condition is met.\n",
         "# By default, max_train_secs is set to -1.\n",
         "\n",
         "max_steps = 1000\n",
@@ -910,7 +989,7 @@
         "    enable_caching=False,\n",
         ")\n",
         "\n",
-        "pipeline_job.run()"
+        "pipeline_job.run(service_account=SERVICE_ACCOUNT)"
       ]
     },
     {
@@ -950,9 +1029,9 @@
       "source": [
         "## Customize TabNet HyperparameterTuningJob configuration and create pipeline\n",
         "\n",
-        "To get the best set of hyperparameters for your dataset, we recommend running a HyperparameterTuningJob.\n",
+        "To get the best set of hyperparameters for your dataset, you recommend running a HyperparameterTuningJob.\n",
         "\n",
-        "Hyperparameters that can be tuned are set in the optional `study_spec_parameters_override` parameter. We provide a helper function called `get_tabnet_study_spec_parameters_override` to get these hyperparameters. You provide `dataset_size_bucket` (one of 'small' (< 1M rows), 'medium' (1M - 100M rows), or 'large' (> 100M rows)), `training_budget_bucket` (one of 'small' (< \\\\$600), 'medium' (\\\\$600 - \\\\$2400), or 'large' (> \\\\$2400)), and `prediction_type` and Vertex AI returns a list of hyperparameters and ranges. `study_spec_parameters_override` can be empty or one or more of these hyperparameters can be specified. For hyperparameters not specified in `study_spec_parameters_override`, we set ranges in the pipeline. For a full list of hyperparameters available for tuning, see [here](https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-1.0.23/google_cloud_pipeline_components.experimental.automl.tabular.html#google_cloud_pipeline_components.experimental.automl.tabular.utils.get_tabnet_trainer_pipeline_and_parameters).\n",
+        "Hyperparameters that can be tuned are set in the optional `study_spec_parameters_override` parameter. you provide a helper function called `get_tabnet_study_spec_parameters_override` to get these hyperparameters. You provide `dataset_size_bucket` (one of 'small' (< 1M rows), 'medium' (1M - 100M rows), or 'large' (> 100M rows)), `training_budget_bucket` (one of 'small' (< \\\\$600), 'medium' (\\\\$600 - \\\\$2400), or 'large' (> \\\\$2400)), and `prediction_type` and Vertex AI returns a list of hyperparameters and ranges. `study_spec_parameters_override` can be empty or one or more of these hyperparameters can be specified. For hyperparameters not specified in `study_spec_parameters_override`, you set ranges in the pipeline. For a full list of hyperparameters available for tuning, see [here](https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-1.0.23/google_cloud_pipeline_components.experimental.automl.tabular.html#google_cloud_pipeline_components.experimental.automl.tabular.utils.get_tabnet_trainer_pipeline_and_parameters).\n",
         "\n",
         "In addition to hyperparameters, HyperparameterTuningJob takes the following values in the example below:\n",
         "\n",
@@ -968,7 +1047,7 @@
         "\n",
         "For a full list of HyperparameterTuningJob parameters, see [here](https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-1.0.23/google_cloud_pipeline_components.experimental.automl.tabular.html#google_cloud_pipeline_components.experimental.automl.tabular.utils.get_tabnet_hyperparameter_tuning_job_pipeline_and_parameters).\n",
         "\n",
-        "Multiple trials can be configured. The pipeline returns the best trial based on the metric configured in `study_spec_metrics`. In the example below, we return the trial with the lowest loss value. "
+        "Multiple trials can be configured. The pipeline returns the best trial based on the metric configured in `study_spec_metrics`. In the example below, you return the trial with the lowest loss value. "
       ]
     },
     {
@@ -1000,7 +1079,7 @@
         "study_spec_metric_goal = \"MINIMIZE\"\n",
         "\n",
         "# max_steps and/or max_train_secs must be set. If both are\n",
-        "# specified, training will stop after either condition is met.\n",
+        "# specified, training stop after either condition is met.\n",
         "# By default, max_train_secs is set to -1 and max_steps is set to\n",
         "# an appropriate range given dataset_size and training budget.\n",
         "study_spec_parameters_override = (\n",
@@ -1052,7 +1131,7 @@
         "    enable_caching=False,\n",
         ")\n",
         "\n",
-        "pipeline_job.run()"
+        "pipeline_job.run(service_account=SERVICE_ACCOUNT)"
       ]
     },
     {
@@ -1077,6 +1156,7 @@
         "    pipeline_job_id\n",
         ").gca_resource.job_detail.task_details\n",
         "HPT_JOB_MODEL = get_model_name(pipeline_job_id)\n",
+        "\n",
         "print(\"model uri:\", get_model_uri(tabnet_hpt_pipeline_task_details))\n",
         "print(\n",
         "    \"model artifacts:\",\n",

notebooks/official/tensorboard/README.md

@@ -20,3 +20,14 @@ The steps performed include:
 * Create a custom container image with your customized training code.
 * Setup service account and Google Cloud Storage buckets.
 * Create & launch your custom training job with your custom container.
+
+[Profile model training performance using Profiler](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/tensorboard/tensorboard_profiler_custom_training.ipynb)
+
+Learn how to enable Vertex AI TensorBoard Profiler for custom training jobs.
+
+The steps performed include:
+
+* Setup a service account and a Cloud Storage bucket
+* Create a TensorBoard instance
+* Create and run a custom training job that enables TensorBoard Profiler
+* View the TensorBoard Profiler dashboard to debug your model training performance

notebooks/official/tensorboard/tensorboard_custom_training_with_custom_container.ipynb

@@ -202,9 +202,29 @@
         "if IS_WORKBENCH_NOTEBOOK:\n",
         "    USER_FLAG = \"--user\"\n",
         "\n",
-        "! pip3 install google-cloud-aiplatform {USER_FLAG} -q\n",
+        "! pip3 install google-cloud-aiplatform {USER_FLAG} -q\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "b24902cde81b"
+      },
+      "source": [
+        "### Restart the kernel\n",
         "\n",
-        "# Automatically restart kernel after installs\n",
+        "Once you've installed the additional packages, you need to restart the notebook kernel so it can find the packages."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "c61d171395d7"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
         "\n",
         "if not os.getenv(\"IS_TESTING\"):\n",
         "    # Automatically restart kernel after installs\n",
@@ -214,21 +234,13 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "r_dA3M6UJELw"
-      },
-      "source": [
-        "## Before you begin"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "1Dunp1YrhPYo"
       },
       "source": [
+        "## Before you begin\n",
         "### Set up your Google Cloud project\n",
         "\n",
         "**The following steps are required, regardless of your notebook environment.**\n",
@@ -266,6 +278,17 @@
         "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
       ]
     },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cde8e0876d62"
+      },
+      "outputs": [],
+      "source": [
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -274,15 +297,11 @@
       },
       "outputs": [],
       "source": [
-        "PROJECT_ID = \"\"\n",
-        "\n",
-        "import os\n",
-        "\n",
-        "# Get your Google Cloud project ID from gcloud\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    shell_output = !gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
+        "    # Get your GCP project id from gcloud\n",
+        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
         "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID: \", PROJECT_ID)"
+        "    print(\"Project ID:\", PROJECT_ID)"
       ]
     },
     {
@@ -302,8 +321,7 @@
       },
       "outputs": [],
       "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
-        "    PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
+        "! gcloud config set project $PROJECT_ID"
       ]
     },
     {
@@ -312,16 +330,9 @@
         "id": "K-KuU54IaVz5"
       },
       "source": [
-        "### Timestamp"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "doEJxrvsaWyt"
-      },
-      "source": [
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append it onto the name of resources you create in this tutorial."
+        "#### UUID\n",
+        "\n",
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
       ]
     },
     {
@@ -332,9 +343,16 @@
       },
       "outputs": [],
       "source": [
-        "from datetime import datetime\n",
+        "import random\n",
+        "import string\n",
         "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+        "\n",
+        "# Generate a uuid of a specifed length(default=8)\n",
+        "def generate_uuid(length: int = 8) -> str:\n",
+        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
+        "\n",
+        "\n",
+        "UUID = generate_uuid()"
       ]
     },
     {
@@ -343,7 +361,18 @@
         "id": "Ee3vBgvdhgTb"
       },
       "source": [
-        "### Set your region"
+        "#### Region\n",
+        "\n",
+        "You can also change the `REGION` variable, which is used for operations\n",
+        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. It is recommended that you choose the region closest to you.\n",
+        "\n",
+        "- Americas: `us-central1`\n",
+        "- Europe: `europe-west4`\n",
+        "- Asia Pacific: `asia-east1`\n",
+        "\n",
+        "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
+        "\n",
+        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
       ]
     },
     {
@@ -355,6 +384,7 @@
       "outputs": [],
       "source": [
         "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
         "if REGION == \"[your-region]\":\n",
         "    REGION = \"us-central1\""
       ]
@@ -365,16 +395,47 @@
         "id": "KuNRbXkIijp6"
       },
       "source": [
-        "### Login to your Google Cloud account"
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
+        "authenticated."
       ]
     },
     {
-      "cell_type": "code",
-      "execution_count": null,
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "f40aa139740f"
+      },
+      "source": [
+        "**If you are using Colab**, run the cell below and follow the instructions\n",
+        "when prompted to authenticate your account via oAuth.\n",
+        "\n",
+        "**Otherwise**, follow these steps:\n",
+        "\n",
+        "1. In the Cloud Console, go to the [**Create service account key**\n",
+        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
+        "\n",
+        "2. Click **Create service account**.\n",
+        "\n",
+        "3. In the **Service account name** field, enter a name, and\n",
+        "   click **Create**.\n",
+        "\n",
+        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
+        "into the filter box, and select\n",
+        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "\n",
+        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
+        "local environment.\n",
+        "\n",
+        "6. Enter the path to your service account key as the\n",
+        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
+      ]
+    },
+    {
+      "cell_type": "markdown",
       "metadata": {
         "id": "P9vQxUzfirCV"
       },
-      "outputs": [],
       "source": [
         "# The Google Cloud Notebook product has specific requirements\n",
         "import os\n",
@@ -430,7 +491,7 @@
         "BUCKET_URI = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}\n",
         "\n",
         "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + UUID"
       ]
     },
     {
@@ -517,10 +578,25 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "0j1NWIQEJI5i"
+        "id": "4eaef8c7be0e"
       },
       "source": [
-        "## Create Docker repository"
+        "### Enable Artifact Registry API\n",
+        "First, you must enable the Artifact Registry API service for your project.\n",
+        "\n",
+        "Learn more about [Enabling service\n",
+        "   page](https://cloud.google.com/artifact-registry/docs/enable-service)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "d03035c8fb6f"
+      },
+      "outputs": [],
+      "source": [
+        "!gcloud services enable artifactregistry.googleapis.com"
       ]
     },
     {
@@ -529,6 +605,8 @@
         "id": "hNmHMIyjBzxx"
       },
       "source": [
+        "### Create Docker repository\n",
+        "\n",
         "Create a Docker repository named `DOCKER_REPOSITORY` in your `REGION`.\n",
         "This docker repository will be deleted in the clearning up section in the end."
       ]
@@ -548,7 +626,7 @@
         "    or DOCKER_REPOSITORY is None\n",
         "    or DOCKER_REPOSITORY == \"[your-docker-repository-name]\"\n",
         "):\n",
-        "    DOCKER_REPOSITORY = \"tb-docker-repo-\" + PROJECT_ID + \"-\" + TIMESTAMP\n",
+        "    DOCKER_REPOSITORY = \"tb-docker-repo-\" + PROJECT_ID + \"-\" + UUID\n",
         "\n",
         "print(\"Docker repository to create:\", DOCKER_REPOSITORY)"
       ]
@@ -561,18 +639,9 @@
       },
       "outputs": [],
       "source": [
-        "! gcloud artifacts repositories create $DOCKER_REPOSITORY --project={PROJECT_ID} \\\n",
+        "! gcloud  artifacts repositories create  $DOCKER_REPOSITORY --project={PROJECT_ID} \\\n",
         "--repository-format=docker \\\n",
-        "--location={REGION} --description=\"Repository for TensorBoard Custom Training Job\""
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "V0pBSC0rDlvq"
-      },
-      "source": [
-        "Verify your Docker repository is created successfully."
+        "--location={REGION} --description=\"Repository for TensorBoard Custom Training Job\" "
       ]
     },
     {
@@ -614,6 +683,7 @@
         "id": "jUcVG77dKmPn"
       },
       "source": [
+        "### Create a training code\n",
         "Write your own training code in task.py file. You can use the following code as an example."
       ]
     },
@@ -753,7 +823,9 @@
         "id": "DK2E1xz8Q7Q-"
       },
       "source": [
-        "Build your container image using `gcloud builds` from your training code and `Dockerfile`. Note that this step may take a few minutes."
+        "Build your container image using `gcloud builds` from your training code and `Dockerfile`. \n",
+        "\n",
+        "*Note* that this step may take a few minutes."
       ]
     },
     {
@@ -773,21 +845,14 @@
         "! gcloud builds submit --project {PROJECT_ID} --region={REGION} --tag {IMAGE_URI} --timeout=20m"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "hwXxa4Qgnh4Y"
-      },
-      "source": [
-        "## Setup service account and permissions"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "7qXFUiHLoFRw"
       },
       "source": [
+        "## Setup service account and permissions\n",
+        "\n",
         "A service account will be used to create custom training job. If you do not want to use your project's Compute Engine service account, set SERVICE_ACCOUNT to another service account ID. You can create a service account by following the [instruction](https://cloud.google.com/iam/docs/creating-managing-service-accounts#creating)."
       ]
     },
@@ -799,7 +864,7 @@
       },
       "outputs": [],
       "source": [
-        "SERVICE_ACCOUNT = \"[your-service-account]\"  # @param {type:\"string\"}"
+        "SERVICE_ACCOUNT = \"[your-service-account]\""
       ]
     },
     {
@@ -810,6 +875,9 @@
       },
       "outputs": [],
       "source": [
+        "import sys\n",
+        "\n",
+        "IS_COLAB = \"google.colab\" in sys.modules\n",
         "if (\n",
         "    SERVICE_ACCOUNT == \"\"\n",
         "    or SERVICE_ACCOUNT is None\n",
@@ -832,37 +900,13 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "UlDhuciOt5vo"
+        "id": "c7798d69970b"
       },
       "outputs": [],
       "source": [
-        "# Grant Cloud Storage permission.\n",
-        "! gcloud projects add-iam-policy-binding {PROJECT_ID} \\\n",
-        "   --member=serviceAccount:{SERVICE_ACCOUNT} \\\n",
-        "   --role=roles/storage.admin"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "lTKVB71soRyr"
-      },
-      "outputs": [],
-      "source": [
-        "# Grant AI Platform permission.\n",
-        "! gcloud projects add-iam-policy-binding {PROJECT_ID} \\\n",
-        "   --member=serviceAccount:{SERVICE_ACCOUNT} \\\n",
-        "   --role=roles/aiplatform.user"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "IaQjIPvuKLwW"
-      },
-      "source": [
-        "## Create a custom training job with your container"
+        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
+        "\n",
+        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
       ]
     },
     {
@@ -871,6 +915,7 @@
         "id": "svUGBOow_Obj"
       },
       "source": [
+        "## Create a custom training job with your container\n",
         "Create a TensorBoard instnace to be used by the custom training job."
       ]
     },
@@ -889,7 +934,7 @@
         "    or TENSORBOARD_NAME is None\n",
         "    or TENSORBOARD_NAME == \"[your-tensorboard-name]\"\n",
         "):\n",
-        "    TENSORBOARD_NAME = PROJECT_ID + \"-tb-\" + TIMESTAMP\n",
+        "    TENSORBOARD_NAME = PROJECT_ID + \"-tb-\" + UUID\n",
         "\n",
         "tensorboard = aiplatform.Tensorboard.create(\n",
         "    display_name=TENSORBOARD_NAME, project=PROJECT_ID, location=REGION\n",
@@ -915,7 +960,7 @@
       },
       "outputs": [],
       "source": [
-        "JOB_NAME = \"tensorboard-example-job-{}\".format(TIMESTAMP)\n",
+        "JOB_NAME = \"tensorboard-example-job-{}\".format(UUID)\n",
         "BASE_OUTPUT_DIR = \"{}/{}\".format(BUCKET_URI, JOB_NAME)\n",
         "\n",
         "job = aiplatform.CustomContainerTrainingJob(\n",
@@ -964,9 +1009,6 @@
       },
       "outputs": [],
       "source": [
-        "# Delete GCS bucket.\n",
-        "! gsutil -m rm -r {BUCKET_URI}\n",
-        "\n",
         "# Delete docker repository.\n",
         "! gcloud artifacts repositories delete $DOCKER_REPOSITORY --project {PROJECT_ID} --location {REGION} --quiet\n",
         "\n",
@@ -974,7 +1016,12 @@
         "! gcloud ai tensorboards delete {TENSORBOARD_RESOURCE_NAME}\n",
         "\n",
         "# Delete custom job.\n",
-        "job.delete()"
+        "job.delete()\n",
+        "\n",
+        "# Delete GCS bucket.\n",
+        "delete_bucket = False\n",
+        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "    ! gsutil -m rm -r $BUCKET_URI"
       ]
     }
   ],

notebooks/official/tensorboard/tensorboard_profiler_custom_training.ipynb

@@ -0,0 +1,942 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "l2mMvIUG9meX"
+      },
+      "source": [
+        "# Profile model training performance using Profiler\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/tensorboard/tensorboard_profiler_custom_training.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/tensorboard/tensorboard_profiler_custom_training.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/tensorboard/tensorboard_profiler_custom_training.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "  </td>                                                                                               \n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ur8xi4C7S06n"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2022 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tvgnzT1CKxrO"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "Vertex AI TensorBoard Profiler lets you monitor and optimize your model training performance by helping you understand the resource consumption of training operations. This tutorial demonstrates how to enable Vertex AI TensorBoard Profiler so you can debug model training performance for your custom training jobs.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dmfmQL6w84pS"
+      },
+      "source": [
+        "### Objective\n",
+        "\n",
+        "In this tutorial, you learn how to enable Vertex AI TensorBoard Profiler for custom training jobs.\n",
+        "\n",
+        "This tutorial uses the following Google Cloud AI services:\n",
+        "\n",
+        "- `Vertex AI Training`\n",
+        "- `Vertex AI TensorBoard`\n",
+        "\n",
+        "The steps performed include:\n",
+        "\n",
+        "- Setup a service account and a Cloud Storage bucket\n",
+        "- Create a TensorBoard instance\n",
+        "- Create and run a custom training job that enables TensorBoard Profiler\n",
+        "- View the TensorBoard Profiler dashboard to debug your model training performance\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "zfXf0r-K81Y-"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the [mnist dataset](https://www.tensorflow.org/datasets/catalog/mnist) from [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/overview).\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "I3KFLvpq87rs"
+      },
+      "source": [
+        "### Costs \n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI\n",
+        "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
+        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
+        "Calculator](https://cloud.google.com/products/calculator/)\n",
+        "to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ze4-nDLfK4pw"
+      },
+      "source": [
+        "## Installation\n",
+        "\n",
+        "Install the following packages required to execute this notebook. "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2b4ef9b72d43"
+      },
+      "outputs": [],
+      "source": [
+        "! pip3 install --user --upgrade google-cloud-aiplatform --quiet"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "aUw6ibN-n5Za"
+      },
+      "source": [
+        "### Colab only: Uncomment the following cell to restart the kernel."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "FM12wbWhn7w0"
+      },
+      "outputs": [],
+      "source": [
+        "# Automatically restart kernel after installs so that your environment can access the new packages\n",
+        "# import IPython\n",
+        "\n",
+        "# app = IPython.Application.instance()\n",
+        "# app.kernel.do_shutdown(True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "LgFWLeJfoGQu"
+      },
+      "source": [
+        "## Before you begin\n",
+        "\n",
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "2. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "3. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com). \n",
+        "\n",
+        "4. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8ckyxpX_oSzD"
+      },
+      "source": [
+        "#### Set your project ID\n",
+        "\n",
+        "**If you don't know your project ID**, try the following:\n",
+        "* Run `gcloud config list`.\n",
+        "* Run `gcloud projects list`.\n",
+        "* See the support page: [Locate the project ID](https://support.google.com/googleapi/answer/7014113)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "zY8DKBoVoVy3"
+      },
+      "outputs": [],
+      "source": [
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# Set the project id\n",
+        "! gcloud config set project {PROJECT_ID}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "mSQjVQmMosMl"
+      },
+      "source": [
+        "#### Region\n",
+        "\n",
+        "You can also change the `REGION` variable used by Vertex AI. Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Se9FWWhLotvB"
+      },
+      "outputs": [],
+      "source": [
+        "REGION = \"us-central1\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "IfJRIMBpo5Pg"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "Depending on your Jupyter environment, you may have to manually authenticate. Follow the relevant instructions below."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "acFN0s3So9-Y"
+      },
+      "source": [
+        "**1. Vertex AI Workbench**\n",
+        "* Do nothing as you are already authenticated."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dQ_mNwuapE5T"
+      },
+      "source": [
+        "**2. Local JupyterLab instance, uncomment and run:**"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cR_MzpknpGgM"
+      },
+      "outputs": [],
+      "source": [
+        "# ! gcloud auth login"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "h-MuVI_ypJfw"
+      },
+      "source": [
+        "**3. Colab, uncomment and run:**"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "BeaQlCwMpQUT"
+      },
+      "outputs": [],
+      "source": [
+        "# from google.colab import auth\n",
+        "# auth.authenticate_user()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3ivZkPUjpaFz"
+      },
+      "source": [
+        "**4. Setup service account and permissions**\n",
+        "\n",
+        "A service account will be used to create custom training jobs. If you do not want to use your project's Compute Engine service account, set SERVICE_ACCOUNT to another service account ID. You can create a service account by following the [instructions](https://cloud.google.com/iam/docs/creating-managing-service-accounts#creating)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "vYE3b942wza4"
+      },
+      "outputs": [],
+      "source": [
+        "SERVICE_ACCOUNT = \"[your-service-account]\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "WWIxsCJFCg5Z"
+      },
+      "outputs": [],
+      "source": [
+        "# Grant Cloud Storage permission.\n",
+        "! gcloud projects add-iam-policy-binding $PROJECT_ID \\\n",
+        "        --member=\"serviceAccount:$SERVICE_ACCOUNT\" \\\n",
+        "        --role=\"roles/storage.admin\" \\\n",
+        "        --quiet"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "S_8_5jm-Gk6w"
+      },
+      "outputs": [],
+      "source": [
+        "# Grant AI Platform permission.\n",
+        "! gcloud projects add-iam-policy-binding $PROJECT_ID \\\n",
+        "        --member=\"serviceAccount:$SERVICE_ACCOUNT\" \\\n",
+        "        --role=\"roles/aiplatform.user\" \\\n",
+        "        --quiet"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "OKtKGmr9pfr6"
+      },
+      "source": [
+        "### Create a Cloud Storage bucket\n",
+        "\n",
+        "Create a storage bucket to store intermediate artifacts such as datasets."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "In3aQanwYjFB"
+      },
+      "outputs": [],
+      "source": [
+        "BUCKET_URI = \"gs://your-bucket-name-unique\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "GOaOsIjxp0oB"
+      },
+      "source": [
+        "**Only if your bucket doesn't already exist**: Run the following cell to create your Cloud Storage bucket."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Wn5QiIl2p16e"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ankcS-vtp7Wv"
+      },
+      "source": [
+        "### Import libraries"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "WffSImMvp-Po"
+      },
+      "outputs": [],
+      "source": [
+        "from google.cloud import aiplatform"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "OMrAJ8RGqBQu"
+      },
+      "source": [
+        "### Initialize Vertex AI SDK for Python\n",
+        "\n",
+        "Initialize the Vertex AI SDK for Python for your project and corresponding bucket."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "AWRzBFExqERG"
+      },
+      "outputs": [],
+      "source": [
+        "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=BUCKET_URI)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MYzX9ZgjsyyY"
+      },
+      "source": [
+        "### Enable Artifact Registry API\n",
+        "\n",
+        "First, you must enable the Artifact Registry API service for your project.\n",
+        "\n",
+        "Learn more about [Enabling service](https://cloud.google.com/artifact-registry/docs/enable-service)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "bG4Rpt-As55f"
+      },
+      "outputs": [],
+      "source": [
+        "! gcloud services enable artifactregistry.googleapis.com --quiet"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "-ayTbNdi62_t"
+      },
+      "source": [
+        "### Create a TensorBoard instance\n",
+        "\n",
+        "A Vertex AI TensorBoard instance, which is a regionalized resource storing your Vertex AI TensorBoard experiments, must be created before the experiments can be visualized. You can create multiple instances in a project. You can use command  `gcloud ai tensorboards list` to get a list of your existing TensorBoard instances."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9c3QrDTZdaxk"
+      },
+      "source": [
+        "#### Set your TensorBoard instance display name\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "azlwb__AX8gs"
+      },
+      "outputs": [],
+      "source": [
+        "TENSORBOARD_NAME = \"your-tensorboard-unique\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "vJrWKK0mY7H7"
+      },
+      "source": [
+        "#### Create a TensorBoard instance\n",
+        "\n",
+        "If you don't have a TensorBoard instance, create one by running the following cell:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "JqVNsRFrc_78"
+      },
+      "outputs": [],
+      "source": [
+        "tensorboard = aiplatform.Tensorboard.create(\n",
+        "    display_name=TENSORBOARD_NAME, project=PROJECT_ID, location=REGION\n",
+        ")\n",
+        "\n",
+        "TENSORBOARD_INSTANCE_NAME = tensorboard.resource_name\n",
+        "\n",
+        "print(\"TensorBoard instance name:\", TENSORBOARD_INSTANCE_NAME)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "yoR29gW2S24w"
+      },
+      "source": [
+        "## Train a model\n",
+        "\n",
+        "To train a model using your custom training code, choose one of the following options:\n",
+        "\n",
+        "- **Prebuilt container**: Load your custom training code as a Python package to a prebuilt container image from Google Cloud.\n",
+        "\n",
+        "- **Custom container**: Create your own container image that contains your custom training code.\n",
+        "\n",
+        "In this tutorial, we will train a custom model using a custom container."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "OrpUIkAIs_uQ"
+      },
+      "source": [
+        "### Create a private Docker repository\n",
+        "\n",
+        "Your first step is to create your own Docker repository in Google Artifact Registry."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "0amu4063tDnG"
+      },
+      "outputs": [],
+      "source": [
+        "DOCKER_REPOSITORY = f\"{PROJECT_ID}-repo-unique\"\n",
+        "\n",
+        "! gcloud artifacts repositories create {DOCKER_REPOSITORY} \\\n",
+        "    --repository-format=docker \\\n",
+        "    --location={REGION} \\\n",
+        "    --description=\"Repository for TensorBoard Custom Training Job\" \\\n",
+        "    --quiet\n",
+        "\n",
+        "! gcloud artifacts repositories list"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MPqbvhO6tK_e"
+      },
+      "source": [
+        "### Configure authentication to your private Docker repository\n",
+        "\n",
+        "Before you push or pull container images, configure Docker to use the `gcloud` command-line tool to authenticate requests to `Artifact Registry` for your region."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "yAeCXZiStPCX"
+      },
+      "outputs": [],
+      "source": [
+        "import sys\n",
+        "\n",
+        "IS_COLAB = \"google.colab\" in sys.modules\n",
+        "\n",
+        "if not IS_COLAB:\n",
+        "    ! gcloud auth configure-docker {REGION}-docker.pkg.dev --quiet"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "pSXrzCWi-bTD"
+      },
+      "source": [
+        "### Create a custom container image and push to your private Docker repository\n",
+        "\n",
+        "First, you create a training script file and a docker file.\n",
+        "\n",
+        "Create a directory for all of your training code."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ZYO55-ZK-lm2"
+      },
+      "outputs": [],
+      "source": [
+        "PYTHON_PACKAGE_APPLICATION_DIR = \"trainer\"\n",
+        "\n",
+        "!mkdir -p $PYTHON_PACKAGE_APPLICATION_DIR"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "hyAwgsoQmaYI"
+      },
+      "source": [
+        "#### Prepare the training script\n",
+        "\n",
+        "Your training code must be configured to write TensorBoard logs to a Cloud Storage bucket, the location of which Vertex AI Training automatically makes available through a predefined environment variable, `AIP_TENSORBOARD_LOG_DIR`.\n",
+        "\n",
+        "This can usually be done by providing `os.environ['AIP_TENSORBOARD_LOG_DIR']` as the log directory to the open source TensorBoard log writing APIs. \n",
+        "\n",
+        "For example, in TensorFlow 2.x, you can use following code to create a tensorboard_callback: \n",
+        "\n",
+        "    tensorboard_callback = tf.keras.callbacks.TensorBoard( \n",
+        "      log_dir=os.environ['AIP_TENSORBOARD_LOG_DIR'], \n",
+        "      histogram_freq=1) \n",
+        "`AIP_TENSORBOARD_LOG_DIR` is in the `BASE_OUTPUT_DIR` that you provide when creating the custom training job.\n",
+        "\n",
+        "To enable Vertex AI TensorBoard Profiler for your training job, add the following to your training script:\n",
+        "\n",
+        "Add the cloud_profiler import at your top level imports:\n",
+        "\n",
+        "    from google.cloud.aiplatform.training_utils import cloud_profiler\n",
+        "\n",
+        "\n",
+        "Initialize the cloud_profiler plugin by adding:\n",
+        "\n",
+        "\n",
+        "    cloud_profiler.init()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "8JCgWW7Au1w8"
+      },
+      "outputs": [],
+      "source": [
+        "%%writefile trainer/task.py\n",
+        "\n",
+        "import tensorflow as tf\n",
+        "import argparse\n",
+        "import os\n",
+        "import sys, traceback\n",
+        "from google.cloud.aiplatform.training_utils import cloud_profiler\n",
+        "\n",
+        "\"\"\"Train an mnist model and use cloud_profiler for profiling.\"\"\"\n",
+        "\n",
+        "def _create_model():\n",
+        "    model = tf.keras.models.Sequential(\n",
+        "        [\n",
+        "            tf.keras.layers.Flatten(input_shape=(28, 28)),\n",
+        "            tf.keras.layers.Dense(128, activation=\"relu\"),\n",
+        "            tf.keras.layers.Dropout(0.2),\n",
+        "            tf.keras.layers.Dense(10),\n",
+        "        ]\n",
+        "    )\n",
+        "    return model\n",
+        "\n",
+        "\n",
+        "def main(args):\n",
+        "    print('Loading and preprocessing data ...')\n",
+        "    mnist = tf.keras.datasets.mnist\n",
+        "\n",
+        "    (x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
+        "    x_train, x_test = x_train / 255.0, x_test / 255.0\n",
+        "\n",
+        "    print('Creating and training model ...')\n",
+        "\n",
+        "    model = _create_model()\n",
+        "    model.compile(\n",
+        "      optimizer=\"adam\",\n",
+        "      loss=tf.keras.losses.sparse_categorical_crossentropy,\n",
+        "      metrics=[\"accuracy\"],\n",
+        "    )\n",
+        "\n",
+        "    # Initialize the profiler.\n",
+        "    print('Initialize the profiler ...')\n",
+        "        \n",
+        "    try:\n",
+        "        cloud_profiler.init()\n",
+        "    except:\n",
+        "        ex_type, ex_value, ex_traceback = sys.exc_info()\n",
+        "        print(\"*** Unexpected:\", ex_type.__name__, ex_value)\n",
+        "        traceback.print_tb(ex_traceback, limit=10, file=sys.stdout)\n",
+        "    \n",
+        "    print('The profiler initiated.')\n",
+        "\n",
+        "    log_dir = \"logs\"\n",
+        "    if 'AIP_TENSORBOARD_LOG_DIR' in os.environ:\n",
+        "      log_dir = os.environ['AIP_TENSORBOARD_LOG_DIR']\n",
+        "\n",
+        "    print('Setting up the TensorBoard callback ...')\n",
+        "    tensorboard_callback = tf.keras.callbacks.TensorBoard(\n",
+        "        log_dir=log_dir,\n",
+        "        histogram_freq=1)\n",
+        "\n",
+        "    print('Training model ...')\n",
+        "    model.fit(\n",
+        "        x_train,\n",
+        "        y_train,\n",
+        "        epochs=args.epochs,\n",
+        "        verbose=0,\n",
+        "        callbacks=[tensorboard_callback],\n",
+        "    )\n",
+        "    print('Training completed.')\n",
+        "\n",
+        "    print('Saving model ...')\n",
+        "\n",
+        "    model_dir = \"model\"\n",
+        "    if 'AIP_MODEL_DIR' in os.environ:\n",
+        "      model_dir = os.environ['AIP_MODEL_DIR']\n",
+        "    tf.saved_model.save(model, model_dir)\n",
+        "\n",
+        "    print('Model saved at ' + model_dir)\n",
+        "\n",
+        "\n",
+        "if __name__ == \"__main__\":\n",
+        "    parser = argparse.ArgumentParser()\n",
+        "    parser.add_argument(\n",
+        "        \"--epochs\", type=int, default=100, help=\"Number of epochs to run model.\"\n",
+        "    )\n",
+        "    \n",
+        "    args = parser.parse_args()\n",
+        "    main(args)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "LiOEvgkA_90f"
+      },
+      "source": [
+        "#### Prepare the Dockerfile\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "aKcS7yzcAXWf"
+      },
+      "outputs": [],
+      "source": [
+        "%%writefile Dockerfile\n",
+        "# Specifies base image and tag\n",
+        "FROM us-docker.pkg.dev/vertex-ai/training/tf-cpu.2-9:latest\n",
+        "WORKDIR /root\n",
+        "\n",
+        "# Installs additional packages as you need.\n",
+        "RUN pip3 install google-cloud-aiplatform[cloud_profiler]\n",
+        "\n",
+        "# Copies the trainer code to the docker image.\n",
+        "RUN mkdir /root/trainer\n",
+        "COPY trainer/task.py /root/trainer/task.py\n",
+        "\n",
+        "# Sets up the entry point to invoke the trainer.\n",
+        "ENTRYPOINT [\"python\", \"-m\", \"trainer.task\"]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ihYFahRAr6sj"
+      },
+      "source": [
+        "#### Build a custom container image and push to your private Docker repository"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "pDs1qDLDA27F"
+      },
+      "outputs": [],
+      "source": [
+        "IMAGE_NAME = \"tensorboard-custom-container\"\n",
+        "IMAGE_URI = f\"{REGION}-docker.pkg.dev/{PROJECT_ID}/{DOCKER_REPOSITORY}/{IMAGE_NAME}\"\n",
+        "\n",
+        "! gcloud builds submit --project {PROJECT_ID} --region={REGION} --tag {IMAGE_URI} --timeout=60m --quiet"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "k4e6OYmimqTR"
+      },
+      "source": [
+        "### Create and run the custom training job\n",
+        "\n",
+        "Configure a [custom job](https://cloud.google.com/vertex-ai/docs/training/create-custom-job) with the custom container image."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "jd6P-lQhFBPg"
+      },
+      "outputs": [],
+      "source": [
+        "JOB_NAME = \"tensorboard-job-unique\"\n",
+        "\n",
+        "job = aiplatform.CustomContainerTrainingJob(\n",
+        "    display_name=JOB_NAME, container_uri=IMAGE_URI\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "51hKGTbU32Eg"
+      },
+      "source": [
+        "#### Run the custom training job\n",
+        "\n",
+        "Next, you run the custom job to start the training job by invoking the method `run`, with the following parameters:\n",
+        "\n",
+        "- `args`: The command-line arguments to pass to the training script.\n",
+        "   - `--epochs` : The number of epochs for training.\n",
+        "- `replica_count`: The number of compute instances for training (replica_count = 1 is single node training).\n",
+        "- `machine_type`: The machine type for the compute instances.\n",
+        "- `tensorboard`: The TensorBoard instance.\n",
+        "- `service_account`: The service account.\n",
+        "- `sync`: Whether to block until completion of the job."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "OC6_4KeI4Fit"
+      },
+      "outputs": [],
+      "source": [
+        "base_output_dir = \"{}/{}\".format(BUCKET_URI, JOB_NAME)\n",
+        "MACHINE_TYPE = \"n1-standard-4\"\n",
+        "EPOCHS = 2\n",
+        "training_args = [\n",
+        "    \"--epochs=\" + str(EPOCHS),\n",
+        "]\n",
+        "\n",
+        "job.run(\n",
+        "    args=training_args,\n",
+        "    replica_count=1,\n",
+        "    machine_type=MACHINE_TYPE,\n",
+        "    base_output_dir=base_output_dir,\n",
+        "    tensorboard=TENSORBOARD_INSTANCE_NAME,\n",
+        "    service_account=SERVICE_ACCOUNT,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "JkEe2Nb_85UD"
+      },
+      "source": [
+        "## View the TensorBoard Profiler dashboard\n",
+        "\n",
+        "When the custom job state switches to `Running`, you can access the Vertex AI TensorBoard Profiler dashboard through the Custom jobs page or the Experiments page on the Google Cloud console. \n",
+        "\n",
+        "The Google Cloud guide to [Profile model training performance using Profiler](https://cloud.google.com/vertex-ai/docs/experiments/tensorboard-profiler) provides detailed instructions for accessing the Vertex AI TensorBoard Profiler dashboard and capturing a profiling session. \n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "TpV-iwP9qw9c"
+      },
+      "source": [
+        "## Cleaning up\n",
+        "\n",
+        "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
+        "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
+        "\n",
+        "Otherwise, you can delete the individual resources you created in this tutorial:\n",
+        "\n",
+        "- Docker repository\n",
+        "- Training job\n",
+        "- TensorBoard instance\n",
+        "- Cloud Storage bucket\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "WR-ZhQ9XwpRI"
+      },
+      "outputs": [],
+      "source": [
+        "delete_tensorboard = True\n",
+        "delete_bucket = False\n",
+        "\n",
+        "# Delete docker repository.\n",
+        "! gcloud artifacts repositories delete $DOCKER_REPOSITORY --project {PROJECT_ID} --location {REGION} --quiet\n",
+        "\n",
+        "job.delete()\n",
+        "\n",
+        "if delete_tensorboard:\n",
+        "    tensorboard.delete()\n",
+        "\n",
+        "if delete_bucket and \"BUCKET_URI\" in globals():\n",
+        "    ! gsutil -m rm -r $BUCKET_URI"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "tensorboard_profiler_custom_training.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/official/training/hyperparameter_tuning_tensorflow.ipynb

@@ -34,18 +34,18 @@
         "<table align=\"left\">\n",
         "\n",
         "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/notebook_template.ipynb\">\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/training/hyperparameter_tuning_tensorflow.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/notebook_template.ipynb\">\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/training/hyperparameter_tuning_tensorflow.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/notebook_template.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/training/hyperparameter_tuning_tensorflow.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",

notebooks/official/workbench/ads_targetting/training-multi-class-classification-model-for-ads-targeting-usecase.ipynb

@@ -71,9 +71,15 @@
         "id": "1bea2b6e9b25"
       },
       "source": [
-        "## Objective\n",
+        "### Objective\n",
         "\n",
-        "This tutorial demonstrates how to collect data from BigQuery, preprocess it, and train a multi-class classification model on an E-commerce dataset. The steps performed include the following:\n",
+        "In this tutorial, you learn how to collect data from BigQuery, preprocess it, and train a multi-class classification model on an e-commerce dataset. \n",
+        "\n",
+        "This tutorial uses the following Google Cloud ML services and resources:\n",
+        "\n",
+        "- BigQuery\n",
+        "\n",
+        "The steps performed include:\n",
         "\n",
         "- Fetch the required data from BigQuery\n",
         "- Preprocess the data\n",
@@ -191,19 +197,9 @@
         "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
         "USER_FLAG = \"\"\n",
         "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\""
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "95826791kXT_"
-      },
-      "outputs": [],
-      "source": [
-        "! pip3 install {USER_FLAG} --upgrade pandas-gbq 'google-cloud-bigquery[bqstorage,pandas]' tensorflow sklearn protobuf==3.20.1 -q \\\n",
-        "                                     "
+        "    USER_FLAG = \"--user\"\n",
+        "\n",
+        "! pip3 install {USER_FLAG} --upgrade pandas-gbq 'google-cloud-bigquery[bqstorage,pandas]' tensorflow scikit-learn protobuf==3.20.3 -q"
       ]
     },
     {
@@ -236,6 +232,39 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "a47846030fef"
+      },
+      "source": [
+        "## Before you begin"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "71b29797836c"
+      },
+      "source": [
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. <a href=\"https://console.cloud.google.com/cloud-resource-manager\" target=\"_blank\">Select or create a Google Cloud project</a>. When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. <a href=\"https://cloud.google.com/billing/docs/how-to/modify-project\" target=\"_blank\">Make sure that billing is enabled for your project</a>.\n",
+        "\n",
+        "1. <a href=\"https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com\" target=\"_blank\">Enable the Vertex AI API</a>.\n",
+        "\n",
+        "1. If you are running this notebook locally, you will need to install the <a href=\"https://cloud.google.com/sdk\" target=\"_blank\">Cloud SDK</a>.\n",
+        "\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -251,7 +280,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "684595f229b3"
+        "id": "3c8049930470"
       },
       "outputs": [],
       "source": [
@@ -262,7 +291,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "5bf9979b96ff"
+        "id": "a36c4b991a39"
       },
       "outputs": [],
       "source": [
@@ -277,7 +306,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "07-xo93jlC6l"
+        "id": "684595f229b3"
       },
       "outputs": [],
       "source": [
@@ -312,7 +341,7 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "REGION = \"us-central1\"  # @param {type: \"string\"}\n",
         "\n",
         "if REGION == \"[your-region]\":\n",
         "    REGION = \"us-central1\""
@@ -324,9 +353,9 @@
         "id": "b2b04f364669"
       },
       "source": [
-        "#### Timestamp\n",
+        "#### UUID\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append it onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a UUID for each instance session, and append it onto the name of resources you create in this tutorial."
       ]
     },
     {
@@ -337,9 +366,16 @@
       },
       "outputs": [],
       "source": [
-        "from datetime import datetime\n",
+        "import random\n",
+        "import string\n",
         "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+        "\n",
+        "# Generate a uuid of a specifed length(default=8)\n",
+        "def generate_uuid(length: int = 8) -> str:\n",
+        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
+        "\n",
+        "\n",
+        "UUID = generate_uuid()"
       ]
     },
     {
@@ -445,7 +481,7 @@
       "outputs": [],
       "source": [
         "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
         "    BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
@@ -509,6 +545,7 @@
         "import warnings\n",
         "\n",
         "import pandas as pd\n",
+        "from google.cloud.bigquery import Client\n",
         "from sklearn.model_selection import train_test_split\n",
         "from sklearn.preprocessing import StandardScaler\n",
         "from tensorflow.keras import Sequential\n",
@@ -526,15 +563,7 @@
       "source": [
         "## Tutorial\n",
         "\n",
-        "### Fetch the data from BigQuery \n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "5c07be8840ae"
-      },
-      "source": [
+        "### Fetch the data from BigQuery \n",
         "If you are using ***Vertex AI Workbench managed notebooks instance***, below cell which starts with \"#@bigquery\" will be a SQL Query. If you are using Vertex AI Workbench user managed notebooks instance or Colab it will be a markdown cell."
       ]
     },
@@ -621,6 +650,17 @@
         "*Note: By default the data is loaded into a `df` variable, though this can be changed before executing the cell if required.*"
       ]
     },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3d44a10b6884"
+      },
+      "outputs": [],
+      "source": [
+        "client = Client(project=PROJECT_ID)"
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -629,12 +669,6 @@
       },
       "outputs": [],
       "source": [
-        "# The following two lines are only necessary to run once.\n",
-        "# Comment out otherwise for speed-up.\n",
-        "from google.cloud.bigquery import Client\n",
-        "\n",
-        "client = Client(project=PROJECT_ID)\n",
-        "\n",
         "query = \"\"\"WITH traindata AS (\n",
         "SELECT b.* except(ad_event_id, user_id), c.* except(id), d.* except(keyword_id, ad_id), a.amount, a.device_type, e.name\n",
         "FROM `looker-private-demo.ecomm.ad_events` a\n",
@@ -664,7 +698,7 @@
       },
       "source": [
         "### Preprocess the data\n",
-        "Select the necessary columns from the E-commerce data and divide them based on their type (numerical/categorical)."
+        "Select the necessary columns from the e-commerce data and divide them based on their type (numerical/categorical)."
       ]
     },
     {
@@ -688,21 +722,13 @@
         "num_cols = [\"age\", \"cpc_bid_amount\", \"quality_score\", \"amount\"]"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "9bd71de0d37e"
-      },
-      "source": [
-        "#### Select top three campaigns"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "ace612851261"
       },
       "source": [
+        "#### Select top three campaigns\n",
         "From the current dataset, only the top three campaigns will be chosen to target the users. All the relevant information about the advertisement and the user who purchased an item after seeing the advertisement is available in the dataframe already. "
       ]
     },
@@ -723,7 +749,7 @@
         "id": "f89106348ffe"
       },
       "source": [
-        "Encode the target variable."
+        "#### Encode the target variable."
       ]
     },
     {
@@ -743,15 +769,7 @@
         "id": "c2d5338b1b95"
       },
       "source": [
-        "#### One-hot encode the categorical variables"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "8902f763d1ca"
-      },
-      "source": [
+        "#### One-hot encode the categorical variables\n",
         "After one-hot encoding, the first level-column is dropped to avoid the [dummy-variable trap](https://en.wikipedia.org/wiki/Dummy_variable_(statistics)) scenario. This process is called *dummy-encoding*."
       ]
     },
@@ -786,7 +804,7 @@
         "id": "3abf027eda2d"
       },
       "source": [
-        "#### Split the data into train and test."
+        "#### Split the data into train and test"
       ]
     },
     {
@@ -811,7 +829,7 @@
         "id": "d1a32b9d9640"
       },
       "source": [
-        "#### Scale the data."
+        "#### Scale the data"
       ]
     },
     {
@@ -834,7 +852,7 @@
       },
       "source": [
         "### Train a TensorFlow model\n",
-        "#### Convert the target column to a categorical encoded colum (one-hot encoded)."
+        "Convert the target column to a categorical encoded colum (one-hot encoded)."
       ]
     },
     {
@@ -855,7 +873,7 @@
         "id": "3dd0014a7e1d"
       },
       "source": [
-        "#### Define hyperparameters for model training. \n",
+        "#### Define hyperparameters for model training\n",
         "\n",
         "*Note: Comment or remove the parameters from the following cell if they are provided already as an input parameter through the executor feature.*"
       ]
@@ -880,7 +898,7 @@
         "id": "406b731f576b"
       },
       "source": [
-        "#### Define the architecture and compile the model."
+        "#### Define the architecture and compile the model"
       ]
     },
     {
@@ -920,7 +938,7 @@
         "id": "4ab12c34f258"
       },
       "source": [
-        "#### Fit the model."
+        "#### Train the model"
       ]
     },
     {
@@ -934,22 +952,13 @@
         "history = model.fit(X_train, y_train_categ, epochs=50, verbose=1)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "51a2d0b52df3"
-      },
-      "source": [
-        "### Run the model on test data\n"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "f08445f2cd02"
       },
       "source": [
-        "#### Evaluate the model on test data."
+        "### Evaluate the model on test data."
       ]
     },
     {
@@ -970,7 +979,7 @@
         "id": "81ef0e081340"
       },
       "source": [
-        "**Please note that executor feature is available only in Vertex AI Workbench managed notebooks**"
+        "**Note:**  Please note that executor feature is available only in Vertex AI Workbench managed notebooks"
       ]
     },
     {
@@ -1045,7 +1054,7 @@
         "id": "29c0ca2a517a"
       },
       "source": [
-        "## Clean up\n",
+        "## Cleaning up\n",
         "\n",
         "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
         "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",

notebooks/official/workbench/demand_forecasting/forecasting-retail-demand.ipynb

@@ -75,7 +75,7 @@
         "id": "dbab58d4ae1a"
       },
       "source": [
-        "## Objective\n",
+        "### Objective\n",
         "In this tutorial, you learn how to build ARIMA (Autoregressive integrated moving average) model from BigQuery ML on retail data\n",
         "\n",
         "This tutorial uses the following Google Cloud ML services:\n",
@@ -97,7 +97,7 @@
         "id": "26a00a419045"
       },
       "source": [
-        "## Dataset \n",
+        "### Dataset \n",
         "\n",
         "This notebook uses the BigQuery public retail data set.\n",
         "The data covers 10 US stores and includes item level, department, product categories, and store details. In addition, it has explanatory variables such as price and gross margin. "
@@ -109,7 +109,7 @@
         "id": "17e0532066d7"
       },
       "source": [
-        "## Costs\n",
+        "### Costs\n",
         "This tutorial uses the following billable components of Google Cloud:\n",
         "\n",
         "* Vertex AI\n",
@@ -170,7 +170,7 @@
         "id": "oH0bZDCmp930"
       },
       "source": [
-        "### Install additional packages\n"
+        "## Install additional packages\n"
       ]
     },
     {
@@ -192,18 +192,9 @@
         "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
         "USER_FLAG = \"\"\n",
         "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\""
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "qkHGZ4xYp933"
-      },
-      "outputs": [],
-      "source": [
-        "! pip3 install {USER_FLAG} --upgrade pandas-gbq 'google-cloud-bigquery[bqstorage,pandas]' sklearn \n"
+        "    USER_FLAG = \"--user\"\n",
+        "\n",
+        "! pip3 install {USER_FLAG} --upgrade pandas-gbq 'google-cloud-bigquery[bqstorage,pandas]' scikit-learn"
       ]
     },
     {
@@ -261,9 +252,9 @@
         "\n",
         "1. [Enable the Vertex AI, Cloud Storage, and Compute Engine APIs](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component,storage-component.googleapis.com). \n",
         "\n",
-        "1. [Configure your Google Cloud project for Vertex Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/configure-project).\n",
+        "1. [Configure your Google Cloud project for Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/configure-project).\n",
         "\n",
-        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
         "\n",
         "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
         "Cloud SDK uses the right project for all the commands in this notebook.\n",
@@ -325,9 +316,9 @@
         "id": "07fc8daffbf9"
       },
       "source": [
-        "#### Timestamp\n",
+        "#### UUID\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append it onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
       ]
     },
     {
@@ -338,9 +329,16 @@
       },
       "outputs": [],
       "source": [
-        "from datetime import datetime\n",
+        "import random\n",
+        "import string\n",
         "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+        "\n",
+        "# Generate a uuid of a specifed length(default=8)\n",
+        "def generate_uuid(length: int = 8) -> str:\n",
+        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
+        "\n",
+        "\n",
+        "UUID = generate_uuid()"
       ]
     },
     {
@@ -433,7 +431,7 @@
         "id": "b81a2c71fa3a"
       },
       "source": [
-        "Load the required libraries."
+        "**Load the required libraries.**"
       ]
     },
     {
@@ -444,8 +442,6 @@
       },
       "outputs": [],
       "source": [
-        "import datetime\n",
-        "\n",
         "import matplotlib.pyplot as plt\n",
         "import pandas as pd\n",
         "from google.cloud import bigquery\n",
@@ -467,7 +463,7 @@
         "id": "40902aa0f1de"
       },
       "source": [
-        "Set the name for the table"
+        "**Set the name for the table**"
       ]
     },
     {
@@ -491,7 +487,7 @@
         "id": "36d3a8aec700"
       },
       "source": [
-        "Create a BigQuery datatset"
+        "**Create a BigQuery datatset**"
       ]
     },
     {
@@ -502,7 +498,7 @@
       },
       "outputs": [],
       "source": [
-        "dataset_id = \"demandforecasting\" + \"_\" + TIMESTAMP"
+        "dataset_id = \"demandforecasting\" + \"_\" + UUID"
       ]
     },
     {
@@ -533,7 +529,7 @@
         "id": "RxnaBh4sp93_"
       },
       "source": [
-        "(**Optional**)If you are using Vertex AI Workbench managed notebooks instance, once the results from BigQuery are displayed in the above cell, click the **Query and load as DataFrame** button and execute the generated code stub to fetch the data into the current notebook as a dataframe.\n",
+        "(**Optional**)If you are using Vertex AI Workbench managed notebooks instance, once the results from BigQuery are displayed in the below cell, click the **Query and load as DataFrame** button and execute the generated code stub to fetch the data into the current notebook as a dataframe.\n",
         "\n",
         "*Note: By default the data is loaded into a `df` variable, though this can be changed before executing the cell if required.*"
       ]
@@ -564,15 +560,7 @@
         "id": "7002b223b2b5"
       },
       "source": [
-        "## Explore the Data\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "6a3ee00b6b9f"
-      },
-      "source": [
+        "## Explore the Data\n",
         "View the data that is stored in the public BigQuery dataset."
       ]
     },
@@ -617,7 +605,7 @@
         "id": "6d52803e14f2"
       },
       "source": [
-        "Create a view named `important_fields` using only the `transaction_timestamp` and `line_items` fields, where the store ID is 10."
+        "**Create a view named `important_fields` using only the `transaction_timestamp` and `line_items` fields, where the store ID is 10.**"
       ]
     },
     {
@@ -659,7 +647,7 @@
         "id": "a9ca8a8bbf82"
       },
       "source": [
-        "Look at the data in the `important_fields` view."
+        "**Look at the data in the `important_fields` view.**"
       ]
     },
     {
@@ -685,17 +673,8 @@
         "\"\"\".format(\n",
         "    dataset_id=dataset_id\n",
         ")\n",
-        "query_job = client.query(query)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "LTHxGXsGp94B"
-      },
-      "outputs": [],
-      "source": [
+        "query_job = client.query(query)\n",
+        "\n",
         "query_job.to_dataframe()"
       ]
     },
@@ -705,7 +684,7 @@
         "id": "5f03b43de905"
       },
       "source": [
-        "Convert the `transaction_timestamp` field into a date."
+        "**Convert the `transaction_timestamp` field into a date.**"
       ]
     },
     {
@@ -747,7 +726,7 @@
         "id": "f0babc4a23fa"
       },
       "source": [
-        "View the data and check the `date` field values."
+        "**View the data and check the `date` field values.**"
       ]
     },
     {
@@ -773,17 +752,8 @@
         "\"\"\".format(\n",
         "    dataset_id=dataset_id\n",
         ")\n",
-        "query_job = client.query(query)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "3mtjBiiPp94D"
-      },
-      "outputs": [],
-      "source": [
+        "query_job = client.query(query)\n",
+        "\n",
         "query_job.to_dataframe()"
       ]
     },
@@ -793,7 +763,7 @@
         "id": "990d04eab2e1"
       },
       "source": [
-        "Load the data into a dataframe."
+        "**Load the data into a dataframe.**"
       ]
     },
     {
@@ -813,7 +783,7 @@
         "id": "f25cf5322fbc"
       },
       "source": [
-        "Check the data types of your dataframe's fields."
+        "**Check the data types of your dataframe's fields.**"
       ]
     },
     {
@@ -875,7 +845,7 @@
         "id": "6c677762b34a"
       },
       "source": [
-        "View the data."
+        "**View the data.**"
       ]
     },
     {
@@ -901,17 +871,8 @@
         "\"\"\".format(\n",
         "    dataset_id=dataset_id\n",
         ")\n",
-        "query_job = client.query(query)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "cTdONqeAp94F"
-      },
-      "outputs": [],
-      "source": [
+        "query_job = client.query(query)\n",
+        "\n",
         "query_job.to_dataframe()"
       ]
     },
@@ -921,7 +882,7 @@
         "id": "7acc57476f1c"
       },
       "source": [
-        "Remove the extra columns to keep only `date` and `product_id`."
+        "**Remove the extra columns to keep only `date` and `product_id`.**"
       ]
     },
     {
@@ -963,7 +924,7 @@
         "id": "4f9d72c483e8"
       },
       "source": [
-        "View the data."
+        "**View the data.**"
       ]
     },
     {
@@ -989,17 +950,8 @@
         "\"\"\".format(\n",
         "    dataset_id=dataset_id\n",
         ")\n",
-        "query_job = client.query(query)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "Ao0EHdchp94G"
-      },
-      "outputs": [],
-      "source": [
+        "query_job = client.query(query)\n",
+        "\n",
         "query_job.to_dataframe()"
       ]
     },
@@ -1009,7 +961,7 @@
         "id": "7c0c4245acb7"
       },
       "source": [
-        "Count the sales of a product for each date."
+        "**Count the sales of a product for each date.**"
       ]
     },
     {
@@ -1068,17 +1020,8 @@
         "\"\"\".format(\n",
         "    dataset_id=dataset_id\n",
         ")\n",
-        "query_job = client.query(query)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "7gLx8lkYp94H"
-      },
-      "outputs": [],
-      "source": [
+        "query_job = client.query(query)\n",
+        "\n",
         "query_job.to_dataframe()"
       ]
     },
@@ -1088,7 +1031,7 @@
         "id": "722fd013c28c"
       },
       "source": [
-        "Create a view for the five products that have sold the most units over the entire date range."
+        "**Create a view for the five products that have sold the most units over the entire date range.**"
       ]
     },
     {
@@ -1181,17 +1124,8 @@
         "\"\"\".format(\n",
         "    dataset_id=dataset_id\n",
         ")\n",
-        "query_job = client.query(query)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "aj8MMWkQp94I"
-      },
-      "outputs": [],
-      "source": [
+        "query_job = client.query(query)\n",
+        "\n",
         "query_job.to_dataframe()"
       ]
     },
@@ -1201,7 +1135,7 @@
         "id": "292855967806"
       },
       "source": [
-        "Load the data into a dataframe."
+        "**Load the data into a dataframe and view the data.**"
       ]
     },
     {
@@ -1212,27 +1146,8 @@
       },
       "outputs": [],
       "source": [
-        "df = query_job.to_dataframe()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "6ebc0bb8ab19"
-      },
-      "source": [
-        "View the data."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "8d11dc64ae2a"
-      },
-      "outputs": [],
-      "source": [
-        "df"
+        "df = query_job.to_dataframe()\n",
+        "print(df)"
       ]
     },
     {
@@ -1241,7 +1156,7 @@
         "id": "5c78641e2881"
       },
       "source": [
-        "Check the data types of your dataframe's fields."
+        "**Check the data types of your dataframe's fields.**"
       ]
     },
     {
@@ -1261,7 +1176,7 @@
         "id": "ef761dcd0109"
       },
       "source": [
-        "Convert the `date` field's data type to `datetime`."
+        "**Convert the `date` field's data type to `datetime`.**"
       ]
     },
     {
@@ -1285,7 +1200,7 @@
         "\n",
         "To construct a dataframe with `0` values for the `sales_count` field, on dates in which products were not sold, determine the minimum and maximum dates so that you know which dates need `0` values.\n",
         "\n",
-        "First, get the earliest (minimum) date."
+        "**First, get the earliest (minimum) date.**"
       ]
     },
     {
@@ -1311,17 +1226,8 @@
         "\"\"\".format(\n",
         "    dataset_id=dataset_id\n",
         ")\n",
-        "query_job = client.query(query)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "e6FG8Ohcp94K"
-      },
-      "outputs": [],
-      "source": [
+        "query_job = client.query(query)\n",
+        "\n",
         "query_job.to_dataframe()"
       ]
     },
@@ -1331,7 +1237,7 @@
         "id": "fbb934199011"
       },
       "source": [
-        "Get the latest (maximum) date."
+        "**Get the latest (maximum) date.**"
       ]
     },
     {
@@ -1357,17 +1263,8 @@
         "\"\"\".format(\n",
         "    dataset_id=dataset_id\n",
         ")\n",
-        "query_job = client.query(query)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "swjUOJmcp94K"
-      },
-      "outputs": [],
-      "source": [
+        "query_job = client.query(query)\n",
+        "\n",
         "query_job.to_dataframe()"
       ]
     },
@@ -1377,7 +1274,7 @@
         "id": "247a08d866ef"
       },
       "source": [
-        "Add the full date range of values to a dataframe."
+        "**Add the full date range of values to a dataframe.**"
       ]
     },
     {
@@ -1397,7 +1294,7 @@
         "id": "b9d3afc9f7cd"
       },
       "source": [
-        "Get a description of the `dates` dataframe."
+        "**Get a description of the `dates` dataframe.**"
       ]
     },
     {
@@ -1417,7 +1314,7 @@
         "id": "49eb81d44b65"
       },
       "source": [
-        "View the data for one of the products, sorted by date, to show that many dates are not present in the dataset."
+        "**View the data for one of the products, sorted by date, to show that many dates are not present in the dataset.**"
       ]
     },
     {
@@ -1468,17 +1365,8 @@
         "    \"int\"\n",
         ")  # convert sales_count column to integer\n",
         "print(\"data after converting for a product with product_id 20552\")\n",
-        "df1"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "4074c59c4fcc"
-      },
-      "outputs": [],
-      "source": [
+        "print(df1)\n",
+        "\n",
         "df2 = (\n",
         "    pd.merge(\n",
         "        df.loc[df[\"product_id\"] == 13596],\n",
@@ -1496,17 +1384,8 @@
         "df2[\"sales_count\"] = df2[\"sales_count\"].astype(\n",
         "    \"int\"\n",
         ")  # convert sales_count column to integer\n",
-        "df2"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "589ffdbf42f1"
-      },
-      "outputs": [],
-      "source": [
+        "print(df2)\n",
+        "\n",
         "df3 = (\n",
         "    pd.merge(\n",
         "        df.loc[df[\"product_id\"] == 23641],\n",
@@ -1524,17 +1403,8 @@
         "df3[\"sales_count\"] = df3[\"sales_count\"].astype(\n",
         "    \"int\"\n",
         ")  # convert sales_count column to integer\n",
-        "df3"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "df19f886f7b7"
-      },
-      "outputs": [],
-      "source": [
+        "print(df3)\n",
+        "\n",
         "df4 = (\n",
         "    pd.merge(\n",
         "        df.loc[df[\"product_id\"] == 28305],\n",
@@ -1552,17 +1422,8 @@
         "df4[\"sales_count\"] = df4[\"sales_count\"].astype(\n",
         "    \"int\"\n",
         ")  # convert sales_count column to integer\n",
-        "df4"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "2bde5882369e"
-      },
-      "outputs": [],
-      "source": [
+        "print(df4)\n",
+        "\n",
         "df5 = (\n",
         "    pd.merge(\n",
         "        df.loc[df[\"product_id\"] == 20547],\n",
@@ -1580,7 +1441,7 @@
         "df5[\"sales_count\"] = df5[\"sales_count\"].astype(\n",
         "    \"int\"\n",
         ")  # convert sales_count column to integer\n",
-        "df5"
+        "print(df5)"
       ]
     },
     {
@@ -1589,7 +1450,7 @@
         "id": "6a2d033b733e"
       },
       "source": [
-        "Merge all five dataframes into one new dataframe"
+        "**Merge all five dataframes into one new dataframe.**"
       ]
     },
     {
@@ -1611,7 +1472,7 @@
         "id": "4a75fa1dc8dc"
       },
       "source": [
-        "Reset the index of the dataframe."
+        "**Reset the index of the dataframe.**"
       ]
     },
     {
@@ -1622,18 +1483,8 @@
       },
       "outputs": [],
       "source": [
-        "new_df.reset_index(inplace=True, drop=True)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "5312dfe1be15"
-      },
-      "outputs": [],
-      "source": [
-        "new_df"
+        "new_df.reset_index(inplace=True, drop=True)\n",
+        "print(new_df)"
       ]
     },
     {
@@ -1642,7 +1493,7 @@
         "id": "97e1289b2106"
       },
       "source": [
-        "View the five product IDs."
+        "**View the five product IDs.**"
       ]
     },
     {
@@ -1688,7 +1539,7 @@
         "id": "aa06b0e893cb"
       },
       "source": [
-        "Plot `sales_count` over time, for each product."
+        "**Plot `sales_count` over time, for each product.**"
       ]
     },
     {
@@ -1772,7 +1623,7 @@
         "id": "755a811fbad5"
       },
       "source": [
-        "List the data types for the `new_df` dataframe."
+        "**List the data types for the `new_df` dataframe.**"
       ]
     },
     {
@@ -1792,7 +1643,7 @@
         "id": "ff06b56ffa7a"
       },
       "source": [
-        "Create a new BigQuery table out of the `new_df` dataframe."
+        "**Create a new BigQuery table out of the `new_df` dataframe.**"
       ]
     },
     {
@@ -1832,7 +1683,7 @@
         "id": "c2e0e9aa67cd"
       },
       "source": [
-        "Create a training dataset by setting a date range that limits the data being used."
+        "**Create a training dataset by setting a date range that limits the data being used.**"
       ]
     },
     {
@@ -1873,7 +1724,7 @@
         "id": "2f7d9d2d4229"
       },
       "source": [
-        "Select the original data for plotting."
+        "**Select the original data for plotting.**"
       ]
     },
     {
@@ -1899,7 +1750,7 @@
       "source": [
         "## Modeling with BigQuery and the ARIMA model\n",
         "\n",
-        "Create an ARIMA model using the training data."
+        "**Create an ARIMA model using the training data.**"
       ]
     },
     {
@@ -1934,7 +1785,7 @@
         "id": "c45e18a773ad"
       },
       "source": [
-        "Train the ARIMA model."
+        "**Train the ARIMA model.**"
       ]
     },
     {
@@ -2008,7 +1859,7 @@
         "id": "801528e3e2c7"
       },
       "source": [
-        "Load the data into a dataframe named `dfforecast`."
+        "**Load the data into a dataframe named `dfforecast`.**"
       ]
     },
     {
@@ -2043,7 +1894,7 @@
         "id": "1e0549381849"
       },
       "source": [
-        "View the first few rows."
+        "**View the first few rows.**"
       ]
     },
     {
@@ -2074,7 +1925,7 @@
         "id": "6fa36b762a33"
       },
       "source": [
-        "Clean the historical and forecasted values for plotting."
+        "**Clean the historical and forecasted values for plotting.**"
       ]
     },
     {
@@ -2102,7 +1953,7 @@
         "id": "d40a95ad0616"
       },
       "source": [
-        "Plot the historical and forecast data.\n"
+        "**Plot the historical and forecast data.**\n"
       ]
     },
     {