Improve download instructions

.cloud-build/cleanup/cleanup-requirements.txt

@@ -1,2 +1 @@
-ratemate
 google-cloud-aiplatform

.cloud-build/cleanup/cleanup.py

@@ -5,8 +5,6 @@ from resource_cleanup_manager import (
     ModelResourceCleanupManager,
     EndpointResourceCleanupManager,
     ResourceCleanupManager,
-    MatchingEngineIndexEndpointResourceCleanupManager,
-    MatchingEngineIndexResourceCleanupManager,
 )
 
 rate_limit = RateLimit(max_count=25, per=60, greedy=False)
@@ -42,12 +40,10 @@ if is_dry_run:
     print("Starting cleanup in dry run mode...")
 
 # List of all cleanup managers
-managers: List[ResourceCleanupManager] = [
+managers = [
     DatasetResourceCleanupManager(),
     EndpointResourceCleanupManager(),
     ModelResourceCleanupManager(),  # ModelResourceCleanupManager must follow EndpointResourceCleanupManager due to deployed models blocking model deletion.
-    MatchingEngineIndexEndpointResourceCleanupManager(),
-    MatchingEngineIndexResourceCleanupManager(),
 ]
 
 run_cleanup_managers(managers=managers, is_dry_run=is_dry_run)

.cloud-build/cleanup/resource_cleanup_manager.py

@@ -109,11 +109,3 @@ class EndpointResourceCleanupManager(VertexAIResourceCleanupManager):
 
 class ModelResourceCleanupManager(VertexAIResourceCleanupManager):
     vertex_ai_resource = aiplatform.Model
-
-
-class MatchingEngineIndexResourceCleanupManager(VertexAIResourceCleanupManager):
-    vertex_ai_resource = aiplatform.MatchingEngineIndex
-
-
-class MatchingEngineIndexEndpointResourceCleanupManager(VertexAIResourceCleanupManager):
-    vertex_ai_resource = aiplatform.MatchingEngineIndexEndpoint

.cloud-build/execute_changed_notebooks_cli.py

@@ -68,12 +68,6 @@ parser.add_argument(
     help="A service account. This is used to inject a variable value into the notebook before running. This is not the account that will run the notebook.",
     required=True,
 )
-parser.add_argument(
-    "--variable_vpc_network",
-    type=str,
-    help="The full VPC network name. See https://cloud.google.com/compute/docs/networks-and-firewalls#networks. Format is projects/{project}/global/networks/{network}, where {project} is a project number, as in '12345', and {network} is network name. See <https://cloud.google.com/compute/docs/reference/rest/v1/networks/insert> for details. This is used to inject a variable value into the notebook before running.",
-    required=False,
-)
 parser.add_argument(
     "--staging_bucket",
     type=str,
@@ -120,11 +114,10 @@ execute_changed_notebooks_helper.process_and_execute_notebooks(
     container_uri=args.container_uri,
     staging_bucket=args.staging_bucket,
     artifacts_bucket=args.artifacts_bucket,
-    should_parallelize=args.should_parallelize,
-    timeout=args.timeout,
     variable_project_id=args.variable_project_id,
     variable_region=args.variable_region,
     variable_service_account=args.variable_service_account,
-    variable_vpc_network=args.variable_vpc_network,
     private_pool_id=args.private_pool_id,
+    should_parallelize=args.should_parallelize,
+    timeout=args.timeout,
 )

.cloud-build/execute_changed_notebooks_helper.py

@@ -17,16 +17,13 @@ import concurrent
 import dataclasses
 import datetime
 import functools
-import json
 import git
 import operator
 import os
 import pathlib
 import re
 import subprocess
-import utils
 from typing import List, Optional
-from utils import util
 
 import execute_notebook_helper
 import execute_notebook_remote
@@ -38,7 +35,6 @@ from utils import NotebookProcessors, util
 
 # A buffer so that workers finish before the orchestrating job
 WORKER_TIMEOUT_BUFFER_IN_SECONDS: int = 60 * 60
-PYTHON_VERSION = "3.9"  # Set default python version
 
 
 def format_timedelta(delta: datetime.timedelta) -> str:
@@ -70,23 +66,14 @@ class NotebookExecutionResult:
     log_url: str
     output_uri: str
     build_id: str
-    logs_bucket: str
     error_message: Optional[str]
 
-    @property
-    def output_uri_web(self) -> Optional[str]:
-        if self.output_uri.startswith("gs://"):
-            return f"https://storage.googleapis.com/{self.output_uri[5:]}"
-        else:
-            return None
-
 
 def _process_notebook(
     notebook_path: str,
     variable_project_id: str,
     variable_region: str,
     variable_service_account: str,
-    variable_vpc_network: Optional[str],
 ):
     # Read notebook
     with open(notebook_path) as f:
@@ -99,10 +86,8 @@ def _process_notebook(
             "PROJECT_ID": variable_project_id,
             "REGION": variable_region,
             "SERVICE_ACCOUNT": variable_service_account,
-            "VPC_NETWORK": variable_vpc_network,
         },
     )
-    unique_strings_preprocessor = NotebookProcessors.UniqueStringsPreprocessor()
 
     # Use no-execute preprocessor
     (
@@ -111,41 +96,11 @@ def _process_notebook(
     ) = remove_no_execute_cells_preprocessor.preprocess(nb)
 
     (nb, resources) = update_variables_preprocessor.preprocess(nb, resources)
-    (nb, resources) = unique_strings_preprocessor.preprocess(nb, resources)
 
     with open(notebook_path, mode="w", encoding="utf-8") as new_file:
         nbformat.write(nb, new_file)
 
 
-def _get_notebook_python_version(notebook_path: str) -> str:
-    """
-    Get the python version for running the notebook if it is specified in
-    the notebook.
-    """
-    python_version = PYTHON_VERSION
-
-    # Load the notebook
-    file = open(notebook_path)
-    src = file.read()
-    nb_json = json.loads(src)
-
-    # Iterate over the cells in the ipynb
-    for cell in nb_json["cells"]:
-        if cell["cell_type"] == "markdown":
-            markdown = str.join("", cell["source"])
-
-            # Look for the python version specification pattern
-            re_match = re.search(
-                "python version = (\d\.\d)", markdown, flags=re.IGNORECASE
-            )
-            if re_match:
-                # get the version number
-                python_version = re_match.group(1)
-                break
-
-    return python_version
-
-
 def _create_tag(filepath: str) -> str:
     tag = os.path.basename(os.path.normpath(filepath))
     tag = re.sub("[^0-9a-zA-Z_.-]+", "-", tag)
@@ -166,9 +121,8 @@ def process_and_execute_notebook(
     variable_project_id: str,
     variable_region: str,
     variable_service_account: str,
-    variable_vpc_network: Optional[str],
     private_pool_id: Optional[str],
-    deadline: datetime.datetime,
+    deadline: datetime,
     notebook: str,
     should_get_tail_logs: bool = False,
 ) -> NotebookExecutionResult:
@@ -176,13 +130,6 @@ def process_and_execute_notebook(
 
     print(f"Running notebook: {notebook}")
 
-    # Handle empty strings
-    if not variable_vpc_network:
-        variable_vpc_network = None
-
-    if not private_pool_id:
-        private_pool_id = None
-
     # Create paths
     notebook_output_uri = "/".join([artifacts_bucket, pathlib.Path(notebook).name])
 
@@ -196,7 +143,6 @@ def process_and_execute_notebook(
         output_uri=notebook_output_uri,
         log_url="",
         build_id="",
-        logs_bucket="",
         error_message=None,
     )
 
@@ -204,19 +150,12 @@ def process_and_execute_notebook(
     time_start = datetime.datetime.now()
     operation = None
     try:
-        # Get the python version for running the notebook if specified
-        notebook_exec_python_version = _get_notebook_python_version(
-            notebook_path=notebook
-        )
-        print(f"Running notebook with python {notebook_exec_python_version}")
-
         # Pre-process notebook by substituting variable names
         _process_notebook(
             notebook_path=notebook,
             variable_project_id=variable_project_id,
             variable_region=variable_region,
             variable_service_account=variable_service_account,
-            variable_vpc_network=variable_vpc_network,
         )
 
         # Upload the pre-processed code to a GCS bucket
@@ -236,13 +175,11 @@ def process_and_execute_notebook(
             private_pool_id=private_pool_id,
             private_pool_region=variable_region,
             timeout_in_seconds=timeout_in_seconds,
-            python_version=notebook_exec_python_version,
         )
 
         operation_metadata = BuildOperationMetadata(mapping=operation.metadata)
         result.build_id = operation_metadata.build.id
         result.log_url = operation_metadata.build.log_url
-        result.logs_bucket = operation_metadata.build.logs_bucket
 
         # Block and wait for the result
         operation_result = operation.result()
@@ -322,8 +259,8 @@ def get_changed_notebooks(
             notebooks = []
     else:
         print(f"Looking for all notebooks.")
-        notebooks_str = subprocess.check_output(["git", "ls-files"] + test_paths)
-        notebooks = notebooks_str.decode("utf-8").split("\n")
+        notebooks = subprocess.check_output(["git", "ls-files"] + test_paths)
+        notebooks = notebooks.decode("utf-8").split("\n")
 
     notebooks = [notebook for notebook in notebooks if notebook.endswith(".ipynb")]
     notebooks = [notebook for notebook in notebooks if len(notebook) > 0]
@@ -342,13 +279,12 @@ def process_and_execute_notebooks(
     container_uri: str,
     staging_bucket: str,
     artifacts_bucket: str,
-    should_parallelize: bool,
-    timeout: int,
     variable_project_id: str,
     variable_region: str,
     variable_service_account: str,
-    variable_vpc_network: Optional[str] = None,
-    private_pool_id: Optional[str] = None,
+    private_pool_id: Optional[str],
+    should_parallelize: bool,
+    timeout: int,
 ):
     """
     Run the notebooks that exist under the folders defined in the test_paths_file.
@@ -384,7 +320,7 @@ def process_and_execute_notebooks(
         seconds=max(timeout - WORKER_TIMEOUT_BUFFER_IN_SECONDS, 0)
     )
 
-    if len(notebooks) >= 1:
+    if len(notebooks) > 1:
         notebook_execution_results: List[NotebookExecutionResult] = []
 
         print(f"Found {len(notebooks)} modified notebooks: {notebooks}")
@@ -406,7 +342,6 @@ def process_and_execute_notebooks(
                             variable_project_id,
                             variable_region,
                             variable_service_account,
-                            variable_vpc_network,
                             private_pool_id,
                             deadline,
                         ),
@@ -422,7 +357,6 @@ def process_and_execute_notebooks(
                     variable_project_id=variable_project_id,
                     variable_region=variable_region,
                     variable_service_account=variable_service_account,
-                    variable_vpc_network=variable_vpc_network,
                     private_pool_id=private_pool_id,
                     deadline=deadline,
                     notebook=notebook,
@@ -448,47 +382,13 @@ def process_and_execute_notebooks(
                         format_timedelta(result.duration),
                         result.log_url,
                         result.output_uri,
-                        result.output_uri_web,
-                        result.logs_bucket,
                     ]
                     for result in results_sorted
                 ],
-                headers=[
-                    "build_tag",
-                    "status",
-                    "duration",
-                    "log_url",
-                    "output_uri",
-                    "output_uri_web",
-                    "logs_bucket",
-                ],
+                headers=["build_tag", "status", "duration", "log_url", "output_url"],
             )
         )
 
-        if len(notebooks) == 1:
-            print("=" * 100)
-            print("The notebook execution build log:\n")
-            print("=" * 100)
-
-            build_id = results_sorted[0].build_id
-            logs_bucket_name = (results_sorted[0].logs_bucket).removeprefix("gs://")
-            log_file_name = f"log-{build_id}.txt"
-
-            log_contents = util.download_blob_into_memory(
-                bucket_name=logs_bucket_name,
-                blob_name=log_file_name,
-                download_as_text=True,
-            )
-
-            # Remove extra steps from the log
-            match = re.search("starting Step #4", log_contents, flags=re.IGNORECASE)
-
-            if match is not None:
-                match_index = match.span()[0]
-                print(log_contents[match_index:])
-            else:
-                print(log_contents)
-
         print("\n=== END RESULTS===\n")
 
         total_notebook_duration = functools.reduce(
@@ -504,5 +404,24 @@ def process_and_execute_notebooks(
         # Raise error if any notebooks failed
         if not all([result.is_pass for result in results_sorted]):
             raise RuntimeError("Notebook failures detected. See logs for details")
+
+    elif len(notebooks) == 1:
+        notebook = notebooks[0]
+
+        # Pre-process notebook by substituting variable names
+        _process_notebook(
+            notebook_path=notebook,
+            variable_project_id=variable_project_id,
+            variable_region=variable_region,
+            variable_service_account=variable_service_account,
+        )
+
+        execute_notebook_helper.execute_notebook(
+            notebook_source=notebook,
+            output_file_or_uri="/".join(
+                [artifacts_bucket, pathlib.Path(notebook).name]
+            ),
+            should_log_output=True,
+        )
     else:
         print("No notebooks modified in this pull request.")

.cloud-build/execute_notebook_helper.py

@@ -26,9 +26,6 @@ from utils import util
 
 # This script is used to execute a notebook and write out the output notebook.
 
-# This is used to force papermill to use this kernel to run the notebook instead of any defined inside the notebook itself
-DEFAULT_KERNEL_NAME = "python3"
-
 
 def execute_notebook(
     notebook_source: str,
@@ -53,17 +50,6 @@ def execute_notebook(
 
     execution_exception = None
 
-    print("\n=== DOWNLOAD EXECUTED NOTEBOOK ===\n")
-    print(f"Please debug the executed notebook by downloading the executed notebook:")
-
-    print("Option 1. Using gsutil. Run the following command in your terminal.")
-    print(f'\tgsutil cp "{output_file_or_uri}" .')
-
-    print("Option 2. Using this link.")
-    print(f"\thttps://storage.googleapis.com/{output_file_or_uri[5:]}")
-
-    print("\n======\n")
-
     # Execute notebook
     try:
         # Execute notebook
@@ -72,7 +58,6 @@ def execute_notebook(
             output_path=notebook_source,
             progress_bar=should_log_output,
             request_save_on_cell_execute=should_log_output,
-            kernel_name=DEFAULT_KERNEL_NAME,
             log_output=should_log_output,
             stdout_file=sys.stdout if should_log_output else None,
             stderr_file=sys.stderr if should_log_output else None,
@@ -86,6 +71,10 @@ def execute_notebook(
             util.upload_file(notebook_source, remote_file_path=output_file_or_uri)
 
             print("\n=== EXECUTION FINISHED ===\n")
+            print(
+                f"Please debug the executed notebook by downloading: {output_file_or_uri}"
+            )
+            print("\n======\n")
         else:
             # Create directories if they don't exist
             if not os.path.exists(os.path.dirname(output_file_or_uri)):

.cloud-build/execute_notebook_remote.py

@@ -40,7 +40,6 @@ def execute_notebook_remote(
     private_pool_region: Optional[str],
     tag: Optional[str],
     timeout_in_seconds: Optional[int] = None,
-    python_version: Optional[str] = None
 ) -> operation.Operation:
     """Create and execute a single notebook on Google Cloud Build"""
     # Load build steps from YAML
@@ -51,12 +50,8 @@ def execute_notebook_remote(
         "_PYTHON_IMAGE": container_uri,
         "_NOTEBOOK_GCS_URI": notebook_uri,
         "_NOTEBOOK_OUTPUT_GCS_URI": notebook_output_uri,
-        "_PYTHON_VERSION" : f"python{python_version}"
     }
 
-    if python_version is not None:
-      substitutions["_PYTHON_VERSION"] = "python" + python_version
-
     build = cloudbuild_v1.Build()
 
     options: Optional[client_options.ClientOptions] = None

.cloud-build/notebook-execution-test-cloudbuild-single.yaml

@@ -4,35 +4,43 @@ steps:
     entrypoint: /bin/sh
     args:
     - -c
-    - 'gcloud config list --quiet'
+    - 'gcloud config list'
   # Check the Python version
   - name: ${_PYTHON_IMAGE}
     entrypoint: /bin/sh
     args:
     - -c
-    - ${_PYTHON_VERSION} .cloud-build/CheckPythonVersion.py -q
+    - python3 .cloud-build/CheckPythonVersion.py
   # Create a virtual environment
   - name: ${_PYTHON_IMAGE}
     entrypoint: /bin/sh
     args:
     - -c
-    - ${_PYTHON_VERSION} -m venv workspace/env
+    - python3 -m venv workspace/env
   # Install Python dependencies
   - name: ${_PYTHON_IMAGE}
     entrypoint: /bin/sh
     args:
-    - -c
+    - -c 
     - . workspace/env/bin/activate &&
-      python -m pip -q install -U pip &&
-      python -m pip -q install -U -r .cloud-build/requirements.txt
-  # Install Python dependencies and run testing script
+      python3 -m pip install -U pip &&
+      python3 -m pip install -U -r .cloud-build/requirements.txt
+  # pip freeze
   - name: ${_PYTHON_IMAGE}
     entrypoint: /bin/sh
     args:
     - -c
     - |
       . workspace/env/bin/activate &&
-      python .cloud-build/execute_notebook_cli.py --notebook_source "${_NOTEBOOK_GCS_URI}" --output_file_or_uri "${_NOTEBOOK_OUTPUT_GCS_URI}"
+      python3 -m pip freeze    
+  # Install Python dependencies and run testing script
+  - name: ${_PYTHON_IMAGE}
+    entrypoint: /bin/sh
+    args:
+    - -c
+    - | 
+      . workspace/env/bin/activate &&
+      python3 .cloud-build/execute_notebook_cli.py --notebook_source "${_NOTEBOOK_GCS_URI}" --output_file_or_uri "${_NOTEBOOK_OUTPUT_GCS_URI}"
     env:
     - 'IS_TESTING=1'
-timeout: 86400s
+timeout: 86400s

.cloud-build/notebook-execution-test-cloudbuild.yaml

@@ -4,16 +4,16 @@ steps:
     entrypoint: /bin/sh
     args:
     - -c
-    - gcloud config list --quiet
+    - gcloud config list
   # Check the Python version
   - name: ${_PYTHON_IMAGE}
     entrypoint: /bin/sh
     args:
     - -c
-    - python3 .cloud-build/CheckPythonVersion.py -q
+    - python3 .cloud-build/CheckPythonVersion.py
   # Fetch full repo for diff purposes
   - name: gcr.io/cloud-builders/git
-    args: [fetch, --unshallow, --quiet]
+    args: [fetch, --unshallow]
   # Create a virtual environment
   - name: ${_PYTHON_IMAGE}
     entrypoint: /bin/sh
@@ -24,22 +24,30 @@ steps:
   - name: ${_PYTHON_IMAGE}
     entrypoint: /bin/sh
     args:
-    - -c
+    - -c 
     - . workspace/env/bin/activate &&
-      python3 -m pip -q install -U pip &&
-      python3 -m pip -q install -U -r .cloud-build/requirements.txt
-  # Install Python dependencies and run testing script
-  # TODO: Only pass in private_pool_id if it is set
+      python3 -m pip install -U pip &&
+      python3 -m pip install -U -r .cloud-build/requirements.txt
+  # pip freeze
   - name: ${_PYTHON_IMAGE}
     entrypoint: /bin/sh
     args:
     - -c
     - |
       . workspace/env/bin/activate &&
-      python3 .cloud-build/execute_changed_notebooks_cli.py --test_paths_file "${_TEST_PATHS_FILE}" --base_branch "${_FORCED_BASE_BRANCH}" --container_uri ${_PYTHON_IMAGE} --staging_bucket ${_GCS_STAGING_BUCKET}  --artifacts_bucket ${_GCS_STAGING_BUCKET}/executed_notebooks/PR_${_PR_NUMBER}/BUILD_${BUILD_ID} --variable_project_id ${PROJECT_ID} --variable_region ${_GCP_REGION} --variable_service_account ${_GCP_SERVICE_ACCOUNT} --variable_vpc_network "${_GPC_VPC_NETWORK_NAME}" `if [ ! -z "${_PRIVATE_POOL_NAME}" ]; then echo "--private_pool_id ${_PRIVATE_POOL_NAME}"; fi`
+      python3 -m pip freeze
+  # Install Python dependencies and run testing script
+  # TODO: Only pass in private_pool_id if it is set
+  - name: ${_PYTHON_IMAGE}
+    entrypoint: /bin/sh
+    args:
+    - -c
+    - | 
+      . workspace/env/bin/activate &&
+      python3 .cloud-build/execute_changed_notebooks_cli.py --test_paths_file "${_TEST_PATHS_FILE}" --base_branch "${_FORCED_BASE_BRANCH}" --container_uri ${_PYTHON_IMAGE} --staging_bucket ${_GCS_STAGING_BUCKET}  --artifacts_bucket ${_GCS_STAGING_BUCKET}/executed_notebooks/PR_${_PR_NUMBER}/BUILD_${BUILD_ID} --variable_project_id ${PROJECT_ID} --variable_region ${_GCP_REGION} --variable_service_account ${_GCP_SERVICE_ACCOUNT} `if [ ! -z "${_PRIVATE_POOL_NAME}" ]; then echo "--private_pool_id ${_PRIVATE_POOL_NAME}"; fi`
     env:
     - 'IS_TESTING=1'
 timeout: 86400s
 options:
   pool:
-    name: ${_PRIVATE_POOL_NAME}
+    name: ${_PRIVATE_POOL_NAME}

.cloud-build/test_notebook_vm.txt

@@ -1,5 +1,5 @@
 notebooks/official/vizier/gapic-vizier-multi-objective-optimization.ipynb
 notebooks/official/pipelines/lightweight_functions_component_io_kfp.ipynb
+notebooks/official/matching_engine/intro-swivel.ipynb
 notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-locally-trained-models.ipynb
-notebooks/official/custom/custom-tabular-bq-managed-dataset.ipynb
-.cloud-build/tests/python_version_test.ipynb
+notebooks/official/pipelines/metrics_viz_run_compare_kfp.ipynb

.cloud-build/test_single_notebook_vm.txt

@@ -1 +0,0 @@
-notebooks/official/custom/custom-tabular-bq-managed-dataset.ipynb

.cloud-build/tests/python_version_test.ipynb

@@ -1,61 +0,0 @@
-{
-  "cells": [
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "57a3d44ed8a8"
-      },
-      "source": [
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**_NOTE_**: This notebook has been tested in the following environment:\n",
-        "\n",
-        "* Python version = 3.7\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI API and Compute Engine API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component).\n",
-        "\n",
-        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "c6516f90311b"
-      },
-      "outputs": [],
-      "source": [
-        "# test if the right python version is being used\n",
-        "import sys\n",
-        "\n",
-        "actual_python_version = f\"{sys.version_info.major}.{sys.version_info.minor}\"\n",
-        "print(f\"Runtime python version: {actual_python_version}\")\n",
-        "\n",
-        "assert actual_python_version == \"3.7\", \"Wrong python version!\""
-      ]
-    }
-  ],
-  "metadata": {
-    "colab": {
-      "name": "python_version_test.ipynb",
-      "toc_visible": true
-    },
-    "kernelspec": {
-      "display_name": "Python 3",
-      "name": "python3"
-    }
-  },
-  "nbformat": 4,
-  "nbformat_minor": 0
-}

.cloud-build/utils/NotebookProcessors.py

@@ -14,8 +14,6 @@
 # limitations under the License.
 
 from typing import Dict
-import random
-import string
 
 from nbconvert.preprocessors import Preprocessor
 
@@ -65,36 +63,3 @@ class UpdateVariablesPreprocessor(Preprocessor):
             executable_cells.append(cell)
         notebook.cells = executable_cells
         return notebook, resources
-
-
-# Generate a uuid of a specifed length
-def generate_uuid(length: int = 8) -> str:
-    return "".join(random.choices(string.ascii_lowercase + string.digits, k=length))
-
-
-class UniqueStringsPreprocessor(Preprocessor):
-    # A preprocessor that replaces strings that end with "-unique" or "_unique" with a uuid.
-
-    @staticmethod
-    def update_unique_strings(content: str):
-        # Replace strings that end with "-unique" or "_unique" with a uuid.
-
-        unique_id = generate_uuid()
-        return (
-            content.replace('-unique"', f'-{unique_id}"')
-            .replace("-unique'", f'-{unique_id}"')
-            .replace('_unique"', f'_{unique_id}"')
-            .replace("_unique'", f'_{unique_id}"')
-        )
-
-    def preprocess(self, notebook, resources=None):
-        executable_cells = []
-        for cell in notebook.cells:
-            if cell.cell_type == "code":
-                cell.source = self.update_unique_strings(
-                    content=cell.source,
-                )
-
-            executable_cells.append(cell)
-        notebook.cells = executable_cells
-        return notebook, resources

.cloud-build/utils/UpdateNotebookVariables.py

@@ -40,3 +40,65 @@ def get_updated_value(content: str, variable_name: str, variable_value: str) ->
         content,
         flags=re.M,
     )
+
+
+def test_update_value():
+    new_content = get_updated_value(
+        content='asdf\nPROJECT_ID = "[your-project-id]" #@param {type:"string"} \nasdf',
+        variable_name="PROJECT_ID",
+        variable_value="sample-project",
+    )
+    assert (
+        new_content
+        == 'asdf\nPROJECT_ID = "sample-project" #@param {type:"string"} \nasdf'
+    )
+
+
+def test_update_value_single_quotes():
+    new_content = get_updated_value(
+        content="PROJECT_ID = '[your-project-id]'",
+        variable_name="PROJECT_ID",
+        variable_value="sample-project",
+    )
+    assert new_content == "PROJECT_ID = 'sample-project'"
+
+
+def test_update_value_avoidance():
+    new_content = get_updated_value(
+        content="PROJECT_ID = shell_output[0] ",
+        variable_name="PROJECT_ID",
+        variable_value="sample-project",
+    )
+    assert new_content == "PROJECT_ID = shell_output[0] "
+
+
+def test_region():
+    new_content = get_updated_value(
+        content='REGION = "[your-region]"  # @param {type:"string"}',
+        variable_name="REGION",
+        variable_value="us-central1",
+    )
+    assert new_content == 'REGION = "us-central1"  # @param {type:"string"}'
+
+
+def test_region_equal_equals_ignore():
+    # Tests that == is ignored
+    new_content = get_updated_value(
+        content='REGION == "[your-region]"  # @param {type:"string"}',
+        variable_name="REGION",
+        variable_value="us-central1",
+    )
+    assert new_content == 'REGION == "[your-region]"  # @param {type:"string"}'
+
+
+def test_service_account():
+    # Tests that == is ignored
+    new_content = get_updated_value(
+        content='SERVICE_ACCOUNT = "[your-service-account]"  # @param {type:"string"}',
+        variable_name="SERVICE_ACCOUNT",
+        variable_value="12345-compute@developer.gserviceaccount.com",
+    )
+    assert (
+        new_content
+        == 'SERVICE_ACCOUNT = "12345-compute@developer.gserviceaccount.com"  # @param {type:"string"}'
+    )

.cloud-build/utils/tests/test_preprocessors.py

@@ -1,14 +0,0 @@
-from utils import NotebookProcessors
-
-
-def test_update_value():
-    # Test that the content was updated
-    preprocessor = NotebookProcessors.UniqueStringsPreprocessor()
-
-    content = 'PROJECT_ID = "your-project-id-unique"'
-
-    new_content = preprocessor.update_unique_strings(content)
-
-    assert new_content != content
-    assert new_content.startswith('PROJECT_ID = "your-project-id-')
-    assert new_content.endswith('"')

.cloud-build/utils/tests/test_update_notebook_variables.py

@@ -1,63 +0,0 @@
-from utils import UpdateNotebookVariables
-
-
-def test_update_value():
-    new_content = UpdateNotebookVariables.get_updated_value(
-        content='asdf\nPROJECT_ID = "[your-project-id]" #@param {type:"string"} \nasdf',
-        variable_name="PROJECT_ID",
-        variable_value="sample-project",
-    )
-    assert (
-        new_content
-        == 'asdf\nPROJECT_ID = "sample-project" #@param {type:"string"} \nasdf'
-    )
-
-
-def test_update_value_single_quotes():
-    new_content = UpdateNotebookVariables.get_updated_value(
-        content="PROJECT_ID = '[your-project-id]'",
-        variable_name="PROJECT_ID",
-        variable_value="sample-project",
-    )
-    assert new_content == "PROJECT_ID = 'sample-project'"
-
-
-def test_update_value_avoidance():
-    new_content = UpdateNotebookVariables.get_updated_value(
-        content="PROJECT_ID = shell_output[0] ",
-        variable_name="PROJECT_ID",
-        variable_value="sample-project",
-    )
-    assert new_content == "PROJECT_ID = shell_output[0] "
-
-
-def test_region():
-    new_content = UpdateNotebookVariables.get_updated_value(
-        content='REGION = "[your-region]"  # @param {type:"string"}',
-        variable_name="REGION",
-        variable_value="us-central1",
-    )
-    assert new_content == 'REGION = "us-central1"  # @param {type:"string"}'
-
-
-def test_region_equal_equals_ignore():
-    # Tests that == is ignored
-    new_content = UpdateNotebookVariables.get_updated_value(
-        content='REGION == "[your-region]"  # @param {type:"string"}',
-        variable_name="REGION",
-        variable_value="us-central1",
-    )
-    assert new_content == 'REGION == "[your-region]"  # @param {type:"string"}'
-
-
-def test_service_account():
-    # Tests that == is ignored
-    new_content = UpdateNotebookVariables.get_updated_value(
-        content='SERVICE_ACCOUNT = "[your-service-account]"  # @param {type:"string"}',
-        variable_name="SERVICE_ACCOUNT",
-        variable_value="12345-compute@developer.gserviceaccount.com",
-    )
-    assert (
-        new_content
-        == 'SERVICE_ACCOUNT = "12345-compute@developer.gserviceaccount.com"  # @param {type:"string"}'
-    )

.cloud-build/utils/util.py

@@ -3,7 +3,7 @@ import subprocess
 import tarfile
 import uuid
 from datetime import datetime
-from typing import Optional, Union
+from typing import Optional
 
 from google.auth import credentials as auth_credentials
 from google.cloud import storage
@@ -58,29 +58,3 @@ def archive_code_and_upload(staging_bucket: str):
     print(f"Uploaded source code archive to {source_archived_file_gcs}")
 
     return source_archived_file_gcs
-
-
-def download_blob_into_memory(
-    bucket_name: str, blob_name: str, download_as_text: Optional[bool] = False
-) -> Union[bytes, str]:
-    """
-    Downloads a blob into memory as byte or as text if
-    download_as_text is set to True.
-    """
-
-    storage_client = storage.Client()
-
-    bucket = storage_client.bucket(bucket_name)
-
-    # Construct a client side representation of a blob.
-    blob = bucket.blob(blob_name)
-
-    # Download the blob content
-    if download_as_text:
-        contents = blob.download_as_text()
-    else:
-        contents = blob.download_as_bytes()
-
-    print(f"Downloaded storage object {blob_name} from bucket {bucket_name}.")
-
-    return contents

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,11 +1,4 @@
-**REQUIRED:** Add a summary of your PR here, typically including why the change is needed and what was changed. Include any design alternatives for discussion purposes.
-
-<br>
---- YOUR PR SUMMARY GOES HERE ---
-<br><br><br>
-
-**REQUIRED:** Fill out the below checklists or remove if irrelevant
-1. If you are opening a PR for `Official Notebooks` under the [notebooks/official](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/notebooks/official) folder, follow this mandatory checklist:
+If you are opening a PR for `Official Notebooks` under the [notebooks/official](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/notebooks/official) folder, follow this mandatory checklist:
 - [ ] Use the [notebook template](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/notebook_template.ipynb) as a starting point.
 - [ ] Follow the style and grammar rules outlined in the above notebook template.
 - [ ] Verify the notebook runs successfully in Colab since the automated tests cannot guarantee this even when it passes.
@@ -14,15 +7,12 @@
 - [ ] This notebook has been added to the [CODEOWNERS](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/CODEOWNERS) file under the `Official Notebooks` section, pointing to the author or the author's team.
 - [ ] The Jupyter notebook cleans up any artifacts it has created (datasets, ML models, endpoints, etc) so as not to eat up unnecessary resources.
 
-<br>
 
-2. If you are opening a PR for `Community Notebooks` under the [notebooks/community](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/notebooks/community) folder:
+If you are opening a PR for `Community Notebooks` under the [notebooks/community](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/notebooks/community) folder:
 - [ ] This notebook has been added to the [CODEOWNERS](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/CODEOWNERS) file under the `Community Notebooks` section, pointing to the author or the author's team.
 - [ ] Passes all the required formatting and linting checks. You can locally test with these [instructions](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/CONTRIBUTING.md#code-quality-checks).
 
-<br>
-
-3. If you are opening a PR for `Community Content` under the [community-content](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/community-content) folder:
+If you are opening a PR for `Community Content` under the [community-content](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/community-content) folder:
 - [ ] Make sure your main `Content Directory Name` is descriptive, informative, and includes some of the key products and attributes of your content, so that it is differentiable from other content
 - [ ] The main content directory has been added to the [CODEOWNERS](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/community-content/CODEOWNERS) file under the `Community Content` section, pointing to the author or the author's team.
 - [ ] Passes all the required formatting and linting checks. You can locally test with these [instructions](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/CONTRIBUTING.md#code-quality-checks).

.github/workflows/linter/Dockerfile

@@ -1,20 +0,0 @@
-# To use this image, run this command with the desired notebook args from the top-level vertex-ai-samples directory:
-# 1. To lint all changed notebooks:
-# docker run -v ${PWD}:/setup/app gcr.io/python-docs-samples-tests/notebook_linter:latest
-# 2. To lint specific notebooks:
-# docker run -v ${PWD}:/setup/app gcr.io/python-docs-samples-tests/notebook_linter:latest notebooks/1.ipynb notebooks/2.ipynb
-
-FROM python:3.10
-
-WORKDIR setup
-
-COPY ./requirements.txt .
-COPY ./run_linter.sh .
-
-# Install dependencies.
-RUN pip install --upgrade pip
-RUN pip install -r requirements.txt
-
-WORKDIR app
-
-ENTRYPOINT ["/setup/run_linter.sh"]

.github/workflows/linter/requirements.txt

@@ -2,9 +2,9 @@ git+https://github.com/tensorflow/docs
 ipython
 jupyter
 nbconvert
-black==22.10.0
-pyupgrade==2.38.4
+black==22.3.0
+pyupgrade==2.34.0
 isort==5.10.1
 flake8==4.0.1
-nbqa==1.5.3
+nbqa==1.4.0
 

.github/workflows/linter/run_linter.sh

@@ -47,22 +47,12 @@ done
 
 echo "Test mode: $is_test"
 
-# Read in user-provided notebooks
-notebooks=()
-for arg in "$@"; do
-    if [[ $arg == *.ipynb ]]; then
-        notebooks+=("$arg")
-    fi
-done
-
 # Only check notebooks in test folders modified in this pull request.
 # Note: Use process substitution to persist the data in the array
-if [ ${#notebooks[@]} -eq 0 ]; then
-    echo "Checking for changed notebooked using git"
-    while read -r file || [ -n "$line" ]; do
-        notebooks+=("$file")
-    done < <(git diff --name-only main... | grep '\.ipynb$')
-fi
+notebooks=()
+while read -r file || [ -n "$line" ]; do
+    notebooks+=("$file")
+done < <(git diff --name-only main... | grep '\.ipynb$')
 
 problematic_notebooks=()
 if [ ${#notebooks[@]} -gt 0 ]; then

community-content/CODEOWNERS

@@ -1,10 +1,7 @@
 *  @vertex-ai-samples-contributors  @GoogleCloudPlatform/cloudml-samples-owners
 /tf_agents_bandits_movie_recommendation_with_kfp_and_vertex_sdk	@yinghsienwu
-/pytorch_pre_built_images_deployment @googleapis/vertex-prediction-team
 /pytorch_text_classification_using_vertex_sdk_and_gcloud  @RajeshThallam
 /pytorch_text_classification_using_vertex_sdk_and_gcloud		@RajeshThallam @ultrons
 /sklearn_text_classification_from_script_using_vertex_sdk       @maxhardt
 /pluto_on_workbench @wkharold
 /cpr-examples @samthrasher
-/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines @Ark-kun
-/pipeline_components @Ark-kun

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_logistic_regression_model_using_Scikit_learn_and_import_to_Vertex_AI/pipeline.component.yaml

@@ -1,83 +0,0 @@
-name: Train tabular classification logistic regression model using Scikit learn pipeline
-metadata:
-  annotations:
-    author: Alexey Volkov <alexey.volkov@ark-kun.com>
-    canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_classification_logistic_regression_model_using_Scikit_learn_and_import_to_Vertex_AI/pipeline.component.yaml
-    sdk: https://cloud-pipelines.net/pipeline-editor/
-implementation:
-  graph:
-    tasks:
-      Download from GCS:
-        componentRef:
-          digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
-        arguments:
-          GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
-        annotations:
-          editor.position: '{"x":40,"y":40,"width":180,"height":40}'
-      Select columns using Pandas on CSV data:
-        componentRef:
-          digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: Data
-              taskId: Download from GCS
-          column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
-        annotations:
-          editor.position: '{"x":40,"y":140,"width":180,"height":54}'
-      Fill all missing values using Pandas on CSV data:
-        componentRef:
-          digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: transformed_table
-              taskId: Select columns using Pandas on CSV data
-              type: CSV
-          replacement_value: '0'
-        annotations:
-          editor.position: '{"x":40,"y":250,"width":180,"height":54}'
-      Binarize column using Pandas on CSV data:
-        componentRef:
-          digest: d699afd4d7cae862708717cc160f4394ed0c04e536e9515923ef1e8865f01d44
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: transformed_table
-              taskId: Fill all missing values using Pandas on CSV data
-              type: CSV
-          column_name: tips
-          predicate: '> 0'
-          new_column_name: class
-        annotations:
-          editor.position: '{"x":40,"y":380,"width":180,"height":54}'
-      Train logistic regression model using scikit learn from CSV:
-        componentRef:
-          digest: a864625a822e4b1c8ef6fe4ae1454fd90f15438f70a6712bb4c30e0dda4d35b7
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/cb44b75c9c062fcc40c2b905b2024b4493dbc62b/components/ML_frameworks/Scikit_learn/Train_logistic_regression_model/from_CSV/component.yaml
-        arguments:
-          dataset:
-            taskOutput:
-              outputName: transformed_table
-              taskId: Binarize column using Pandas on CSV data
-              type: CSV
-          label_column_name: class
-        annotations:
-          editor.position: '{"x":40,"y":510,"width":180,"height":70}'
-      Upload Scikit learn pickle model to Google Cloud Vertex AI:
-        componentRef:
-          digest: 81c91c8d7d21ec97e0872f669d68bd89edea87279d703685db54aa94743bebcd
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/workaround_for_buggy_KFPv2_compiler/component.yaml
-        arguments:
-          model:
-            taskOutput:
-              outputName: model
-              taskId: Train logistic regression model using scikit learn from CSV
-              type: ScikitLearnPickleModel
-        annotations:
-          editor.position: '{"x":40,"y":660,"width":180,"height":70}'
-    outputValues: {}

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_logistic_regression_model_using_Scikit_learn_and_import_to_Vertex_AI/pipeline.py

@@ -1,73 +0,0 @@
-# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
-from kfp import components
-
-# %% Loading components
-download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/storage/download/component.yaml")
-select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml")
-fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
-binarize_column_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Binarize_column/in_CSV_format/component.yaml")
-train_logistic_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/ML_frameworks/Scikit_learn/Train_logistic_regression_model/from_CSV/component.yaml")
-upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/1f5cf6e06409b704064b2086c0a705e4e6b4fcde/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/component.yaml")
-deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
-
-# %% Pipeline definition
-def train_tabular_classification_logistic_regression_model_using_Scikit_learn_pipeline():
-    dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
-    feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]  # Excluded "trip_total"
-    label_column = "tips"
-    # Deploying the model might incur additional costs over time
-    deploy_model = False
-
-    classification_label_column = "class"
-    all_columns = [label_column] + feature_columns
-
-    training_data = download_from_gcs_op(
-        gcs_path=dataset_gcs_uri
-    ).outputs["Data"]
-
-    training_data = select_columns_using_Pandas_on_CSV_data_op(
-        table=training_data,
-        column_names=all_columns,
-    ).outputs["transformed_table"]
-
-    # Cleaning the NaN values.
-    training_data = fill_all_missing_values_using_Pandas_on_CSV_data_op(
-        table=training_data,
-        replacement_value="0",
-        #replacement_type_name="float",
-    ).outputs["transformed_table"]
-
-    classification_training_data = binarize_column_using_Pandas_on_CSV_data_op(
-        table=training_data,
-        column_name=label_column,
-        predicate="> 0",
-        new_column_name=classification_label_column,
-    ).outputs["transformed_table"]
-
-    model = train_logistic_regression_model_using_scikit_learn_from_CSV_op(
-        dataset=classification_training_data,
-        label_column_name=classification_label_column,
-        # Optional:
-        #penalty="l2",
-        #solver="lbfgs",
-        #max_iterations=100,
-        #multi_class_mode="auto",
-        #random_seed=0,
-    ).outputs["model"]
-
-    vertex_model_name = upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op(
-        model=model,
-    ).outputs["model_name"]
-
-    # Deploying the model might incur additional costs over time
-    if deploy_model:
-        sklearn_vertex_endpoint_name = deploy_model_to_endpoint_op(
-            model_name=vertex_model_name,
-        ).outputs["endpoint_name"]
-
-pipeline_func = train_tabular_classification_logistic_regression_model_using_Scikit_learn_pipeline
-
-# %% Pipeline submission
-if __name__ == '__main__':
-    from google.cloud import aiplatform
-    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_model_using_PyTorch_and_import_to_Vertex_AI/pipeline.component.yaml

@@ -1,114 +0,0 @@
-name: Train tabular classification model using PyTorch pipeline
-metadata:
-  annotations:
-    author: Alexey Volkov <alexey.volkov@ark-kun.com>
-    canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_classification_model_using_PyTorch_and_import_to_Vertex_AI/pipeline.component.yaml
-    sdk: https://cloud-pipelines.net/pipeline-editor/
-implementation:
-  graph:
-    tasks:
-      Download from GCS:
-        componentRef:
-          digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
-        arguments:
-          GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
-        annotations:
-          editor.position: '{"x":240,"y":40,"width":180,"height":40}'
-      Select columns using Pandas on CSV data:
-        componentRef:
-          digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: Data
-              taskId: Download from GCS
-          column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
-        annotations:
-          editor.position: '{"x":240,"y":140,"width":180,"height":54}'
-      Fill all missing values using Pandas on CSV data:
-        componentRef:
-          digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: transformed_table
-              taskId: Select columns using Pandas on CSV data
-              type: CSV
-          replacement_value: '0'
-        annotations:
-          editor.position: '{"x":240,"y":250,"width":180,"height":54}'
-      Create fully connected pytorch network:
-        componentRef:
-          digest: d03d8248fd358a0275ec33568ee7dd7dce576cc112b09dfafe2651e4d97e04a9
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1a2ef3eeb77bc278f33cad0dd29008ea2431e191/components/PyTorch/Create_fully_connected_network/component.yaml
-        arguments:
-          input_size: '7'
-          hidden_layer_sizes: '[10]'
-          activation_name: elu
-          output_activation_name: sigmoid
-        annotations:
-          editor.position: '{"x":40,"y":360,"width":180,"height":54}'
-      Binarize column using Pandas on CSV data:
-        componentRef:
-          digest: d699afd4d7cae862708717cc160f4394ed0c04e536e9515923ef1e8865f01d44
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: transformed_table
-              taskId: Fill all missing values using Pandas on CSV data
-              type: CSV
-          column_name: tips
-          predicate: ' > 0'
-          new_column_name: class
-        annotations:
-          editor.position: '{"x":240,"y":360,"width":180,"height":54}'
-      Train pytorch model from csv:
-        componentRef:
-          digest: 40f3185eb61e9727f41a4e0c05dd3d3b44bd802aa0f378cfc31756560033949a
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml
-        arguments:
-          model:
-            taskOutput:
-              outputName: model
-              taskId: Create fully connected pytorch network
-              type: PyTorchScriptModule
-          training_data:
-            taskOutput:
-              outputName: transformed_table
-              taskId: Binarize column using Pandas on CSV data
-              type: CSV
-          label_column_name: class
-          loss_function_name: binary_cross_entropy
-        annotations:
-          editor.position: '{"x":240,"y":490,"width":180,"height":40}'
-      Create PyTorch Model Archive with base handler:
-        componentRef:
-          digest: 8298b5ee1b0f0879f893add4cf352c8dec7cf9e21bb9db134c91a2d046cdb0ec
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/46d51383e6554b7f3ab4fd8cf614d8c2b422fb22/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml
-        arguments:
-          Model:
-            taskOutput:
-              outputName: trained_model
-              taskId: Train pytorch model from csv
-              type: PyTorchScriptModule
-          Model name: model
-          Model version: '1.0'
-        annotations:
-          editor.position: '{"x":240,"y":590,"width":180,"height":54}'
-      Upload PyTorch model archive to Google Cloud Vertex AI:
-        componentRef:
-          digest: 4450212fae7b9001482aca7eb78b28413c205506eccf08a04e7754a8dfa99004
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/workaround_for_buggy_KFPv2_compiler/component.yaml
-        arguments:
-          model_archive:
-            taskOutput:
-              outputName: Model archive
-              taskId: Create PyTorch Model Archive with base handler
-              type: PyTorchModelArchive
-        annotations:
-          editor.position: '{"x":240,"y":720,"width":180,"height":70}'
-    outputValues: {}

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_model_using_PyTorch_and_import_to_Vertex_AI/pipeline.py

@@ -1,95 +0,0 @@
-# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
-from kfp import components
-
-# %% Loading components
-download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/storage/download/component.yaml")
-select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml")
-fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
-binarize_column_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Binarize_column/in_CSV_format/component.yaml")
-create_fully_connected_pytorch_network_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Create_fully_connected_network/component.yaml")
-train_pytorch_model_from_csv_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml")
-create_pytorch_model_archive_with_base_handler_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml")
-upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/component.yaml")
-deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
-
-# %% Pipeline definition
-def train_tabular_classification_model_using_PyTorch_pipeline():
-    dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
-    feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]  # Excluded "trip_total"
-    label_column = "tips"
-    # Deploying the model might incur additional costs over time
-    deploy_model = False
-
-    classification_label_column = "class"
-    all_columns = [label_column] + feature_columns
-
-    training_data = download_from_gcs_op(
-        gcs_path=dataset_gcs_uri
-    ).outputs["Data"]
-
-    training_data = select_columns_using_Pandas_on_CSV_data_op(
-        table=training_data,
-        column_names=all_columns,
-    ).outputs["transformed_table"]
-
-    # Cleaning the NaN values.
-    training_data = fill_all_missing_values_using_Pandas_on_CSV_data_op(
-        table=training_data,
-        replacement_value="0",
-        #replacement_type_name="float",
-    ).outputs["transformed_table"]
-
-    classification_training_data = binarize_column_using_Pandas_on_CSV_data_op(
-        table=training_data,
-        column_name=label_column,
-        predicate=" > 0",
-        new_column_name=classification_label_column,
-    ).outputs["transformed_table"]
-
-    network = create_fully_connected_pytorch_network_op(
-        input_size=len(feature_columns),
-        # Optional:
-        hidden_layer_sizes=[10],
-        activation_name="elu",
-        output_activation_name="sigmoid",
-        # output_size=1,
-    ).outputs["model"]
-
-    model = train_pytorch_model_from_csv_op(
-        model=network,
-        training_data=classification_training_data,
-        label_column_name=classification_label_column,
-        loss_function_name="binary_cross_entropy",
-        # Optional:
-        #number_of_epochs=1,
-        #learning_rate=0.1,
-        #optimizer_name="Adadelta",
-        #optimizer_parameters={},
-        #batch_size=32,
-        #batch_log_interval=100,
-        #random_seed=0,
-    ).outputs["trained_model"]
-
-    model_archive = create_pytorch_model_archive_with_base_handler_op(
-        model=model,
-        # Optional:
-        # model_name="model",
-        # model_version="1.0",
-    ).outputs["Model archive"]
-
-    vertex_model_name = upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op(
-        model_archive=model_archive,
-    ).outputs["model_name"]
-
-    # Deploying the model might incur additional costs over time
-    if deploy_model:
-        vertex_endpoint_name = deploy_model_to_endpoint_op(
-            model_name=vertex_model_name,
-        ).outputs["endpoint_name"]
-
-pipeline_func=train_tabular_classification_model_using_PyTorch_pipeline
-
-# %% Pipeline submission
-if __name__ == '__main__':
-    from google.cloud import aiplatform
-    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_model_using_TensorFlow_and_import_to_Vertex_AI/pipeline.component.yaml

@@ -1,132 +0,0 @@
-name: Train tabular classification model using TensorFlow pipeline
-metadata:
-  annotations:
-    author: Alexey Volkov <alexey.volkov@ark-kun.com>
-    canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_classification_model_using_TensorFlow_and_import_to_Vertex_AI/pipeline.component.yaml
-    sdk: https://cloud-pipelines.net/pipeline-editor/
-implementation:
-  graph:
-    tasks:
-      Download from GCS:
-        componentRef:
-          digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
-        arguments:
-          GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
-        annotations:
-          editor.position: '{"x":40,"y":40,"width":180,"height":40}'
-      Select columns using Pandas on CSV data:
-        componentRef:
-          digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: Data
-              taskId: Download from GCS
-          column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
-        annotations:
-          editor.position: '{"x":40,"y":140,"width":180,"height":54}'
-      Fill all missing values using Pandas on CSV data:
-        componentRef:
-          digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: transformed_table
-              taskId: Select columns using Pandas on CSV data
-              type: CSV
-          replacement_value: '0'
-        annotations:
-          editor.position: '{"x":40,"y":250,"width":180,"height":54}'
-      Binarize column using Pandas on CSV data:
-        componentRef:
-          digest: d699afd4d7cae862708717cc160f4394ed0c04e536e9515923ef1e8865f01d44
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: transformed_table
-              taskId: Fill all missing values using Pandas on CSV data
-              type: CSV
-          column_name: tips
-          predicate: ' > 0'
-          new_column_name: class
-        annotations:
-          editor.position: '{"x":40,"y":370,"width":180,"height":54}'
-      Split rows into subsets:
-        componentRef:
-          digest: a609c3c9196484290f24a1174955f95b27f07a7b458aa5cb8cde28866cb2cb46
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: transformed_table
-              taskId: Binarize column using Pandas on CSV data
-              type: CSV
-          fraction_1: '0.8'
-        annotations:
-          editor.position: '{"x":170,"y":500,"width":180,"height":40}'
-      Create fully connected tensorflow network:
-        componentRef:
-          digest: bfcafbc5ce711b1f69cabf1338212d10d50136a73db9f9f7c984de7b80b4bfb0
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/9ca0f9eecf5f896f65b8538bbd809747052617d1/components/tensorflow/Create_fully_connected_network/component.yaml
-        arguments:
-          input_size: '7'
-          hidden_layer_sizes: '[10]'
-          activation_name: elu
-          output_activation_name: sigmoid
-        annotations:
-          editor.position: '{"x":370,"y":500,"width":180,"height":54}'
-      Train model using Keras on CSV:
-        componentRef:
-          digest: 42ae60c889034dbad74815653e95b4f7d576b5f47f803173e8679c7b54984609
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c504a4010348c50eaaf6d4337586ccc008f4dcef/components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml
-        arguments:
-          training_data:
-            taskOutput:
-              outputName: split_1
-              taskId: Split rows into subsets
-              type: CSV
-          model:
-            taskOutput:
-              outputName: model
-              taskId: Create fully connected tensorflow network
-              type: TensorflowSavedModel
-          label_column_name: class
-          loss_function_name: binary_crossentropy
-          number_of_epochs: '10'
-        annotations:
-          editor.position: '{"x":40,"y":620,"width":180,"height":54}'
-      Upload Tensorflow model to Google Cloud Vertex AI:
-        componentRef:
-          digest: 2e45263ff640b1a688e359b6936e27a81b2407749a84f340af2aa5547e0cb92c
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/workaround_for_buggy_KFPv2_compiler/component.yaml
-        arguments:
-          model:
-            taskOutput:
-              outputName: trained_model
-              taskId: Train model using Keras on CSV
-              type: TensorflowSavedModel
-        annotations:
-          editor.position: '{"x":40,"y":750,"width":180,"height":54}'
-      Predict with TensorFlow model on CSV data:
-        componentRef:
-          digest: 921bb1563e93a78233b8acceab87055b9154ccf5595d056028cf0396ca224cd4
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/59c759ce6f543184e30db6817d2a703879bc0f39/components/tensorflow/Predict/on_CSV/component.yaml
-        arguments:
-          dataset:
-            taskOutput:
-              outputName: split_2
-              taskId: Split rows into subsets
-              type: CSV
-          model:
-            taskOutput:
-              outputName: trained_model
-              taskId: Train model using Keras on CSV
-              type: TensorflowSavedModel
-          label_column_name: class
-        annotations:
-          editor.position: '{"x":240,"y":750,"width":180,"height":54}'
-    outputValues: {}

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_model_using_TensorFlow_and_import_to_Vertex_AI/pipeline.py

@@ -1,106 +0,0 @@
-# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
-from kfp import components
-
-# %% Loading components
-download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/storage/download/component.yaml")
-select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml")
-fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
-binarize_column_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Binarize_column/in_CSV_format/component.yaml")
-split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
-create_fully_connected_tensorflow_network_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Create_fully_connected_network/component.yaml")
-train_model_using_Keras_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml")
-predict_with_TensorFlow_model_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Predict/on_CSV/component.yaml")
-upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/component.yaml")
-deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
-
-# %% Pipeline definition
-def train_tabular_classification_model_using_TensorFlow_pipeline():
-    dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
-    feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]  # Excluded "trip_total"
-    label_column = "tips"
-    training_set_fraction = 0.8
-    # Deploying the model might incur additional costs over time
-    deploy_model = False
-
-    classification_label_column = "class"
-    all_columns = [label_column] + feature_columns
-
-    dataset = download_from_gcs_op(
-        gcs_path=dataset_gcs_uri
-    ).outputs["Data"]
-
-    dataset = select_columns_using_Pandas_on_CSV_data_op(
-        table=dataset,
-        column_names=all_columns,
-    ).outputs["transformed_table"]
-
-    dataset = fill_all_missing_values_using_Pandas_on_CSV_data_op(
-        table=dataset,
-        replacement_value="0",
-        # # Optional:
-        # column_names=None,  # =[...]
-    ).outputs["transformed_table"]
-
-    classification_dataset = binarize_column_using_Pandas_on_CSV_data_op(
-        table=dataset,
-        column_name=label_column,
-        predicate=" > 0",
-        new_column_name=classification_label_column,
-    ).outputs["transformed_table"]
-
-    split_task = split_rows_into_subsets_op(
-        table=classification_dataset,
-        fraction_1=training_set_fraction,
-    )
-    classification_training_data = split_task.outputs["split_1"]
-    classification_testing_data = split_task.outputs["split_2"]
-
-    network = create_fully_connected_tensorflow_network_op(
-        input_size=len(feature_columns),
-        # Optional:
-        hidden_layer_sizes=[10],
-        activation_name="elu",
-        output_activation_name="sigmoid",
-        # output_size=1,
-    ).outputs["model"]
-
-    model = train_model_using_Keras_on_CSV_op(
-        training_data=classification_training_data,
-        model=network,
-        label_column_name=classification_label_column,
-        # Optional:
-        loss_function_name="binary_crossentropy",
-        number_of_epochs=10,
-        #learning_rate=0.1,
-        #optimizer_name="Adadelta",
-        #optimizer_parameters={},
-        #batch_size=32,
-        #metric_names=["mean_absolute_error"],
-        #random_seed=0,
-    ).outputs["trained_model"]
-
-    predictions = predict_with_TensorFlow_model_on_CSV_data_op(
-        dataset=classification_testing_data,
-        model=model,
-        # label_column_name needs to be set when doing prediction on a dataset that has labels
-        label_column_name=classification_label_column,
-        # Optional:
-        # batch_size=1000,
-    ).outputs["predictions"]
-
-    vertex_model_name = upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op(
-        model=model,
-    ).outputs["model_name"]
-
-    # Deploying the model might incur additional costs over time
-    if deploy_model:
-        vertex_endpoint_name = deploy_model_to_endpoint_op(
-            model_name=vertex_model_name,
-        ).outputs["endpoint_name"]
-
-pipeline_func = train_tabular_classification_model_using_TensorFlow_pipeline
-
-# %% Pipeline submission
-if __name__ == '__main__':
-    from google.cloud import aiplatform
-    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_model_using_XGBoost_and_import_to_Vertex_AI/pipeline.component.yaml

@@ -1,115 +0,0 @@
-name: Train tabular classification model using XGBoost pipeline
-metadata:
-  annotations:
-    author: Alexey Volkov <alexey.volkov@ark-kun.com>
-    canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_classification_model_using_XGBoost_and_import_to_Vertex_AI/pipeline.component.yaml
-    sdk: https://cloud-pipelines.net/pipeline-editor/
-implementation:
-  graph:
-    tasks:
-      Download from GCS:
-        componentRef:
-          digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
-        arguments:
-          GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
-        annotations:
-          editor.position: '{"x":40,"y":40,"width":180,"height":40}'
-      Select columns using Pandas on CSV data:
-        componentRef:
-          digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: Data
-              taskId: Download from GCS
-          column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
-        annotations:
-          editor.position: '{"x":40,"y":140,"width":180,"height":54}'
-      Fill all missing values using Pandas on CSV data:
-        componentRef:
-          digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: transformed_table
-              taskId: Select columns using Pandas on CSV data
-              type: CSV
-          replacement_value: '0'
-        annotations:
-          editor.position: '{"x":40,"y":250,"width":180,"height":54}'
-      Binarize column using Pandas on CSV data:
-        componentRef:
-          digest: d699afd4d7cae862708717cc160f4394ed0c04e536e9515923ef1e8865f01d44
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: transformed_table
-              taskId: Fill all missing values using Pandas on CSV data
-              type: CSV
-          column_name: tips
-          predicate: '> 0'
-          new_column_name: class
-        annotations:
-          editor.position: '{"x":40,"y":380,"width":180,"height":54}'
-      Split rows into subsets:
-        componentRef:
-          digest: a609c3c9196484290f24a1174955f95b27f07a7b458aa5cb8cde28866cb2cb46
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: transformed_table
-              taskId: Binarize column using Pandas on CSV data
-              type: CSV
-          fraction_1: '0.8'
-        annotations:
-          editor.position: '{"x":170,"y":510,"width":180,"height":40}'
-      Train XGBoost model on CSV:
-        componentRef:
-          digest: 538c5a01eb38deaf532d619f0bbeaff4efc550fe1f0f776fc06791097b68ceac
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/58d3a47f904f32a64af8403330ba7e2134cae46d/components/XGBoost/Train/component.yaml
-        arguments:
-          training_data:
-            taskOutput:
-              outputName: split_1
-              taskId: Split rows into subsets
-              type: CSV
-          label_column_name: class
-          objective: binary:logistic
-        annotations:
-          editor.position: '{"x":40,"y":630,"width":180,"height":40}'
-      Upload XGBoost model to Google Cloud Vertex AI:
-        componentRef:
-          digest: 5a5a273c403670743820986c03a4175b7cb4595a556524fefcce403656286977
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/workaround_for_buggy_KFPv2_compiler/component.yaml
-        arguments:
-          model:
-            taskOutput:
-              outputName: model
-              taskId: Train XGBoost model on CSV
-              type: XGBoostModel
-        annotations:
-          editor.position: '{"x":40,"y":750,"width":180,"height":54}'
-      Xgboost predict on CSV:
-        componentRef:
-          digest: 0876233a0c7306fefec188bd70f059b46d1fb5aa57be231799570e3bbbdd0d95
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/4694ec97baccf59284c2a1db4aa2250c22291eab/components/XGBoost/Predict/component.yaml
-        arguments:
-          data:
-            taskOutput:
-              outputName: split_2
-              taskId: Split rows into subsets
-              type: CSV
-          model:
-            taskOutput:
-              outputName: model
-              taskId: Train XGBoost model on CSV
-              type: XGBoostModel
-          label_column_name: class
-        annotations:
-          editor.position: '{"x":240,"y":750,"width":180,"height":40}'
-    outputValues: {}

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_model_using_XGBoost_and_import_to_Vertex_AI/pipeline.py

@@ -1,94 +0,0 @@
-# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
-from kfp import components
-
-# %% Loading components
-download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/storage/download/component.yaml")
-select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml")
-fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
-binarize_column_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Binarize_column/in_CSV_format/component.yaml")
-split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
-train_XGBoost_model_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Train/component.yaml")
-xgboost_predict_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Predict/component.yaml")
-upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/component.yaml")
-deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
-
-# %% Pipeline definition
-def train_tabular_classification_model_using_XGBoost_pipeline():
-    dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
-    feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]  # Excluded "trip_total"
-    label_column = "tips"
-    training_set_fraction = 0.8
-    # Deploying the model might incur additional costs over time
-    deploy_model = False
-
-    classification_label_column = "class"
-    all_columns = [label_column] + feature_columns
-
-    dataset = download_from_gcs_op(
-        gcs_path=dataset_gcs_uri
-    ).outputs["Data"]
-
-    dataset = select_columns_using_Pandas_on_CSV_data_op(
-        table=dataset,
-        column_names=all_columns,
-    ).outputs["transformed_table"]
-
-    dataset = fill_all_missing_values_using_Pandas_on_CSV_data_op(
-        table=dataset,
-        replacement_value="0",
-        # # Optional:
-        # column_names=None,  # =[...]
-    ).outputs["transformed_table"]
-
-    classification_dataset = binarize_column_using_Pandas_on_CSV_data_op(
-        table=dataset,
-        column_name=label_column,
-        predicate="> 0",
-        new_column_name=classification_label_column,
-    ).outputs["transformed_table"]
-
-    split_task = split_rows_into_subsets_op(
-        table=classification_dataset,
-        fraction_1=training_set_fraction,
-    )
-    classification_training_data = split_task.outputs["split_1"]
-    classification_testing_data = split_task.outputs["split_2"]
-
-    model = train_XGBoost_model_on_CSV_op(
-        training_data=classification_training_data,
-        label_column_name=classification_label_column,
-        objective="binary:logistic",
-        # Optional:
-        #starting_model=None,
-        #num_iterations=10,
-        #booster_params={},
-        #booster="gbtree",
-        #learning_rate=0.3,
-        #min_split_loss=0,
-        #max_depth=6,
-    ).outputs["model"]
-
-    # Predicting on the testing data
-    predictions = xgboost_predict_on_CSV_op(
-        data=classification_testing_data,
-        model=model,
-        # label_column needs to be set when doing prediction on a dataset that has labels
-        label_column_name=classification_label_column,
-    ).outputs["predictions"]
-
-    vertex_model_name = upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op(
-        model=model,
-    ).outputs["model_name"]
-
-    # Deploying the model might incur additional costs over time
-    if deploy_model:
-        vertex_endpoint_name = deploy_model_to_endpoint_op(
-            model_name=vertex_model_name,
-        ).outputs["endpoint_name"]
-
-pipeline_func = train_tabular_classification_model_using_XGBoost_pipeline
-
-# %% Pipeline submission
-if __name__ == '__main__':
-    from google.cloud import aiplatform
-    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_model_using_all_frameworks_and_import_to_Vertex_AI/pipeline.component.yaml

@@ -1,257 +0,0 @@
-name: Train tabular classification model using all frameworks pipeline
-metadata:
-  annotations:
-    author: Alexey Volkov <alexey.volkov@ark-kun.com>
-    canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_classification_model_using_all_frameworks_and_import_to_Vertex_AI/pipeline.component.yaml
-    sdk: https://cloud-pipelines.net/pipeline-editor/
-implementation:
-  graph:
-    tasks:
-      Download from GCS:
-        componentRef:
-          digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
-        arguments:
-          GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
-        annotations:
-          editor.position: '{"x":550,"y":40,"width":180,"height":40}'
-      Select columns using Pandas on CSV data:
-        componentRef:
-          digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: Data
-              taskId: Download from GCS
-          column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
-        annotations:
-          editor.position: '{"x":550,"y":140,"width":180,"height":54}'
-      Fill all missing values using Pandas on CSV data:
-        componentRef:
-          digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: transformed_table
-              taskId: Select columns using Pandas on CSV data
-              type: CSV
-          replacement_value: '0'
-        annotations:
-          editor.position: '{"x":550,"y":250,"width":180,"height":54}'
-      Binarize column using Pandas on CSV data:
-        componentRef:
-          digest: d699afd4d7cae862708717cc160f4394ed0c04e536e9515923ef1e8865f01d44
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: transformed_table
-              taskId: Fill all missing values using Pandas on CSV data
-              type: CSV
-          column_name: tips
-          predicate: ' > 0'
-          new_column_name: class
-        annotations:
-          editor.position: '{"x":550,"y":380,"width":180,"height":54}'
-      Split rows into subsets:
-        componentRef:
-          digest: a609c3c9196484290f24a1174955f95b27f07a7b458aa5cb8cde28866cb2cb46
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: transformed_table
-              taskId: Binarize column using Pandas on CSV data
-              type: CSV
-          fraction_1: '0.8'
-        annotations:
-          editor.position: '{"x":550,"y":490,"width":180,"height":40}'
-      Create fully connected pytorch network:
-        componentRef:
-          digest: d03d8248fd358a0275ec33568ee7dd7dce576cc112b09dfafe2651e4d97e04a9
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1a2ef3eeb77bc278f33cad0dd29008ea2431e191/components/PyTorch/Create_fully_connected_network/component.yaml
-        arguments:
-          input_size: '7'
-          hidden_layer_sizes: '[10]'
-          activation_name: elu
-          output_activation_name: sigmoid
-        annotations:
-          editor.position: '{"x":380,"y":620,"width":180,"height":54}'
-      Create fully connected tensorflow network:
-        componentRef:
-          digest: bfcafbc5ce711b1f69cabf1338212d10d50136a73db9f9f7c984de7b80b4bfb0
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/9ca0f9eecf5f896f65b8538bbd809747052617d1/components/tensorflow/Create_fully_connected_network/component.yaml
-        arguments:
-          input_size: '7'
-          hidden_layer_sizes: '[10]'
-          activation_name: elu
-          output_activation_name: sigmoid
-        annotations:
-          editor.position: '{"x":40,"y":630,"width":180,"height":54}'
-      Train model using Keras on CSV:
-        componentRef:
-          digest: 42ae60c889034dbad74815653e95b4f7d576b5f47f803173e8679c7b54984609
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c504a4010348c50eaaf6d4337586ccc008f4dcef/components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml
-        arguments:
-          training_data:
-            taskOutput:
-              outputName: split_1
-              taskId: Split rows into subsets
-              type: CSV
-          model:
-            taskOutput:
-              outputName: model
-              taskId: Create fully connected tensorflow network
-              type: TensorflowSavedModel
-          label_column_name: class
-          loss_function_name: binary_crossentropy
-          number_of_epochs: '10'
-        annotations:
-          editor.position: '{"x":40,"y":750,"width":180,"height":54}'
-      Train pytorch model from csv:
-        componentRef:
-          digest: 40f3185eb61e9727f41a4e0c05dd3d3b44bd802aa0f378cfc31756560033949a
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml
-        arguments:
-          model:
-            taskOutput:
-              outputName: model
-              taskId: Create fully connected pytorch network
-              type: PyTorchScriptModule
-          training_data:
-            taskOutput:
-              outputName: split_1
-              taskId: Split rows into subsets
-              type: CSV
-          label_column_name: class
-          loss_function_name: binary_cross_entropy
-        annotations:
-          editor.position: '{"x":380,"y":750,"width":180,"height":40}'
-      Train XGBoost model on CSV:
-        componentRef:
-          digest: 538c5a01eb38deaf532d619f0bbeaff4efc550fe1f0f776fc06791097b68ceac
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/58d3a47f904f32a64af8403330ba7e2134cae46d/components/XGBoost/Train/component.yaml
-        arguments:
-          training_data:
-            taskOutput:
-              outputName: split_1
-              taskId: Split rows into subsets
-              type: CSV
-          label_column_name: class
-          objective: binary:logistic
-        annotations:
-          editor.position: '{"x":720,"y":750,"width":180,"height":40}'
-      Train logistic regression model using scikit learn from CSV:
-        componentRef:
-          digest: a864625a822e4b1c8ef6fe4ae1454fd90f15438f70a6712bb4c30e0dda4d35b7
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/cb44b75c9c062fcc40c2b905b2024b4493dbc62b/components/ML_frameworks/Scikit_learn/Train_logistic_regression_model/from_CSV/component.yaml
-        arguments:
-          dataset:
-            taskOutput:
-              outputName: split_1
-              taskId: Split rows into subsets
-              type: CSV
-          label_column_name: class
-        annotations:
-          editor.position: '{"x":1030,"y":750,"width":180,"height":70}'
-      Predict with TensorFlow model on CSV data:
-        componentRef:
-          digest: 921bb1563e93a78233b8acceab87055b9154ccf5595d056028cf0396ca224cd4
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/59c759ce6f543184e30db6817d2a703879bc0f39/components/tensorflow/Predict/on_CSV/component.yaml
-        arguments:
-          dataset:
-            taskOutput:
-              outputName: split_2
-              taskId: Split rows into subsets
-              type: CSV
-          model:
-            taskOutput:
-              outputName: trained_model
-              taskId: Train model using Keras on CSV
-              type: TensorflowSavedModel
-          label_column_name: class
-        annotations:
-          editor.position: '{"x":160,"y":880,"width":180,"height":54}'
-      Create PyTorch Model Archive with base handler:
-        componentRef:
-          digest: 8298b5ee1b0f0879f893add4cf352c8dec7cf9e21bb9db134c91a2d046cdb0ec
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/46d51383e6554b7f3ab4fd8cf614d8c2b422fb22/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml
-        arguments:
-          Model:
-            taskOutput:
-              outputName: trained_model
-              taskId: Train pytorch model from csv
-              type: PyTorchScriptModule
-          Model name: model
-          Model version: '1.0'
-        annotations:
-          editor.position: '{"x":380,"y":880,"width":180,"height":54}'
-      Xgboost predict on CSV:
-        componentRef:
-          digest: 0876233a0c7306fefec188bd70f059b46d1fb5aa57be231799570e3bbbdd0d95
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/4694ec97baccf59284c2a1db4aa2250c22291eab/components/XGBoost/Predict/component.yaml
-        arguments:
-          data:
-            taskOutput:
-              outputName: split_2
-              taskId: Split rows into subsets
-              type: CSV
-          model:
-            taskOutput:
-              outputName: model
-              taskId: Train XGBoost model on CSV
-              type: XGBoostModel
-          label_column_name: class
-        annotations:
-          editor.position: '{"x":810,"y":880,"width":180,"height":40}'
-      Upload Scikit learn pickle model to Google Cloud Vertex AI:
-        componentRef:
-          digest: 81c91c8d7d21ec97e0872f669d68bd89edea87279d703685db54aa94743bebcd
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/workaround_for_buggy_KFPv2_compiler/component.yaml
-        arguments:
-          model:
-            taskOutput:
-              outputName: model
-              taskId: Train logistic regression model using scikit learn from CSV
-              type: ScikitLearnPickleModel
-        annotations:
-          editor.position: '{"x":1030,"y":880,"width":180,"height":70}'
-      Upload Tensorflow model to Google Cloud Vertex AI:
-        componentRef:
-          digest: 2e45263ff640b1a688e359b6936e27a81b2407749a84f340af2aa5547e0cb92c
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/workaround_for_buggy_KFPv2_compiler/component.yaml
-        arguments:
-          model:
-            taskOutput:
-              outputName: trained_model
-              taskId: Train model using Keras on CSV
-              type: TensorflowSavedModel
-        annotations:
-          editor.position: '{"x":40,"y":1010,"width":180,"height":54}'
-      Upload PyTorch model archive to Google Cloud Vertex AI:
-        componentRef:
-          digest: 4450212fae7b9001482aca7eb78b28413c205506eccf08a04e7754a8dfa99004
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/workaround_for_buggy_KFPv2_compiler/component.yaml
-        arguments:
-          model_archive:
-            taskOutput:
-              outputName: Model archive
-              taskId: Create PyTorch Model Archive with base handler
-              type: PyTorchModelArchive
-        annotations:
-          editor.position: '{"x":380,"y":1010,"width":180,"height":70}'
-      Upload XGBoost model to Google Cloud Vertex AI:
-        componentRef:
-          digest: 5a5a273c403670743820986c03a4175b7cb4595a556524fefcce403656286977
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/workaround_for_buggy_KFPv2_compiler/component.yaml
-        arguments:
-          model:
-            taskOutput:
-              outputName: model
-              taskId: Train XGBoost model on CSV
-              type: XGBoostModel
-        annotations:
-          editor.position: '{"x":720,"y":1010,"width":180,"height":54}'
-    outputValues: {}

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_model_using_all_frameworks_and_import_to_Vertex_AI/pipeline.py

@@ -1,224 +0,0 @@
-# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
-from kfp import components
-
-# %% Loading components
-download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/storage/download/component.yaml")
-select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml")
-fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
-binarize_column_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Binarize_column/in_CSV_format/component.yaml")
-split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
-
-# TensorFlow
-create_fully_connected_tensorflow_network_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Create_fully_connected_network/component.yaml")
-train_model_using_Keras_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml")
-predict_with_TensorFlow_model_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Predict/on_CSV/component.yaml")
-upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/component.yaml")
-
-# PyTorch
-create_fully_connected_pytorch_network_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Create_fully_connected_network/component.yaml")
-train_pytorch_model_from_csv_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml")
-create_pytorch_model_archive_with_base_handler_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml")
-upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/component.yaml")
-
-# XGBoost
-train_XGBoost_model_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Train/component.yaml")
-xgboost_predict_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Predict/component.yaml")
-upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/component.yaml")
-
-# Scikit-learn
-#train_linear_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/1f5cf6e06409b704064b2086c0a705e4e6b4fcde/community-content/pipeline_components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml")
-train_logistic_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/1f5cf6e06409b704064b2086c0a705e4e6b4fcde/community-content/pipeline_components/ML_frameworks/Scikit_learn/Train_logistic_regression_model/from_CSV/component.yaml")
-upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/component.yaml")
-
-# Vertex AI
-deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
-
-# %% Pipeline definition
-def train_tabular_classification_model_using_all_frameworks_pipeline():
-    dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
-    feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]  # Excluded "trip_total"
-    label_column = "tips"
-    training_set_fraction = 0.8
-    # Deploying the model might incur additional costs over time
-    deploy_model = False
-
-    classification_label_column = "class"
-    all_columns = [label_column] + feature_columns
-
-    dataset = download_from_gcs_op(
-        gcs_path=dataset_gcs_uri
-    ).outputs["Data"]
-
-    dataset = select_columns_using_Pandas_on_CSV_data_op(
-        table=dataset,
-        column_names=all_columns,
-    ).outputs["transformed_table"]
-
-    dataset = fill_all_missing_values_using_Pandas_on_CSV_data_op(
-        table=dataset,
-        replacement_value="0",
-        # # Optional:
-        # column_names=None,  # =[...]
-    ).outputs["transformed_table"]
-
-    classification_dataset = binarize_column_using_Pandas_on_CSV_data_op(
-        table=dataset,
-        column_name=label_column,
-        predicate=" > 0",
-        new_column_name=classification_label_column,
-    ).outputs["transformed_table"]
-
-    split_task = split_rows_into_subsets_op(
-        table=classification_dataset,
-        fraction_1=training_set_fraction,
-    )
-    classification_training_data = split_task.outputs["split_1"]
-    classification_testing_data = split_task.outputs["split_2"]
-
-    # TensorFlow
-    tensorflow_network = create_fully_connected_tensorflow_network_op(
-        input_size=len(feature_columns),
-        # Optional:
-        hidden_layer_sizes=[10],
-        activation_name="elu",
-        output_activation_name="sigmoid",
-        # output_size=1,
-    ).outputs["model"]
-
-    tensorflow_model = train_model_using_Keras_on_CSV_op(
-        training_data=classification_training_data,
-        model=tensorflow_network,
-        label_column_name=classification_label_column,
-        # Optional:
-        loss_function_name="binary_crossentropy",
-        number_of_epochs=10,
-        #learning_rate=0.1,
-        #optimizer_name="Adadelta",
-        #optimizer_parameters={},
-        #batch_size=32,
-        #metric_names=["mean_absolute_error"],
-        #random_seed=0,
-    ).outputs["trained_model"]
-
-    tensorflow_predictions = predict_with_TensorFlow_model_on_CSV_data_op(
-        dataset=classification_testing_data,
-        model=tensorflow_model,
-        # label_column_name needs to be set when doing prediction on a dataset that has labels
-        label_column_name=classification_label_column,
-        # Optional:
-        # batch_size=1000,
-    ).outputs["predictions"]
-
-    tensorflow_vertex_model_name = upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op(
-        model=tensorflow_model,
-    ).outputs["model_name"]
-
-    # Deploying the model might incur additional costs over time
-    if deploy_model:
-        tensorflow_vertex_endpoint_name = deploy_model_to_endpoint_op(
-            model_name=tensorflow_vertex_model_name,
-        ).outputs["endpoint_name"]
-
-    # PyTorch
-    pytorch_network = create_fully_connected_pytorch_network_op(
-        input_size=len(feature_columns),
-        # Optional:
-        hidden_layer_sizes=[10],
-        activation_name="elu",
-        output_activation_name="sigmoid",
-        # output_size=1,
-    ).outputs["model"]
-
-    pytorch_model = train_pytorch_model_from_csv_op(
-        model=pytorch_network,
-        training_data=classification_training_data,
-        label_column_name=classification_label_column,
-        loss_function_name="binary_cross_entropy",
-        # Optional:
-        #number_of_epochs=1,
-        #learning_rate=0.1,
-        #optimizer_name="Adadelta",
-        #optimizer_parameters={},
-        #batch_size=32,
-        #batch_log_interval=100,
-        #random_seed=0,
-    ).outputs["trained_model"]
-
-    pytorch_model_archive = create_pytorch_model_archive_with_base_handler_op(
-        model=pytorch_model,
-        # Optional:
-        # model_name="model",
-        # model_version="1.0",
-    ).outputs["Model archive"]
-
-    pytorch_vertex_model_name = upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op(
-        model_archive=pytorch_model_archive,
-    ).outputs["model_name"]
-
-    # Deploying the model might incur additional costs over time
-    if deploy_model:
-        pytorch_vertex_endpoint_name = deploy_model_to_endpoint_op(
-            model_name=pytorch_vertex_model_name,
-        ).outputs["endpoint_name"]
-
-    # XGBoost
-    xgboost_model = train_XGBoost_model_on_CSV_op(
-        training_data=classification_training_data,
-        label_column_name=classification_label_column,
-        objective="binary:logistic",
-        # Optional:
-        #starting_model=None,
-        #num_iterations=10,
-        #booster_params={},
-        #booster="gbtree",
-        #learning_rate=0.3,
-        #min_split_loss=0,
-        #max_depth=6,
-    ).outputs["model"]
-
-    # Predicting on the testing data
-    xgboost_predictions = xgboost_predict_on_CSV_op(
-        data=classification_testing_data,
-        model=xgboost_model,
-        # label_column needs to be set when doing prediction on a dataset that has labels
-        label_column_name=classification_label_column,
-    ).outputs["predictions"]
-
-    xgboost_vertex_model_name = upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op(
-        model=xgboost_model,
-    ).outputs["model_name"]
-
-    # Deploying the model might incur additional costs over time
-    if deploy_model:
-        xgboost_vertex_endpoint_name = deploy_model_to_endpoint_op(
-            model_name=xgboost_vertex_model_name,
-        ).outputs["endpoint_name"]
-
-    # Scikit-learn
-    sklearn_model = train_logistic_regression_model_using_scikit_learn_from_CSV_op(
-        dataset=classification_training_data,
-        label_column_name=classification_label_column,
-        # Optional:
-        #penalty="l2",
-        #solver="lbfgs",
-        #max_iterations=100,
-        #multi_class_mode="auto",
-        #random_seed=0,
-    ).outputs["model"]
-
-    sklearn_vertex_model_name = upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op(
-        model=sklearn_model,
-    ).outputs["model_name"]
-
-    # Deploying the model might incur additional costs over time
-    if deploy_model:
-        sklearn_vertex_endpoint_name = deploy_model_to_endpoint_op(
-            model_name=sklearn_vertex_model_name,
-        ).outputs["endpoint_name"]
-
-pipeline_func=train_tabular_classification_model_using_all_frameworks_pipeline
-
-# %% Pipeline submission
-if __name__ == '__main__':
-    from google.cloud import aiplatform
-    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_regression_linear_model_using_Scikit_learn_and_import_to_Vertex_AI/pipeline.component.yaml

@@ -1,68 +0,0 @@
-name: Train tabular regression linear model using Scikit learn pipeline
-metadata:
-  annotations:
-    author: Alexey Volkov <alexey.volkov@ark-kun.com>
-    canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_regression_linear_model_using_Scikit_learn_and_import_to_Vertex_AI/pipeline.component.yaml
-    sdk: https://cloud-pipelines.net/pipeline-editor/
-implementation:
-  graph:
-    tasks:
-      Download from GCS:
-        componentRef:
-          digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
-        arguments:
-          GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
-        annotations:
-          editor.position: '{"x":40,"y":40,"width":180,"height":40}'
-      Select columns using Pandas on CSV data:
-        componentRef:
-          digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: Data
-              taskId: Download from GCS
-          column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
-        annotations:
-          editor.position: '{"x":40,"y":140,"width":180,"height":54}'
-      Fill all missing values using Pandas on CSV data:
-        componentRef:
-          digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: transformed_table
-              taskId: Select columns using Pandas on CSV data
-              type: CSV
-          replacement_value: '0'
-        annotations:
-          editor.position: '{"x":40,"y":250,"width":180,"height":54}'
-      Train linear regression model using scikit learn from CSV:
-        componentRef:
-          digest: c7fe7912ab0d1fb45d201d452e9ce6be5544e7d8c6d229db7a4b931ff58560f3
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/f807e02b54d4886c65a05f40848fd51c72407f40/components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml
-        arguments:
-          dataset:
-            taskOutput:
-              outputName: transformed_table
-              taskId: Fill all missing values using Pandas on CSV data
-              type: CSV
-          label_column_name: tips
-        annotations:
-          editor.position: '{"x":40,"y":360,"width":180,"height":54}'
-      Upload Scikit learn pickle model to Google Cloud Vertex AI:
-        componentRef:
-          digest: 81c91c8d7d21ec97e0872f669d68bd89edea87279d703685db54aa94743bebcd
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/workaround_for_buggy_KFPv2_compiler/component.yaml
-        arguments:
-          model:
-            taskOutput:
-              outputName: model
-              taskId: Train linear regression model using scikit learn from CSV
-              type: ScikitLearnPickleModel
-        annotations:
-          editor.position: '{"x":40,"y":490,"width":180,"height":70}'
-    outputValues: {}

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_regression_linear_model_using_Scikit_learn_and_import_to_Vertex_AI/pipeline.py

@@ -1,57 +0,0 @@
-# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
-from kfp import components
-
-# %% Loading components
-download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/storage/download/component.yaml")
-select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml")
-fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
-train_linear_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/1f5cf6e06409b704064b2086c0a705e4e6b4fcde/community-content/pipeline_components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml")
-upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/component.yaml")
-deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
-
-# %% Pipeline definition
-def train_tabular_regression_linear_model_using_Scikit_learn_pipeline():
-    dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
-    feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]  # Excluded "trip_total"
-    label_column = "tips"
-    all_columns = [label_column] + feature_columns
-    # Deploying the model might incur additional costs over time
-    deploy_model = False
-
-    training_data = download_from_gcs_op(
-        gcs_path=dataset_gcs_uri
-    ).outputs["Data"]
-
-    training_data = select_columns_using_Pandas_on_CSV_data_op(
-        table=training_data,
-        column_names=all_columns,
-    ).outputs["transformed_table"]
-
-    # Cleaning the NaN values.
-    training_data = fill_all_missing_values_using_Pandas_on_CSV_data_op(
-        table=training_data,
-        replacement_value="0",
-        #replacement_type_name="float",
-    ).outputs["transformed_table"]
-
-    model = train_linear_regression_model_using_scikit_learn_from_CSV_op(
-        dataset=training_data,
-        label_column_name=label_column,
-    ).outputs["model"]
-
-    vertex_model_name = upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op(
-        model=model,
-    ).outputs["model_name"]
-
-    # Deploying the model might incur additional costs over time
-    if deploy_model:
-        sklearn_vertex_endpoint_name = deploy_model_to_endpoint_op(
-            model_name=vertex_model_name,
-        ).outputs["endpoint_name"]
-
-pipeline_func = train_tabular_regression_linear_model_using_Scikit_learn_pipeline
-
-# %% Pipeline submission
-if __name__ == '__main__':
-    from google.cloud import aiplatform
-    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_regression_model_using_PyTorch_and_import_to_Vertex_AI/pipeline.component.yaml

@@ -1,97 +0,0 @@
-name: Train tabular regression model using PyTorch pipeline
-metadata:
-  annotations:
-    author: Alexey Volkov <alexey.volkov@ark-kun.com>
-    canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_regression_model_using_PyTorch_and_import_to_Vertex_AI/pipeline.component.yaml
-    sdk: https://cloud-pipelines.net/pipeline-editor/
-implementation:
-  graph:
-    tasks:
-      Download from GCS:
-        componentRef:
-          digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
-        arguments:
-          GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
-        annotations:
-          editor.position: '{"x":240,"y":40,"width":180,"height":40}'
-      Select columns using Pandas on CSV data:
-        componentRef:
-          digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: Data
-              taskId: Download from GCS
-          column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
-        annotations:
-          editor.position: '{"x":240,"y":130,"width":180,"height":54}'
-      Create fully connected pytorch network:
-        componentRef:
-          digest: d03d8248fd358a0275ec33568ee7dd7dce576cc112b09dfafe2651e4d97e04a9
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1a2ef3eeb77bc278f33cad0dd29008ea2431e191/components/PyTorch/Create_fully_connected_network/component.yaml
-        arguments:
-          input_size: '7'
-          hidden_layer_sizes: '[10]'
-          activation_name: elu
-        annotations:
-          editor.position: '{"x":40,"y":240,"width":180,"height":54}'
-      Fill all missing values using Pandas on CSV data:
-        componentRef:
-          digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: transformed_table
-              taskId: Select columns using Pandas on CSV data
-              type: CSV
-          replacement_value: '0'
-        annotations:
-          editor.position: '{"x":240,"y":240,"width":180,"height":54}'
-      Train pytorch model from csv:
-        componentRef:
-          digest: 40f3185eb61e9727f41a4e0c05dd3d3b44bd802aa0f378cfc31756560033949a
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml
-        arguments:
-          model:
-            taskOutput:
-              outputName: model
-              taskId: Create fully connected pytorch network
-              type: PyTorchScriptModule
-          training_data:
-            taskOutput:
-              outputName: transformed_table
-              taskId: Fill all missing values using Pandas on CSV data
-              type: CSV
-          label_column_name: tips
-        annotations:
-          editor.position: '{"x":240,"y":380,"width":180,"height":40}'
-      Create PyTorch Model Archive with base handler:
-        componentRef:
-          digest: 8298b5ee1b0f0879f893add4cf352c8dec7cf9e21bb9db134c91a2d046cdb0ec
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/46d51383e6554b7f3ab4fd8cf614d8c2b422fb22/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml
-        arguments:
-          Model:
-            taskOutput:
-              outputName: trained_model
-              taskId: Train pytorch model from csv
-              type: PyTorchScriptModule
-          Model name: model
-          Model version: '1.0'
-        annotations:
-          editor.position: '{"x":240,"y":500,"width":180,"height":54}'
-      Upload PyTorch model archive to Google Cloud Vertex AI:
-        componentRef:
-          digest: 4450212fae7b9001482aca7eb78b28413c205506eccf08a04e7754a8dfa99004
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/workaround_for_buggy_KFPv2_compiler/component.yaml
-        arguments:
-          model_archive:
-            taskOutput:
-              outputName: Model archive
-              taskId: Create PyTorch Model Archive with base handler
-              type: PyTorchModelArchive
-        annotations:
-          editor.position: '{"x":240,"y":630,"width":180,"height":70}'
-    outputValues: {}

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_regression_model_using_PyTorch_and_import_to_Vertex_AI/pipeline.py

@@ -1,85 +0,0 @@
-# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
-from kfp import components
-
-# %% Loading components
-download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/storage/download/component.yaml")
-select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml")
-fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
-create_fully_connected_pytorch_network_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Create_fully_connected_network/component.yaml")
-train_pytorch_model_from_csv_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml")
-create_pytorch_model_archive_with_base_handler_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml")
-upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/component.yaml")
-deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
-
-# %% Pipeline definition
-def train_tabular_regression_model_using_PyTorch_pipeline():
-    dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
-    feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]  # Excluded "trip_total"
-    label_column = "tips"
-    all_columns = [label_column] + feature_columns
-    # Deploying the model might incur additional costs over time
-    deploy_model = False
-
-    training_data = download_from_gcs_op(
-        gcs_path=dataset_gcs_uri
-    ).outputs["Data"]
-
-    training_data = select_columns_using_Pandas_on_CSV_data_op(
-        table=training_data,
-        column_names=all_columns,
-    ).outputs["transformed_table"]
-
-    # Cleaning the NaN values.
-    training_data = fill_all_missing_values_using_Pandas_on_CSV_data_op(
-        table=training_data,
-        replacement_value="0",
-        #replacement_type_name="float",
-    ).outputs["transformed_table"]
-
-    network = create_fully_connected_pytorch_network_op(
-        input_size=len(feature_columns),
-        # Optional:
-        hidden_layer_sizes=[10],
-        activation_name="elu",
-        # output_activation_name=None,
-        # output_size=1,
-    ).outputs["model"]
-
-    model = train_pytorch_model_from_csv_op(
-        model=network,
-        training_data=training_data,
-        label_column_name=label_column,
-        # Optional:
-        #loss_function_name="mse_loss",
-        #number_of_epochs=1,
-        #learning_rate=0.1,
-        #optimizer_name="Adadelta",
-        #optimizer_parameters={},
-        #batch_size=32,
-        #batch_log_interval=100,
-        #random_seed=0,
-    ).outputs["trained_model"]
-
-    model_archive = create_pytorch_model_archive_with_base_handler_op(
-        model=model,
-        # Optional:
-        # model_name="model",
-        # model_version="1.0",
-    ).outputs["Model archive"]
-
-    vertex_model_name = upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op(
-        model_archive=model_archive,
-    ).outputs["model_name"]
-
-    # Deploying the model might incur additional costs over time
-    if deploy_model:
-        vertex_endpoint_name = deploy_model_to_endpoint_op(
-            model_name=vertex_model_name,
-        ).outputs["endpoint_name"]
-
-pipeline_func=train_tabular_regression_model_using_PyTorch_pipeline
-
-# %% Pipeline submission
-if __name__ == '__main__':
-    from google.cloud import aiplatform
-    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_regression_model_using_TensorFlow_and_import_to_Vertex_AI/pipeline.component.yaml

@@ -1,116 +0,0 @@
-name: Train tabular regression model using Tensorflow pipeline
-metadata:
-  annotations:
-    author: Alexey Volkov <alexey.volkov@ark-kun.com>
-    canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_regression_model_using_TensorFlow_and_import_to_Vertex_AI/pipeline.component.yaml
-    sdk: https://cloud-pipelines.net/pipeline-editor/
-implementation:
-  graph:
-    tasks:
-      Download from GCS:
-        componentRef:
-          digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
-        arguments:
-          GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
-        annotations:
-          editor.position: '{"x":40,"y":40,"width":180,"height":40}'
-      Select columns using Pandas on CSV data:
-        componentRef:
-          digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: Data
-              taskId: Download from GCS
-          column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
-        annotations:
-          editor.position: '{"x":40,"y":140,"width":180,"height":54}'
-      Fill all missing values using Pandas on CSV data:
-        componentRef:
-          digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: transformed_table
-              taskId: Select columns using Pandas on CSV data
-              type: CSV
-          replacement_value: '0'
-        annotations:
-          editor.position: '{"x":40,"y":250,"width":180,"height":54}'
-      Split rows into subsets:
-        componentRef:
-          digest: a609c3c9196484290f24a1174955f95b27f07a7b458aa5cb8cde28866cb2cb46
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: transformed_table
-              taskId: Fill all missing values using Pandas on CSV data
-              type: CSV
-          fraction_1: '0.8'
-        annotations:
-          editor.position: '{"x":170,"y":380,"width":180,"height":40}'
-      Create fully connected tensorflow network:
-        componentRef:
-          digest: bfcafbc5ce711b1f69cabf1338212d10d50136a73db9f9f7c984de7b80b4bfb0
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/9ca0f9eecf5f896f65b8538bbd809747052617d1/components/tensorflow/Create_fully_connected_network/component.yaml
-        arguments:
-          input_size: '7'
-          hidden_layer_sizes: '[10]'
-          activation_name: elu
-        annotations:
-          editor.position: '{"x":370,"y":380,"width":180,"height":54}'
-      Train model using Keras on CSV:
-        componentRef:
-          digest: 42ae60c889034dbad74815653e95b4f7d576b5f47f803173e8679c7b54984609
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c504a4010348c50eaaf6d4337586ccc008f4dcef/components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml
-        arguments:
-          training_data:
-            taskOutput:
-              outputName: split_1
-              taskId: Split rows into subsets
-              type: CSV
-          model:
-            taskOutput:
-              outputName: model
-              taskId: Create fully connected tensorflow network
-              type: TensorflowSavedModel
-          label_column_name: tips
-          number_of_epochs: '10'
-          metric_names: '["mean_absolute_error"]'
-        annotations:
-          editor.position: '{"x":40,"y":500,"width":180,"height":54}'
-      Upload Tensorflow model to Google Cloud Vertex AI:
-        componentRef:
-          digest: 2e45263ff640b1a688e359b6936e27a81b2407749a84f340af2aa5547e0cb92c
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/workaround_for_buggy_KFPv2_compiler/component.yaml
-        arguments:
-          model:
-            taskOutput:
-              outputName: trained_model
-              taskId: Train model using Keras on CSV
-              type: TensorflowSavedModel
-        annotations:
-          editor.position: '{"x":40,"y":630,"width":180,"height":54}'
-      Predict with TensorFlow model on CSV data:
-        componentRef:
-          digest: 921bb1563e93a78233b8acceab87055b9154ccf5595d056028cf0396ca224cd4
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/59c759ce6f543184e30db6817d2a703879bc0f39/components/tensorflow/Predict/on_CSV/component.yaml
-        arguments:
-          dataset:
-            taskOutput:
-              outputName: split_2
-              taskId: Split rows into subsets
-              type: CSV
-          model:
-            taskOutput:
-              outputName: trained_model
-              taskId: Train model using Keras on CSV
-              type: TensorflowSavedModel
-          label_column_name: tips
-        annotations:
-          editor.position: '{"x":240,"y":630,"width":180,"height":54}'
-    outputValues: {}

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_regression_model_using_TensorFlow_and_import_to_Vertex_AI/pipeline.py

@@ -1,97 +0,0 @@
-# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
-from kfp import components
-
-# %% Loading components
-download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/storage/download/component.yaml")
-select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml")
-fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
-split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
-create_fully_connected_tensorflow_network_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Create_fully_connected_network/component.yaml")
-train_model_using_Keras_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml")
-predict_with_TensorFlow_model_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Predict/on_CSV/component.yaml")
-upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/component.yaml")
-deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
-
-# %% Pipeline definition
-def train_tabular_regression_model_using_Tensorflow_pipeline():
-    dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
-    feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]  # Excluded "trip_total"
-    label_column = "tips"
-    training_set_fraction = 0.8
-    # Deploying the model might incur additional costs over time
-    deploy_model = False
-
-    all_columns = [label_column] + feature_columns
-
-    dataset = download_from_gcs_op(
-        gcs_path=dataset_gcs_uri
-    ).outputs["Data"]
-
-    dataset = select_columns_using_Pandas_on_CSV_data_op(
-        table=dataset,
-        column_names=all_columns,
-    ).outputs["transformed_table"]
-
-    dataset = fill_all_missing_values_using_Pandas_on_CSV_data_op(
-        table=dataset,
-        replacement_value="0",
-        # # Optional:
-        # column_names=None,  # =[...]
-    ).outputs["transformed_table"]
-
-    split_task = split_rows_into_subsets_op(
-        table=dataset,
-        fraction_1=training_set_fraction,
-    )
-    training_data = split_task.outputs["split_1"]
-    testing_data = split_task.outputs["split_2"]
-
-    network = create_fully_connected_tensorflow_network_op(
-        input_size=len(feature_columns),
-        # Optional:
-        hidden_layer_sizes=[10],
-        activation_name="elu",
-        # output_activation_name=None,
-        # output_size=1,
-    ).outputs["model"]
-
-    model = train_model_using_Keras_on_CSV_op(
-        training_data=training_data,
-        model=network,
-        label_column_name=label_column,
-        # Optional:
-        #loss_function_name="mean_squared_error",
-        number_of_epochs=10,
-        #learning_rate=0.1,
-        #optimizer_name="Adadelta",
-        #optimizer_parameters={},
-        #batch_size=32,
-        metric_names=["mean_absolute_error"],
-        #random_seed=0,
-    ).outputs["trained_model"]
-
-    predictions = predict_with_TensorFlow_model_on_CSV_data_op(
-        dataset=testing_data,
-        model=model,
-        # label_column_name needs to be set when doing prediction on a dataset that has labels
-        label_column_name=label_column,
-        # Optional:
-        # batch_size=1000,
-    ).outputs["predictions"]
-
-    vertex_model_name = upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op(
-        model=model,
-    ).outputs["model_name"]
-
-    # Deploying the model might incur additional costs over time
-    if deploy_model:
-        vertex_endpoint_name = deploy_model_to_endpoint_op(
-            model_name=vertex_model_name,
-        ).outputs["endpoint_name"]
-
-pipeline_func=train_tabular_regression_model_using_Tensorflow_pipeline
-
-# %% Pipeline submission
-if __name__ == '__main__':
-    from google.cloud import aiplatform
-    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_regression_model_using_XGBoost_and_import_to_Vertex_AI/pipeline.component.yaml

@@ -1,99 +0,0 @@
-name: Train tabular regression model using XGBoost pipeline
-metadata:
-  annotations:
-    author: Alexey Volkov <alexey.volkov@ark-kun.com>
-    canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_regression_model_using_XGBoost_and_import_to_Vertex_AI/pipeline.component.yaml
-    sdk: https://cloud-pipelines.net/pipeline-editor/
-implementation:
-  graph:
-    tasks:
-      Download from GCS:
-        componentRef:
-          digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
-        arguments:
-          GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
-        annotations:
-          editor.position: '{"x":40,"y":40,"width":180,"height":40}'
-      Select columns using Pandas on CSV data:
-        componentRef:
-          digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: Data
-              taskId: Download from GCS
-          column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
-        annotations:
-          editor.position: '{"x":40,"y":140,"width":180,"height":54}'
-      Fill all missing values using Pandas on CSV data:
-        componentRef:
-          digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: transformed_table
-              taskId: Select columns using Pandas on CSV data
-              type: CSV
-          replacement_value: '0'
-        annotations:
-          editor.position: '{"x":40,"y":250,"width":180,"height":54}'
-      Split rows into subsets:
-        componentRef:
-          digest: a609c3c9196484290f24a1174955f95b27f07a7b458aa5cb8cde28866cb2cb46
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: transformed_table
-              taskId: Fill all missing values using Pandas on CSV data
-              type: CSV
-          fraction_1: '0.8'
-        annotations:
-          editor.position: '{"x":170,"y":360,"width":180,"height":40}'
-      Train XGBoost model on CSV:
-        componentRef:
-          digest: 538c5a01eb38deaf532d619f0bbeaff4efc550fe1f0f776fc06791097b68ceac
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/58d3a47f904f32a64af8403330ba7e2134cae46d/components/XGBoost/Train/component.yaml
-        arguments:
-          training_data:
-            taskOutput:
-              outputName: split_1
-              taskId: Split rows into subsets
-              type: CSV
-          label_column_name: tips
-        annotations:
-          editor.position: '{"x":40,"y":480,"width":180,"height":40}'
-      Upload XGBoost model to Google Cloud Vertex AI:
-        componentRef:
-          digest: 5a5a273c403670743820986c03a4175b7cb4595a556524fefcce403656286977
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/workaround_for_buggy_KFPv2_compiler/component.yaml
-        arguments:
-          model:
-            taskOutput:
-              outputName: model
-              taskId: Train XGBoost model on CSV
-              type: XGBoostModel
-        annotations:
-          editor.position: '{"x":40,"y":600,"width":180,"height":54}'
-      Xgboost predict on CSV:
-        componentRef:
-          digest: 0876233a0c7306fefec188bd70f059b46d1fb5aa57be231799570e3bbbdd0d95
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/4694ec97baccf59284c2a1db4aa2250c22291eab/components/XGBoost/Predict/component.yaml
-        arguments:
-          data:
-            taskOutput:
-              outputName: split_2
-              taskId: Split rows into subsets
-              type: CSV
-          model:
-            taskOutput:
-              outputName: model
-              taskId: Train XGBoost model on CSV
-              type: XGBoostModel
-          label_column_name: tips
-        annotations:
-          editor.position: '{"x":240,"y":600,"width":180,"height":40}'
-    outputValues: {}

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_regression_model_using_XGBoost_and_import_to_Vertex_AI/pipeline.py

@@ -1,85 +0,0 @@
-# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
-from kfp import components
-
-# %% Loading components
-download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/storage/download/component.yaml")
-select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml")
-fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
-split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
-train_XGBoost_model_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Train/component.yaml")
-xgboost_predict_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Predict/component.yaml")
-upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/component.yaml")
-deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
-
-# %% Pipeline definition
-def train_tabular_regression_model_using_XGBoost_pipeline():
-    dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
-    feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]  # Excluded "trip_total"
-    label_column = "tips"
-    training_set_fraction = 0.8
-    # Deploying the model might incur additional costs over time
-    deploy_model = False
-
-    all_columns = [label_column] + feature_columns
-
-    dataset = download_from_gcs_op(
-        gcs_path=dataset_gcs_uri
-    ).outputs["Data"]
-
-    dataset = select_columns_using_Pandas_on_CSV_data_op(
-        table=dataset,
-        column_names=all_columns,
-    ).outputs["transformed_table"]
-
-    dataset = fill_all_missing_values_using_Pandas_on_CSV_data_op(
-        table=dataset,
-        replacement_value="0",
-        # # Optional:
-        # column_names=None,  # =[...]
-    ).outputs["transformed_table"]
-
-    split_task = split_rows_into_subsets_op(
-        table=dataset,
-        fraction_1=training_set_fraction,
-    )
-    training_data = split_task.outputs["split_1"]
-    testing_data = split_task.outputs["split_2"]
-
-    model = train_XGBoost_model_on_CSV_op(
-        training_data=training_data,
-        label_column_name=label_column,
-        # Optional:
-        #starting_model=None,
-        #num_iterations=10,
-        #booster_params={},
-        #objective="reg:squarederror",
-        #booster="gbtree",
-        #learning_rate=0.3,
-        #min_split_loss=0,
-        #max_depth=6,
-    ).outputs["model"]
-
-    # Predicting on the testing data
-    predictions = xgboost_predict_on_CSV_op(
-        data=testing_data,
-        model=model,
-        # label_column needs to be set when doing prediction on a dataset that has labels
-        label_column_name=label_column,
-    ).outputs["predictions"]
-
-    vertex_model_name = upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op(
-        model=model,
-    ).outputs["model_name"]
-
-    # Deploying the model might incur additional costs over time
-    if deploy_model:
-        vertex_endpoint_name = deploy_model_to_endpoint_op(
-            model_name=vertex_model_name,
-        ).outputs["endpoint_name"]
-
-pipeline_func = train_tabular_regression_model_using_XGBoost_pipeline
-
-# %% Pipeline submission
-if __name__ == '__main__':
-    from google.cloud import aiplatform
-    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_regression_model_using_all_frameworks_and_import_to_Vertex_AI/pipeline.component.yaml

@@ -1,238 +0,0 @@
-name: Train tabular regression model using all frameworks pipeline
-metadata:
-  annotations:
-    author: Alexey Volkov <alexey.volkov@ark-kun.com>
-    canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_regression_model_using_all_frameworks_and_import_to_Vertex_AI/pipeline.component.yaml
-    sdk: https://cloud-pipelines.net/pipeline-editor/
-implementation:
-  graph:
-    tasks:
-      Download from GCS:
-        componentRef:
-          digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
-        arguments:
-          GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
-        annotations:
-          editor.position: '{"x":550,"y":40,"width":180,"height":40}'
-      Select columns using Pandas on CSV data:
-        componentRef:
-          digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: Data
-              taskId: Download from GCS
-          column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
-        annotations:
-          editor.position: '{"x":550,"y":140,"width":180,"height":54}'
-      Fill all missing values using Pandas on CSV data:
-        componentRef:
-          digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: transformed_table
-              taskId: Select columns using Pandas on CSV data
-              type: CSV
-          replacement_value: '0'
-        annotations:
-          editor.position: '{"x":550,"y":250,"width":180,"height":54}'
-      Split rows into subsets:
-        componentRef:
-          digest: a609c3c9196484290f24a1174955f95b27f07a7b458aa5cb8cde28866cb2cb46
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: transformed_table
-              taskId: Fill all missing values using Pandas on CSV data
-              type: CSV
-          fraction_1: '0.8'
-        annotations:
-          editor.position: '{"x":550,"y":360,"width":180,"height":40}'
-      Create fully connected pytorch network:
-        componentRef:
-          digest: d03d8248fd358a0275ec33568ee7dd7dce576cc112b09dfafe2651e4d97e04a9
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1a2ef3eeb77bc278f33cad0dd29008ea2431e191/components/PyTorch/Create_fully_connected_network/component.yaml
-        arguments:
-          input_size: '7'
-          hidden_layer_sizes: '[10]'
-          activation_name: elu
-        annotations:
-          editor.position: '{"x":380,"y":490,"width":180,"height":54}'
-      Create fully connected tensorflow network:
-        componentRef:
-          digest: bfcafbc5ce711b1f69cabf1338212d10d50136a73db9f9f7c984de7b80b4bfb0
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/9ca0f9eecf5f896f65b8538bbd809747052617d1/components/tensorflow/Create_fully_connected_network/component.yaml
-        arguments:
-          input_size: '7'
-          hidden_layer_sizes: '[10]'
-          activation_name: elu
-        annotations:
-          editor.position: '{"x":40,"y":500,"width":180,"height":54}'
-      Train model using Keras on CSV:
-        componentRef:
-          digest: 42ae60c889034dbad74815653e95b4f7d576b5f47f803173e8679c7b54984609
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c504a4010348c50eaaf6d4337586ccc008f4dcef/components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml
-        arguments:
-          training_data:
-            taskOutput:
-              outputName: split_1
-              taskId: Split rows into subsets
-              type: CSV
-          model:
-            taskOutput:
-              outputName: model
-              taskId: Create fully connected tensorflow network
-              type: TensorflowSavedModel
-          label_column_name: tips
-          number_of_epochs: '10'
-          metric_names: '["mean_absolute_error"]'
-        annotations:
-          editor.position: '{"x":40,"y":620,"width":180,"height":54}'
-      Train pytorch model from csv:
-        componentRef:
-          digest: 40f3185eb61e9727f41a4e0c05dd3d3b44bd802aa0f378cfc31756560033949a
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml
-        arguments:
-          model:
-            taskOutput:
-              outputName: model
-              taskId: Create fully connected pytorch network
-              type: PyTorchScriptModule
-          training_data:
-            taskOutput:
-              outputName: split_1
-              taskId: Split rows into subsets
-              type: CSV
-          label_column_name: tips
-        annotations:
-          editor.position: '{"x":380,"y":620,"width":180,"height":40}'
-      Train XGBoost model on CSV:
-        componentRef:
-          digest: 538c5a01eb38deaf532d619f0bbeaff4efc550fe1f0f776fc06791097b68ceac
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/58d3a47f904f32a64af8403330ba7e2134cae46d/components/XGBoost/Train/component.yaml
-        arguments:
-          training_data:
-            taskOutput:
-              outputName: split_1
-              taskId: Split rows into subsets
-              type: CSV
-          label_column_name: tips
-        annotations:
-          editor.position: '{"x":720,"y":620,"width":180,"height":40}'
-      Train linear regression model using scikit learn from CSV:
-        componentRef:
-          digest: c7fe7912ab0d1fb45d201d452e9ce6be5544e7d8c6d229db7a4b931ff58560f3
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/f807e02b54d4886c65a05f40848fd51c72407f40/components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml
-        arguments:
-          dataset:
-            taskOutput:
-              outputName: split_1
-              taskId: Split rows into subsets
-              type: CSV
-          label_column_name: tips
-        annotations:
-          editor.position: '{"x":1030,"y":620,"width":180,"height":54}'
-      Predict with TensorFlow model on CSV data:
-        componentRef:
-          digest: 921bb1563e93a78233b8acceab87055b9154ccf5595d056028cf0396ca224cd4
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/59c759ce6f543184e30db6817d2a703879bc0f39/components/tensorflow/Predict/on_CSV/component.yaml
-        arguments:
-          dataset:
-            taskOutput:
-              outputName: split_2
-              taskId: Split rows into subsets
-              type: CSV
-          model:
-            taskOutput:
-              outputName: trained_model
-              taskId: Train model using Keras on CSV
-              type: TensorflowSavedModel
-          label_column_name: tips
-        annotations:
-          editor.position: '{"x":160,"y":750,"width":180,"height":54}'
-      Create PyTorch Model Archive with base handler:
-        componentRef:
-          digest: 8298b5ee1b0f0879f893add4cf352c8dec7cf9e21bb9db134c91a2d046cdb0ec
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/46d51383e6554b7f3ab4fd8cf614d8c2b422fb22/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml
-        arguments:
-          Model:
-            taskOutput:
-              outputName: trained_model
-              taskId: Train pytorch model from csv
-              type: PyTorchScriptModule
-          Model name: model
-          Model version: '1.0'
-        annotations:
-          editor.position: '{"x":380,"y":750,"width":180,"height":54}'
-      Xgboost predict on CSV:
-        componentRef:
-          digest: 0876233a0c7306fefec188bd70f059b46d1fb5aa57be231799570e3bbbdd0d95
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/4694ec97baccf59284c2a1db4aa2250c22291eab/components/XGBoost/Predict/component.yaml
-        arguments:
-          data:
-            taskOutput:
-              outputName: split_2
-              taskId: Split rows into subsets
-              type: CSV
-          model:
-            taskOutput:
-              outputName: model
-              taskId: Train XGBoost model on CSV
-              type: XGBoostModel
-          label_column_name: tips
-        annotations:
-          editor.position: '{"x":810,"y":750,"width":180,"height":40}'
-      Upload Scikit learn pickle model to Google Cloud Vertex AI:
-        componentRef:
-          digest: 81c91c8d7d21ec97e0872f669d68bd89edea87279d703685db54aa94743bebcd
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/workaround_for_buggy_KFPv2_compiler/component.yaml
-        arguments:
-          model:
-            taskOutput:
-              outputName: model
-              taskId: Train linear regression model using scikit learn from CSV
-              type: ScikitLearnPickleModel
-        annotations:
-          editor.position: '{"x":1030,"y":750,"width":180,"height":70}'
-      Upload Tensorflow model to Google Cloud Vertex AI:
-        componentRef:
-          digest: 2e45263ff640b1a688e359b6936e27a81b2407749a84f340af2aa5547e0cb92c
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/workaround_for_buggy_KFPv2_compiler/component.yaml
-        arguments:
-          model:
-            taskOutput:
-              outputName: trained_model
-              taskId: Train model using Keras on CSV
-              type: TensorflowSavedModel
-        annotations:
-          editor.position: '{"x":40,"y":880,"width":180,"height":54}'
-      Upload PyTorch model archive to Google Cloud Vertex AI:
-        componentRef:
-          digest: 4450212fae7b9001482aca7eb78b28413c205506eccf08a04e7754a8dfa99004
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/workaround_for_buggy_KFPv2_compiler/component.yaml
-        arguments:
-          model_archive:
-            taskOutput:
-              outputName: Model archive
-              taskId: Create PyTorch Model Archive with base handler
-              type: PyTorchModelArchive
-        annotations:
-          editor.position: '{"x":380,"y":880,"width":180,"height":70}'
-      Upload XGBoost model to Google Cloud Vertex AI:
-        componentRef:
-          digest: 5a5a273c403670743820986c03a4175b7cb4595a556524fefcce403656286977
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/workaround_for_buggy_KFPv2_compiler/component.yaml
-        arguments:
-          model:
-            taskOutput:
-              outputName: model
-              taskId: Train XGBoost model on CSV
-              type: XGBoostModel
-        annotations:
-          editor.position: '{"x":720,"y":880,"width":180,"height":54}'
-    outputValues: {}

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_regression_model_using_all_frameworks_and_import_to_Vertex_AI/pipeline.py

@@ -1,208 +0,0 @@
-# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
-from kfp import components
-
-# %% Loading components
-download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/storage/download/component.yaml")
-select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml")
-fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
-split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
-
-# TensorFlow
-create_fully_connected_tensorflow_network_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Create_fully_connected_network/component.yaml")
-train_model_using_Keras_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml")
-predict_with_TensorFlow_model_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Predict/on_CSV/component.yaml")
-upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/component.yaml")
-
-# PyTorch
-create_fully_connected_pytorch_network_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Create_fully_connected_network/component.yaml")
-train_pytorch_model_from_csv_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml")
-create_pytorch_model_archive_with_base_handler_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml")
-upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/component.yaml")
-
-# XGBoost
-train_XGBoost_model_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Train/component.yaml")
-xgboost_predict_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Predict/component.yaml")
-upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/component.yaml")
-
-# Scikit-learn
-train_linear_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/1f5cf6e06409b704064b2086c0a705e4e6b4fcde/community-content/pipeline_components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml")
-upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/component.yaml")
-
-# Vertex AI
-deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
-
-# %% Pipeline definition
-def train_tabular_regression_model_using_all_frameworks_pipeline():
-    dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
-    feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]  # Excluded "trip_total"
-    label_column = "tips"
-    training_set_fraction = 0.8
-    # Deploying the model might incur additional costs over time
-    deploy_model = False
-
-    all_columns = [label_column] + feature_columns
-
-    dataset = download_from_gcs_op(
-        gcs_path=dataset_gcs_uri
-    ).outputs["Data"]
-
-    dataset = select_columns_using_Pandas_on_CSV_data_op(
-        table=dataset,
-        column_names=all_columns,
-    ).outputs["transformed_table"]
-
-    dataset = fill_all_missing_values_using_Pandas_on_CSV_data_op(
-        table=dataset,
-        replacement_value="0",
-        # # Optional:
-        # column_names=None,  # =[...]
-    ).outputs["transformed_table"]
-
-    split_task = split_rows_into_subsets_op(
-        table=dataset,
-        fraction_1=training_set_fraction,
-    )
-    training_data = split_task.outputs["split_1"]
-    testing_data = split_task.outputs["split_2"]
-
-    # TensorFlow
-    tensorflow_network = create_fully_connected_tensorflow_network_op(
-        input_size=len(feature_columns),
-        # Optional:
-        hidden_layer_sizes=[10],
-        activation_name="elu",
-        # output_activation_name=None,
-        # output_size=1,
-    ).outputs["model"]
-
-    tensorflow_model = train_model_using_Keras_on_CSV_op(
-        training_data=training_data,
-        model=tensorflow_network,
-        label_column_name=label_column,
-        # Optional:
-        #loss_function_name="mean_squared_error",
-        number_of_epochs=10,
-        #learning_rate=0.1,
-        #optimizer_name="Adadelta",
-        #optimizer_parameters={},
-        #batch_size=32,
-        metric_names=["mean_absolute_error"],
-        #random_seed=0,
-    ).outputs["trained_model"]
-
-    tensorflow_predictions = predict_with_TensorFlow_model_on_CSV_data_op(
-        dataset=testing_data,
-        model=tensorflow_model,
-        # label_column_name needs to be set when doing prediction on a dataset that has labels
-        label_column_name=label_column,
-        # Optional:
-        # batch_size=1000,
-    ).outputs["predictions"]
-
-    tensorflow_vertex_model_name = upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op(
-        model=tensorflow_model,
-    ).outputs["model_name"]
-
-    # Deploying the model might incur additional costs over time
-    if deploy_model:
-        tensorflow_vertex_endpoint_name = deploy_model_to_endpoint_op(
-            model_name=tensorflow_vertex_model_name,
-        ).outputs["endpoint_name"]
-
-    # PyTorch
-    pytorch_network = create_fully_connected_pytorch_network_op(
-        input_size=len(feature_columns),
-        # Optional:
-        hidden_layer_sizes=[10],
-        activation_name="elu",
-        # output_activation_name=None,
-        # output_size=1,
-    ).outputs["model"]
-
-    pytorch_model = train_pytorch_model_from_csv_op(
-        model=pytorch_network,
-        training_data=training_data,
-        label_column_name=label_column,
-        # Optional:
-        #loss_function_name="mse_loss",
-        #number_of_epochs=1,
-        #learning_rate=0.1,
-        #optimizer_name="Adadelta",
-        #optimizer_parameters={},
-        #batch_size=32,
-        #batch_log_interval=100,
-        #random_seed=0,
-    ).outputs["trained_model"]
-
-    pytorch_model_archive = create_pytorch_model_archive_with_base_handler_op(
-        model=pytorch_model,
-        # Optional:
-        # model_name="model",
-        # model_version="1.0",
-    ).outputs["Model archive"]
-
-    pytorch_vertex_model_name = upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op(
-        model_archive=pytorch_model_archive,
-    ).outputs["model_name"]
-
-    # Deploying the model might incur additional costs over time
-    if deploy_model:
-        pytorch_vertex_endpoint_name = deploy_model_to_endpoint_op(
-            model_name=pytorch_vertex_model_name,
-        ).outputs["endpoint_name"]
-
-    # XGBoost
-    xgboost_model = train_XGBoost_model_on_CSV_op(
-        training_data=training_data,
-        label_column_name=label_column,
-        # Optional:
-        #starting_model=None,
-        #num_iterations=10,
-        #booster_params={},
-        #objective="reg:squarederror",
-        #booster="gbtree",
-        #learning_rate=0.3,
-        #min_split_loss=0,
-        #max_depth=6,
-    ).outputs["model"]
-
-    # Predicting on the testing data
-    xgboost_predictions = xgboost_predict_on_CSV_op(
-        data=testing_data,
-        model=xgboost_model,
-        # label_column needs to be set when doing prediction on a dataset that has labels
-        label_column_name=label_column,
-    ).outputs["predictions"]
-
-    xgboost_vertex_model_name = upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op(
-        model=xgboost_model,
-    ).outputs["model_name"]
-
-    # Deploying the model might incur additional costs over time
-    if deploy_model:
-        xgboost_vertex_endpoint_name = deploy_model_to_endpoint_op(
-            model_name=xgboost_vertex_model_name,
-        ).outputs["endpoint_name"]
-
-    # Scikit-learn
-    sklearn_model = train_linear_regression_model_using_scikit_learn_from_CSV_op(
-        dataset=training_data,
-        label_column_name=label_column,
-    ).outputs["model"]
-
-    sklearn_vertex_model_name = upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op(
-        model=sklearn_model,
-    ).outputs["model_name"]
-
-    # Deploying the model might incur additional costs over time
-    if deploy_model:
-        sklearn_vertex_endpoint_name = deploy_model_to_endpoint_op(
-            model_name=sklearn_vertex_model_name,
-        ).outputs["endpoint_name"]
-
-pipeline_func=train_tabular_regression_model_using_all_frameworks_pipeline
-
-# %% Pipeline submission
-if __name__ == '__main__':
-    from google.cloud import aiplatform
-    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

community-content/cpr-examples/timm_serving/.gitignore

@@ -2,5 +2,4 @@ cpr_model_server.py
 entrypoint.py
 state_dict.pth
 config.json
-**/__pycache__
-!testdata/**
+**/__pycache__

community-content/cpr-examples/timm_serving/README.md

@@ -2,7 +2,7 @@
 
 ## About CPR
 
-CPR ([custom prediction routines](https://github.com/googleapis/python-aiplatform/blob/main/google/cloud/aiplatform/prediction/README.md)) is a framework designed by Google Cloud developers to make it easier to combine machine learning models with custom preprocessing and postprocessing logic in a real-time serving application. 
+CPR ([custom prediction routines](https://github.com/googleapis/python-aiplatform/blob/custom-prediction-routine/google/cloud/aiplatform/prediction/README.md)) is a framework designed by Google Cloud developers to make it easier to combine machine learning models with custom preprocessing and postprocessing logic in a real-time serving application. 
 
 ## Using this example
 
@@ -34,23 +34,6 @@ Finally, install the Python modules required to build and run the model server:
 pip install -r requirements.txt
 ```
 
-### Auth
-
-This example uses Google Cloud Storage for hosting model artifacts and Artifact Registry to store the container image. 
-You'll need to authorize yourself before you can interact with these.
-
-First, log in to GCP with application default credentials:
-```sh
-gcloud auth application-default login
-```
-
-Next, if you haven't done so already, set up the [gcloud credential helper](https://cloud.google.com/artifact-registry/docs/docker/authentication)
-for the Artifact Registry region where you intend to host the image.  
-```
-gcloud auth configure-docker <region>-docker.pkg.dev
-```
-
-
 ### Predictor
 
 The `TimmPredictor` class in `timm_serving/predictor.py` implements most of the important logic for the server.

community-content/cpr-examples/timm_serving/config.py

@@ -60,9 +60,9 @@ class CPRConfig(object):
     image: str = "timm_predictor:latest"
     artifact_local_dir: str = ""
     region: str = "us-central1"
-    project_id: str = "<your project ID here>"
+    project_id: str = "samthrasher-experimental"
     repository: str = "cpr-images"
-    artifact_gcs_dir: str = "gs://<your bucket ID here>/timm-vit224/"
+    artifact_gcs_dir: str = "gs://samthrasher-cpr-example/timm-vit224/"
     model_name: str = ""
     endpoint_name: str = ""
     machine_type: str = "n1-standard-2"

community-content/cpr-examples/timm_serving/requirements.txt

@@ -5,4 +5,4 @@ timm==0.5.4
 smart_open==6.0.0
 
 google-cloud-storage>=1.26.0,<2.0.0dev
-google-cloud-aiplatform[prediction]>=1.16.0
+google-cloud-aiplatform[prediction] @ git+https://github.com/googleapis/python-aiplatform.git@custom-prediction-routine

community-content/cpr-examples/timm_serving/test.py

@@ -70,10 +70,7 @@ class PredictorUnitTests(absltest.TestCase):
     def setUp(self):
         super().setUp()
         self.config = CPRConfig()
-        try:
-            self.config.load()
-        except FileNotFoundError:
-            logging.info("No saved config file found, using default values.")
+        self.config.load()
         self.predictor = predictor.TimmPredictor()
 
     def test_load_from_saved_state_dict_ok(self):
@@ -173,10 +170,7 @@ class ServerEndToEndTests(absltest.TestCase):
     def setUp(self):
         super().setUp()
         self.config = CPRConfig()
-        try:
-            self.config.load()
-        except FileNotFoundError:
-            logging.info("No saved config file found, using default values.")
+        self.config.load()
         self.local_model = cpr.LocalModel(
             serving_container_spec=aiplatform.gapic.ModelContainerSpec(
                 image_uri=self.config.image

community-content/cpr-examples/timm_serving/testdata/bad_model_1/state_dict.pth

@@ -1 +0,0 @@
-blah

community-content/pipeline_components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml

@@ -1,64 +0,0 @@
-name: Train linear regression model using scikit learn from CSV
-metadata:
-  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml'}
-inputs:
-- {name: dataset, type: CSV}
-- {name: label_column_name, type: String}
-outputs:
-- {name: model, type: ScikitLearnPickleModel}
-implementation:
-  container:
-    image: python:3.9
-    command:
-    - sh
-    - -c
-    - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
-      'scikit-learn==1.0.2' 'pandas==1.4.3' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3
-      -m pip install --quiet --no-warn-script-location 'scikit-learn==1.0.2' 'pandas==1.4.3'
-      --user) && "$0" "$@"
-    - sh
-    - -ec
-    - |
-      program_path=$(mktemp)
-      printf "%s" "$0" > "$program_path"
-      python3 -u "$program_path" "$@"
-    - |
-      def _make_parent_dirs_and_return_path(file_path: str):
-          import os
-          os.makedirs(os.path.dirname(file_path), exist_ok=True)
-          return file_path
-
-      def train_linear_regression_model_using_scikit_learn_from_CSV(
-          dataset_path,
-          model_path,
-          label_column_name,
-      ):
-          import pandas
-          import pickle
-          from sklearn import linear_model
-
-          df = pandas.read_csv(dataset_path)
-          model = linear_model.LinearRegression()
-          model.fit(
-              X=df.drop(columns=label_column_name),
-              y=df[label_column_name],
-          )
-
-          with open(model_path, "wb") as f:
-              pickle.dump(model, f)
-
-      import argparse
-      _parser = argparse.ArgumentParser(prog='Train linear regression model using scikit learn from CSV', description='')
-      _parser.add_argument("--dataset", dest="dataset_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--label-column-name", dest="label_column_name", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--model", dest="model_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
-      _parsed_args = vars(_parser.parse_args())
-
-      _outputs = train_linear_regression_model_using_scikit_learn_from_CSV(**_parsed_args)
-    args:
-    - --dataset
-    - {inputPath: dataset}
-    - --label-column-name
-    - {inputValue: label_column_name}
-    - --model
-    - {outputPath: model}

community-content/pipeline_components/ML_frameworks/Scikit_learn/Train_logistic_regression_model/from_CSV/component.yaml

@@ -1,163 +0,0 @@
-name: Train logistic regression model using scikit learn from CSV
-description: Train logistic regression model using Scikit-learn
-metadata:
-  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/ML_frameworks/Scikit_learn/Train_logistic_regression_model/from_CSV/component.yaml'}
-inputs:
-- {name: dataset, type: CSV}
-- {name: label_column_name, type: String}
-- {name: penalty, type: String, default: l2, optional: true}
-- {name: solver, type: String, default: lbfgs, optional: true}
-- {name: max_iterations, type: Integer, default: '100', optional: true}
-- {name: multi_class_mode, type: String, default: auto, optional: true}
-- {name: random_seed, type: Integer, default: '0', optional: true}
-outputs:
-- {name: model, type: ScikitLearnPickleModel}
-- {name: model_parameters, type: JsonObject}
-implementation:
-  container:
-    image: python:3.9
-    command:
-    - sh
-    - -c
-    - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
-      'scikit-learn==1.0.2' 'pandas==1.4.3' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3
-      -m pip install --quiet --no-warn-script-location 'scikit-learn==1.0.2' 'pandas==1.4.3'
-      --user) && "$0" "$@"
-    - sh
-    - -ec
-    - |
-      program_path=$(mktemp)
-      printf "%s" "$0" > "$program_path"
-      python3 -u "$program_path" "$@"
-    - |
-      def _make_parent_dirs_and_return_path(file_path: str):
-          import os
-          os.makedirs(os.path.dirname(file_path), exist_ok=True)
-          return file_path
-
-      def train_logistic_regression_model_using_scikit_learn_from_CSV(
-          dataset_path,
-          model_path,
-          label_column_name,
-          penalty = "l2", # l1, l2, elasticnet, none
-          solver = "lbfgs", # newton-cg, lbfgs, liblinear, sag, saga
-          max_iterations = 100,
-          multi_class_mode = "auto", # auto, ovr, multinomial
-          random_seed = 0,
-      ):
-          """Train logistic regression model using Scikit-learn
-
-          See https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html
-          """
-          import json
-          import pandas
-          import pickle
-          from sklearn import linear_model
-
-          df = pandas.read_csv(dataset_path)
-          model = linear_model.LogisticRegression(
-              penalty=penalty,
-              #dual=False,
-              #tol=1e-4,
-              #C=1.0,
-              #fit_intercept=True,
-              #intercept_scaling=1,
-              #class_weight=None,
-              random_state=random_seed,
-              solver=solver,
-              max_iter=max_iterations,
-              multi_class=multi_class_mode,
-              #l1_ratio=None,
-              verbose=1,
-          )
-
-          model_parameters = model.get_params()
-          model_parameters_json = json.dumps(model_parameters, indent=2)
-          print("Model parameters:")
-          print(model_parameters_json)
-          print()
-
-          model.fit(
-              X=df.drop(columns=label_column_name),
-              y=df[label_column_name],
-          )
-
-          with open(model_path, "wb") as f:
-              pickle.dump(model, f)
-
-          return (model_parameters_json,)
-
-      def _serialize_json(obj) -> str:
-          if isinstance(obj, str):
-              return obj
-          import json
-          def default_serializer(obj):
-              if hasattr(obj, 'to_struct'):
-                  return obj.to_struct()
-              else:
-                  raise TypeError("Object of type '%s' is not JSON serializable and does not have .to_struct() method." % obj.__class__.__name__)
-          return json.dumps(obj, default=default_serializer, sort_keys=True)
-
-      import argparse
-      _parser = argparse.ArgumentParser(prog='Train logistic regression model using scikit learn from CSV', description='Train logistic regression model using Scikit-learn')
-      _parser.add_argument("--dataset", dest="dataset_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--label-column-name", dest="label_column_name", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--penalty", dest="penalty", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--solver", dest="solver", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--max-iterations", dest="max_iterations", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--multi-class-mode", dest="multi_class_mode", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--random-seed", dest="random_seed", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--model", dest="model_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("----output-paths", dest="_output_paths", type=str, nargs=1)
-      _parsed_args = vars(_parser.parse_args())
-      _output_files = _parsed_args.pop("_output_paths", [])
-
-      _outputs = train_logistic_regression_model_using_scikit_learn_from_CSV(**_parsed_args)
-
-      _output_serializers = [
-          _serialize_json,
-
-      ]
-
-      import os
-      for idx, output_file in enumerate(_output_files):
-          try:
-              os.makedirs(os.path.dirname(output_file))
-          except OSError:
-              pass
-          with open(output_file, 'w') as f:
-              f.write(_output_serializers[idx](_outputs[idx]))
-    args:
-    - --dataset
-    - {inputPath: dataset}
-    - --label-column-name
-    - {inputValue: label_column_name}
-    - if:
-        cond: {isPresent: penalty}
-        then:
-        - --penalty
-        - {inputValue: penalty}
-    - if:
-        cond: {isPresent: solver}
-        then:
-        - --solver
-        - {inputValue: solver}
-    - if:
-        cond: {isPresent: max_iterations}
-        then:
-        - --max-iterations
-        - {inputValue: max_iterations}
-    - if:
-        cond: {isPresent: multi_class_mode}
-        then:
-        - --multi-class-mode
-        - {inputValue: multi_class_mode}
-    - if:
-        cond: {isPresent: random_seed}
-        then:
-        - --random-seed
-        - {inputValue: random_seed}
-    - --model
-    - {outputPath: model}
-    - '----output-paths'
-    - {outputPath: model_parameters}

community-content/pipeline_components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml

@@ -1,41 +0,0 @@
-name: Create PyTorch Model Archive with base handler
-inputs:
-- {name: Model, type: PyTorchScriptModule}
-- {name: Model name, type: String, default: model}
-- {name: Model version, type: String, default: "1.0"}
-outputs:
-- {name: Model archive, type: PyTorchModelArchive}
-metadata:
-  annotations:
-    author: Alexey Volkov <alexey.volkov@ark-kun.com>
-    canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml'
-implementation:
-  container:
-    image: pytorch/torchserve:0.6.0-cpu
-    command:
-    - bash
-    - -exc
-    - |
-      model_path=$0
-      model_name=$1
-      model_version=$2
-      output_model_archive_path=$3
-
-      mkdir -p "$(dirname "$output_model_archive_path")"
-
-      # TODO: Use the built-in base_handler once my fix is merged: https://github.com/pytorch/serve/pull/1682
-      echo '
-      from ts.torch_handler import base_handler
-      class BaseHandler(base_handler.BaseHandler):
-          pass
-      ' > base_handler.py  # torch-model-archiver needs the handler to have .py extension
-      torch-model-archiver --model-name "$model_name" --version "$model_version" --serialized-file "$model_path" --handler base_handler.py
-
-      # torch-model-archiver does not allow specifying the output path, but always writes to "${model_name}.<format>"
-      expected_model_archive_path="${model_name}.mar"
-      mv "$expected_model_archive_path" "$output_model_archive_path"
-
-    - {inputPath: Model}
-    - {inputValue: Model name}
-    - {inputValue: Model version}
-    - {outputPath: Model archive}

community-content/pipeline_components/PyTorch/Create_fully_connected_network/component.yaml

@@ -1,117 +0,0 @@
-name: Create fully connected pytorch network
-description: Creates fully-connected network in PyTorch ScriptModule format
-metadata:
-  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/PyTorch/Create_fully_connected_network/component.yaml'}
-inputs:
-- {name: input_size, type: Integer}
-- {name: hidden_layer_sizes, type: JsonArray, default: '[]', optional: true}
-- {name: output_size, type: Integer, default: '1', optional: true}
-- {name: activation_name, type: String, default: relu, optional: true}
-- {name: output_activation_name, type: String, optional: true}
-- {name: random_seed, type: Integer, default: '0', optional: true}
-outputs:
-- {name: model, type: PyTorchScriptModule}
-implementation:
-  container:
-    image: pytorch/pytorch:1.7.1-cuda11.0-cudnn8-runtime
-    command:
-    - sh
-    - -ec
-    - |
-      program_path=$(mktemp)
-      printf "%s" "$0" > "$program_path"
-      python3 -u "$program_path" "$@"
-    - |
-      def _make_parent_dirs_and_return_path(file_path: str):
-          import os
-          os.makedirs(os.path.dirname(file_path), exist_ok=True)
-          return file_path
-
-      def create_fully_connected_pytorch_network(
-          input_size,
-          model_path,
-          hidden_layer_sizes = [],
-          output_size = 1,
-          activation_name = 'relu',
-          output_activation_name = None,
-          random_seed = 0,
-      ):
-          '''Creates fully-connected network in PyTorch ScriptModule format'''
-          import torch
-          torch.manual_seed(random_seed)
-
-          activation = getattr(torch, activation_name, None) or getattr(torch.nn.functional, activation_name, None)
-          if not activation:
-              raise ValueError(f'Activation "{activation_name}" was not found.')
-
-          class ActivationLayer(torch.nn.Module):
-              def forward(self, input):
-                  return activation(input)
-
-          layers = []
-          prev_layer_size = input_size
-          for layer_size in hidden_layer_sizes:
-              layer = torch.nn.Linear(prev_layer_size, layer_size)
-              prev_layer_size = layer_size
-              layers.append(layer)
-              layers.append(ActivationLayer())
-
-          # Adding the output layer
-          layers.append(torch.nn.Linear(prev_layer_size, output_size))
-
-          # Adding the optional activation after the output layer
-          if output_activation_name:
-              output_activation = getattr(torch, output_activation_name, None) or getattr(torch.nn.functional, output_activation_name, None)
-              class OutputActivationLayer(torch.nn.Module):
-                  def forward(self, input):
-                      return output_activation(input)
-              layers.append(OutputActivationLayer())
-
-          network = torch.nn.Sequential(*layers)
-          script_module = torch.jit.script(network)
-          print(script_module)
-          script_module.save(model_path)
-
-      import json
-      import argparse
-      _parser = argparse.ArgumentParser(prog='Create fully connected pytorch network', description='Creates fully-connected network in PyTorch ScriptModule format')
-      _parser.add_argument("--input-size", dest="input_size", type=int, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--hidden-layer-sizes", dest="hidden_layer_sizes", type=json.loads, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--output-size", dest="output_size", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--activation-name", dest="activation_name", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--output-activation-name", dest="output_activation_name", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--random-seed", dest="random_seed", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--model", dest="model_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
-      _parsed_args = vars(_parser.parse_args())
-
-      _outputs = create_fully_connected_pytorch_network(**_parsed_args)
-    args:
-    - --input-size
-    - {inputValue: input_size}
-    - if:
-        cond: {isPresent: hidden_layer_sizes}
-        then:
-        - --hidden-layer-sizes
-        - {inputValue: hidden_layer_sizes}
-    - if:
-        cond: {isPresent: output_size}
-        then:
-        - --output-size
-        - {inputValue: output_size}
-    - if:
-        cond: {isPresent: activation_name}
-        then:
-        - --activation-name
-        - {inputValue: activation_name}
-    - if:
-        cond: {isPresent: output_activation_name}
-        then:
-        - --output-activation-name
-        - {inputValue: output_activation_name}
-    - if:
-        cond: {isPresent: random_seed}
-        then:
-        - --random-seed
-        - {inputValue: random_seed}
-    - --model
-    - {outputPath: model}

community-content/pipeline_components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml

@@ -1,209 +0,0 @@
-name: Train pytorch model from csv
-description: Trains PyTorch model
-metadata:
-  annotations:
-    author: Alexey Volkov <alexey.volkov@ark-kun.com>
-    canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml'
-inputs:
-- {name: model, type: PyTorchScriptModule}
-- {name: training_data, type: CSV}
-- {name: label_column_name, type: String}
-- {name: loss_function_name, type: String, default: mse_loss, optional: true}
-- {name: number_of_epochs, type: Integer, default: '1', optional: true}
-- {name: learning_rate, type: Float, default: '0.1', optional: true}
-- {name: optimizer_name, type: String, default: Adadelta, optional: true}
-- {name: optimizer_parameters, type: JsonObject, optional: true}
-- {name: batch_size, type: Integer, default: '32', optional: true}
-- {name: batch_log_interval, type: Integer, default: '100', optional: true}
-- {name: random_seed, type: Integer, default: '0', optional: true}
-outputs:
-- {name: trained_model, type: PyTorchScriptModule}
-implementation:
-  container:
-    image: pytorch/pytorch:1.7.1-cuda11.0-cudnn8-runtime
-    command:
-    - sh
-    - -c
-    - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
-      'pandas==1.4.3' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet
-      --no-warn-script-location 'pandas==1.4.3' --user) && "$0" "$@"
-    - sh
-    - -ec
-    - |
-      program_path=$(mktemp)
-      printf "%s" "$0" > "$program_path"
-      python3 -u "$program_path" "$@"
-    - |
-      def _make_parent_dirs_and_return_path(file_path: str):
-          import os
-          os.makedirs(os.path.dirname(file_path), exist_ok=True)
-          return file_path
-
-      def train_pytorch_model_from_csv(
-          model_path,
-          training_data_path,
-          trained_model_path,
-          label_column_name,
-          loss_function_name = 'mse_loss',
-          number_of_epochs = 1,
-          learning_rate = 0.1,
-          optimizer_name = 'Adadelta',
-          optimizer_parameters = None,
-          batch_size = 32,
-          batch_log_interval = 100,
-          random_seed = 0,
-      ):
-          '''Trains PyTorch model'''
-          import pandas
-          import torch
-
-          torch.manual_seed(random_seed)
-
-          use_cuda = torch.cuda.is_available()
-          device = torch.device("cuda" if use_cuda else "cpu")
-
-          model = torch.jit.load(model_path)
-          model.to(device)
-          model.train()
-
-          optimizer_class = getattr(torch.optim, optimizer_name, None)
-          if not optimizer_class:
-              raise ValueError(f'Optimizer "{optimizer_name}" was not found.')
-
-          optimizer_parameters = optimizer_parameters or {}
-          optimizer_parameters['lr'] = learning_rate
-          optimizer = optimizer_class(model.parameters(), **optimizer_parameters)
-
-          loss_function = getattr(torch, loss_function_name, None) or getattr(torch.nn, loss_function_name, None) or getattr(torch.nn.functional, loss_function_name, None)
-          if not loss_function:
-              raise ValueError(f'Loss function "{loss_function_name}" was not found.')
-
-          class CsvDataset(torch.utils.data.Dataset):
-
-              def __init__(self, file_path, label_column_name, drop_nan_columns_or_rows = 'columns'):
-                  dataframe = pandas.read_csv(file_path).convert_dtypes()
-                  # Preventing error: default_collate: batch must contain tensors, numpy arrays, numbers, dicts or lists; found object
-                  if drop_nan_columns_or_rows == 'columns':
-                      non_nan_data = dataframe.dropna(axis='columns')
-                      removed_columns = set(dataframe.columns) - set(non_nan_data.columns)
-                      if removed_columns:
-                          print('Skipping columns with NaNs: ' + str(removed_columns))
-                      dataframe = non_nan_data
-                  if drop_nan_columns_or_rows == 'rows':
-                      non_nan_data = dataframe.dropna(axis='index')
-                      number_of_removed_rows = len(dataframe) - len(non_nan_data)
-                      if number_of_removed_rows:
-                          print(f'Skipped {number_of_removed_rows} rows with NaNs.')
-                      dataframe = non_nan_data
-                  numerical_data = dataframe.select_dtypes(include='number')
-                  non_numerical_data = dataframe.select_dtypes(exclude='number')
-                  if not non_numerical_data.empty:
-                      print('Skipping non-number columns:')
-                      print(non_numerical_data.dtypes)
-                  self._dataframe = dataframe
-                  self.labels = numerical_data[[label_column_name]]
-                  self.features = numerical_data.drop(columns=[label_column_name])
-
-              def __len__(self):
-                  return len(self._dataframe)
-
-              def __getitem__(self, index):
-                  return [self.features.loc[index].to_numpy(dtype='float32'), self.labels.loc[index].to_numpy(dtype='float32')]
-
-          dataset = CsvDataset(
-              file_path=training_data_path,
-              label_column_name=label_column_name,
-          )
-          train_loader = torch.utils.data.DataLoader(
-              dataset=dataset,
-              batch_size=batch_size,
-              shuffle=True,
-          )
-
-          last_full_batch_loss = None
-          for epoch in range(1, number_of_epochs + 1):
-              for batch_idx, (data, target) in enumerate(train_loader):
-                  data, target = data.to(device), target.to(device)
-                  optimizer.zero_grad()
-                  output = model(data)
-                  loss = loss_function(output, target)
-                  loss.backward()
-                  optimizer.step()
-                  if len(data) == batch_size:
-                      last_full_batch_loss = loss.item()
-                  if batch_idx % batch_log_interval == 0:
-                      print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
-                          epoch, batch_idx * len(data), len(train_loader.dataset),
-                          100. * batch_idx / len(train_loader), loss.item()))
-              print(f'Training epoch {epoch} completed. Last full batch loss: {last_full_batch_loss:.6f}')
-
-          # print(optimizer.state_dict())
-          model.save(trained_model_path)
-
-      import json
-      import argparse
-      _parser = argparse.ArgumentParser(prog='Train pytorch model from csv', description='Trains PyTorch model')
-      _parser.add_argument("--model", dest="model_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--training-data", dest="training_data_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--label-column-name", dest="label_column_name", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--loss-function-name", dest="loss_function_name", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--number-of-epochs", dest="number_of_epochs", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--learning-rate", dest="learning_rate", type=float, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--optimizer-name", dest="optimizer_name", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--optimizer-parameters", dest="optimizer_parameters", type=json.loads, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--batch-size", dest="batch_size", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--batch-log-interval", dest="batch_log_interval", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--random-seed", dest="random_seed", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--trained-model", dest="trained_model_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
-      _parsed_args = vars(_parser.parse_args())
-
-      _outputs = train_pytorch_model_from_csv(**_parsed_args)
-    args:
-    - --model
-    - {inputPath: model}
-    - --training-data
-    - {inputPath: training_data}
-    - --label-column-name
-    - {inputValue: label_column_name}
-    - if:
-        cond: {isPresent: loss_function_name}
-        then:
-        - --loss-function-name
-        - {inputValue: loss_function_name}
-    - if:
-        cond: {isPresent: number_of_epochs}
-        then:
-        - --number-of-epochs
-        - {inputValue: number_of_epochs}
-    - if:
-        cond: {isPresent: learning_rate}
-        then:
-        - --learning-rate
-        - {inputValue: learning_rate}
-    - if:
-        cond: {isPresent: optimizer_name}
-        then:
-        - --optimizer-name
-        - {inputValue: optimizer_name}
-    - if:
-        cond: {isPresent: optimizer_parameters}
-        then:
-        - --optimizer-parameters
-        - {inputValue: optimizer_parameters}
-    - if:
-        cond: {isPresent: batch_size}
-        then:
-        - --batch-size
-        - {inputValue: batch_size}
-    - if:
-        cond: {isPresent: batch_log_interval}
-        then:
-        - --batch-log-interval
-        - {inputValue: batch_log_interval}
-    - if:
-        cond: {isPresent: random_seed}
-        then:
-        - --random-seed
-        - {inputValue: random_seed}
-    - --trained-model
-    - {outputPath: trained_model}

community-content/pipeline_components/XGBoost/Predict/component.yaml

@@ -1,110 +0,0 @@
-name: Xgboost predict on CSV
-description: Makes predictions using a trained XGBoost model.
-metadata:
-  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/XGBoost/Predict/component.yaml'}
-inputs:
-- {name: data, type: CSV, description: Feature data in Apache Parquet format.}
-- {name: model, type: XGBoostModel, description: Trained model in binary XGBoost format.}
-- {name: label_column_name, type: String, description: Optional. Name of the column
-    containing the label data that is excluded during the prediction., optional: true}
-outputs:
-- {name: predictions, description: Model predictions.}
-implementation:
-  container:
-    image: python:3.10
-    command:
-    - sh
-    - -c
-    - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
-      'xgboost==1.6.1' 'pandas==1.4.3' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3
-      -m pip install --quiet --no-warn-script-location 'xgboost==1.6.1' 'pandas==1.4.3'
-      --user) && "$0" "$@"
-    - sh
-    - -ec
-    - |
-      program_path=$(mktemp)
-      printf "%s" "$0" > "$program_path"
-      python3 -u "$program_path" "$@"
-    - |
-      def _make_parent_dirs_and_return_path(file_path: str):
-          import os
-          os.makedirs(os.path.dirname(file_path), exist_ok=True)
-          return file_path
-
-      def xgboost_predict_on_CSV(
-          data_path,
-          model_path,
-          predictions_path,
-          label_column_name = None,
-      ):
-          """Makes predictions using a trained XGBoost model.
-
-          Args:
-              data_path: Feature data in Apache Parquet format.
-              model_path: Trained model in binary XGBoost format.
-              predictions_path: Model predictions.
-              label_column_name: Optional. Name of the column containing the label data that is excluded during the prediction.
-
-          Annotations:
-              author: Alexey Volkov <alexey.volkov@ark-kun.com>
-          """
-          from pathlib import Path
-
-          import numpy
-          import pandas
-          import xgboost
-
-          df = pandas.read_csv(
-              data_path,
-          ).convert_dtypes()
-          print("Evaluation data information:")
-          df.info(verbose=True)
-          # Converting column types that XGBoost does not support
-          for column_name, dtype in df.dtypes.items():
-              if dtype in ["string", "object"]:
-                  print(f"Treating the {dtype.name} column '{column_name}' as categorical.")
-                  df[column_name] = df[column_name].astype("category")
-                  print(f"Inferred {len(df[column_name].cat.categories)} categories for the '{column_name}' column.")
-              # Working around the XGBoost issue with nullable floats: https://github.com/dmlc/xgboost/issues/8213
-              if pandas.api.types.is_float_dtype(dtype):
-                  # Converting from "Float64" to "float64"
-                  df[column_name] = df[column_name].astype(dtype.name.lower())
-          print("Final evaluation data information:")
-          df.info(verbose=True)
-
-          if label_column_name is not None:
-              df = df.drop(columns=[label_column_name])
-
-          testing_data = xgboost.DMatrix(
-              data=df,
-              enable_categorical=True,
-          )
-
-          model = xgboost.Booster(model_file=model_path)
-
-          predictions = model.predict(testing_data)
-
-          Path(predictions_path).parent.mkdir(parents=True, exist_ok=True)
-          numpy.savetxt(predictions_path, predictions)
-
-      import argparse
-      _parser = argparse.ArgumentParser(prog='Xgboost predict on CSV', description='Makes predictions using a trained XGBoost model.')
-      _parser.add_argument("--data", dest="data_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--model", dest="model_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--label-column-name", dest="label_column_name", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--predictions", dest="predictions_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
-      _parsed_args = vars(_parser.parse_args())
-
-      _outputs = xgboost_predict_on_CSV(**_parsed_args)
-    args:
-    - --data
-    - {inputPath: data}
-    - --model
-    - {inputPath: model}
-    - if:
-        cond: {isPresent: label_column_name}
-        then:
-        - --label-column-name
-        - {inputValue: label_column_name}
-    - --predictions
-    - {outputPath: predictions}

community-content/pipeline_components/XGBoost/Train/component.yaml

@@ -1,241 +0,0 @@
-name: Train XGBoost model on CSV
-description: Trains an XGBoost model.
-metadata:
-  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/XGBoost/Train/component.yaml'}
-inputs:
-- {name: training_data, type: CSV, description: Training data in CSV format.}
-- {name: label_column_name, type: String, description: Name of the column containing
-    the label data.}
-- {name: starting_model, type: XGBoostModel, description: Existing trained model to
-    start from (in the binary XGBoost format)., optional: true}
-- {name: num_iterations, type: Integer, description: Number of boosting iterations.,
-  default: '10', optional: true}
-- name: objective
-  type: String
-  description: |-
-    The learning task and the corresponding learning objective.
-    See https://xgboost.readthedocs.io/en/latest/parameter.html#learning-task-parameters
-    The most common values are:
-    "reg:squarederror" - Regression with squared loss (default).
-    "reg:logistic" - Logistic regression.
-    "binary:logistic" - Logistic regression for binary classification, output probability.
-    "binary:logitraw" - Logistic regression for binary classification, output score before logistic transformation
-    "rank:pairwise" - Use LambdaMART to perform pairwise ranking where the pairwise loss is minimized
-    "rank:ndcg" - Use LambdaMART to perform list-wise ranking where Normalized Discounted Cumulative Gain (NDCG) is maximized
-  default: reg:squarederror
-  optional: true
-- {name: booster, type: String, description: 'The booster to use. Can be `gbtree`,
-    `gblinear` or `dart`; `gbtree` and `dart` use tree based models while `gblinear`
-    uses linear functions.', default: gbtree, optional: true}
-- {name: learning_rate, type: Float, description: 'Step size shrinkage used in update
-    to prevents overfitting. Range: [0,1].', default: '0.3', optional: true}
-- name: min_split_loss
-  type: Float
-  description: |-
-    Minimum loss reduction required to make a further partition on a leaf node of the tree.
-    The larger `min_split_loss` is, the more conservative the algorithm will be. Range: [0,Inf].
-  default: '0'
-  optional: true
-- name: max_depth
-  type: Integer
-  description: |-
-    Maximum depth of a tree. Increasing this value will make the model more complex and more likely to overfit.
-    0 indicates no limit on depth. Range: [0,Inf].
-  default: '6'
-  optional: true
-- {name: booster_params, type: JsonObject, description: 'Parameters for the booster.
-    See https://xgboost.readthedocs.io/en/latest/parameter.html', optional: true}
-outputs:
-- {name: model, type: XGBoostModel, description: Trained model in the binary XGBoost
-    format.}
-- {name: model_config, type: XGBoostModelConfig, description: The internal parameter
-    configuration of Booster as a JSON string.}
-implementation:
-  container:
-    image: python:3.10
-    command:
-    - sh
-    - -c
-    - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
-      'xgboost==1.6.1' 'pandas==1.4.3' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3
-      -m pip install --quiet --no-warn-script-location 'xgboost==1.6.1' 'pandas==1.4.3'
-      --user) && "$0" "$@"
-    - sh
-    - -ec
-    - |
-      program_path=$(mktemp)
-      printf "%s" "$0" > "$program_path"
-      python3 -u "$program_path" "$@"
-    - |
-      def _make_parent_dirs_and_return_path(file_path: str):
-          import os
-          os.makedirs(os.path.dirname(file_path), exist_ok=True)
-          return file_path
-
-      def train_XGBoost_model_on_CSV(
-          training_data_path,
-          model_path,
-          model_config_path,
-          label_column_name,
-          starting_model_path = None,
-          num_iterations = 10,
-          # Booster parameters
-          objective = "reg:squarederror",
-          booster = "gbtree",
-          learning_rate = 0.3,
-          min_split_loss = 0,
-          max_depth = 6,
-          booster_params = None,
-      ):
-          """Trains an XGBoost model.
-
-          Args:
-              training_data_path: Training data in CSV format.
-              model_path: Trained model in the binary XGBoost format.
-              model_config_path: The internal parameter configuration of Booster as a JSON string.
-              starting_model_path: Existing trained model to start from (in the binary XGBoost format).
-              label_column_name: Name of the column containing the label data.
-              num_iterations: Number of boosting iterations.
-              booster_params: Parameters for the booster. See https://xgboost.readthedocs.io/en/latest/parameter.html
-              objective: The learning task and the corresponding learning objective.
-                  See https://xgboost.readthedocs.io/en/latest/parameter.html#learning-task-parameters
-                  The most common values are:
-                  "reg:squarederror" - Regression with squared loss (default).
-                  "reg:logistic" - Logistic regression.
-                  "binary:logistic" - Logistic regression for binary classification, output probability.
-                  "binary:logitraw" - Logistic regression for binary classification, output score before logistic transformation
-                  "rank:pairwise" - Use LambdaMART to perform pairwise ranking where the pairwise loss is minimized
-                  "rank:ndcg" - Use LambdaMART to perform list-wise ranking where Normalized Discounted Cumulative Gain (NDCG) is maximized
-              booster: The booster to use. Can be `gbtree`, `gblinear` or `dart`; `gbtree` and `dart` use tree based models while `gblinear` uses linear functions.
-              learning_rate: Step size shrinkage used in update to prevents overfitting. Range: [0,1].
-              min_split_loss: Minimum loss reduction required to make a further partition on a leaf node of the tree.
-                  The larger `min_split_loss` is, the more conservative the algorithm will be. Range: [0,Inf].
-              max_depth: Maximum depth of a tree. Increasing this value will make the model more complex and more likely to overfit.
-                  0 indicates no limit on depth. Range: [0,Inf].
-
-          Annotations:
-              author: Alexey Volkov <alexey.volkov@ark-kun.com>
-          """
-          import pandas
-          import xgboost
-
-          df = pandas.read_csv(
-              training_data_path,
-          ).convert_dtypes()
-          print("Training data information:")
-          df.info(verbose=True)
-          # Converting column types that XGBoost does not support
-          for column_name, dtype in df.dtypes.items():
-              if dtype in ["string", "object"]:
-                  print(f"Treating the {dtype.name} column '{column_name}' as categorical.")
-                  df[column_name] = df[column_name].astype("category")
-                  print(f"Inferred {len(df[column_name].cat.categories)} categories for the '{column_name}' column.")
-              # Working around the XGBoost issue with nullable floats: https://github.com/dmlc/xgboost/issues/8213
-              if pandas.api.types.is_float_dtype(dtype):
-                  # Converting from "Float64" to "float64"
-                  df[column_name] = df[column_name].astype(dtype.name.lower())
-          print()
-          print("Final training data information:")
-          df.info(verbose=True)
-
-          training_data = xgboost.DMatrix(
-              data=df.drop(columns=[label_column_name]),
-              label=df[[label_column_name]],
-              enable_categorical=True,
-          )
-
-          booster_params = booster_params or {}
-          booster_params.setdefault("objective", objective)
-          booster_params.setdefault("booster", booster)
-          booster_params.setdefault("learning_rate", learning_rate)
-          booster_params.setdefault("min_split_loss", min_split_loss)
-          booster_params.setdefault("max_depth", max_depth)
-
-          starting_model = None
-          if starting_model_path:
-              starting_model = xgboost.Booster(model_file=starting_model_path)
-
-          print()
-          print("Training the model:")
-          model = xgboost.train(
-              params=booster_params,
-              dtrain=training_data,
-              num_boost_round=num_iterations,
-              xgb_model=starting_model,
-              evals=[(training_data, "training_data")],
-          )
-
-          # Saving the model in binary format
-          model.save_model(model_path)
-
-          model_config_str = model.save_config()
-          with open(model_config_path, "w") as model_config_file:
-              model_config_file.write(model_config_str)
-
-      import json
-      import argparse
-      _parser = argparse.ArgumentParser(prog='Train XGBoost model on CSV', description='Trains an XGBoost model.')
-      _parser.add_argument("--training-data", dest="training_data_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--label-column-name", dest="label_column_name", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--starting-model", dest="starting_model_path", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--num-iterations", dest="num_iterations", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--objective", dest="objective", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--booster", dest="booster", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--learning-rate", dest="learning_rate", type=float, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--min-split-loss", dest="min_split_loss", type=float, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--max-depth", dest="max_depth", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--booster-params", dest="booster_params", type=json.loads, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--model", dest="model_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--model-config", dest="model_config_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
-      _parsed_args = vars(_parser.parse_args())
-
-      _outputs = train_XGBoost_model_on_CSV(**_parsed_args)
-    args:
-    - --training-data
-    - {inputPath: training_data}
-    - --label-column-name
-    - {inputValue: label_column_name}
-    - if:
-        cond: {isPresent: starting_model}
-        then:
-        - --starting-model
-        - {inputPath: starting_model}
-    - if:
-        cond: {isPresent: num_iterations}
-        then:
-        - --num-iterations
-        - {inputValue: num_iterations}
-    - if:
-        cond: {isPresent: objective}
-        then:
-        - --objective
-        - {inputValue: objective}
-    - if:
-        cond: {isPresent: booster}
-        then:
-        - --booster
-        - {inputValue: booster}
-    - if:
-        cond: {isPresent: learning_rate}
-        then:
-        - --learning-rate
-        - {inputValue: learning_rate}
-    - if:
-        cond: {isPresent: min_split_loss}
-        then:
-        - --min-split-loss
-        - {inputValue: min_split_loss}
-    - if:
-        cond: {isPresent: max_depth}
-        then:
-        - --max-depth
-        - {inputValue: max_depth}
-    - if:
-        cond: {isPresent: booster_params}
-        then:
-        - --booster-params
-        - {inputValue: booster_params}
-    - --model
-    - {outputPath: model}
-    - --model-config
-    - {outputPath: model_config}

community-content/pipeline_components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml

@@ -1,204 +0,0 @@
-name: Split rows into subsets
-description: Splits the data table according to the split fractions.
-metadata:
-  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml'}
-inputs:
-- {name: table, type: CSV}
-- {name: fraction_1, type: Float, description: 'The proportion of the lines to put
-    into the 1st split. Range: [0, 1]'}
-- name: fraction_2
-  type: Float
-  description: |-
-    The proportion of the lines to put into the 2nd split. Range: [0, 1]
-    If fraction_2 is not specified, then fraction_2 = 1 - fraction_1.
-    The remaining lines go to the 3rd split (if any).
-  optional: true
-- {name: random_seed, type: Integer, default: '0', optional: true}
-outputs:
-- {name: split_1, type: CSV}
-- {name: split_2, type: CSV}
-- {name: split_3, type: CSV}
-- {name: split_1_count, type: Integer}
-- {name: split_2_count, type: Integer}
-- {name: split_3_count, type: Integer}
-implementation:
-  container:
-    image: python:3.9
-    command:
-    - sh
-    - -ec
-    - |
-      program_path=$(mktemp)
-      printf "%s" "$0" > "$program_path"
-      python3 -u "$program_path" "$@"
-    - |
-      def _make_parent_dirs_and_return_path(file_path: str):
-          import os
-          os.makedirs(os.path.dirname(file_path), exist_ok=True)
-          return file_path
-
-      def split_rows_into_subsets(
-          table_path,
-          split_1_path,
-          split_2_path,
-          split_3_path,
-          fraction_1,
-          fraction_2 = None,
-          random_seed = 0,
-      ):
-          """Splits the data table according to the split fractions.
-
-          Args:
-              fraction_1: The proportion of the lines to put into the 1st split. Range: [0, 1]
-              fraction_2: The proportion of the lines to put into the 2nd split. Range: [0, 1]
-                  If fraction_2 is not specified, then fraction_2 = 1 - fraction_1.
-                  The remaining lines go to the 3rd split (if any).
-          """
-          import random
-
-          random.seed(random_seed)
-
-          SHUFFLE_BUFFER_SIZE = 10000
-
-          num_splits = 3
-
-          if fraction_1 < 0 or fraction_1 > 1:
-              raise ValueError("fraction_1 must be in between 0 and 1.")
-
-          if fraction_2 is None:
-              fraction_2 = 1 - fraction_1
-          if fraction_2 < 0 or fraction_2 > 1:
-              raise ValueError("fraction_2 must be in between 0 and 1.")
-
-          fraction_3 = 1 - fraction_1 - fraction_2
-
-          fractions = [
-              fraction_1,
-              fraction_2,
-              fraction_3,
-          ]
-
-          assert sum(fractions) == 1
-
-          written_line_counts = [0] * num_splits
-
-          output_files = [
-              open(split_1_path, "wb"),
-              open(split_2_path, "wb"),
-              open(split_3_path, "wb"),
-          ]
-
-          with open(table_path, "rb") as input_file:
-              # Writing the headers
-              header_line = input_file.readline()
-              for output_file in output_files:
-                  output_file.write(header_line)
-
-              while True:
-                  line_buffer = []
-                  for i in range(SHUFFLE_BUFFER_SIZE):
-                      line = input_file.readline()
-                      if not line:
-                          break
-                      line_buffer.append(line)
-
-                  # We need to exactly partition the lines between the output files
-                  # To overcome possible systematic bias, we could calculate the total numbers
-                  # of lines written to each file and take that into account.
-                  num_read_lines = len(line_buffer)
-                  number_of_lines_for_files = [0] * num_splits
-                  # List that will have the index of the destination file for each line
-                  file_index_for_line = []
-                  remaining_lines = num_read_lines
-                  remaining_fraction = 1
-                  for i in range(num_splits):
-                      number_of_lines_for_file = (
-                          round(remaining_lines * (fractions[i] / remaining_fraction))
-                          if remaining_fraction > 0
-                          else 0
-                      )
-                      number_of_lines_for_files[i] = number_of_lines_for_file
-                      remaining_lines -= number_of_lines_for_file
-                      remaining_fraction -= fractions[i]
-                      file_index_for_line.extend([i] * number_of_lines_for_file)
-
-                  assert remaining_lines == 0, f"{remaining_lines}"
-                  assert len(file_index_for_line) == num_read_lines
-
-                  random.shuffle(file_index_for_line)
-
-                  for i in range(num_read_lines):
-                      output_files[file_index_for_line[i]].write(line_buffer[i])
-                      written_line_counts[file_index_for_line[i]] += 1
-
-                  # Exit if the file ended before we were able to fully fill the buffer
-                  if len(line_buffer) != SHUFFLE_BUFFER_SIZE:
-                      break
-
-          for output_file in output_files:
-              output_file.close()
-
-          return written_line_counts
-
-      def _serialize_int(int_value: int) -> str:
-          if isinstance(int_value, str):
-              return int_value
-          if not isinstance(int_value, int):
-              raise TypeError('Value "{}" has type "{}" instead of int.'.format(str(int_value), str(type(int_value))))
-          return str(int_value)
-
-      import argparse
-      _parser = argparse.ArgumentParser(prog='Split rows into subsets', description='Splits the data table according to the split fractions.')
-      _parser.add_argument("--table", dest="table_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--fraction-1", dest="fraction_1", type=float, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--fraction-2", dest="fraction_2", type=float, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--random-seed", dest="random_seed", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--split-1", dest="split_1_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--split-2", dest="split_2_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--split-3", dest="split_3_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("----output-paths", dest="_output_paths", type=str, nargs=3)
-      _parsed_args = vars(_parser.parse_args())
-      _output_files = _parsed_args.pop("_output_paths", [])
-
-      _outputs = split_rows_into_subsets(**_parsed_args)
-
-      _output_serializers = [
-          _serialize_int,
-          _serialize_int,
-          _serialize_int,
-
-      ]
-
-      import os
-      for idx, output_file in enumerate(_output_files):
-          try:
-              os.makedirs(os.path.dirname(output_file))
-          except OSError:
-              pass
-          with open(output_file, 'w') as f:
-              f.write(_output_serializers[idx](_outputs[idx]))
-    args:
-    - --table
-    - {inputPath: table}
-    - --fraction-1
-    - {inputValue: fraction_1}
-    - if:
-        cond: {isPresent: fraction_2}
-        then:
-        - --fraction-2
-        - {inputValue: fraction_2}
-    - if:
-        cond: {isPresent: random_seed}
-        then:
-        - --random-seed
-        - {inputValue: random_seed}
-    - --split-1
-    - {outputPath: split_1}
-    - --split-2
-    - {outputPath: split_2}
-    - --split-3
-    - {outputPath: split_3}
-    - '----output-paths'
-    - {outputPath: split_1_count}
-    - {outputPath: split_2_count}
-    - {outputPath: split_3_count}

community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml

@@ -1,241 +0,0 @@
-name: Deploy model to endpoint for Google Cloud Vertex AI Model
-description: Deploys Google Cloud Vertex AI Model to a Google Cloud Vertex AI Endpoint.
-metadata:
-  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/KFPv2_hell/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/workaround_for_buggy_KFPv2_compiler/component.yaml'}
-inputs:
-- {name: model_name, type: String, description: Full resource name of a Google Cloud
-    Vertex AI Model}
-- name: endpoint_name
-  type: String
-  description: |-
-    Optional. Full name of Google Cloud Vertex Endpoint. A new
-    endpoint is created if the name is not passed.
-  optional: true
-- name: machine_type
-  type: String
-  description: |-
-    The type of the machine. See the [list of machine types
-    supported for prediction
-    ](https://cloud.google.com/vertex-ai/docs/predictions/configure-compute#machine-types).
-    Defaults to "n1-standard-2"
-  default: n1-standard-2
-  optional: true
-- name: min_replica_count
-  type: Integer
-  description: |-
-    Optional. The minimum number of machine replicas this deployed
-    model will be always deployed on. If traffic against it increases,
-    it may dynamically be deployed onto more replicas, and as traffic
-    decreases, some of these extra replicas may be freed.
-  default: '1'
-  optional: true
-- name: max_replica_count
-  type: Integer
-  description: |-
-    Optional. The maximum number of replicas this deployed model may
-    be deployed on when the traffic against it increases. If requested
-    value is too large, the deployment will error, but if deployment
-    succeeds then the ability to scale the model to that many replicas
-    is guaranteed (barring service outages). If traffic against the
-    deployed model increases beyond what its replicas at maximum may
-    handle, a portion of the traffic will be dropped. If this value
-    is not provided, the smaller value of min_replica_count or 1 will
-    be used.
-  default: '1'
-  optional: true
-- name: accelerator_type
-  type: String
-  description: |-
-    Optional. Hardware accelerator type. Must also set accelerator_count if used.
-    One of ACCELERATOR_TYPE_UNSPECIFIED, NVIDIA_TESLA_K80, NVIDIA_TESLA_P100,
-    NVIDIA_TESLA_V100, NVIDIA_TESLA_P4, NVIDIA_TESLA_T4
-  optional: true
-- {name: accelerator_count, type: Integer, description: Optional. The number of accelerators
-    to attach to a worker replica., optional: true}
-outputs:
-- {name: endpoint_name, type: String}
-- {name: endpoint_dict, type: JsonObject}
-implementation:
-  container:
-    image: python:3.9
-    command:
-    - sh
-    - -c
-    - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
-      'google-cloud-aiplatform==1.7.0' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3
-      -m pip install --quiet --no-warn-script-location 'google-cloud-aiplatform==1.7.0'
-      --user) && "$0" "$@"
-    - sh
-    - -ec
-    - |
-      program_path=$(mktemp)
-      printf "%s" "$0" > "$program_path"
-      python3 -u "$program_path" "$@"
-    - |
-      def deploy_model_to_endpoint_for_Google_Cloud_Vertex_AI_Model(
-          model_name,
-          endpoint_name = None,
-          machine_type = "n1-standard-2",
-          min_replica_count = 1,
-          max_replica_count = 1,
-          accelerator_type = None,
-          accelerator_count = None,
-          #
-          # Uncomment when anyone requests these:
-          # deployed_model_display_name: str = None,
-          # traffic_percentage: int = 0,
-          # traffic_split: dict = None,
-          # service_account: str = None,
-          # explanation_metadata: "google.cloud.aiplatform_v1.types.explanation_metadata.ExplanationMetadata" = None,
-          # explanation_parameters: "google.cloud.aiplatform_v1.types.explanation.ExplanationParameters" = None,
-          #
-          # encryption_spec_key_name: str = None,
-      ):
-          """Deploys Google Cloud Vertex AI Model to a Google Cloud Vertex AI Endpoint.
-
-          Args:
-              model_name: Full resource name of a Google Cloud Vertex AI Model
-              endpoint_name: Optional. Full name of Google Cloud Vertex Endpoint. A new
-                  endpoint is created if the name is not passed.
-              machine_type: The type of the machine. See the [list of machine types
-                  supported for prediction
-                  ](https://cloud.google.com/vertex-ai/docs/predictions/configure-compute#machine-types).
-                  Defaults to "n1-standard-2"
-              min_replica_count (int):
-                  Optional. The minimum number of machine replicas this deployed
-                  model will be always deployed on. If traffic against it increases,
-                  it may dynamically be deployed onto more replicas, and as traffic
-                  decreases, some of these extra replicas may be freed.
-              max_replica_count (int):
-                  Optional. The maximum number of replicas this deployed model may
-                  be deployed on when the traffic against it increases. If requested
-                  value is too large, the deployment will error, but if deployment
-                  succeeds then the ability to scale the model to that many replicas
-                  is guaranteed (barring service outages). If traffic against the
-                  deployed model increases beyond what its replicas at maximum may
-                  handle, a portion of the traffic will be dropped. If this value
-                  is not provided, the smaller value of min_replica_count or 1 will
-                  be used.
-              accelerator_type (str):
-                  Optional. Hardware accelerator type. Must also set accelerator_count if used.
-                  One of ACCELERATOR_TYPE_UNSPECIFIED, NVIDIA_TESLA_K80, NVIDIA_TESLA_P100,
-                  NVIDIA_TESLA_V100, NVIDIA_TESLA_P4, NVIDIA_TESLA_T4
-              accelerator_count (int):
-                  Optional. The number of accelerators to attach to a worker replica.
-          """
-          import json
-          from google.cloud import aiplatform
-
-          model = aiplatform.Model(model_name=model_name)
-
-          if endpoint_name:
-              endpoint = aiplatform.Endpoint(endpoint_name=endpoint_name)
-          else:
-              endpoint_display_name = model.display_name[:118] + "_endpoint"
-              endpoint = aiplatform.Endpoint.create(
-                  display_name=endpoint_display_name,
-                  project=model.project,
-                  location=model.location,
-                  # encryption_spec_key_name=encryption_spec_key_name,
-                  labels={"component-source": "github-com-ark-kun-pipeline-components"},
-              )
-
-          endpoint = model.deploy(
-              endpoint=endpoint,
-              # deployed_model_display_name=deployed_model_display_name,
-              machine_type=machine_type,
-              min_replica_count=min_replica_count,
-              max_replica_count=max_replica_count,
-              accelerator_type=accelerator_type,
-              accelerator_count=accelerator_count,
-              # service_account=service_account,
-              # explanation_metadata=explanation_metadata,
-              # explanation_parameters=explanation_parameters,
-              # encryption_spec_key_name=encryption_spec_key_name,
-          )
-
-          endpoint_json = json.dumps(endpoint.to_dict(), indent=2)
-          print(endpoint_json)
-          return (endpoint.resource_name, endpoint_json)
-
-      def _serialize_json(obj) -> str:
-          if isinstance(obj, str):
-              return obj
-          import json
-          def default_serializer(obj):
-              if hasattr(obj, 'to_struct'):
-                  return obj.to_struct()
-              else:
-                  raise TypeError("Object of type '%s' is not JSON serializable and does not have .to_struct() method." % obj.__class__.__name__)
-          return json.dumps(obj, default=default_serializer, sort_keys=True)
-
-      def _serialize_str(str_value: str) -> str:
-          if not isinstance(str_value, str):
-              raise TypeError('Value "{}" has type "{}" instead of str.'.format(str(str_value), str(type(str_value))))
-          return str_value
-
-      import argparse
-      _parser = argparse.ArgumentParser(prog='Deploy model to endpoint for Google Cloud Vertex AI Model', description='Deploys Google Cloud Vertex AI Model to a Google Cloud Vertex AI Endpoint.')
-      _parser.add_argument("--model-name", dest="model_name", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--endpoint-name", dest="endpoint_name", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--machine-type", dest="machine_type", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--min-replica-count", dest="min_replica_count", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--max-replica-count", dest="max_replica_count", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--accelerator-type", dest="accelerator_type", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--accelerator-count", dest="accelerator_count", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("----output-paths", dest="_output_paths", type=str, nargs=2)
-      _parsed_args = vars(_parser.parse_args())
-      _output_files = _parsed_args.pop("_output_paths", [])
-
-      _outputs = deploy_model_to_endpoint_for_Google_Cloud_Vertex_AI_Model(**_parsed_args)
-
-      _output_serializers = [
-          _serialize_str,
-          _serialize_json,
-
-      ]
-
-      import os
-      for idx, output_file in enumerate(_output_files):
-          try:
-              os.makedirs(os.path.dirname(output_file))
-          except OSError:
-              pass
-          with open(output_file, 'w') as f:
-              f.write(_output_serializers[idx](_outputs[idx]))
-    args:
-    - --model-name
-    - {inputValue: model_name}
-    - if:
-        cond: {isPresent: endpoint_name}
-        then:
-        - --endpoint-name
-        - {inputValue: endpoint_name}
-    - if:
-        cond: {isPresent: machine_type}
-        then:
-        - --machine-type
-        - {inputValue: machine_type}
-    - if:
-        cond: {isPresent: min_replica_count}
-        then:
-        - --min-replica-count
-        - {inputValue: min_replica_count}
-    - if:
-        cond: {isPresent: max_replica_count}
-        then:
-        - --max-replica-count
-        - {inputValue: max_replica_count}
-    - if:
-        cond: {isPresent: accelerator_type}
-        then:
-        - --accelerator-type
-        - {inputValue: accelerator_type}
-    - if:
-        cond: {isPresent: accelerator_count}
-        then:
-        - --accelerator-count
-        - {inputValue: accelerator_count}
-    - '----output-paths'
-    - {outputPath: endpoint_name}
-    - {outputPath: endpoint_dict}

community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/component.yaml

@@ -1,297 +0,0 @@
-name: Upload PyTorch model archive to Google Cloud Vertex AI
-metadata:
-  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/KFPv2_hell/components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/workaround_for_buggy_KFPv2_compiler/component.yaml'}
-inputs:
-- {name: model_archive, type: PyTorchModelArchive}
-- {name: torchserve_version, type: String, default: 0.6.0, optional: true}
-- name: use_gpu
-  type: Boolean
-  default: "False"
-  optional: true
-- {name: display_name, type: String, optional: true}
-- {name: description, type: String, optional: true}
-- {name: project, type: String, optional: true}
-- {name: location, type: String, optional: true}
-- {name: labels, type: JsonObject, optional: true}
-- {name: staging_bucket, type: String, optional: true}
-outputs:
-- {name: model_name, type: String}
-- {name: model_dict, type: JsonObject}
-implementation:
-  container:
-    image: python:3.9
-    command:
-    - sh
-    - -c
-    - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
-      'google-cloud-aiplatform==1.13.1' 'google-cloud-build==3.8.3' || PIP_DISABLE_PIP_VERSION_CHECK=1
-      python3 -m pip install --quiet --no-warn-script-location 'google-cloud-aiplatform==1.13.1'
-      'google-cloud-build==3.8.3' --user) && "$0" "$@"
-    - sh
-    - -ec
-    - |
-      program_path=$(mktemp)
-      printf "%s" "$0" > "$program_path"
-      python3 -u "$program_path" "$@"
-    - |
-      def upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI(
-          model_archive_path,
-          torchserve_version = "0.6.0",
-          use_gpu = False,
-
-          display_name = None,
-          description = None,
-
-          # Uncomment when anyone requests these:
-          # instance_schema_uri: str = None,
-          # parameters_schema_uri: str = None,
-          # prediction_schema_uri: str = None,
-          # explanation_metadata: "google.cloud.aiplatform_v1.types.explanation_metadata.ExplanationMetadata" = None,
-          # explanation_parameters: "google.cloud.aiplatform_v1.types.explanation.ExplanationParameters" = None,
-
-          project = None,
-          location = None,
-          labels = None,
-          # encryption_spec_key_name: str = None,
-          staging_bucket = None,
-      ):
-          import json
-          import os
-          from google.cloud import aiplatform
-
-          if not location:
-              location = os.environ.get("CLOUD_ML_REGION")
-
-          if not labels:
-              labels = {}
-          labels["component-source"] = "github-com-ark-kun-pipeline-components"
-
-          container_image_tag = torchserve_version + "-" + ("gpu" if use_gpu else "cpu")
-          container_image_uri = f"pytorch/torchserve:{container_image_tag}"
-
-          # Vertex Endpoints refuse to support non-Google container registries.
-          # We have to work around this to reduce user frustration
-          # TODO: Remove this code when Vertex Endpoints service starts supporting other container registries.
-          def copy_container_image(
-              src_container_image_uri,
-              dst_container_image_uri,
-              project_id,
-          ):
-              from google.cloud.devtools import cloudbuild
-              from google import protobuf
-              build_client = cloudbuild.CloudBuildClient()
-              build_config = cloudbuild.Build(
-                  images=[dst_container_image_uri],
-                  steps=[
-                      cloudbuild.BuildStep(
-                          name="gcr.io/cloud-builders/docker",
-                          entrypoint="bash",
-                          args=[
-                              "-exc",
-                              'docker pull --quiet "$0" && docker tag "$0" "$1"',
-                              src_container_image_uri,
-                              dst_container_image_uri,
-                          ],
-                      ),
-                  ],
-                  timeout=protobuf.duration_pb2.Duration(
-                      seconds=1800,
-                  ),
-              )
-              build_operation = build_client.create_build(
-                  project_id=project_id,
-                  build=build_config,
-              )
-              try:
-                  result = build_operation.result()
-              except:
-                  print(f"Logs are available at [{build_operation.metadata.build.log_url}].")
-                  raise
-              return result
-
-          project_id = aiplatform.initializer.global_config.project
-          mirrored_container_uri = f"gcr.io/{project_id}/container_mirror/{container_image_uri}"
-          # FIX: Only mirror when image does not exist
-          # docker does is unable to get the registry data from inside container (it cannot connecto to docker socket):
-          # docker.errors.DockerException: Error while fetching server API version: ('Connection aborted.', FileNotFoundError(2, 'No such file or directory'))
-          # import docker
-          # try:
-          #     docker_client = docker.from_env()
-          #     docker_client.images.get_registry_data(mirrored_container_uri)
-          # except docker.errors.NotFound:
-          if True:
-              print(f"Mirroring {container_image_uri} to {mirrored_container_uri}")
-              copy_container_image(
-                  src_container_image_uri=container_image_uri,
-                  dst_container_image_uri=mirrored_container_uri,
-                  project_id=project_id,
-              )
-          container_image_uri = mirrored_container_uri
-          # End of container image mirroring code
-
-          model_archive_file_name = os.path.basename(model_archive_path)
-          model_archive_dir = os.path.dirname(model_archive_path)
-
-          model = aiplatform.Model.upload(
-              # FIX: Use public image or mirror the official image
-              #serving_container_image_uri="gcr.io/avolkov-31337/mirror/pytorch/torchserve",
-              serving_container_image_uri=container_image_uri,
-              artifact_uri=model_archive_dir,
-              serving_container_command=[
-                  "bash",
-                  "-exc",
-                  '''
-      model_archive_uri="$0"
-      #model_archive_local_path=$(mktemp --suffix ".mar")
-      # For some reason the model must already be inside the model-store directory.
-      model_archive_local_path=./model-store/model.mar
-
-      # Downloading the model archive from GCS
-      # TODO: Fix gsutil bugs (requires project ID, has auth issues) and use gsutil instead.
-      # gsutil cp "$model_archive_uri" "$model_archive_local_path"
-      pip install google-cloud-storage
-      python -c '
-      import sys
-      from google.cloud import storage
-
-      model_archive_uri = sys.argv[1]
-      model_archive_local_path = sys.argv[2]
-
-      storage_client = storage.Client()
-      blob = storage.Blob.from_string(uri=model_archive_uri, client=storage_client)
-      blob.download_to_filename(filename=model_archive_local_path)
-      ' "$model_archive_uri" "$model_archive_local_path"
-
-      #Note: config.properties is owned by root. Our user is not root.
-      echo "
-      service_envelope=json
-      # Needed for external access
-      inference_address=http://0.0.0.0:8080
-      management_address=http://0.0.0.0:8081
-      " > config2.properties
-      torchserve --start --foreground --no-config-snapshots --models main-model="$model_archive_local_path" --model-store ./model-store/ --ts-config config2.properties
-                  ''',
-                  "$(AIP_STORAGE_URI)/" + model_archive_file_name,
-              ],
-              serving_container_predict_route="/predictions/main-model",
-              #serving_container_predict_route="/v1/models/main-model:predict",
-              serving_container_health_route="/ping",
-              serving_container_ports=[8080],
-
-              display_name=display_name,
-              description=description,
-
-              # instance_schema_uri=instance_schema_uri,
-              # parameters_schema_uri=parameters_schema_uri,
-              # prediction_schema_uri=prediction_schema_uri,
-              # explanation_metadata=explanation_metadata,
-              # explanation_parameters=explanation_parameters,
-
-              project=project,
-              location=location,
-              labels=labels,
-              # encryption_spec_key_name=encryption_spec_key_name,
-              staging_bucket=staging_bucket,
-          )
-          model_json = json.dumps(model.to_dict(), indent=2)
-          print(model_json)
-          return (model.resource_name, model_json)
-
-      def _deserialize_bool(s) -> bool:
-          from distutils.util import strtobool
-          return strtobool(s) == 1
-
-      def _serialize_json(obj) -> str:
-          if isinstance(obj, str):
-              return obj
-          import json
-          def default_serializer(obj):
-              if hasattr(obj, 'to_struct'):
-                  return obj.to_struct()
-              else:
-                  raise TypeError("Object of type '%s' is not JSON serializable and does not have .to_struct() method." % obj.__class__.__name__)
-          return json.dumps(obj, default=default_serializer, sort_keys=True)
-
-      def _serialize_str(str_value: str) -> str:
-          if not isinstance(str_value, str):
-              raise TypeError('Value "{}" has type "{}" instead of str.'.format(str(str_value), str(type(str_value))))
-          return str_value
-
-      import json
-      import argparse
-      _parser = argparse.ArgumentParser(prog='Upload PyTorch model archive to Google Cloud Vertex AI', description='')
-      _parser.add_argument("--model-archive", dest="model_archive_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--torchserve-version", dest="torchserve_version", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--use-gpu", dest="use_gpu", type=_deserialize_bool, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--display-name", dest="display_name", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--description", dest="description", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--project", dest="project", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--location", dest="location", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--labels", dest="labels", type=json.loads, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--staging-bucket", dest="staging_bucket", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("----output-paths", dest="_output_paths", type=str, nargs=2)
-      _parsed_args = vars(_parser.parse_args())
-      _output_files = _parsed_args.pop("_output_paths", [])
-
-      _outputs = upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI(**_parsed_args)
-
-      _output_serializers = [
-          _serialize_str,
-          _serialize_json,
-
-      ]
-
-      import os
-      for idx, output_file in enumerate(_output_files):
-          try:
-              os.makedirs(os.path.dirname(output_file))
-          except OSError:
-              pass
-          with open(output_file, 'w') as f:
-              f.write(_output_serializers[idx](_outputs[idx]))
-    args:
-    - --model-archive
-    - {inputPath: model_archive}
-    - if:
-        cond: {isPresent: torchserve_version}
-        then:
-        - --torchserve-version
-        - {inputValue: torchserve_version}
-    - if:
-        cond: {isPresent: use_gpu}
-        then:
-        - --use-gpu
-        - {inputValue: use_gpu}
-    - if:
-        cond: {isPresent: display_name}
-        then:
-        - --display-name
-        - {inputValue: display_name}
-    - if:
-        cond: {isPresent: description}
-        then:
-        - --description
-        - {inputValue: description}
-    - if:
-        cond: {isPresent: project}
-        then:
-        - --project
-        - {inputValue: project}
-    - if:
-        cond: {isPresent: location}
-        then:
-        - --location
-        - {inputValue: location}
-    - if:
-        cond: {isPresent: labels}
-        then:
-        - --labels
-        - {inputValue: labels}
-    - if:
-        cond: {isPresent: staging_bucket}
-        then:
-        - --staging-bucket
-        - {inputValue: staging_bucket}
-    - '----output-paths'
-    - {outputPath: model_name}
-    - {outputPath: model_dict}

community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/component.yaml

@@ -1,181 +0,0 @@
-name: Upload Scikit learn pickle model to Google Cloud Vertex AI
-metadata:
-  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/KFPv2_hell/components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/workaround_for_buggy_KFPv2_compiler/component.yaml'}
-inputs:
-- {name: model, type: ScikitLearnPickleModel}
-- {name: sklearn_version, type: String, optional: true}
-- {name: display_name, type: String, optional: true}
-- {name: description, type: String, optional: true}
-- {name: project, type: String, optional: true}
-- {name: location, type: String, optional: true}
-- {name: labels, type: JsonObject, optional: true}
-- {name: staging_bucket, type: String, optional: true}
-outputs:
-- {name: model_name, type: String}
-- {name: model_dict, type: JsonObject}
-implementation:
-  container:
-    image: python:3.9
-    command:
-    - sh
-    - -c
-    - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
-      'google-cloud-aiplatform==1.16.0' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3
-      -m pip install --quiet --no-warn-script-location 'google-cloud-aiplatform==1.16.0'
-      --user) && "$0" "$@"
-    - sh
-    - -ec
-    - |
-      program_path=$(mktemp)
-      printf "%s" "$0" > "$program_path"
-      python3 -u "$program_path" "$@"
-    - |
-      def upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI(
-          model_path,
-          sklearn_version = None,
-
-          display_name = None,
-          description = None,
-
-          # Uncomment when anyone requests these:
-          # instance_schema_uri: str = None,
-          # parameters_schema_uri: str = None,
-          # prediction_schema_uri: str = None,
-          # explanation_metadata: "google.cloud.aiplatform_v1.types.explanation_metadata.ExplanationMetadata" = None,
-          # explanation_parameters: "google.cloud.aiplatform_v1.types.explanation.ExplanationParameters" = None,
-
-          project = None,
-          location = None,
-          labels = None,
-          # encryption_spec_key_name: str = None,
-          staging_bucket = None,
-      ):
-          import json
-          import os
-          import shutil
-          import tempfile
-          from google.cloud import aiplatform
-
-          if not location:
-              location = os.environ.get("CLOUD_ML_REGION")
-
-          if not labels:
-              labels = {}
-          labels["component-source"] = "github-com-ark-kun-pipeline-components"
-
-          # The serving container decides the model type based on the model file extension.
-          # So we need to rename the mode file (e.g. /tmp/inputs/model/data) to *.pkl
-          _, renamed_model_path = tempfile.mkstemp(suffix=".pkl")
-          shutil.copyfile(src=model_path, dst=renamed_model_path)
-
-          model = aiplatform.Model.upload_scikit_learn_model_file(
-              model_file_path=renamed_model_path,
-              sklearn_version=sklearn_version,
-
-              display_name=display_name,
-              description=description,
-
-              # instance_schema_uri=instance_schema_uri,
-              # parameters_schema_uri=parameters_schema_uri,
-              # prediction_schema_uri=prediction_schema_uri,
-              # explanation_metadata=explanation_metadata,
-              # explanation_parameters=explanation_parameters,
-
-              project=project,
-              location=location,
-              labels=labels,
-              # encryption_spec_key_name=encryption_spec_key_name,
-              staging_bucket=staging_bucket,
-          )
-          model_json = json.dumps(model.to_dict(), indent=2)
-          print(model_json)
-          return (model.resource_name, model_json)
-
-      def _serialize_json(obj) -> str:
-          if isinstance(obj, str):
-              return obj
-          import json
-          def default_serializer(obj):
-              if hasattr(obj, 'to_struct'):
-                  return obj.to_struct()
-              else:
-                  raise TypeError("Object of type '%s' is not JSON serializable and does not have .to_struct() method." % obj.__class__.__name__)
-          return json.dumps(obj, default=default_serializer, sort_keys=True)
-
-      def _serialize_str(str_value: str) -> str:
-          if not isinstance(str_value, str):
-              raise TypeError('Value "{}" has type "{}" instead of str.'.format(str(str_value), str(type(str_value))))
-          return str_value
-
-      import json
-      import argparse
-      _parser = argparse.ArgumentParser(prog='Upload Scikit learn pickle model to Google Cloud Vertex AI', description='')
-      _parser.add_argument("--model", dest="model_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--sklearn-version", dest="sklearn_version", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--display-name", dest="display_name", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--description", dest="description", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--project", dest="project", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--location", dest="location", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--labels", dest="labels", type=json.loads, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--staging-bucket", dest="staging_bucket", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("----output-paths", dest="_output_paths", type=str, nargs=2)
-      _parsed_args = vars(_parser.parse_args())
-      _output_files = _parsed_args.pop("_output_paths", [])
-
-      _outputs = upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI(**_parsed_args)
-
-      _output_serializers = [
-          _serialize_str,
-          _serialize_json,
-
-      ]
-
-      import os
-      for idx, output_file in enumerate(_output_files):
-          try:
-              os.makedirs(os.path.dirname(output_file))
-          except OSError:
-              pass
-          with open(output_file, 'w') as f:
-              f.write(_output_serializers[idx](_outputs[idx]))
-    args:
-    - --model
-    - {inputPath: model}
-    - if:
-        cond: {isPresent: sklearn_version}
-        then:
-        - --sklearn-version
-        - {inputValue: sklearn_version}
-    - if:
-        cond: {isPresent: display_name}
-        then:
-        - --display-name
-        - {inputValue: display_name}
-    - if:
-        cond: {isPresent: description}
-        then:
-        - --description
-        - {inputValue: description}
-    - if:
-        cond: {isPresent: project}
-        then:
-        - --project
-        - {inputValue: project}
-    - if:
-        cond: {isPresent: location}
-        then:
-        - --location
-        - {inputValue: location}
-    - if:
-        cond: {isPresent: labels}
-        then:
-        - --labels
-        - {inputValue: labels}
-    - if:
-        cond: {isPresent: staging_bucket}
-        then:
-        - --staging-bucket
-        - {inputValue: staging_bucket}
-    - '----output-paths'
-    - {outputPath: model_name}
-    - {outputPath: model_dict}

community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/component.yaml

@@ -1,190 +0,0 @@
-name: Upload Tensorflow model to Google Cloud Vertex AI
-metadata:
-  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/KFPv2_hell/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/workaround_for_buggy_KFPv2_compiler/component.yaml'}
-inputs:
-- {name: model, type: TensorflowSavedModel}
-- {name: tensorflow_version, type: String, optional: true}
-- name: use_gpu
-  type: Boolean
-  default: "False"
-  optional: true
-- {name: display_name, type: String, optional: true}
-- {name: description, type: String, optional: true}
-- {name: project, type: String, optional: true}
-- {name: location, type: String, optional: true}
-- {name: labels, type: JsonObject, optional: true}
-- {name: staging_bucket, type: String, optional: true}
-outputs:
-- {name: model_name, type: String}
-- {name: model_dict, type: JsonObject}
-implementation:
-  container:
-    image: python:3.9
-    command:
-    - sh
-    - -c
-    - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
-      'google-cloud-aiplatform==1.16.0' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3
-      -m pip install --quiet --no-warn-script-location 'google-cloud-aiplatform==1.16.0'
-      --user) && "$0" "$@"
-    - sh
-    - -ec
-    - |
-      program_path=$(mktemp)
-      printf "%s" "$0" > "$program_path"
-      python3 -u "$program_path" "$@"
-    - |
-      def upload_Tensorflow_model_to_Google_Cloud_Vertex_AI(
-          model_path,
-          tensorflow_version = None,
-          use_gpu = False,
-
-          display_name = None,
-          description = None,
-
-          # Uncomment when anyone requests these:
-          # instance_schema_uri: str = None,
-          # parameters_schema_uri: str = None,
-          # prediction_schema_uri: str = None,
-          # explanation_metadata: "google.cloud.aiplatform_v1.types.explanation_metadata.ExplanationMetadata" = None,
-          # explanation_parameters: "google.cloud.aiplatform_v1.types.explanation.ExplanationParameters" = None,
-
-          project = None,
-          location = None,
-          labels = None,
-          # encryption_spec_key_name: str = None,
-          staging_bucket = None,
-      ):
-          import json
-          import os
-          from google.cloud import aiplatform
-
-          if not location:
-              location = os.environ.get("CLOUD_ML_REGION")
-
-          if not labels:
-              labels = {}
-          labels["component-source"] = "github-com-ark-kun-pipeline-components"
-
-          model = aiplatform.Model.upload_tensorflow_saved_model(
-              saved_model_dir=model_path,
-              tensorflow_version=tensorflow_version,
-              use_gpu=use_gpu,
-
-              display_name=display_name,
-              description=description,
-
-              # instance_schema_uri=instance_schema_uri,
-              # parameters_schema_uri=parameters_schema_uri,
-              # prediction_schema_uri=prediction_schema_uri,
-              # explanation_metadata=explanation_metadata,
-              # explanation_parameters=explanation_parameters,
-
-              project=project,
-              location=location,
-              labels=labels,
-              # encryption_spec_key_name=encryption_spec_key_name,
-              staging_bucket=staging_bucket,
-          )
-          model_json = json.dumps(model.to_dict(), indent=2)
-          print(model_json)
-          return (model.resource_name, model_json)
-
-      def _deserialize_bool(s) -> bool:
-          from distutils.util import strtobool
-          return strtobool(s) == 1
-
-      def _serialize_json(obj) -> str:
-          if isinstance(obj, str):
-              return obj
-          import json
-          def default_serializer(obj):
-              if hasattr(obj, 'to_struct'):
-                  return obj.to_struct()
-              else:
-                  raise TypeError("Object of type '%s' is not JSON serializable and does not have .to_struct() method." % obj.__class__.__name__)
-          return json.dumps(obj, default=default_serializer, sort_keys=True)
-
-      def _serialize_str(str_value: str) -> str:
-          if not isinstance(str_value, str):
-              raise TypeError('Value "{}" has type "{}" instead of str.'.format(str(str_value), str(type(str_value))))
-          return str_value
-
-      import json
-      import argparse
-      _parser = argparse.ArgumentParser(prog='Upload Tensorflow model to Google Cloud Vertex AI', description='')
-      _parser.add_argument("--model", dest="model_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--tensorflow-version", dest="tensorflow_version", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--use-gpu", dest="use_gpu", type=_deserialize_bool, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--display-name", dest="display_name", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--description", dest="description", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--project", dest="project", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--location", dest="location", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--labels", dest="labels", type=json.loads, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--staging-bucket", dest="staging_bucket", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("----output-paths", dest="_output_paths", type=str, nargs=2)
-      _parsed_args = vars(_parser.parse_args())
-      _output_files = _parsed_args.pop("_output_paths", [])
-
-      _outputs = upload_Tensorflow_model_to_Google_Cloud_Vertex_AI(**_parsed_args)
-
-      _output_serializers = [
-          _serialize_str,
-          _serialize_json,
-
-      ]
-
-      import os
-      for idx, output_file in enumerate(_output_files):
-          try:
-              os.makedirs(os.path.dirname(output_file))
-          except OSError:
-              pass
-          with open(output_file, 'w') as f:
-              f.write(_output_serializers[idx](_outputs[idx]))
-    args:
-    - --model
-    - {inputPath: model}
-    - if:
-        cond: {isPresent: tensorflow_version}
-        then:
-        - --tensorflow-version
-        - {inputValue: tensorflow_version}
-    - if:
-        cond: {isPresent: use_gpu}
-        then:
-        - --use-gpu
-        - {inputValue: use_gpu}
-    - if:
-        cond: {isPresent: display_name}
-        then:
-        - --display-name
-        - {inputValue: display_name}
-    - if:
-        cond: {isPresent: description}
-        then:
-        - --description
-        - {inputValue: description}
-    - if:
-        cond: {isPresent: project}
-        then:
-        - --project
-        - {inputValue: project}
-    - if:
-        cond: {isPresent: location}
-        then:
-        - --location
-        - {inputValue: location}
-    - if:
-        cond: {isPresent: labels}
-        then:
-        - --labels
-        - {inputValue: labels}
-    - if:
-        cond: {isPresent: staging_bucket}
-        then:
-        - --staging-bucket
-        - {inputValue: staging_bucket}
-    - '----output-paths'
-    - {outputPath: model_name}
-    - {outputPath: model_dict}

community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/component.yaml

@@ -1,181 +0,0 @@
-name: Upload XGBoost model to Google Cloud Vertex AI
-metadata:
-  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/workaround_for_buggy_KFPv2_compiler/component.yaml'}
-inputs:
-- {name: model, type: XGBoostModel}
-- {name: xgboost_version, type: String, optional: true}
-- {name: display_name, type: String, optional: true}
-- {name: description, type: String, optional: true}
-- {name: project, type: String, optional: true}
-- {name: location, type: String, optional: true}
-- {name: labels, type: JsonObject, optional: true}
-- {name: staging_bucket, type: String, optional: true}
-outputs:
-- {name: model_name, type: String}
-- {name: model_dict, type: JsonObject}
-implementation:
-  container:
-    image: python:3.9
-    command:
-    - sh
-    - -c
-    - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
-      'google-cloud-aiplatform==1.16.0' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3
-      -m pip install --quiet --no-warn-script-location 'google-cloud-aiplatform==1.16.0'
-      --user) && "$0" "$@"
-    - sh
-    - -ec
-    - |
-      program_path=$(mktemp)
-      printf "%s" "$0" > "$program_path"
-      python3 -u "$program_path" "$@"
-    - |
-      def upload_XGBoost_model_to_Google_Cloud_Vertex_AI(
-          model_path,
-          xgboost_version = None,
-
-          display_name = None,
-          description = None,
-
-          # Uncomment when anyone requests these:
-          # instance_schema_uri: str = None,
-          # parameters_schema_uri: str = None,
-          # prediction_schema_uri: str = None,
-          # explanation_metadata: "google.cloud.aiplatform_v1.types.explanation_metadata.ExplanationMetadata" = None,
-          # explanation_parameters: "google.cloud.aiplatform_v1.types.explanation.ExplanationParameters" = None,
-
-          project = None,
-          location = None,
-          labels = None,
-          # encryption_spec_key_name: str = None,
-          staging_bucket = None,
-      ):
-          import json
-          import os
-          import shutil
-          import tempfile
-          from google.cloud import aiplatform
-
-          if not location:
-              location = os.environ.get("CLOUD_ML_REGION")
-
-          if not labels:
-              labels = {}
-          labels["component-source"] = "github-com-ark-kun-pipeline-components"
-
-          # The serving container decides the model type based on the model file extension.
-          # So we need to rename the mode file (e.g. /tmp/inputs/model/data) to *.pkl
-          _, renamed_model_path = tempfile.mkstemp(suffix=".pkl")
-          shutil.copyfile(src=model_path, dst=renamed_model_path)
-
-          model = aiplatform.Model.upload_xgboost_model_file(
-              model_file_path=renamed_model_path,
-              xgboost_version=xgboost_version,
-
-              display_name=display_name,
-              description=description,
-
-              # instance_schema_uri=instance_schema_uri,
-              # parameters_schema_uri=parameters_schema_uri,
-              # prediction_schema_uri=prediction_schema_uri,
-              # explanation_metadata=explanation_metadata,
-              # explanation_parameters=explanation_parameters,
-
-              project=project,
-              location=location,
-              labels=labels,
-              # encryption_spec_key_name=encryption_spec_key_name,
-              staging_bucket=staging_bucket,
-          )
-          model_json = json.dumps(model.to_dict(), indent=2)
-          print(model_json)
-          return (model.resource_name, model_json)
-
-      def _serialize_json(obj) -> str:
-          if isinstance(obj, str):
-              return obj
-          import json
-          def default_serializer(obj):
-              if hasattr(obj, 'to_struct'):
-                  return obj.to_struct()
-              else:
-                  raise TypeError("Object of type '%s' is not JSON serializable and does not have .to_struct() method." % obj.__class__.__name__)
-          return json.dumps(obj, default=default_serializer, sort_keys=True)
-
-      def _serialize_str(str_value: str) -> str:
-          if not isinstance(str_value, str):
-              raise TypeError('Value "{}" has type "{}" instead of str.'.format(str(str_value), str(type(str_value))))
-          return str_value
-
-      import json
-      import argparse
-      _parser = argparse.ArgumentParser(prog='Upload XGBoost model to Google Cloud Vertex AI', description='')
-      _parser.add_argument("--model", dest="model_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--xgboost-version", dest="xgboost_version", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--display-name", dest="display_name", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--description", dest="description", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--project", dest="project", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--location", dest="location", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--labels", dest="labels", type=json.loads, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--staging-bucket", dest="staging_bucket", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("----output-paths", dest="_output_paths", type=str, nargs=2)
-      _parsed_args = vars(_parser.parse_args())
-      _output_files = _parsed_args.pop("_output_paths", [])
-
-      _outputs = upload_XGBoost_model_to_Google_Cloud_Vertex_AI(**_parsed_args)
-
-      _output_serializers = [
-          _serialize_str,
-          _serialize_json,
-
-      ]
-
-      import os
-      for idx, output_file in enumerate(_output_files):
-          try:
-              os.makedirs(os.path.dirname(output_file))
-          except OSError:
-              pass
-          with open(output_file, 'w') as f:
-              f.write(_output_serializers[idx](_outputs[idx]))
-    args:
-    - --model
-    - {inputPath: model}
-    - if:
-        cond: {isPresent: xgboost_version}
-        then:
-        - --xgboost-version
-        - {inputValue: xgboost_version}
-    - if:
-        cond: {isPresent: display_name}
-        then:
-        - --display-name
-        - {inputValue: display_name}
-    - if:
-        cond: {isPresent: description}
-        then:
-        - --description
-        - {inputValue: description}
-    - if:
-        cond: {isPresent: project}
-        then:
-        - --project
-        - {inputValue: project}
-    - if:
-        cond: {isPresent: location}
-        then:
-        - --location
-        - {inputValue: location}
-    - if:
-        cond: {isPresent: labels}
-        then:
-        - --labels
-        - {inputValue: labels}
-    - if:
-        cond: {isPresent: staging_bucket}
-        then:
-        - --staging-bucket
-        - {inputValue: staging_bucket}
-    - '----output-paths'
-    - {outputPath: model_name}
-    - {outputPath: model_dict}

community-content/pipeline_components/google-cloud/storage/download/component.yaml

@@ -1,35 +0,0 @@
-name: Download from GCS
-inputs:
-- {name: GCS path, type: String}
-outputs:
-- {name: Data}
-metadata:
-  annotations:
-    author: Alexey Volkov <alexey.volkov@ark-kun.com>
-    canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml'
-implementation:
-    container:
-        image: google/cloud-sdk
-        command:
-        - bash # Pattern comparison only works in Bash
-        - -ex
-        - -c
-        - |
-            if [ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then
-                gcloud auth activate-service-account --key-file="${GOOGLE_APPLICATION_CREDENTIALS}"
-            fi
-
-            uri="$0"
-            output_path="$1"
-
-            # Checking whether the URI points to a single blob, a directory or a URI pattern
-            # URI points to a blob when that URI does not end with slash and listing that URI only yields the same URI
-            if [[ "$uri" != */ ]] && (gsutil ls "$uri" | grep --fixed-strings --line-regexp "$uri"); then
-                mkdir -p "$(dirname "$output_path")"
-                gsutil -m cp -r "$uri" "$output_path"
-            else
-                mkdir -p "$output_path" # When source path is a directory, gsutil requires the destination to also be a directory
-                gsutil -m rsync -r "$uri" "$output_path" # gsutil cp has different path handling than Linux cp. It always puts the source directory (name) inside the destination directory. gsutil rsync does not have that problem.
-            fi
-        - inputValue: GCS path
-        - outputPath: Data

community-content/pipeline_components/pandas/Binarize_column/in_CSV_format/component.yaml

@@ -1,113 +0,0 @@
-name: Binarize column using Pandas on CSV data
-metadata:
-  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/pandas/Binarize_column/in_CSV_format/component.yaml'}
-inputs:
-- {name: table, type: CSV}
-- {name: column_name, type: String}
-- {name: predicate, type: String, default: '> 0', optional: true}
-- {name: new_column_name, type: String, optional: true}
-- name: keep_original_column
-  type: Boolean
-  default: "False"
-  optional: true
-outputs:
-- {name: transformed_table, type: CSV}
-implementation:
-  container:
-    image: python:3.9
-    command:
-    - sh
-    - -c
-    - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
-      'pandas==1.4.3' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet
-      --no-warn-script-location 'pandas==1.4.3' --user) && "$0" "$@"
-    - sh
-    - -ec
-    - |
-      program_path=$(mktemp)
-      printf "%s" "$0" > "$program_path"
-      python3 -u "$program_path" "$@"
-    - |
-      def _make_parent_dirs_and_return_path(file_path: str):
-          import os
-          os.makedirs(os.path.dirname(file_path), exist_ok=True)
-          return file_path
-
-      def binarize_column_using_Pandas_on_CSV_data(
-          table_path,
-          transformed_table_path,
-          column_name,
-          predicate = "> 0",
-          new_column_name = None,
-          keep_original_column = False,
-      ):
-          import pandas
-
-          df = pandas.read_csv(table_path).convert_dtypes()
-          original_series = df[column_name]
-
-          # Dynamically executing the predicate code
-          # Variable namespace for code execution
-          namespace = dict(x=original_series)
-          # I though that there should be no space before `predicate` so that "dot" predicate methods like ".between(min, max)" work.
-          # However Python allows spaces before dot: `df .isna()`.
-          # So having a space is not a problem
-          transform_code = f"""new_series_boolean = x {predicate}"""
-          # Note: exec() takes no keyword arguments
-          # exec(__source=transform_code, __globals=namespace)
-          exec(transform_code, namespace)
-          new_series_boolean = namespace["new_series_boolean"]
-
-          # There are multiple ways to convert boolean column to integer.
-          # .apply(int) might be faster. https://stackoverflow.com/a/49804868/1497385
-          # TODO: Do a proper benchmark.
-          new_series = new_series_boolean.apply(int)
-          # new_series = new_series_boolean.astype(int)
-          # new_series = new_series_boolean.replace({False: 0, True: 1})
-
-          if new_column_name:
-              df.insert(loc=0, column=new_column_name, value=new_series)
-              if not keep_original_column:
-                  df = df.drop(columns=[column_name])
-          else:
-              df[column_name] = new_series
-
-          df.to_csv(transformed_table_path, index=False)
-
-      def _deserialize_bool(s) -> bool:
-          from distutils.util import strtobool
-          return strtobool(s) == 1
-
-      import argparse
-      _parser = argparse.ArgumentParser(prog='Binarize column using Pandas on CSV data', description='')
-      _parser.add_argument("--table", dest="table_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--column-name", dest="column_name", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--predicate", dest="predicate", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--new-column-name", dest="new_column_name", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--keep-original-column", dest="keep_original_column", type=_deserialize_bool, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--transformed-table", dest="transformed_table_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
-      _parsed_args = vars(_parser.parse_args())
-
-      _outputs = binarize_column_using_Pandas_on_CSV_data(**_parsed_args)
-    args:
-    - --table
-    - {inputPath: table}
-    - --column-name
-    - {inputValue: column_name}
-    - if:
-        cond: {isPresent: predicate}
-        then:
-        - --predicate
-        - {inputValue: predicate}
-    - if:
-        cond: {isPresent: new_column_name}
-        then:
-        - --new-column-name
-        - {inputValue: new_column_name}
-    - if:
-        cond: {isPresent: keep_original_column}
-        then:
-        - --keep-original-column
-        - {inputValue: keep_original_column}
-    - --transformed-table
-    - {outputPath: transformed_table}

community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml

@@ -1,75 +0,0 @@
-name: Fill all missing values using Pandas on CSV data
-metadata:
-  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml'}
-inputs:
-- {name: table, type: CSV}
-- {name: replacement_value, type: String, default: '0', optional: true}
-- {name: column_names, type: JsonArray, optional: true}
-outputs:
-- {name: transformed_table, type: CSV}
-implementation:
-  container:
-    image: python:3.9
-    command:
-    - sh
-    - -c
-    - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
-      'pandas==1.4.1' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet
-      --no-warn-script-location 'pandas==1.4.1' --user) && "$0" "$@"
-    - sh
-    - -ec
-    - |
-      program_path=$(mktemp)
-      printf "%s" "$0" > "$program_path"
-      python3 -u "$program_path" "$@"
-    - |
-      def _make_parent_dirs_and_return_path(file_path: str):
-          import os
-          os.makedirs(os.path.dirname(file_path), exist_ok=True)
-          return file_path
-
-      def fill_all_missing_values_using_Pandas_on_CSV_data(
-          table_path,
-          transformed_table_path,
-          replacement_value = "0",
-          column_names = None,
-      ):
-          import pandas
-
-          df = pandas.read_csv(
-              table_path,
-              dtype="string",
-          )
-
-          for column_name in column_names or df.columns:
-              df[column_name] = df[column_name].fillna(value=replacement_value)
-
-          df.to_csv(
-              transformed_table_path, index=False,
-          )
-
-      import json
-      import argparse
-      _parser = argparse.ArgumentParser(prog='Fill all missing values using Pandas on CSV data', description='')
-      _parser.add_argument("--table", dest="table_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--replacement-value", dest="replacement_value", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--column-names", dest="column_names", type=json.loads, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--transformed-table", dest="transformed_table_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
-      _parsed_args = vars(_parser.parse_args())
-
-      _outputs = fill_all_missing_values_using_Pandas_on_CSV_data(**_parsed_args)
-    args:
-    - --table
-    - {inputPath: table}
-    - if:
-        cond: {isPresent: replacement_value}
-        then:
-        - --replacement-value
-        - {inputValue: replacement_value}
-    - if:
-        cond: {isPresent: column_names}
-        then:
-        - --column-names
-        - {inputValue: column_names}
-    - --transformed-table
-    - {outputPath: transformed_table}

community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml

@@ -1,59 +0,0 @@
-name: Select columns using Pandas on CSV data
-metadata:
-  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/pandas/Select_columns/in_CSV_format/component.yaml'}
-inputs:
-- {name: table, type: CSV}
-- {name: column_names, type: JsonArray}
-outputs:
-- {name: transformed_table, type: CSV}
-implementation:
-  container:
-    image: python:3.9
-    command:
-    - sh
-    - -c
-    - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
-      'pandas==1.4.2' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet
-      --no-warn-script-location 'pandas==1.4.2' --user) && "$0" "$@"
-    - sh
-    - -ec
-    - |
-      program_path=$(mktemp)
-      printf "%s" "$0" > "$program_path"
-      python3 -u "$program_path" "$@"
-    - |
-      def _make_parent_dirs_and_return_path(file_path: str):
-          import os
-          os.makedirs(os.path.dirname(file_path), exist_ok=True)
-          return file_path
-
-      def select_columns_using_Pandas_on_CSV_data(
-          table_path,
-          transformed_table_path,
-          column_names,
-      ):
-          import pandas
-
-          df = pandas.read_csv(
-              table_path,
-              dtype="string",
-          )
-          df = df[column_names]
-          df.to_csv(transformed_table_path, index=False)
-
-      import json
-      import argparse
-      _parser = argparse.ArgumentParser(prog='Select columns using Pandas on CSV data', description='')
-      _parser.add_argument("--table", dest="table_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--column-names", dest="column_names", type=json.loads, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--transformed-table", dest="transformed_table_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
-      _parsed_args = vars(_parser.parse_args())
-
-      _outputs = select_columns_using_Pandas_on_CSV_data(**_parsed_args)
-    args:
-    - --table
-    - {inputPath: table}
-    - --column-names
-    - {inputValue: column_names}
-    - --transformed-table
-    - {outputPath: transformed_table}

community-content/pipeline_components/tensorflow/Create_fully_connected_network/component.yaml

@@ -1,102 +0,0 @@
-name: Create fully connected tensorflow network
-description: Creates fully-connected network in Tensorflow SavedModel format
-metadata:
-  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/tensorflow/Create_fully_connected_network/component.yaml'}
-inputs:
-- {name: input_size, type: Integer}
-- {name: hidden_layer_sizes, type: JsonArray, default: '[]', optional: true}
-- {name: output_size, type: Integer, default: '1', optional: true}
-- {name: activation_name, type: String, default: relu, optional: true}
-- {name: output_activation_name, type: String, optional: true}
-- {name: random_seed, type: Integer, default: '0', optional: true}
-outputs:
-- {name: model, type: TensorflowSavedModel}
-implementation:
-  container:
-    image: tensorflow/tensorflow:2.7.0
-    command:
-    - sh
-    - -ec
-    - |
-      program_path=$(mktemp)
-      printf "%s" "$0" > "$program_path"
-      python3 -u "$program_path" "$@"
-    - |
-      def _make_parent_dirs_and_return_path(file_path: str):
-          import os
-          os.makedirs(os.path.dirname(file_path), exist_ok=True)
-          return file_path
-
-      def create_fully_connected_tensorflow_network(
-          input_size,
-          model_path,
-          hidden_layer_sizes = [],
-          output_size = 1,
-          activation_name = "relu",
-          output_activation_name = None,
-          random_seed = 0,
-      ):
-          """Creates fully-connected network in Tensorflow SavedModel format"""
-          import tensorflow as tf
-          tf.random.set_seed(seed=random_seed)
-
-          model = tf.keras.models.Sequential()
-          model.add(tf.keras.Input(shape=(input_size,)))
-          for layer_size in hidden_layer_sizes:
-              model.add(tf.keras.layers.Dense(units=layer_size, activation=activation_name))
-          # The last layer is left without activation
-          model.add(tf.keras.layers.Dense(units=output_size, activation=output_activation_name))
-
-          print(model.summary())
-
-          # Using tf.keras.models.save_model instead of tf.saved_model.save to prevent downstream error:
-          #tf.saved_model.save(model, model_path)
-          # ValueError: Unable to create a Keras model from this SavedModel.
-          # This SavedModel was created with `tf.saved_model.save`, and lacks the Keras metadata.
-          # Please save your Keras model by calling `model.save`or `tf.keras.models.save_model`.
-          # See https://github.com/keras-team/keras/issues/16451
-          tf.keras.models.save_model(model, model_path)
-
-      import json
-      import argparse
-      _parser = argparse.ArgumentParser(prog='Create fully connected tensorflow network', description='Creates fully-connected network in Tensorflow SavedModel format')
-      _parser.add_argument("--input-size", dest="input_size", type=int, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--hidden-layer-sizes", dest="hidden_layer_sizes", type=json.loads, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--output-size", dest="output_size", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--activation-name", dest="activation_name", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--output-activation-name", dest="output_activation_name", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--random-seed", dest="random_seed", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--model", dest="model_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
-      _parsed_args = vars(_parser.parse_args())
-
-      _outputs = create_fully_connected_tensorflow_network(**_parsed_args)
-    args:
-    - --input-size
-    - {inputValue: input_size}
-    - if:
-        cond: {isPresent: hidden_layer_sizes}
-        then:
-        - --hidden-layer-sizes
-        - {inputValue: hidden_layer_sizes}
-    - if:
-        cond: {isPresent: output_size}
-        then:
-        - --output-size
-        - {inputValue: output_size}
-    - if:
-        cond: {isPresent: activation_name}
-        then:
-        - --activation-name
-        - {inputValue: activation_name}
-    - if:
-        cond: {isPresent: output_activation_name}
-        then:
-        - --output-activation-name
-        - {inputValue: output_activation_name}
-    - if:
-        cond: {isPresent: random_seed}
-        then:
-        - --random-seed
-        - {inputValue: random_seed}
-    - --model
-    - {outputPath: model}

community-content/pipeline_components/tensorflow/Predict/on_CSV/component.yaml

@@ -1,100 +0,0 @@
-name: Predict with TensorFlow model on CSV data
-metadata:
-  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/tensorflow/Predict/on_CSV/component.yaml'}
-inputs:
-- {name: dataset, type: CSV}
-- {name: model, type: TensorflowSavedModel}
-- {name: label_column_name, type: String, optional: true}
-- {name: batch_size, type: Integer, default: '1000', optional: true}
-outputs:
-- {name: predictions}
-implementation:
-  container:
-    image: tensorflow/tensorflow:2.9.1
-    command:
-    - sh
-    - -ec
-    - |
-      program_path=$(mktemp)
-      printf "%s" "$0" > "$program_path"
-      python3 -u "$program_path" "$@"
-    - |
-      def _make_parent_dirs_and_return_path(file_path: str):
-          import os
-          os.makedirs(os.path.dirname(file_path), exist_ok=True)
-          return file_path
-
-      def predict_with_TensorFlow_model_on_CSV_data(
-          dataset_path,
-          model_path,
-          predictions_path,
-          label_column_name = None,
-          batch_size = 1000,
-      ):
-          import numpy
-          import tensorflow as tf
-
-          model = tf.saved_model.load(export_dir=model_path)
-
-          dataset = tf.data.experimental.make_csv_dataset(
-              file_pattern=dataset_path,
-              batch_size=batch_size,
-              label_name=label_column_name,
-              header=True,
-              num_epochs=1,
-              shuffle=False,
-              ignore_errors=False,
-          )
-
-          def stack_feature_batches(features_batch):
-              # Need to stack individual feature columns to create a single feature tensor
-              # Need to cast all column tensor types to float to prevent errors.
-              list_of_feature_batches = list(
-                  tf.cast(x=feature_batch, dtype=tf.float32)
-                  for feature_batch in features_batch.values()
-              )
-              return tf.stack(list_of_feature_batches, axis=-1)
-
-          def transform_features_and_drop_labels(features_batch, labels_batch):
-              return stack_feature_batches(features_batch)
-
-          dataset_map_fn = (
-              transform_features_and_drop_labels
-              if label_column_name
-              else stack_feature_batches
-          )
-
-          dataset = dataset.map(dataset_map_fn)
-
-          with open(predictions_path, "w") as predictions_file:
-              for features_batch in dataset:
-                  predictions_tensor = model(features_batch)
-                  numpy.savetxt(predictions_file, predictions_tensor.numpy())
-
-      import argparse
-      _parser = argparse.ArgumentParser(prog='Predict with TensorFlow model on CSV data', description='')
-      _parser.add_argument("--dataset", dest="dataset_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--model", dest="model_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--label-column-name", dest="label_column_name", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--batch-size", dest="batch_size", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--predictions", dest="predictions_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
-      _parsed_args = vars(_parser.parse_args())
-
-      _outputs = predict_with_TensorFlow_model_on_CSV_data(**_parsed_args)
-    args:
-    - --dataset
-    - {inputPath: dataset}
-    - --model
-    - {inputPath: model}
-    - if:
-        cond: {isPresent: label_column_name}
-        then:
-        - --label-column-name
-        - {inputValue: label_column_name}
-    - if:
-        cond: {isPresent: batch_size}
-        then:
-        - --batch-size
-        - {inputValue: batch_size}
-    - --predictions
-    - {outputPath: predictions}

community-content/pipeline_components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml

@@ -1,170 +0,0 @@
-name: Train model using Keras on CSV
-metadata:
-  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml'}
-inputs:
-- {name: training_data, type: CSV}
-- {name: model, type: TensorflowSavedModel}
-- {name: label_column_name, type: String}
-- {name: loss_function_name, type: String, default: mean_squared_error, optional: true}
-- {name: number_of_epochs, type: Integer, default: '1', optional: true}
-- {name: learning_rate, type: Float, default: '0.1', optional: true}
-- {name: optimizer_name, type: String, default: Adadelta, optional: true}
-- {name: optimizer_parameters, type: JsonObject, optional: true}
-- {name: batch_size, type: Integer, default: '32', optional: true}
-- {name: metric_names, type: JsonArray, optional: true}
-- {name: random_seed, type: Integer, default: '0', optional: true}
-outputs:
-- {name: trained_model, type: TensorflowSavedModel}
-implementation:
-  container:
-    image: tensorflow/tensorflow:2.8.0
-    command:
-    - sh
-    - -ec
-    - |
-      program_path=$(mktemp)
-      printf "%s" "$0" > "$program_path"
-      python3 -u "$program_path" "$@"
-    - |
-      def _make_parent_dirs_and_return_path(file_path: str):
-          import os
-          os.makedirs(os.path.dirname(file_path), exist_ok=True)
-          return file_path
-
-      def train_model_using_Keras_on_CSV(
-          training_data_path,
-          model_path,
-          trained_model_path,
-          label_column_name,
-          loss_function_name = "mean_squared_error",
-          number_of_epochs = 1,
-          learning_rate = 0.1,
-          optimizer_name = "Adadelta",
-          optimizer_parameters = None,
-          batch_size = 32,
-          metric_names = None,
-          random_seed = 0,
-      ):
-          import tensorflow as tf
-          tf.random.set_seed(seed=random_seed)
-
-          # Loading model using Keras. Model loaded using TensorFlow does not have .fit.
-          #model = tf.saved_model.load(export_dir=model_path)
-          keras_model = tf.keras.models.load_model(filepath=model_path)
-
-          optimizer_parameters = optimizer_parameters or {}
-          optimizer_parameters["learning_rate"] = learning_rate
-          optimizer_config = {
-              "class_name": optimizer_name,
-              "config": optimizer_parameters,
-          }
-          optimizer = tf.keras.optimizers.get(optimizer_config)
-          loss = tf.keras.losses.get(loss_function_name)
-
-          training_dataset = tf.data.experimental.make_csv_dataset(
-              file_pattern=training_data_path,
-              batch_size=batch_size,
-              label_name=label_column_name,
-              header=True,
-              # Need to specify num_epochs=1 otherwise the training becomes infinite
-              num_epochs=1,
-              shuffle=True,
-              shuffle_seed=random_seed,
-              ignore_errors=True,
-          )
-          def stack_feature_batches(features_batch, labels_batch):
-              # Need to stack individual feature columns to create a single feature tensor
-              # Need to cast all column tensor types to float to prevent error:
-              # TypeError: Tensors in list passed to 'values' of 'Pack' Op have types [int32, float32, float32, int32, int32] that don't all match.
-              list_of_feature_batches = list(tf.cast(x=feature_batch, dtype=tf.float32) for feature_batch in features_batch.values())
-              return tf.stack(list_of_feature_batches, axis=-1), labels_batch
-
-          training_dataset = training_dataset.map(stack_feature_batches)
-
-          # Need to compile the model to prevent error:
-          # ValueError: No gradients provided for any variable: [..., ...].
-          keras_model.compile(
-              optimizer=optimizer,
-              loss=loss,
-              metrics=metric_names,
-          )
-          keras_model.fit(
-              training_dataset,
-              epochs=number_of_epochs,
-          )
-
-          # Using tf.keras.models.save_model instead of tf.saved_model.save to prevent downstream error:
-          #tf.saved_model.save(keras_model, trained_model_path)
-          # ValueError: Unable to create a Keras model from this SavedModel.
-          # This SavedModel was created with `tf.saved_model.save`, and lacks the Keras metadata.
-          # Please save your Keras model by calling `model.save`or `tf.keras.models.save_model`.
-          # See https://github.com/keras-team/keras/issues/16451
-          tf.keras.models.save_model(keras_model, trained_model_path)
-
-      import json
-      import argparse
-      _parser = argparse.ArgumentParser(prog='Train model using Keras on CSV', description='')
-      _parser.add_argument("--training-data", dest="training_data_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--model", dest="model_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--label-column-name", dest="label_column_name", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--loss-function-name", dest="loss_function_name", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--number-of-epochs", dest="number_of_epochs", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--learning-rate", dest="learning_rate", type=float, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--optimizer-name", dest="optimizer_name", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--optimizer-parameters", dest="optimizer_parameters", type=json.loads, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--batch-size", dest="batch_size", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--metric-names", dest="metric_names", type=json.loads, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--random-seed", dest="random_seed", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--trained-model", dest="trained_model_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
-      _parsed_args = vars(_parser.parse_args())
-
-      _outputs = train_model_using_Keras_on_CSV(**_parsed_args)
-    args:
-    - --training-data
-    - {inputPath: training_data}
-    - --model
-    - {inputPath: model}
-    - --label-column-name
-    - {inputValue: label_column_name}
-    - if:
-        cond: {isPresent: loss_function_name}
-        then:
-        - --loss-function-name
-        - {inputValue: loss_function_name}
-    - if:
-        cond: {isPresent: number_of_epochs}
-        then:
-        - --number-of-epochs
-        - {inputValue: number_of_epochs}
-    - if:
-        cond: {isPresent: learning_rate}
-        then:
-        - --learning-rate
-        - {inputValue: learning_rate}
-    - if:
-        cond: {isPresent: optimizer_name}
-        then:
-        - --optimizer-name
-        - {inputValue: optimizer_name}
-    - if:
-        cond: {isPresent: optimizer_parameters}
-        then:
-        - --optimizer-parameters
-        - {inputValue: optimizer_parameters}
-    - if:
-        cond: {isPresent: batch_size}
-        then:
-        - --batch-size
-        - {inputValue: batch_size}
-    - if:
-        cond: {isPresent: metric_names}
-        then:
-        - --metric-names
-        - {inputValue: metric_names}
-    - if:
-        cond: {isPresent: random_seed}
-        then:
-        - --random-seed
-        - {inputValue: random_seed}
-    - --trained-model
-    - {outputPath: trained_model}

community-content/pytorch_efficient_training/README.md

@@ -1,33 +0,0 @@
-# PyTorch Efficient Training Examples
-
-This folder provides PyTorch efficient training examples using ResNet-50 and ImageNet data.
-
-## Requirements
-
-```shell
-pip install --upgrade pip
-pip install -r requirements.txt
-```
-
-## Description
-
-* resnet.py - Train ResNet-50 on single GPU.
-* resnet_dp.py - Train ResNet-50 on single node multiple GPUs with `DataParallel` strategy.
-* resnet_ddp.py - Train ResNet-50 on single node multiple GPUs with `DistributedDataParallel` strategy.
-* resnet_ddp_wds.py - Train ResNet-50 on single node multiple GPUs with `DistributedDataParallel` strategy and `Webdataset`.
-* resnet_fsdp.py - Train ResNet-50 on single node multiple GPUs with `FullyShardedDataParallel` strategy.
-* resnet_fsdp_wds.py - Train ResNet-50 on single node multiple GPUs with `FullyShardedDataParallel` strategy and `Webdataset`.
-* shard_imagenet.py - Shard ImagNet individual files into `tar` files.
-
-## Benchmark
-
-When run the benchmark on Nvidia T4 GPUs using ImageNet validation dataset, you can get the result like:
-Strategy               | Seconds/Epoch - Local Data | Seconds/Epoch - Cloud Data
----------------------- | -------------------------- | --------------------------
-On 1 GPU               | 489                        | 804 (2x slower)
-On 4 GPUs (DP)         | 157                        | 738 (5x slower)
-On 4 GPUs (DDP)        | 134                        | 432 (3x slower)
-On 4 GPUs (DDP + WDS)  | 131                        | 133 (same performance)
-On 4 GPUs (FSDP)       | 139                        | 353 (3x slower)
-On 4 GPUs (FSDP + WDS) | 138                        | 135 (same performance)
-

community-content/pytorch_efficient_training/requirements.txt

@@ -1 +0,0 @@
-webdataset == 0.2.26

community-content/pytorch_efficient_training/resnet.py

@@ -1,197 +0,0 @@
-# Copyright 2022 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the \"License\");
-# you may not use this file except in compliance with the License.\n",
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an \"AS IS\" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Train resnet on single GPU."""
-
-import argparse
-import time
-
-from PIL import Image
-import torch
-from torch import nn
-import torchmetrics
-import torchvision
-from torchvision.models import resnet50
-
-
-class ImageFolder(torchvision.datasets.ImageFolder):
-  """Class for loading imagenet."""
-
-  def __init__(self, image_list_file, transform=None, target_transform=None):
-    self.samples = self._make_dataset(image_list_file)
-    self.loader = self._loader
-
-    self.imgs = self.samples
-    self.targets = [s[1] for s in self.samples]
-
-    self.transform = transform
-    self.target_transform = target_transform
-
-  def _make_dataset(self, image_list_file):
-    items = []
-    with open(image_list_file, 'r') as f:
-      for line in f:
-        item = line.strip().split(' ')
-        items.append((item[0], int(item[1])))
-    return items
-
-  def _loader(self, image_path):
-    with open(image_path, 'rb') as f:
-      img = Image.open(f)
-      img = img.convert('RGB')
-      return img
-
-
-def train(model, device, dataloader, optimizer):
-  model.train()
-  for image, target in dataloader:
-    image, target = image.to(device), target.to(device)
-    pred = model(image)
-    # pred.shape (N, C), target.shape (N)
-    loss = nn.functional.cross_entropy(pred, target)
-    optimizer.zero_grad()
-    loss.backward()
-    optimizer.step()
-  return loss
-
-
-def evaluate(model, device, dataloader, metric):
-  model.eval()
-  with torch.no_grad():
-    for image, target in dataloader:
-      image, target = image.to(device), target.to(device)
-      pred = model(image)
-      metric.update(pred, target)
-  accuracy = metric.compute()
-  metric.reset()
-  return accuracy
-
-
-def run_training(args):
-  """Run training and evaluation."""
-  # Create model.
-  model = resnet50(weights=None)
-  model = model.to(args.device)
-
-  # Create train dataloader.
-  train_dataset = ImageFolder(
-      image_list_file=args.train_data_path,
-      transform=torchvision.transforms.Compose([
-          torchvision.transforms.RandomResizedCrop(224),
-          torchvision.transforms.RandomHorizontalFlip(),
-          torchvision.transforms.ToTensor(),
-          torchvision.transforms.Normalize(
-              mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
-      ]))
-  train_dataloader = torch.utils.data.DataLoader(
-      dataset=train_dataset,
-      batch_size=args.train_batch_size,
-      shuffle=True,
-      num_workers=args.dataloader_num_workers,
-      pin_memory=True)
-  print(f'Train dataloader | samples: {len(train_dataloader.dataset)}, '
-        f'num workers: {train_dataloader.num_workers}, '
-        f'batch size: {args.train_batch_size}, '
-        f'batches/epoch: {len(train_dataloader)}')
-
-  # Create eval dataloader.
-  eval_dataset = ImageFolder(
-      image_list_file=args.eval_data_path,
-      transform=torchvision.transforms.Compose([
-          torchvision.transforms.Resize(256),
-          torchvision.transforms.CenterCrop(224),
-          torchvision.transforms.ToTensor(),
-          torchvision.transforms.Normalize(
-              mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
-      ]))
-  eval_dataloader = torch.utils.data.DataLoader(
-      dataset=eval_dataset,
-      batch_size=args.eval_batch_size,
-      shuffle=False,
-      num_workers=args.dataloader_num_workers,
-      pin_memory=True,
-      drop_last=True)
-  print(f'Eval dataloader | samples: {len(eval_dataloader.dataset)}, '
-        f'num workers: {eval_dataloader.num_workers}, '
-        f'batch size: {args.eval_batch_size}, '
-        f'batches/epoch: {len(eval_dataloader)}')
-
-  # Optimizer.
-  optimizer = torch.optim.SGD(model.parameters(), 0.1)
-
-  # Main loop.
-  metric = torchmetrics.classification.Accuracy(top_k=1).to(args.device)
-  for epoch in range(1, args.epochs + 1):
-    print(f'Running epoch {epoch}')
-
-    start = time.time()
-    train(model, args.device, train_dataloader, optimizer)
-    end = time.time()
-    print(f'Training finished in {(end - start):>0.3f} seconds')
-
-    start = time.time()
-    evaluate(model, args.device, eval_dataloader, metric)
-    end = time.time()
-    print(f'Evaluation finished in {(end - start):>0.3f} seconds')
-  print('Done')
-
-
-def create_args():
-  """Create main args."""
-  parser = argparse.ArgumentParser(
-      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-  parser.add_argument(
-      '--epochs',
-      default=1,
-      type=int,
-      help='number of total epochs to run')
-  parser.add_argument(
-      '--dataloader_num_workers',
-      default=2,
-      type=int,
-      help='number of workders for dataloader')
-  parser.add_argument(
-      '--train_data_path',
-      default='',
-      type=str,
-      help='path to training data')
-  parser.add_argument(
-      '--train_batch_size',
-      default=32,
-      type=int,
-      help='batch size for training')
-  parser.add_argument(
-      '--eval_data_path',
-      default='',
-      type=str,
-      help='path to evaluation data')
-  parser.add_argument(
-      '--eval_batch_size',
-      default=32,
-      type=int,
-      help='batch size for evaluation')
-  args = parser.parse_args()
-  return args
-
-
-def main():
-  args = create_args()
-  args.device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
-
-  print('Launch job on 1 GPU')
-  run_training(args)
-
-
-if __name__ == '__main__':
-  main()

community-content/pytorch_efficient_training/resnet_ddp.py

@@ -1,234 +0,0 @@
-# Copyright 2022 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the \"License\");
-# you may not use this file except in compliance with the License.\n",
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an \"AS IS\" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Train resnet on multiple GPUs with DDP."""
-
-import argparse
-import os
-import time
-
-from PIL import Image
-import torch
-from torch import nn
-import torch.distributed as dist
-import torch.multiprocessing as mp
-import torchmetrics
-import torchvision
-from torchvision.models import resnet50
-
-
-class ImageFolder(torchvision.datasets.ImageFolder):
-  """Class for loading imagenet."""
-
-  def __init__(self, image_list_file, transform=None, target_transform=None):
-    self.samples = self._make_dataset(image_list_file)
-    self.loader = self._loader
-
-    self.imgs = self.samples
-    self.targets = [s[1] for s in self.samples]
-
-    self.transform = transform
-    self.target_transform = target_transform
-
-  def _make_dataset(self, image_list_file):
-    items = []
-    with open(image_list_file, 'r') as f:
-      for line in f:
-        item = line.strip().split(' ')
-        items.append((item[0], int(item[1])))
-    return items
-
-  def _loader(self, image_path):
-    with open(image_path, 'rb') as f:
-      img = Image.open(f)
-      img = img.convert('RGB')
-      return img
-
-
-def train(model, device, dataloader, optimizer):
-  model.train()
-  for image, target in dataloader:
-    image = image.to(device, non_blocking=True)
-    target = target.to(device, non_blocking=True)
-    pred = model(image)
-    # pred.shape (N, C), target.shape (N)
-    loss = nn.functional.cross_entropy(pred, target)
-    optimizer.zero_grad()
-    loss.backward()
-    optimizer.step()
-  return loss
-
-
-def evaluate(model, device, dataloader, metric):
-  model.eval()
-  with torch.no_grad():
-    for image, target in dataloader:
-      image = image.to(device, non_blocking=True)
-      target = target.to(device, non_blocking=True)
-      pred = model(image)
-      metric.update(pred, target)
-  accuracy = metric.compute()
-  metric.reset()
-  return accuracy
-
-
-def worker(gpu, args):
-  """Run training and evaluation."""
-  # Init process group.
-  print(f'Initiating process {gpu}')
-  dist.init_process_group(
-      backend='nccl',
-      init_method='env://',
-      world_size=args.gpus,
-      rank=gpu)
-
-  # Create model.
-  model = resnet50(weights=None)
-  torch.cuda.set_device(gpu)
-  model.to(args.device)
-  model = torch.nn.SyncBatchNorm.convert_sync_batchnorm(model)
-  model = nn.parallel.DistributedDataParallel(model, device_ids=[gpu])
-
-  # Create train dataloader.
-  train_dataset = ImageFolder(
-      image_list_file=args.train_data_path,
-      transform=torchvision.transforms.Compose([
-          torchvision.transforms.RandomResizedCrop(224),
-          torchvision.transforms.RandomHorizontalFlip(),
-          torchvision.transforms.ToTensor(),
-          torchvision.transforms.Normalize(
-              mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
-      ]))
-  train_sampler = torch.utils.data.distributed.DistributedSampler(
-      train_dataset, num_replicas=args.gpus, rank=gpu)
-  train_dataloader = torch.utils.data.DataLoader(
-      dataset=train_dataset,
-      batch_size=args.train_batch_size,
-      shuffle=False,
-      num_workers=args.dataloader_num_workers,
-      pin_memory=True,
-      sampler=train_sampler)
-  if gpu == 0:
-    print(f'Train dataloader | samples: {len(train_dataloader.dataset)}, '
-          f'num workers: {train_dataloader.num_workers}, '
-          f'global batch size: {args.train_batch_size * args.gpus}, '
-          f'batches/epoch: {len(train_dataloader)}')
-
-  # Create eval dataloader.
-  eval_dataset = ImageFolder(
-      image_list_file=args.eval_data_path,
-      transform=torchvision.transforms.Compose([
-          torchvision.transforms.Resize(256),
-          torchvision.transforms.CenterCrop(224),
-          torchvision.transforms.ToTensor(),
-          torchvision.transforms.Normalize(
-              mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
-      ]))
-  eval_sampler = torch.utils.data.distributed.DistributedSampler(
-      eval_dataset, num_replicas=args.gpus, rank=gpu)
-  eval_dataloader = torch.utils.data.DataLoader(
-      dataset=eval_dataset,
-      batch_size=args.eval_batch_size,
-      shuffle=False,
-      num_workers=args.dataloader_num_workers,
-      pin_memory=True,
-      drop_last=True,
-      sampler=eval_sampler)
-  if gpu == 0:
-    print(f'Eval dataloader | samples: {len(eval_dataloader.dataset)}, '
-          f'num workers: {eval_dataloader.num_workers}, '
-          f'batch size: {args.eval_batch_size}, '
-          f'batches/epoch: {len(eval_dataloader)}')
-
-  # Optimizer.
-  optimizer = torch.optim.SGD(model.parameters(), 0.1)
-
-  # Main loop.
-  metric = torchmetrics.classification.Accuracy(top_k=1).to(args.device)
-  for epoch in range(1, args.epochs + 1):
-    if gpu == 0:
-      print(f'Running epoch {epoch}')
-    train_sampler.set_epoch(epoch)
-
-    start = time.time()
-    train(model, args.device, train_dataloader, optimizer)
-    end = time.time()
-    if gpu == 0:
-      print(f'Training finished in {(end - start):>0.3f} seconds')
-
-    start = time.time()
-    evaluate(model, args.device, eval_dataloader, metric)
-    end = time.time()
-    if gpu == 0:
-      print(f'Evaluation finished in {(end - start):>0.3f} seconds')
-
-  if gpu == 0:
-    print('Done')
-
-
-def create_args():
-  """Create main args."""
-  parser = argparse.ArgumentParser(
-      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-  parser.add_argument(
-      '--gpus',
-      default=4,
-      type=int,
-      help='number of gpus to use')
-  parser.add_argument(
-      '--epochs',
-      default=1,
-      type=int,
-      help='number of total epochs to run')
-  parser.add_argument(
-      '--dataloader_num_workers',
-      default=2,
-      type=int,
-      help='number of workders for dataloader')
-  parser.add_argument(
-      '--train_data_path',
-      default='',
-      type=str,
-      help='path to training data')
-  parser.add_argument(
-      '--train_batch_size',
-      default=32,
-      type=int,
-      help='batch size for training per gpu')
-  parser.add_argument(
-      '--eval_data_path',
-      default='',
-      type=str,
-      help='path to evaluation data')
-  parser.add_argument(
-      '--eval_batch_size',
-      default=32,
-      type=int,
-      help='batch size for evaluation per gpu')
-  args = parser.parse_args()
-  return args
-
-
-def main():
-  args = create_args()
-  os.environ['MASTER_ADDR'] = 'localhost'
-  os.environ['MASTER_PORT'] = '8888'
-
-  args.device = 'cuda' if torch.cuda.is_available() else 'cpu'
-  print(f'Launch job on {args.gpus} GPUs with DDP')
-  mp.spawn(worker, nprocs=args.gpus, args=(args,))
-
-
-if __name__ == '__main__':
-  main()

community-content/pytorch_efficient_training/resnet_ddp_wds.py

@@ -1,249 +0,0 @@
-# Copyright 2022 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the \"License\");
-# you may not use this file except in compliance with the License.\n",
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an \"AS IS\" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Train resnet on multiple GPUs with DDP."""
-
-import argparse
-import functools
-import itertools
-import math
-import os
-import time
-
-import torch
-from torch import nn
-import torch.distributed as dist
-import torch.multiprocessing as mp
-import torchmetrics
-from torchvision.models import resnet50
-from torchvision.transforms import transforms
-import webdataset as wds
-
-
-def wds_split(src, rank, world_size):
-  """Shards split function for webdataset."""
-  # The context of caller of this function is within multiple processes
-  # (by DDP world_size) and multiple workers (by dataloader_num_workers).
-  # So we totally have (world_size * num_workers) workers for processing data.
-  # NOTE: Raw data should be sharded to enough shards to make sure one process
-  # can handle at least one shard, otherwise the process may hang.
-  worker_id = 0
-  num_workers = 1
-  worker_info = torch.utils.data.get_worker_info()
-  if worker_info:
-    worker_id = worker_info.id
-    num_workers = worker_info.num_workers
-  for s in itertools.islice(src, rank * num_workers + worker_id, None,
-                            world_size * num_workers):
-    yield s
-
-
-def identity(x):
-  return x
-
-
-def create_wds_dataloader(rank, args, mode):
-  """Create webdataset dataset and dataloader."""
-  if mode == 'train':
-    transform = transforms.Compose([
-        transforms.RandomResizedCrop(224),
-        transforms.RandomHorizontalFlip(),
-        transforms.ToTensor(),
-        transforms.Normalize(
-            mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
-    ])
-    data_path = args.train_data_path
-    data_size = args.train_data_size
-    batch_size_local = args.train_batch_size
-    batch_size_global = args.train_batch_size * args.gpus
-    # Since webdataset disallows partial batch, we pad the last batch for train.
-    batches = int(math.ceil(data_size / batch_size_global))
-  else:
-    transform = transforms.Compose([
-        transforms.Resize(256),
-        transforms.CenterCrop(224),
-        transforms.ToTensor(),
-        transforms.Normalize(
-            mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
-    ])
-    data_path = args.eval_data_path
-    data_size = args.eval_data_size
-    batch_size_local = args.eval_batch_size
-    batch_size_global = args.eval_batch_size * args.gpus
-    # Since webdataset disallows partial batch, we drop the last batch for eval.
-    batches = int(data_size / batch_size_global)
-
-  dataset = wds.DataPipeline(
-      wds.SimpleShardList(data_path),
-      functools.partial(wds_split, rank=rank, world_size=args.gpus),
-      wds.tarfile_to_samples(),
-      wds.decode('pil'),
-      wds.to_tuple('jpg;png;jpeg cls'),
-      wds.map_tuple(transform, identity),
-      wds.batched(batch_size_local, partial=False),
-  )
-  num_workers = args.dataloader_num_workers
-  dataloader = wds.WebLoader(
-      dataset=dataset,
-      batch_size=None,
-      shuffle=False,
-      num_workers=num_workers,
-      persistent_workers=True if num_workers > 0 else False,
-      pin_memory=True).repeat(nbatches=batches)
-  print(f'{mode} dataloader | samples: {data_size}, '
-        f'num_workers: {num_workers}, '
-        f'local batch size: {batch_size_local}, '
-        f'global batch size: {batch_size_global}, '
-        f'batches: {batches}')
-  return dataloader
-
-
-def train(model, device, dataloader, optimizer):
-  model.train()
-  for image, target in dataloader:
-    image = image.to(device, non_blocking=True)
-    target = target.to(device, non_blocking=True)
-    pred = model(image)
-    # pred.shape (N, C), target.shape (N)
-    loss = nn.functional.cross_entropy(pred, target)
-    optimizer.zero_grad()
-    loss.backward()
-    optimizer.step()
-  return loss
-
-
-def evaluate(model, device, dataloader, metric):
-  model.eval()
-  with torch.no_grad():
-    for image, target in dataloader:
-      image = image.to(device, non_blocking=True)
-      target = target.to(device, non_blocking=True)
-      pred = model(image)
-      metric.update(pred, target)
-  accuracy = metric.compute()
-  metric.reset()
-  return accuracy
-
-
-def worker(gpu, args):
-  """Run training and evaluation."""
-  # Init process group.
-  print(f'Initiating process {gpu}')
-  dist.init_process_group(
-      backend='nccl',
-      init_method='env://',
-      world_size=args.gpus,
-      rank=gpu)
-
-  # Create model.
-  model = resnet50(weights=None)
-  torch.cuda.set_device(gpu)
-  model.to(args.device)
-  model = torch.nn.SyncBatchNorm.convert_sync_batchnorm(model)
-  model = nn.parallel.DistributedDataParallel(model, device_ids=[gpu])
-
-  # Create dataloader.
-  train_dataloader = create_wds_dataloader(gpu, args, 'train')
-  eval_dataloader = create_wds_dataloader(gpu, args, 'eval')
-
-  # Optimizer.
-  optimizer = torch.optim.SGD(model.parameters(), 0.1)
-
-  # Main loop.
-  metric = torchmetrics.classification.Accuracy(top_k=1).to(args.device)
-  for epoch in range(1, args.epochs + 1):
-    if gpu == 0:
-      print(f'Running epoch {epoch}')
-
-    start = time.time()
-    train(model, args.device, train_dataloader, optimizer)
-    end = time.time()
-    if gpu == 0:
-      print(f'Training finished in {(end - start):>0.3f} seconds')
-
-    start = time.time()
-    evaluate(model, args.device, eval_dataloader, metric)
-    end = time.time()
-    if gpu == 0:
-      print(f'Evaluation finished in {(end - start):>0.3f} seconds')
-
-  if gpu == 0:
-    print('Done')
-
-
-def create_args():
-  """Create main args."""
-  parser = argparse.ArgumentParser(
-      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-  parser.add_argument(
-      '--gpus',
-      default=4,
-      type=int,
-      help='number of gpus to use')
-  parser.add_argument(
-      '--epochs',
-      default=1,
-      type=int,
-      help='number of total epochs to run')
-  parser.add_argument(
-      '--dataloader_num_workers',
-      default=2,
-      type=int,
-      help='number of workders for dataloader')
-  parser.add_argument(
-      '--train_data_path',
-      default='',
-      type=str,
-      help='path to training data')
-  parser.add_argument(
-      '--train_batch_size',
-      default=32,
-      type=int,
-      help='batch size for training per gpu')
-  parser.add_argument(
-      '--train_data_size',
-      default=50000,
-      type=int,
-      help='data size for training')
-  parser.add_argument(
-      '--eval_data_path',
-      default='',
-      type=str,
-      help='path to evaluation data')
-  parser.add_argument(
-      '--eval_batch_size',
-      default=32,
-      type=int,
-      help='batch size for evaluation per gpu')
-  parser.add_argument(
-      '--eval_data_size',
-      default=50000,
-      type=int,
-      help='data size for evaluation')
-  args = parser.parse_args()
-  return args
-
-
-def main():
-  args = create_args()
-  os.environ['MASTER_ADDR'] = 'localhost'
-  os.environ['MASTER_PORT'] = '8888'
-
-  args.device = 'cuda' if torch.cuda.is_available() else 'cpu'
-  print(f'Launch job on {args.gpus} GPUs with DDP')
-  mp.spawn(worker, nprocs=args.gpus, args=(args,))
-
-
-if __name__ == '__main__':
-  main()

community-content/pytorch_efficient_training/resnet_dp.py

@@ -1,207 +0,0 @@
-# Copyright 2022 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the \"License\");
-# you may not use this file except in compliance with the License.\n",
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an \"AS IS\" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Train resnet on multiple GPUs with DP."""
-
-import argparse
-import time
-
-from PIL import Image
-import torch
-from torch import nn
-import torchmetrics
-import torchvision
-from torchvision.models import resnet50
-
-
-class ImageFolder(torchvision.datasets.ImageFolder):
-  """Class for loading imagenet."""
-
-  def __init__(self, image_list_file, transform=None, target_transform=None):
-    self.samples = self._make_dataset(image_list_file)
-    self.loader = self._loader
-
-    self.imgs = self.samples
-    self.targets = [s[1] for s in self.samples]
-
-    self.transform = transform
-    self.target_transform = target_transform
-
-  def _make_dataset(self, image_list_file):
-    items = []
-    with open(image_list_file, 'r') as f:
-      for line in f:
-        item = line.strip().split(' ')
-        items.append((item[0], int(item[1])))
-    return items
-
-  def _loader(self, image_path):
-    with open(image_path, 'rb') as f:
-      img = Image.open(f)
-      img = img.convert('RGB')
-      return img
-
-
-def train(model, device, dataloader, optimizer):
-  model.train()
-  for image, target in dataloader:
-    image, target = image.to(device), target.to(device)
-    pred = model(image)
-    # pred.shape (N, C), target.shape (N)
-    loss = nn.functional.cross_entropy(pred, target)
-    optimizer.zero_grad()
-    loss.backward()
-    optimizer.step()
-  return loss
-
-
-def evaluate(model, device, dataloader, metric):
-  model.eval()
-  with torch.no_grad():
-    for image, target in dataloader:
-      image, target = image.to(device), target.to(device)
-      pred = model(image)
-      metric.update(pred, target)
-  accuracy = metric.compute()
-  metric.reset()
-  return accuracy
-
-
-def run_training(args):
-  """Run training and evaluation."""
-  # Create model.
-  model = resnet50(weights=None)
-  model = nn.DataParallel(model)
-  model = model.to(args.device)
-
-  # Create train dataloader.
-  train_dataset = ImageFolder(
-      image_list_file=args.train_data_path,
-      transform=torchvision.transforms.Compose([
-          torchvision.transforms.RandomResizedCrop(224),
-          torchvision.transforms.RandomHorizontalFlip(),
-          torchvision.transforms.ToTensor(),
-          torchvision.transforms.Normalize(
-              mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
-      ]))
-  train_dataloader = torch.utils.data.DataLoader(
-      dataset=train_dataset,
-      batch_size=args.train_batch_size,
-      shuffle=True,
-      num_workers=args.dataloader_num_workers,
-      pin_memory=True)
-  print(f'Train dataloader | samples: {len(train_dataloader.dataset)}, '
-        f'num workers: {train_dataloader.num_workers}, '
-        f'global batch size: {args.train_batch_size}, '
-        f'batches/epoch: {len(train_dataloader)}')
-
-  # Create eval dataloader.
-  eval_dataset = ImageFolder(
-      image_list_file=args.eval_data_path,
-      transform=torchvision.transforms.Compose([
-          torchvision.transforms.Resize(256),
-          torchvision.transforms.CenterCrop(224),
-          torchvision.transforms.ToTensor(),
-          torchvision.transforms.Normalize(
-              mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
-      ]))
-  eval_dataloader = torch.utils.data.DataLoader(
-      dataset=eval_dataset,
-      batch_size=args.eval_batch_size,
-      shuffle=False,
-      num_workers=args.dataloader_num_workers,
-      pin_memory=True,
-      drop_last=True)
-  print(f'Eval dataloader | samples: {len(eval_dataloader.dataset)}, '
-        f'num workers: {eval_dataloader.num_workers}, '
-        f'global batch size: {args.eval_batch_size}, '
-        f'batches/epoch: {len(eval_dataloader)}')
-
-  # Optimizer.
-  optimizer = torch.optim.SGD(model.parameters(), 0.1)
-
-  # Main loop.
-  metric = torchmetrics.classification.Accuracy(top_k=1).to(args.device)
-  for epoch in range(1, args.epochs + 1):
-    print(f'Running epoch {epoch}')
-
-    start = time.time()
-    train(model, args.device, train_dataloader, optimizer)
-    end = time.time()
-    print(f'Training finished in {(end - start):>0.3f} seconds')
-
-    start = time.time()
-    evaluate(model, args.device, eval_dataloader, metric)
-    end = time.time()
-    print(f'Evaluation finished in {(end - start):>0.3f} seconds')
-  print('Done')
-
-
-def create_args():
-  """Create main args."""
-  parser = argparse.ArgumentParser(
-      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-  parser.add_argument(
-      '--gpus',
-      default=4,
-      type=int,
-      help='number of gpus to use')
-  parser.add_argument(
-      '--epochs',
-      default=1,
-      type=int,
-      help='number of total epochs to run')
-  parser.add_argument(
-      '--dataloader_num_workers',
-      default=2,
-      type=int,
-      help='number of workders for dataloader')
-  parser.add_argument(
-      '--train_data_path',
-      default='',
-      type=str,
-      help='path to training data')
-  parser.add_argument(
-      '--train_batch_size',
-      default=32,
-      type=int,
-      help='batch size for training per gpu')
-  parser.add_argument(
-      '--eval_data_path',
-      default='',
-      type=str,
-      help='path to evaluation data')
-  parser.add_argument(
-      '--eval_batch_size',
-      default=32,
-      type=int,
-      help='batch size for evaluation per gpu')
-  args = parser.parse_args()
-  return args
-
-
-def main():
-  args = create_args()
-
-  args.device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
-  args.train_batch_size *= args.gpus
-  args.eval_batch_size *= args.gpus
-  args.dataloader_num_workers *= args.gpus
-
-  print(f'Launch job on {args.gpus} GPU with nn.DataParallel')
-  run_training(args)
-
-
-if __name__ == '__main__':
-  main()

community-content/pytorch_efficient_training/resnet_fsdp.py

@@ -1,242 +0,0 @@
-# Copyright 2022 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the \"License\");
-# you may not use this file except in compliance with the License.\n",
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an \"AS IS\" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Train resnet on multiple GPUs with FSDP."""
-
-import argparse
-import functools
-import os
-import time
-
-from PIL import Image
-import torch
-from torch import nn
-import torch.distributed as dist
-from torch.distributed.fsdp import FullyShardedDataParallel as FSDP
-from torch.distributed.fsdp.wrap import size_based_auto_wrap_policy
-import torch.multiprocessing as mp
-import torchmetrics
-import torchvision
-from torchvision.models import resnet50
-
-
-class ImageFolder(torchvision.datasets.ImageFolder):
-  """Class for loading imagenet."""
-
-  def __init__(self, image_list_file, transform=None, target_transform=None):
-    self.samples = self._make_dataset(image_list_file)
-    self.loader = self._loader
-
-    self.imgs = self.samples
-    self.targets = [s[1] for s in self.samples]
-
-    self.transform = transform
-    self.target_transform = target_transform
-
-  def _make_dataset(self, image_list_file):
-    items = []
-    with open(image_list_file, 'r') as f:
-      for line in f:
-        item = line.strip().split(' ')
-        items.append((item[0], int(item[1])))
-    return items
-
-  def _loader(self, image_path):
-    with open(image_path, 'rb') as f:
-      img = Image.open(f)
-      img = img.convert('RGB')
-      return img
-
-
-def train(model, device, dataloader, optimizer):
-  model.train()
-  for image, target in dataloader:
-    image = image.to(device, non_blocking=True)
-    target = target.to(device, non_blocking=True)
-    pred = model(image)
-    # pred.shape (N, C), target.shape (N)
-    loss = nn.functional.cross_entropy(pred, target)
-    optimizer.zero_grad()
-    loss.backward()
-    optimizer.step()
-  return loss
-
-
-def evaluate(model, device, dataloader, metric):
-  model.eval()
-  with torch.no_grad():
-    for image, target in dataloader:
-      image = image.to(device, non_blocking=True)
-      target = target.to(device, non_blocking=True)
-      pred = model(image)
-      metric.update(pred, target)
-  accuracy = metric.compute()
-  metric.reset()
-  return accuracy
-
-
-def worker(gpu, args):
-  """Run training and evaluation."""
-  # Init process group.
-  print(f'Initiating process {gpu}')
-  dist.init_process_group(
-      backend='nccl',
-      init_method='env://',
-      world_size=args.gpus,
-      rank=gpu)
-
-  # Create train dataloader.
-  train_dataset = ImageFolder(
-      image_list_file=args.train_data_path,
-      transform=torchvision.transforms.Compose([
-          torchvision.transforms.RandomResizedCrop(224),
-          torchvision.transforms.RandomHorizontalFlip(),
-          torchvision.transforms.ToTensor(),
-          torchvision.transforms.Normalize(
-              mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
-      ]))
-  train_sampler = torch.utils.data.distributed.DistributedSampler(
-      train_dataset, num_replicas=args.gpus, rank=gpu)
-  train_dataloader = torch.utils.data.DataLoader(
-      dataset=train_dataset,
-      batch_size=args.train_batch_size,
-      shuffle=False,
-      num_workers=args.dataloader_num_workers,
-      pin_memory=True,
-      sampler=train_sampler)
-  if gpu == 0:
-    print(f'Train dataloader | samples: {len(train_dataloader.dataset)}, '
-          f'num workers: {train_dataloader.num_workers}, '
-          f'global batch size: {args.train_batch_size * args.gpus}, '
-          f'batches/epoch: {len(train_dataloader)}')
-
-  # Create eval dataloader.
-  eval_dataset = ImageFolder(
-      image_list_file=args.eval_data_path,
-      transform=torchvision.transforms.Compose([
-          torchvision.transforms.Resize(256),
-          torchvision.transforms.CenterCrop(224),
-          torchvision.transforms.ToTensor(),
-          torchvision.transforms.Normalize(
-              mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
-      ]))
-  eval_sampler = torch.utils.data.distributed.DistributedSampler(
-      eval_dataset, num_replicas=args.gpus, rank=gpu)
-  eval_dataloader = torch.utils.data.DataLoader(
-      dataset=eval_dataset,
-      batch_size=args.eval_batch_size,
-      shuffle=False,
-      num_workers=args.dataloader_num_workers,
-      pin_memory=True,
-      drop_last=True,
-      sampler=eval_sampler)
-  if gpu == 0:
-    print(f'Eval dataloader | samples: {len(eval_dataloader.dataset)}, '
-          f'num workers: {eval_dataloader.num_workers}, '
-          f'batch size: {args.eval_batch_size}, '
-          f'batches/epoch: {len(eval_dataloader)}')
-
-  # Wrap policy.
-  my_auto_wrap_policy = functools.partial(
-      size_based_auto_wrap_policy, min_num_params=100)
-  torch.cuda.set_device(gpu)
-
-  # Create model.
-  model = resnet50(weights=None)
-  model.to(args.device)
-  model = FSDP(model, auto_wrap_policy=my_auto_wrap_policy)
-
-  # Optimizer.
-  optimizer = torch.optim.SGD(model.parameters(), 0.1)
-
-  # Main loop.
-  metric = torchmetrics.classification.Accuracy(top_k=1).to(args.device)
-  for epoch in range(1, args.epochs + 1):
-    if gpu == 0:
-      print(f'Running epoch {epoch}')
-    train_sampler.set_epoch(epoch)
-
-    start = time.time()
-    train(model, args.device, train_dataloader, optimizer)
-    end = time.time()
-    if gpu == 0:
-      print(f'Training finished in {(end - start):>0.3f} seconds')
-
-    start = time.time()
-    evaluate(model, args.device, eval_dataloader, metric)
-    end = time.time()
-    if gpu == 0:
-      print(f'Evaluation finished in {(end - start):>0.3f} seconds')
-
-  if gpu == 0:
-    print('Done')
-  dist.destroy_process_group()
-
-
-def create_args():
-  """Create main args."""
-  parser = argparse.ArgumentParser(
-      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-  parser.add_argument(
-      '--gpus',
-      default=4,
-      type=int,
-      help='number of gpus to use')
-  parser.add_argument(
-      '--epochs',
-      default=2,
-      type=int,
-      help='number of total epochs to run')
-  parser.add_argument(
-      '--dataloader_num_workers',
-      default=2,
-      type=int,
-      help='number of workders for dataloader')
-  parser.add_argument(
-      '--train_data_path',
-      default='',
-      type=str,
-      help='path to training data')
-  parser.add_argument(
-      '--train_batch_size',
-      default=32,
-      type=int,
-      help='batch size for training per gpu')
-  parser.add_argument(
-      '--eval_data_path',
-      default='',
-      type=str,
-      help='path to evaluation data')
-  parser.add_argument(
-      '--eval_batch_size',
-      default=32,
-      type=int,
-      help='batch size for evaluation per gpu')
-  args = parser.parse_args()
-  return args
-
-
-def main():
-  args = create_args()
-
-  os.environ['MASTER_ADDR'] = 'localhost'
-  os.environ['MASTER_PORT'] = '8888'
-
-  args.device = 'cuda' if torch.cuda.is_available() else 'cpu'
-  print(f'Launch job on {args.gpus} GPUs with FSDP')
-  mp.spawn(worker, nprocs=args.gpus, args=(args,))
-
-
-if __name__ == '__main__':
-  main()

community-content/pytorch_efficient_training/resnet_fsdp_wds.py

@@ -1,240 +0,0 @@
-"""Train resnet on multiple GPUs with DDP."""
-
-import argparse
-import functools
-import itertools
-import math
-import os
-import time
-
-import torch
-from torch import nn
-import torch.distributed as dist
-from torch.distributed.fsdp import FullyShardedDataParallel as FSDP
-from torch.distributed.fsdp.wrap import size_based_auto_wrap_policy
-import torch.multiprocessing as mp
-import torchmetrics
-from torchvision.models import resnet50
-from torchvision.transforms import transforms
-import webdataset as wds
-
-
-def wds_split(src, rank, world_size):
-  """Shards split function for webdataset."""
-  # The context of caller of this function is within multiple processes
-  # (by DDP world_size) and multiple workers (by dataloader_num_workers).
-  # So we totally have (world_size * num_workers) workers for processing data.
-  # NOTE: Raw data should be sharded to enough shards to make sure one process
-  # can handle at least one shard, otherwise the process may hang.
-  worker_id = 0
-  num_workers = 1
-  worker_info = torch.utils.data.get_worker_info()
-  if worker_info:
-    worker_id = worker_info.id
-    num_workers = worker_info.num_workers
-  for s in itertools.islice(src, rank * num_workers + worker_id, None,
-                            world_size * num_workers):
-    yield s
-
-
-def identity(x):
-  return x
-
-
-def create_wds_dataloader(rank, args, mode):
-  """Create webdataset dataset and dataloader."""
-  if mode == 'train':
-    transform = transforms.Compose([
-        transforms.RandomResizedCrop(224),
-        transforms.RandomHorizontalFlip(),
-        transforms.ToTensor(),
-        transforms.Normalize(
-            mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
-    ])
-    data_path = args.train_data_path
-    data_size = args.train_data_size
-    batch_size_local = args.train_batch_size
-    batch_size_global = args.train_batch_size * args.gpus
-    # Since webdataset disallows partial batch, we pad the last batch for train.
-    batches = int(math.ceil(data_size / batch_size_global))
-  else:
-    transform = transforms.Compose([
-        transforms.Resize(256),
-        transforms.CenterCrop(224),
-        transforms.ToTensor(),
-        transforms.Normalize(
-            mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
-    ])
-    data_path = args.eval_data_path
-    data_size = args.eval_data_size
-    batch_size_local = args.eval_batch_size
-    batch_size_global = args.eval_batch_size * args.gpus
-    # Since webdataset disallows partial batch, we drop the last batch for eval.
-    batches = int(data_size / batch_size_global)
-
-  dataset = wds.DataPipeline(
-      wds.SimpleShardList(data_path),
-      functools.partial(wds_split, rank=rank, world_size=args.gpus),
-      wds.tarfile_to_samples(),
-      wds.decode('pil'),
-      wds.to_tuple('jpg;png;jpeg cls'),
-      wds.map_tuple(transform, identity),
-      wds.batched(batch_size_local, partial=False),
-  )
-  num_workers = args.dataloader_num_workers
-  dataloader = wds.WebLoader(
-      dataset=dataset,
-      batch_size=None,
-      shuffle=False,
-      num_workers=num_workers,
-      persistent_workers=True if num_workers > 0 else False,
-      pin_memory=True).repeat(nbatches=batches)
-  print(f'{mode} dataloader | samples: {data_size}, '
-        f'num_workers: {num_workers}, '
-        f'local batch size: {batch_size_local}, '
-        f'global batch size: {batch_size_global}, '
-        f'batches: {batches}')
-  return dataloader
-
-
-def train(model, device, dataloader, optimizer):
-  model.train()
-  for image, target in dataloader:
-    image = image.to(device, non_blocking=True)
-    target = target.to(device, non_blocking=True)
-    pred = model(image)
-    # pred.shape (N, C), target.shape (N)
-    loss = nn.functional.cross_entropy(pred, target)
-    optimizer.zero_grad()
-    loss.backward()
-    optimizer.step()
-  return loss
-
-
-def evaluate(model, device, dataloader, metric):
-  model.eval()
-  with torch.no_grad():
-    for image, target in dataloader:
-      image = image.to(device, non_blocking=True)
-      target = target.to(device, non_blocking=True)
-      pred = model(image)
-      metric.update(pred, target)
-  accuracy = metric.compute()
-  metric.reset()
-  return accuracy
-
-
-def worker(gpu, args):
-  """Run training and evaluation."""
-  # Init process group.
-  print(f'Initiating process {gpu}')
-  dist.init_process_group(
-      backend='nccl',
-      init_method='env://',
-      world_size=args.gpus,
-      rank=gpu)
-
-  # Create dataloader.
-  train_dataloader = create_wds_dataloader(gpu, args, 'train')
-  eval_dataloader = create_wds_dataloader(gpu, args, 'eval')
-
-  # Wrap policy.
-  my_auto_wrap_policy = functools.partial(
-      size_based_auto_wrap_policy, min_num_params=100)
-  torch.cuda.set_device(gpu)
-
-  # Create model.
-  model = resnet50(weights=None)
-  model.to(args.device)
-  model = FSDP(model, auto_wrap_policy=my_auto_wrap_policy)
-
-  # Optimizer.
-  optimizer = torch.optim.SGD(model.parameters(), 0.1)
-
-  # Main loop.
-  metric = torchmetrics.classification.Accuracy(top_k=1).to(args.device)
-  for epoch in range(1, args.epochs + 1):
-    if gpu == 0:
-      print(f'Running epoch {epoch}')
-
-    start = time.time()
-    train(model, args.device, train_dataloader, optimizer)
-    end = time.time()
-    if gpu == 0:
-      print(f'Training finished in {(end - start):>0.3f} seconds')
-
-    start = time.time()
-    evaluate(model, args.device, eval_dataloader, metric)
-    end = time.time()
-    if gpu == 0:
-      print(f'Evaluation finished in {(end - start):>0.3f} seconds')
-
-  if gpu == 0:
-    print('Done')
-
-
-def create_args():
-  """Create main args."""
-  parser = argparse.ArgumentParser(
-      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-  parser.add_argument(
-      '--gpus',
-      default=4,
-      type=int,
-      help='number of gpus to use')
-  parser.add_argument(
-      '--epochs',
-      default=2,
-      type=int,
-      help='number of total epochs to run')
-  parser.add_argument(
-      '--dataloader_num_workers',
-      default=2,
-      type=int,
-      help='number of workders for dataloader')
-  parser.add_argument(
-      '--train_data_path',
-      default='',
-      type=str,
-      help='path to training data')
-  parser.add_argument(
-      '--train_batch_size',
-      default=32,
-      type=int,
-      help='batch size for training per gpu')
-  parser.add_argument(
-      '--train_data_size',
-      default=50000,
-      type=int,
-      help='data size for training')
-  parser.add_argument(
-      '--eval_data_path',
-      default='',
-      type=str,
-      help='path to evaluation data')
-  parser.add_argument(
-      '--eval_batch_size',
-      default=32,
-      type=int,
-      help='batch size for evaluation per gpu')
-  parser.add_argument(
-      '--eval_data_size',
-      default=50000,
-      type=int,
-      help='data size for evaluation')
-  args = parser.parse_args()
-  return args
-
-
-def main():
-  args = create_args()
-  os.environ['MASTER_ADDR'] = 'localhost'
-  os.environ['MASTER_PORT'] = '8888'
-
-  args.device = 'cuda' if torch.cuda.is_available() else 'cpu'
-  print(f'Launch job on {args.gpus} GPUs with FSDP')
-  mp.spawn(worker, nprocs=args.gpus, args=(args,))
-
-
-if __name__ == '__main__':
-  main()

community-content/pytorch_efficient_training/shard_imagenet.py

@@ -1,98 +0,0 @@
-# Copyright 2022 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the \"License\");
-# you may not use this file except in compliance with the License.\n",
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an \"AS IS\" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-r"""Main function to shard ImageNet dataset.
-
-Example usage:
-python3 -u shard_imagenet.py \
---image_list_file=/home/jupyter/data/imagenet/train_list.txt \
---output_pattern=/home/jupyter/data/imagenet/validation-%06d.tar
-"""
-
-import argparse
-import os
-import random
-import webdataset as wds  # version: 0.2.26
-
-
-# NOTE: only supports writing to local path,
-# need gcsfuse mounting if want to write to gcs bucket.
-def write_shards(args):
-  """Shard individual data files."""
-  output_dir = os.path.dirname(args.output_pattern)
-  if not os.path.isdir(output_dir):
-    os.makedirs(output_dir)
-
-  items = []
-  # Image list file is a text file, each line is a pair (image_path, label).
-  with open(args.image_list_file, 'r') as f:
-    for line in f:
-      item = line.strip().split(' ')
-      items.append((item[0], int(item[1])))
-  # Shuffle items to avoid any large sequences of a single class
-  # in the dataset.
-  random.shuffle(items)
-
-  def _read_image(image_path):
-    with open(image_path, 'rb') as f:
-      return f.read()
-
-  with wds.ShardWriter(pattern=args.output_pattern,
-                       maxcount=args.max_images_per_shard,
-                       maxsize=args.max_bytes_per_shard) as sink:
-    for i, (image_path, target) in enumerate(items):
-      key = str(i)
-      image = _read_image(image_path)
-      sample = {'__key__': key, 'jpg': image, 'cls': target}
-      sink.write(sample)
-    if len(items) != sink.total:
-      raise ValueError('Items read {} != items written {}'.format(
-          len(items), sink.total))
-
-
-def create_args():
-  """Creates arg parser."""
-  parser = argparse.ArgumentParser(
-      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-  parser.add_argument(
-      '--image_list_file',
-      default='',
-      type=str,
-      help='path to image list file')
-  parser.add_argument(
-      '--output_pattern',
-      default='',
-      type=str,
-      help='the pattern for output shards, like /path/to/train-%06d.tar')
-  parser.add_argument(
-      '--max_images_per_shard',
-      default=10 * 1024,
-      type=int,
-      help='max number of images per shard')
-  parser.add_argument(
-      '--max_bytes_per_shard',
-      default=300 * 1024 * 1024,
-      type=int,
-      help='max bytes per shard')
-  args = parser.parse_args()
-  return args
-
-
-def main():
-  args = create_args()
-  write_shards(args)
-
-
-if __name__ == '__main__':
-  main()

community-content/pytorch_image_classification_distributed_data_parallel_training_with_vertex_sdk/multi_node_ddp_gloo_vertex_training_with_custom_container.ipynb

@@ -0,0 +1,474 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "a6b56b1c7b76"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2021 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "c414a395a19b"
+      },
+      "source": [
+        "# PyTorch Image Classification Multi-Node Distributed Data Parallel Training on CPU using Vertex Training with Custom Container"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "b98238e32cf7"
+      },
+      "source": [
+        "<table align=\"left\">\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/community-content/pytorch_image_classification_distributed_data_parallel_training_with_vertex_sdk/multi_node_ddp_gloo_vertex_training_with_custom_container.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "03d216c7f7b1"
+      },
+      "source": [
+        "## Setup"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "c5ac73516218"
+      },
+      "outputs": [],
+      "source": [
+        "PROJECT_ID = \"YOUR PROJECT ID\"\n",
+        "BUCKET_NAME = \"gs://YOUR BUCKET NAME\"\n",
+        "REGION = \"YOUR REGION\"\n",
+        "SERVICE_ACCOUNT = \"YOUR SERVICE ACCOUNT\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "0b5ae674177e"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil ls -al $BUCKET_NAME"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "19a9b3bdd553"
+      },
+      "outputs": [],
+      "source": [
+        "content_name = \"pt-img-cls-multi-node-ddp-cust-cont\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "57bf6f8b4361"
+      },
+      "source": [
+        "## Local Training"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "e5d8a3443da0"
+      },
+      "outputs": [],
+      "source": [
+        "! ls trainer"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "07f79309472d"
+      },
+      "outputs": [],
+      "source": [
+        "! cat trainer/requirements.txt"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "e16cd8bb7483"
+      },
+      "outputs": [],
+      "source": [
+        "! pip install -r trainer/requirements.txt"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "0b8a210718c4"
+      },
+      "outputs": [],
+      "source": [
+        "! cat trainer/task.py"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "c0c6e7dfb3c6"
+      },
+      "outputs": [],
+      "source": [
+        "%run trainer/task.py --epochs 5 --no-cuda --local-mode"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "31dfdeede587"
+      },
+      "outputs": [],
+      "source": [
+        "! ls ./tmp"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "48d56ec621cc"
+      },
+      "outputs": [],
+      "source": [
+        "! rm -rf ./tmp"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8f3ea1210749"
+      },
+      "source": [
+        "## Vertex Training using Vertex SDK and Custom Container"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "93002a20a2a6"
+      },
+      "source": [
+        "### Build Custom Container"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "4130ce43fd08"
+      },
+      "outputs": [],
+      "source": [
+        "hostname = \"gcr.io\"\n",
+        "image_name = content_name\n",
+        "tag = \"latest\"\n",
+        "\n",
+        "custom_container_image_uri = f\"{hostname}/{PROJECT_ID}/{image_name}:{tag}\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2f1fc5b05240"
+      },
+      "outputs": [],
+      "source": [
+        "! cd trainer && docker build -t $custom_container_image_uri -f Dockerfile ."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b4f274f499ac"
+      },
+      "outputs": [],
+      "source": [
+        "! docker run --rm $custom_container_image_uri --epochs 5 --no-cuda --local-mode"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ee1a0a06d0b4"
+      },
+      "outputs": [],
+      "source": [
+        "! docker push $custom_container_image_uri"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cb763be12fc9"
+      },
+      "outputs": [],
+      "source": [
+        "! gcloud container images list --repository $hostname/$PROJECT_ID"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "10c8cc6b3334"
+      },
+      "source": [
+        "### Initialize Vertex SDK"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "1a12348169fa"
+      },
+      "outputs": [],
+      "source": [
+        "! pip install -r requirements.txt"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "42e981cefe41"
+      },
+      "outputs": [],
+      "source": [
+        "from google.cloud import aiplatform\n",
+        "\n",
+        "aiplatform.init(\n",
+        "    project=PROJECT_ID,\n",
+        "    staging_bucket=BUCKET_NAME,\n",
+        "    location=REGION,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "73c92c9298e9"
+      },
+      "source": [
+        "### Create a Vertex Tensorboard Instance"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "bde509558cd5"
+      },
+      "outputs": [],
+      "source": [
+        "content_name = content_name + \"-cpu\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6d7908c0083c"
+      },
+      "outputs": [],
+      "source": [
+        "tensorboard = aiplatform.Tensorboard.create(\n",
+        "    display_name=content_name,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "a1f0a4f54037"
+      },
+      "source": [
+        "#### Option: Use a Previously Created Vertex Tensorboard Instance\n",
+        "\n",
+        "```\n",
+        "tensorboard_name = \"Your Tensorboard Resource Name or Tensorboard ID\"\n",
+        "tensorboard = aiplatform.Tensorboard(tensorboard_name=tensorboard_name)\n",
+        "```"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "a4cac84e04ac"
+      },
+      "source": [
+        "### Run a Vertex SDK CustomContainerTrainingJob"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "f92e8fdd44ee"
+      },
+      "outputs": [],
+      "source": [
+        "display_name = content_name\n",
+        "gcs_output_uri_prefix = f\"{BUCKET_NAME}/{display_name}\"\n",
+        "\n",
+        "replica_count = 4\n",
+        "machine_type = \"n1-standard-4\"\n",
+        "\n",
+        "args = [\n",
+        "    \"--backend\",\n",
+        "    \"gloo\",\n",
+        "    \"--no-cuda\",\n",
+        "    \"--batch-size\",\n",
+        "    \"128\",\n",
+        "    \"--epochs\",\n",
+        "    \"25\",\n",
+        "]"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ae4c57df7e07"
+      },
+      "outputs": [],
+      "source": [
+        "custom_container_training_job = aiplatform.CustomContainerTrainingJob(\n",
+        "    display_name=display_name,\n",
+        "    container_uri=custom_container_image_uri,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "35cf3ecdf0df"
+      },
+      "outputs": [],
+      "source": [
+        "custom_container_training_job.run(\n",
+        "    args=args,\n",
+        "    base_output_dir=gcs_output_uri_prefix,\n",
+        "    replica_count=replica_count,\n",
+        "    machine_type=machine_type,\n",
+        "    tensorboard=tensorboard.resource_name,\n",
+        "    service_account=SERVICE_ACCOUNT,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "49d10dded73b"
+      },
+      "outputs": [],
+      "source": [
+        "print(f\"Custom Training Job Name: {custom_container_training_job.resource_name}\")\n",
+        "print(f\"GCS Output URI Prefix: {gcs_output_uri_prefix}\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "78398f52807b"
+      },
+      "source": [
+        "### Training Output Artifact"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "fc74422de1d1"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil ls $gcs_output_uri_prefix"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5e99a6a05b10"
+      },
+      "source": [
+        "## Clean Up Artifact"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b0c1b3f7466b"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil rm -rf $gcs_output_uri_prefix"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "multi_node_ddp_gloo_vertex_training_with_custom_container.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

community-content/pytorch_image_classification_distributed_data_parallel_training_with_vertex_sdk/multi_node_ddp_nccl_vertex_training_with_custom_container.ipynb

@@ -0,0 +1,347 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "a6b56b1c7b76"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2021 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "20a5ea0081d0"
+      },
+      "source": [
+        "# PyTorch Image Classification Multi-Node Distributed Data Parallel Training on GPU using Vertex Training with Custom Container"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8752d4a255fb"
+      },
+      "source": [
+        "<table align=\"left\">\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/community-content/pytorch_image_classification_distributed_data_parallel_training_with_vertex_sdk/multi_node_ddp_nccl_vertex_training_with_custom_container.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "03d216c7f7b1"
+      },
+      "source": [
+        "## Setup"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "c5ac73516218"
+      },
+      "outputs": [],
+      "source": [
+        "PROJECT_ID = \"YOUR PROJECT ID\"\n",
+        "BUCKET_NAME = \"gs://YOUR BUCKET NAME\"\n",
+        "REGION = \"YOUR REGION\"\n",
+        "SERVICE_ACCOUNT = \"YOUR SERVICE ACCOUNT\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "0b5ae674177e"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil ls -al $BUCKET_NAME"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "19a9b3bdd553"
+      },
+      "outputs": [],
+      "source": [
+        "content_name = \"pt-img-cls-multi-node-ddp-cust-cont\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5307fe28b633"
+      },
+      "source": [
+        "## Vertex Training using Vertex SDK and Custom Container"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "46cb58c7fbf9"
+      },
+      "source": [
+        "### Built Custom Container"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "97e66e9f9bab"
+      },
+      "outputs": [],
+      "source": [
+        "hostname = \"gcr.io\"\n",
+        "image_name = content_name\n",
+        "tag = \"latest\"\n",
+        "\n",
+        "custom_container_image_uri = f\"{hostname}/{PROJECT_ID}/{image_name}:{tag}\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ae9b29c4773f"
+      },
+      "source": [
+        "### Initialize Vertex SDK"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "dc1e84d5dec2"
+      },
+      "outputs": [],
+      "source": [
+        "! pip install -r requirements.txt"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6964be27b98e"
+      },
+      "outputs": [],
+      "source": [
+        "from google.cloud import aiplatform\n",
+        "\n",
+        "aiplatform.init(\n",
+        "    project=PROJECT_ID,\n",
+        "    staging_bucket=BUCKET_NAME,\n",
+        "    location=REGION,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "594a91f438f2"
+      },
+      "source": [
+        "### Create a Vertex Tensorboard Instance"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "93134273261e"
+      },
+      "outputs": [],
+      "source": [
+        "content_name = content_name + \"-gpu\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "c2bd82dbcd9b"
+      },
+      "outputs": [],
+      "source": [
+        "tensorboard = aiplatform.Tensorboard.create(\n",
+        "    display_name=content_name,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ebc593c6472e"
+      },
+      "source": [
+        "#### Option: Use a Previously Created Vertex Tensorboard Instance\n",
+        "\n",
+        "```\n",
+        "tensorboard_name = \"Your Tensorboard Resource Name or Tensorboard ID\"\n",
+        "tensorboard = aiplatform.Tensorboard(tensorboard_name=tensorboard_name)\n",
+        "```"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0769e8e34c2f"
+      },
+      "source": [
+        "### Run a Vertex SDK CustomContainerTrainingJob"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "023f33ece826"
+      },
+      "outputs": [],
+      "source": [
+        "display_name = content_name\n",
+        "gcs_output_uri_prefix = f\"{BUCKET_NAME}/{display_name}\"\n",
+        "\n",
+        "replica_count = 1\n",
+        "machine_type = \"n1-standard-4\"\n",
+        "accelerator_count = 4\n",
+        "accelerator_type = \"NVIDIA_TESLA_K80\"\n",
+        "\n",
+        "args = [\n",
+        "    \"--backend\",\n",
+        "    \"nccl\",\n",
+        "    \"--batch-size\",\n",
+        "    \"128\",\n",
+        "    \"--epochs\",\n",
+        "    \"25\",\n",
+        "]"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "d4b599e726ef"
+      },
+      "outputs": [],
+      "source": [
+        "custom_container_training_job = aiplatform.CustomContainerTrainingJob(\n",
+        "    display_name=display_name,\n",
+        "    container_uri=custom_container_image_uri,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "81321e3bdf7f"
+      },
+      "outputs": [],
+      "source": [
+        "custom_container_training_job.run(\n",
+        "    args=args,\n",
+        "    base_output_dir=gcs_output_uri_prefix,\n",
+        "    replica_count=replica_count,\n",
+        "    machine_type=machine_type,\n",
+        "    accelerator_count=accelerator_count,\n",
+        "    accelerator_type=accelerator_type,\n",
+        "    tensorboard=tensorboard.resource_name,\n",
+        "    service_account=SERVICE_ACCOUNT,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "5100712c2c4c"
+      },
+      "outputs": [],
+      "source": [
+        "print(f\"Custom Training Job Name: {custom_container_training_job.resource_name}\")\n",
+        "print(f\"GCS Output URI Prefix: {gcs_output_uri_prefix}\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "f9b77676e5a6"
+      },
+      "source": [
+        "### Training Output Artifact"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "0e171ce95ace"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil ls $gcs_output_uri_prefix"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "cf1b74a12b87"
+      },
+      "source": [
+        "## Clean Up Artifact"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "a0b15089c341"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil rm -rf $gcs_output_uri_prefix"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "multi_node_ddp_nccl_vertex_training_with_custom_container.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

community-content/pytorch_pre_built_images_deployment/README.md

@@ -1,30 +0,0 @@
-# PyTorch Deployment on Google Cloud: Text Classification
-
-**This is an Experimental release**, covered by the Pre-GA Offerings Terms of your Google Cloud Platform [Terms of Service](https://cloud.google.com/terms).
-
-Experiments are focused on validating a prototype and are not guaranteed to be released. They are not intended for production use or covered by any SLA, support obligation, or deprecation policy and might be subject to backward-incompatible changes.
-
-**Kindly drop us a note before you run any scale tests.**
-
-**Do not hesitate to contact vertexai-prediction-preview-feedback@google.com if you have any questions or run into any issues.**
-
-The projects need to be added to the allowlist in order to deploy PyTorch models using Vertex AI Prediction pre-built PyTorch images. If you are interested in the feature, please send an email to vertexai-prediction-preview-feedback@google.com to provide your project numbers OR project ids.
-
-## Overview
-
-In the PyTorch on Google Cloud series of blog posts, we aim to share how to deploy PyTorch models at scale on [Vertex AI](https://cloud.google.com/vertex-ai).
-
-This tutorial on text classification shows how to deploy a PyTorch based text classification model on [Vertex AI](https://cloud.google.com/vertex-ai/docs/start/client-libraries#python) using Vertex SDK and [`gcloud ai`](https://cloud.google.com/sdk/gcloud/reference/beta/ai).
-
-## Notebooks
-
-| <h4>Notebook</h4> | <h4>Description</h4>                       |
-| :-------- | :------- |
-| [pytorch-text-classification-vertex-ai-deploy.ipynb](./pytorch-text-classification-vertex-ai-deploy.ipynb) | Notebook to show deploying a PyTorch model on Vertex AI |
-
-## Folders
-
-
-| <h4>Folder Name</h4> | <h4>Description</h4>                       |
-| :-------- | :------- |
-| [`predictor`](./predictor) | Folder with custom prediction handler to deploy a PyTorch model to Vertex Prediction. In the [notebook](./pytorch-text-classification-vertex-ai-deploy.ipynb), this folder is used for deploying a PyTorch model on Vertex AI using Vertex Prediction pre-built PyTorch images |

community-content/pytorch_pre_built_images_deployment/predictor/custom_handler.py

@@ -1,91 +0,0 @@
-
-import os
-import json
-import logging
-
-import torch
-from transformers import AutoModelForSequenceClassification, AutoTokenizer
-from ts.torch_handler.base_handler import BaseHandler
-
-logger = logging.getLogger(__name__)
-
-
-class TransformersClassifierHandler(BaseHandler):
-    """
-    The handler takes an input string and returns the classification text 
-    based on the serialized transformers checkpoint.
-    """
-    def __init__(self):
-        super(TransformersClassifierHandler, self).__init__()
-        self.initialized = False
-
-    def initialize(self, ctx):
-        """ Loads the model.pt file and initialized the model object.
-        Instantiates Tokenizer for preprocessor to use
-        Loads labels to name mapping file for post-processing inference response
-        """
-        self.manifest = ctx.manifest
-
-        properties = ctx.system_properties
-        model_dir = properties.get("model_dir")
-        self.device = torch.device("cuda:" + str(properties.get("gpu_id")) if torch.cuda.is_available() else "cpu")
-
-        # Read model serialize/pt file
-        serialized_file = self.manifest["model"]["serializedFile"]
-        model_pt_path = os.path.join(model_dir, serialized_file)
-        if not os.path.isfile(model_pt_path):
-            raise RuntimeError("Missing the model.pt or pytorch_model.bin file")
-        
-        # Load model
-        self.model = AutoModelForSequenceClassification.from_pretrained(model_dir)
-        self.model.to(self.device)
-        self.model.eval()
-        logger.debug('Transformer model from path {0} loaded successfully'.format(model_dir))
-        
-        # Ensure to use the same tokenizer used during training
-        self.tokenizer = AutoTokenizer.from_pretrained('bert-base-cased')
-
-        # Read the mapping file, index to object name
-        mapping_file_path = os.path.join(model_dir, "index_to_name.json")
-
-        if os.path.isfile(mapping_file_path):
-            with open(mapping_file_path) as f:
-                self.mapping = json.load(f)
-        else:
-            logger.warning('Missing the index_to_name.json file. Inference output will default.')
-            self.mapping = {"0": "Negative",  "1": "Positive"}
-
-        self.initialized = True
-
-    def preprocess(self, data):
-        """ Preprocessing input request by tokenizing
-            Extend with your own preprocessing steps as needed
-        """
-        text = data[0].get("data")
-        if text is None:
-            text = data[0].get("body")
-        sentences = text.decode('utf-8')
-        logger.info("Received text: '%s'", sentences)
-
-        # Tokenize the texts
-        tokenizer_args = ((sentences,))
-        inputs = self.tokenizer(*tokenizer_args,
-                                padding='max_length',
-                                max_length=128,
-                                truncation=True,
-                                return_tensors = "pt")
-        return inputs
-
-    def inference(self, inputs):
-        """ Predict the class of a text using a trained transformer model.
-        """
-        prediction = self.model(inputs['input_ids'].to(self.device))[0].argmax().item()
-
-        if self.mapping:
-            prediction = self.mapping[str(prediction)]
-
-        logger.info("Model predicted: '%s'", prediction)
-        return [prediction]
-
-    def postprocess(self, inference_output):
-        return inference_output

community-content/pytorch_pre_built_images_deployment/predictor/index_to_name.json

@@ -1,5 +0,0 @@
-
-{
-    "0": "Negative", 
-    "1": "Positive"
-}

community-content/pytorch_text_classification_using_vertex_sdk_and_gcloud/pytorch-text-classification-vertex-ai-train-tune-deploy.ipynb

@@ -658,8 +658,8 @@
       },
       "outputs": [],
       "source": [
-        "dataset = load_dataset(\"imdb\")\n",
-        "dataset"
+        "datasets = load_dataset(\"imdb\")\n",
+        "datasets"
       ]
     },
     {
@@ -668,7 +668,7 @@
         "id": "RzfPtOMoIrIu"
       },
       "source": [
-        "The `dataset` object itself is [`DatasetDict`](https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasetdict), which contains one key for the training, validation and test set."
+        "The `datasets` object itself is [`DatasetDict`](https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasetdict), which contains one key for the training, validation and test set."
       ]
     },
     {
@@ -681,12 +681,12 @@
       "source": [
         "print(\n",
         "    \"Total # of rows in training dataset {} and size {:5.2f} MB\".format(\n",
-        "        dataset[\"train\"].shape[0], dataset[\"train\"].size_in_bytes / (1024 * 1024)\n",
+        "        datasets[\"train\"].shape[0], datasets[\"train\"].size_in_bytes / (1024 * 1024)\n",
         "    )\n",
         ")\n",
         "print(\n",
         "    \"Total # of rows in test dataset {} and size {:5.2f} MB\".format(\n",
-        "        dataset[\"test\"].shape[0], dataset[\"test\"].size_in_bytes / (1024 * 1024)\n",
+        "        datasets[\"test\"].shape[0], datasets[\"test\"].size_in_bytes / (1024 * 1024)\n",
         "    )\n",
         ")"
       ]
@@ -708,7 +708,7 @@
       },
       "outputs": [],
       "source": [
-        "dataset[\"train\"][0]"
+        "datasets[\"train\"][0]"
       ]
     },
     {
@@ -728,7 +728,7 @@
       },
       "outputs": [],
       "source": [
-        "label_list = dataset[\"train\"].unique(\"label\")\n",
+        "label_list = datasets[\"train\"].unique(\"label\")\n",
         "label_list"
       ]
     },
@@ -779,7 +779,7 @@
       },
       "outputs": [],
       "source": [
-        "show_random_elements(dataset[\"train\"])"
+        "show_random_elements(datasets[\"train\"])"
       ]
     },
     {
@@ -883,7 +883,7 @@
       },
       "outputs": [],
       "source": [
-        "example = dataset[\"train\"][4]\n",
+        "example = datasets[\"train\"][4]\n",
         "print(example)"
       ]
     },
@@ -920,7 +920,7 @@
       "source": [
         "# Dataset loading repeated here to make this cell idempotent\n",
         "# Since we are over-writing datasets variable\n",
-        "dataset = load_dataset(\"imdb\")\n",
+        "datasets = load_dataset(\"imdb\")\n",
         "\n",
         "# Mapping labels to ids\n",
         "# NOTE: We can extract this automatically but the `Unique` method of the datasets\n",
@@ -948,7 +948,7 @@
         "\n",
         "\n",
         "# apply preprocessing function to input examples\n",
-        "dataset = dataset.map(preprocess_function, batched=True, load_from_cache_file=True)"
+        "datasets = datasets.map(preprocess_function, batched=True, load_from_cache_file=True)"
       ]
     },
     {
@@ -1091,8 +1091,8 @@
         "trainer = Trainer(\n",
         "    model,\n",
         "    args,\n",
-        "    train_dataset=dataset[\"train\"],\n",
-        "    eval_dataset=dataset[\"test\"],\n",
+        "    train_dataset=datasets[\"train\"],\n",
+        "    eval_dataset=datasets[\"test\"],\n",
         "    data_collator=default_data_collator,\n",
         "    tokenizer=tokenizer,\n",
         "    compute_metrics=compute_metrics,\n",

notebooks/ci_notebook_ingestion.md

@@ -1,102 +0,0 @@
-# Administrative Howto notes on CI Notebook Ingestion
-
-
-This readme covers administrative actions that are performed on an as-needed basis.
-
-## Team: vertex-ai-owners
-
-Members of the vertex-ai-owners (git team) have administrative privileges. 
-
-
-### Viewing members
-
-1. Goto the repo
-2. From top-level menu, select: (Settings -> Collaborators and Teams)[https://github.com/GoogleCloudPlatform/vertex-ai-samples/settings/access]
-
-
-### Adding a new member
-
-If another member needs to be added:
-   - Have the new member make a request to join the team.
-   - vertex-ai-owners with the `Maintainer` tag may add the new member.
-
-   
-## Executing CI notebook ingestion checks on a PR
-   
-### Killing a stuck PR
-
-If the CI notebook ingestion test is stuck (not terminating), you can kill the process by:
-
-1. Goto the PR
-2. Under checks, find the entry: vertex-ai-notebook-execution-test (python-docs-samples-tests) In progress —> Summary
-3. Select Details
-4. At bottom of details page, select: View more details on Google Cloud Build
-5. In Cloud Build history page, select Cancel on the top menu bar.
-
-### Restart a PR test
-
-There are two ways to restart the CI notebook ingestion tests on an open PR.
-
-1. In Cloud Build history page, select Rebuild on the top menu bar.
-2. or, in a comment in the PR enter: /gcbrun
-
-## Bypassing CI notebook ingestion checks on a PR
-
-We strongly discourage this, unless there is a compelling reason that would impact the integrity of the quality process.
-
-There are two ways of doing this. In both cases, you do:
-
-1. Goto the repo
-2. From top-level menu, select: (Settings -> Branches)[https://github.com/GoogleCloudPlatform/vertex-ai-samples/settings/branches]
-3. Under Branch Protection Rules, select the `main` branch.
-
-### Allowing a member to disable requirements for merging
-
-Specific member(s) can be assigned the ability to override requirements and merge a PR, by:
-
-1. Select Edit for the `main` branch in Branch Protection Rules.
-2. Find the entry "Allow specified actors to bypass required pull requests".
-3. Under this entry, add the member's git LDAP.
-4. Select SAVE.
-5. The "Squash and Merge" button will now be enabled on all PRs viewed by that member.
-
-### Temporarily disable checks.
-
-You can disable requirement checks temporarily on all PRs. 
-
-1. Select Edit for the `main` branch in Branch Protection Rules.
-2. Uncheck:
-  - Require approvals
-  - Require review from Code Owners
-  - Require status checks to pass before merging
-3. Select SAVE
-4. Now all members will see a green "Squash and Merge" on all PRs viewed by that member.
-
-To reverse, recheck the settings you unchecked above.
-
-## Linting
-
-To execute the identical lint image locally, from the CI notebook ingestion checks, do:
-
-1. Goto the corresponding local folder in the repo.
-2. Run: `docker run -v ${PWD}:/setup/app gcr.io/python-docs-samples-tests/notebook_linter:latest <your_notebooks>`
-
-## Install dependency issues
-
-Some packages (and combinations) have dependencies that fail on the virgin VM image used for the CI notebook ingestion test.
-
-### TFDV
-
-If the notebook installs and uses tensorflow_data_validation, install as follows:
-
-! pip3 install -q {USER_FLAG} google-cloud-aiplatform \
-                              tensorflow-data-validation \
-                              protobuf==3.20.3
-
-! pip3 install -q {USER_FLAG} cachetools==5.2.0
-
-
-
-
-   
-   

notebooks/community/CODEOWNERS

@@ -7,33 +7,23 @@
 /gapic @andrewferlitsch
 /gapic/custom/showcase_custom_image_classification_online_explain_example_based_api.ipynb @inardini
 /ml_ops @andrewferlitsch
-/model_monitoring/* @andrewferlitsch
+/model_monitoring/* @mco-gh
 /structured_data/rapid_prototyping_* @rafael-carvalho
 
 /managed_notebooks/
-/bigquery_ml/ @polong
+/sdk/SDK_FBProphet_Forecasting_Online.ipynb @brianchunkang
 /pipelines/google_cloud_pipeline_components_TPU_model_train_upload_deploy.ipynb @brianchunkang
 /explainable_ai/SDK_Custom_Container_XAI.ipynb @brianchunkang
 /matching_engine/sdk_matching_engine_for_indexing.ipynb @ivanmkc
 /matching_engine/matching_engine_for_indexing.ipynb @yinghsienwu
-/matching_engine/stream_update_for_matching_engine.ipynb @peterping666
 /sdk/pytorch_lightning_custom_container_training.ipynb @brianchunkang
-/sdk/sdk_pytorch_torchrun_custom_container_training_imagenet.ipynb @brianchunkang
 /tensorboard @yfang1
 /feature_store @nayaknishant @morgandu
 /prediction @googleapis/vertex-prediction-team
 /vertex_endpoints/tf_hub_obj_detection/deploy_tfhub_object_detection_on_vertex_endpoints.ipynb @entrpn
-/vertex_endpoints/find_ideal_machine_type/find_ideal_machine_type/find_ideal_machine_type.ipynb @entrpn
 /vertex_endpoints/nvidia-triton/nvidia-triton-custom-container-prediction.ipynb @RajeshThallam
 /vertex_endpoints/optimized_tensorflow_runtime @vlasenkoalexey
 /notebooks/community/ml_ops/stage2/get_started_with_visionapi_and_automl.ipynb @mansari
 /notebooks/community/neo4j/graph_paysim.ipynb @benofben @laeg
 /notebooks/community/ml_ops/stage1/get_started_with_visionapi_and_vertex_datasets.ipynb @mansari
-/notebooks/community/pipelines/google_cloud_pipeline_components_bqml_pipeline_demand_forecasting.ipynb @inardini
-/notebooks/community/ml_ops/stage2/get_started_vertex_hpt_r_kernel.ipynb @fhirschmann
-/notebooks/community/ml_ops/stage2/get_started_vertex_training_r_using_r_kernel.ipynb @fhirschmann
-/notebooks/community/vertex-ai-samples/notebooks/community/model_registry/vertex_ai_model_registry_bqml_custom_model_versioning.ipynb @inardini
-/notebooks/community/vertex-ai-samples/notebooks/community/model_registry/vertex_ai_model_registry_automl_model_versioning.ipynb @inardini
-/notebooks/community/vizier/conversions_vertex_vizier_and_open_source_vizier.ipynb @halio-g
-/notebooks/community/experiments/vertex_ai_model_experimentation.ipynb @inardini @asobran
-/notebooks/community/pipelines/google_cloud_pipeline_components_bqml_pipeline_anomaly_detection.ipynb @inardini
+/notebooks/community/pipelines/google_cloud_pipeline_components_bqml_pipeline_demand_forecasting.ipynb @inardini

notebooks/community/feature_store/sdk-feature-store-pandas.ipynb

@@ -29,28 +29,18 @@
         "id": "JAPoU8Sm5E6e"
       },
       "source": [
-        "# Using Vertex AI Feature Store with Pandas Dataframe\n",
-        "\n",
         "<table align=\"left\">\n",
-        "    <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/feature_store/sdk-feature-store-pandas.ipynb\">\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/feature_store/sdk-feature-store-pandas.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/feature_store/sdk-feature-store-pandas.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
-        "    \n",
-        "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/feature_store/sdk-feature-store-pandas.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> \n",
-        "        Run in Colab\n",
-        "    </a>\n",
-        "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/feature_store/sdk-feature-store-pandas.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
-        "    </a>\n",
-        "  </td>\n",
         "</table>"
       ]
     },
@@ -62,33 +52,7 @@
       "source": [
         "## Overview\n",
         "\n",
-        "This notebook introduces Pandas support for Feature Store using Vertex AI SDK. For pre-requisites and introduction on Vertex AI SDK and Feature Store native support, please go through this [Colab notebook](https://colab.sandbox.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/feature_store/sdk-feature-store.ipynb). "
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "DxF5JWRVT5PP"
-      },
-      "source": [
-        "### Objective\n",
-        "\n",
-        "In this notebook, you learn how to use `Vertex AI Feature Store` with pandas Dataframe.\n",
-        "\n",
-        "This tutorial uses the following Google Cloud ML services and resources:\n",
-        "\n",
-        "- Vertex AI Feature Store\n",
-        "\n",
-        "The steps performed include:\n",
-        "\n",
-        "- Ingest Feature values from Pandas DataFrame into Feature Store's Entity types.\n",
-        "- Read Entity feature values from Online Feature Store into Pandas DataFrame.\n",
-        "- Batch serve feature values from your Feature Store into Pandas DataFrame.\n",
-        "\n",
-        "You also learn how Vertex AI Feature Store can be useful in the below scenarios:\n",
-        "\n",
-        "- Online serving with updated feature values.\n",
-        "- Point-in-time correctness to fetch feature values for training."
+        "This Colab introduces Pandas support of Vertex AI SDK Feature Store. For pre-requisite and introduction for Vertex AI SDK Feature Store native support, please see this [Colab](https://colab.sandbox.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/feature_store/sdk-feature-store.ipynb). "
       ]
     },
     {
@@ -99,7 +63,27 @@
       "source": [
         "### Dataset\n",
         "\n",
-        "This tutorial is a part of the Feature Store tutorial notebooks. It uses a movie recommendation dataset as an example for demonstrating various functionalities of Feature Store. The original task is to train a model to predict if a user is going to watch a movie, and serve the model online."
+        "This Colab uses a movie recommendation dataset as an example throughout all the sessions. The task is to train a model to predict if a user is going to watch a movie and serve this model online."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "DxF5JWRVT5PP"
+      },
+      "source": [
+        "### Objective\n",
+        "\n",
+        "In this notebook, you will learn how to:\n",
+        "\n",
+        "    * Ingest Feature Values from Pandas DataFrame into featurestore's entity types.\n",
+        "    * Read Entity Feature Values from Online Feature Store into Pandas DataFrame.\n",
+        "    * Batch Serve Feature Values from your featurestore to Pandas DataFrame.\n",
+        "\n",
+        "We will also discuss how Vertex AI Feature Store can be useful in the below scenarios:\n",
+        "\n",
+        "    * online serving with updated feature values\n",
+        "    * point-in-time correctness to fetch feature values for training"
       ]
     },
     {
@@ -113,9 +97,11 @@
         "This tutorial uses billable components of Google Cloud:\n",
         "\n",
         "* Vertex AI\n",
+        "* Cloud BigQuery\n",
         "\n",
         "Learn about [Vertex AI\n",
-        "pricing](https://cloud.google.com/vertex-ai/pricing) and use the [Pricing\n",
+        "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
+        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
         "Calculator](https://cloud.google.com/products/calculator/)\n",
         "to generate a cost estimate based on your projected usage."
       ]
@@ -137,7 +123,7 @@
       "source": [
         "### Install additional packages\n",
         "\n",
-        "To run this notebook, you need to install the following packages for Python."
+        "For this Colab, you need the Vertex SDK for Python."
       ]
     },
     {
@@ -150,23 +136,41 @@
       "source": [
         "import os\n",
         "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
+        "# The Google Cloud Notebook product has specific requirements\n",
+        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
         "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "# Google Cloud Notebook requires dependencies to be installed with '--user'\n",
         "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "    \n",
-        "! pip install -U {USER_FLAG} --upgrade google-cloud-aiplatform \\\n",
-        "                                        google-cloud-bigquery \\\n",
-        "                                        google-cloud-bigquery-storage \\\n",
-        "                                        avro \\\n",
-        "                                        pyarrow \\\n",
-        "                                        pandas -q"
+        "if IS_GOOGLE_CLOUD_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Kd0kgDqVZyRe"
+      },
+      "outputs": [],
+      "source": [
+        "! pip uninstall {USER_FLAG} -y google-cloud-aiplatform\n",
+        "! pip uninstall {USER_FLAG} -y google-cloud-bigquery\n",
+        "! pip uninstall {USER_FLAG} -y google-cloud-bigquery-storage\n",
+        "! pip uninstall {USER_FLAG} -y google-cloud-aiplatform"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "wUswAmpiN2l-"
+      },
+      "outputs": [],
+      "source": [
+        "! pip install {USER_FLAG} --upgrade google-cloud-aiplatform\n",
+        "! pip install {USER_FLAG} --upgrade google-cloud-bigquery\n",
+        "! pip install {USER_FLAG} --upgrade google-cloud-bigquery-storage\n",
+        "! pip install {USER_FLAG} avro"
       ]
     },
     {
@@ -177,7 +181,7 @@
       "source": [
         "### Restart the kernel\n",
         "\n",
-        "After you install the packages, you need to restart the notebook kernel so that it can find the packages."
+        "After you install the SDK, you need to restart the notebook kernel so it can find the packages. You can restart kernel from *Kernel -> Restart Kernel*, or running the following:"
       ]
     },
     {
@@ -215,7 +219,7 @@
         "\n",
         "1. [Enable the Vertex AI API and Compute Engine API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component).\n",
         "\n",
-        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
         "\n",
         "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
         "Cloud SDK uses the right project for all the commands in this notebook.\n",
@@ -234,17 +238,6 @@
         "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "dcdfccf50581"
-      },
-      "outputs": [],
-      "source": [
-        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -253,56 +246,37 @@
       },
       "outputs": [],
       "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
-        "    # Get your GCP project id from gcloud\n",
-        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "import os\n",
+        "\n",
+        "PROJECT_ID = \"\"\n",
+        "\n",
+        "# Get your Google Cloud project ID from gcloud\n",
+        "if not os.getenv(\"IS_TESTING\"):\n",
+        "    shell_output = !gcloud config list --format 'value(core.project)' 2>/dev/null\n",
         "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID:\", PROJECT_ID)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "09021c90b34c"
-      },
-      "outputs": [],
-      "source": [
-        "! gcloud config set project $PROJECT_ID"
+        "    print(\"Project ID: \", PROJECT_ID)"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "f41eda68c379"
+        "id": "qJYoRfYng0XZ"
       },
       "source": [
-        "#### Region\n",
-        "\n",
-        "You can also change the `REGION` variable, which is used for operations\n",
-        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. It is recommended that you choose the region closest to you.\n",
-        "\n",
-        "- Americas: `us-central1`\n",
-        "- Europe: `europe-west4`\n",
-        "- Asia Pacific: `asia-east1`\n",
-        "\n",
-        "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
-        "\n",
-        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
+        "Otherwise, set your project ID here."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "5c615e53149f"
+        "id": "riG_qUokg0XZ"
       },
       "outputs": [],
       "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
+        "    PROJECT_ID = \"\"  # @param {type:\"string\"}\n",
+        "print(\"Project ID: \", PROJECT_ID)"
       ]
     },
     {
@@ -311,29 +285,10 @@
         "id": "dr--iN2kAylZ"
       },
       "source": [
-        "#### UUID\n",
+        "### Authenticate your Google Cloud account\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "4e166d927e36"
-      },
-      "outputs": [],
-      "source": [
-        "import random\n",
-        "import string\n",
-        "\n",
-        "\n",
-        "# Generate a uuid of a specifed length(default=8)\n",
-        "def generate_uuid(length: int = 8) -> str:\n",
-        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
-        "\n",
-        "\n",
-        "UUID = generate_uuid()"
+        "**If you are using Google Cloud Notebooks**, your environment is already\n",
+        "authenticated. Skip this step."
       ]
     },
     {
@@ -342,11 +297,6 @@
         "id": "sBCra4QMA2wR"
       },
       "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
-        "authenticated. \n",
-        "\n",
         "**If you are using Colab**, run the cell below and follow the instructions\n",
         "when prompted to authenticate your account via oAuth.\n",
         "\n",
@@ -364,7 +314,7 @@
         "into the filter box, and select\n",
         "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
         "\n",
-        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
+        "5. Click **Create**. A JSON file that contains your key downloads to your\n",
         "local environment.\n",
         "\n",
         "6. Enter the path to your service account key as the\n",
@@ -379,19 +329,19 @@
       },
       "outputs": [],
       "source": [
+        "import os\n",
+        "import sys\n",
+        "\n",
         "# If you are running this notebook in Colab, run this cell and follow the\n",
         "# instructions to authenticate your GCP account. This provides access to your\n",
         "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
         "# requests.\n",
         "\n",
-        "import os\n",
-        "import sys\n",
+        "# The Google Cloud Notebook product has specific requirements\n",
+        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
         "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = \"google.colab\" in sys.modules\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
+        "# If on Google Cloud Notebooks, then don't execute this code\n",
+        "if not IS_GOOGLE_CLOUD_NOTEBOOK:\n",
         "    if \"google.colab\" in sys.modules:\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
@@ -401,7 +351,7 @@
         "    # path to your service account key and run this cell to authenticate your GCP\n",
         "    # account.\n",
         "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS '[your-service-account-key-path]'"
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
       ]
     },
     {
@@ -410,7 +360,7 @@
         "id": "XoEqT2Y4DJmf"
       },
       "source": [
-        "### Import libraries"
+        "### Import libraries and define constants"
       ]
     },
     {
@@ -424,31 +374,20 @@
         "import datetime\n",
         "\n",
         "import pandas as pd\n",
-        "from avro.datafile import DataFileReader\n",
-        "from avro.io import DatumReader\n",
-        "from google.cloud import aiplatform"
+        "from google.cloud import aiplatform\n",
+        "\n",
+        "REGION = \"\"  # @param {type:\"string\"}\n",
+        "\n",
+        "aiplatform.init(project=PROJECT_ID, location=REGION)"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "138407556b22"
+        "id": "9UvxYyGUimKw"
       },
       "source": [
-        "### Initialize Vertex AI SDK for Python\n",
-        "\n",
-        "Initialize the Vertex AI SDK for Python for your project and region."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "8d2077ffee78"
-      },
-      "outputs": [],
-      "source": [
-        "aiplatform.init(project=PROJECT_ID, location=REGION)"
+        "## Create Feature Store Resources"
       ]
     },
     {
@@ -457,12 +396,11 @@
         "id": "buQBIv3ZL3A0"
       },
       "source": [
-        "## Create a Feature Store\n",
+        "### Create Featurestore\n",
         "\n",
-        "The method to create a Feature Store in Vertex AI returns a\n",
-        "[long-running operation](https://google.aip.dev/151) (LRO). An LRO starts an asynchronous job. LROs are returned for other API methods too, such as updating or deleting a featurestore. \n",
-        "\n",
-        "Running the code cell below creates a featurestore and prints the process' logs."
+        "The method to create a Featurestore returns a\n",
+        "[long-running operation](https://google.aip.dev/151) (LRO). An LRO starts an asynchronous job. LROs are returned for other API\n",
+        "methods too, such as updating or deleting a featurestore. Running the code cell will create a featurestore and print the process log."
       ]
     },
     {
@@ -473,9 +411,9 @@
       },
       "outputs": [],
       "source": [
-        "# Create featurestore\n",
         "movie_predictions_feature_store = aiplatform.Featurestore.create(\n",
-        "    featurestore_id=f\"movie_predictions_{UUID}\", online_store_fixed_node_count=1\n",
+        "    featurestore_id=\"movie_predictions\",\n",
+        "    online_store_fixed_node_count=1,\n",
         ")"
       ]
     },
@@ -485,9 +423,9 @@
         "id": "EpmJq75zXjmT"
       },
       "source": [
-        "## Create Entity types\n",
+        "### Create Entity Types\n",
         "\n",
-        "Entity types can be created within the Featurestore class. Below, you create the `Users` entity type and `Movies` entity type. Process logs are printed in the output for each cell."
+        "Entity types can be created within the Featurestore class. Below, create the Users entity type and Movies entity type. A process log will be printed out."
       ]
     },
     {
@@ -498,7 +436,6 @@
       },
       "outputs": [],
       "source": [
-        "# Create users entity type\n",
         "users_entity_type = movie_predictions_feature_store.create_entity_type(\n",
         "    entity_type_id=\"users\",\n",
         "    description=\"Users entity\",\n",
@@ -513,7 +450,6 @@
       },
       "outputs": [],
       "source": [
-        "# Create movies entity type\n",
         "movies_entity_type = movie_predictions_feature_store.create_entity_type(\n",
         "    entity_type_id=\"movies\",\n",
         "    description=\"Movies entity\",\n",
@@ -526,11 +462,8 @@
         "id": "FJW4q-0jO2Xf"
       },
       "source": [
-        "## Create Features\n",
-        "Features can be created within each entity type. Add defined features to the `Users` entity type and `Movies` entity type by using the following methods.\n",
-        "\n",
-        "### Add features using *create_feature* method\n",
-        "Provide the feature information like id, type and description to the `create_feature` method of entity type."
+        "### Create Features\n",
+        "Features can be created within each entity type. Add defining features to the Users entity type and Movies entity type by using the following methods."
       ]
     },
     {
@@ -541,21 +474,18 @@
       },
       "outputs": [],
       "source": [
-        "# Create age feature\n",
         "users_feature_age = users_entity_type.create_feature(\n",
         "    feature_id=\"age\",\n",
         "    value_type=\"INT64\",\n",
         "    description=\"User age\",\n",
         ")\n",
         "\n",
-        "# Create gender feature\n",
         "users_feature_gender = users_entity_type.create_feature(\n",
         "    feature_id=\"gender\",\n",
         "    value_type=\"STRING\",\n",
         "    description=\"User gender\",\n",
         ")\n",
         "\n",
-        "# Create liked_genres feature\n",
         "users_feature_liked_genres = users_entity_type.create_feature(\n",
         "    feature_id=\"liked_genres\",\n",
         "    value_type=\"STRING_ARRAY\",\n",
@@ -563,18 +493,6 @@
         ")"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "ecb141839033"
-      },
-      "source": [
-        "### Add features using batch method\n",
-        "You can also create features using a config map in a dictionary format and the `batch_create_features` method. This way, you can add multiple features at once. \n",
-        "\n",
-        "Below, you define and create *title*, *genres* and *average_rating* features using the batch method."
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -596,8 +514,17 @@
         "        \"value_type\": \"DOUBLE\",\n",
         "        \"description\": \"The average rating for the movie, range is [1.0-5.0]\",\n",
         "    },\n",
-        "}\n",
-        "\n",
+        "}"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "YhfOKJL_BvuM"
+      },
+      "outputs": [],
+      "source": [
         "movie_features = movies_entity_type.batch_create_features(\n",
         "    feature_configs=movies_feature_configs,\n",
         ")"
@@ -609,15 +536,18 @@
         "id": "K3n5XdK8Xjmw"
       },
       "source": [
-        "## Ingest Feature values into Entity types from dataframes\n",
+        "## Ingest Feature Values into Entity Type from a Pandas DataFrame\n",
         "\n",
-        "You need to ingest feature values into your entity type containing the features. It is so that you can later `read` (online) or `batch serve` (offline) the feature values from the entity type. \n",
-        "\n",
-        "In this step, you learn how to ingest feature values from a Pandas dataframe into an entity type. You can also import feature values from BigQuery or Google Cloud Storage.\n",
-        "\n",
-        "### Get data from source\n",
-        "\n",
-        "Define the public data sources for users and movies and copy them locally into *avro* files."
+        "You need to ingest feature values into your entity type containing the features, so you can later `read` (online) or `batch serve` (offline) the feature values from the entity type. In this step, you will learn how to ingest feature values from a Pandas DataFrame into an entity type. We can also import feature values from BigQuery or Google Cloud Storage.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "BlqJ-QdTcs6W"
+      },
+      "source": [
+        "#### Entity Type Source Files"
       ]
     },
     {
@@ -636,8 +566,17 @@
         ")\n",
         "\n",
         "USERS_AVRO_FN = \"users.avro\"\n",
-        "MOVIES_AVRO_FN = \"movies.avro\"\n",
-        "\n",
+        "MOVIES_AVRO_FN = \"movies.avro\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "KqIH_bS-5OW5"
+      },
+      "outputs": [],
+      "source": [
         "! gsutil cp $GCS_USERS_AVRO_URI $USERS_AVRO_FN\n",
         "! gsutil cp $GCS_MOVIES_AVRO_URI $MOVIES_AVRO_FN"
       ]
@@ -648,9 +587,7 @@
         "id": "Fd6Z0jfR5OW5"
       },
       "source": [
-        "### Load data from avro files \n",
-        "\n",
-        "Load users and movies data from avro files into Pandas dataframes."
+        "#### Load Avro Files into Pandas DataFrames"
       ]
     },
     {
@@ -661,7 +598,10 @@
       },
       "outputs": [],
       "source": [
-        "# Define a class for reading the avro data\n",
+        "from avro.datafile import DataFileReader\n",
+        "from avro.io import DatumReader\n",
+        "\n",
+        "\n",
         "class AvroReader:\n",
         "    def __init__(self, data_file):\n",
         "        self.avro_reader = DataFileReader(open(data_file, \"rb\"), DatumReader())\n",
@@ -679,7 +619,6 @@
       },
       "outputs": [],
       "source": [
-        "# Load users data from avro file\n",
         "users_avro_reader = AvroReader(data_file=USERS_AVRO_FN)\n",
         "users_source_df = users_avro_reader.to_dataframe()\n",
         "print(users_source_df)"
@@ -693,7 +632,6 @@
       },
       "outputs": [],
       "source": [
-        "# Load movies data from avro file\n",
         "movies_avro_reader = AvroReader(data_file=MOVIES_AVRO_FN)\n",
         "movies_source_df = movies_avro_reader.to_dataframe()\n",
         "print(movies_source_df)"
@@ -705,9 +643,7 @@
         "id": "bgb0WGwX5OW6"
       },
       "source": [
-        "### Ingest Feature values into Entity types\n",
-        "\n",
-        "Load the feature values into `users` entity type providing the id fields and time field."
+        "#### Ingest Feature Values into Users Entity Type"
       ]
     },
     {
@@ -732,7 +668,7 @@
         "id": "PCAdQ3cF5OW6"
       },
       "source": [
-        "Load the feature values into `movie` entity type providing the id fields and time field."
+        "#### Ingest Feature Values into Movies Entity Type"
       ]
     },
     {
@@ -757,12 +693,10 @@
         "id": "pIYLZwao5OW6"
       },
       "source": [
-        "## Read/serve Entity's feature values online from Feature Store\n",
+        "## Read/Online Serve Entity's Feature Values from Vertex AI Online Feature Store\n",
         "\n",
         "Feature Store allows [online serving](https://cloud.google.com/vertex-ai/docs/featurestore/serving-online)\n",
-        "which lets you read feature values for small batches of entities. It works well when you want to read values of selected features from an entity or multiple entities in an entity type.\n",
-        "\n",
-        "### Read feature values for users"
+        "which lets you read feature values for small batches of entities. It works well when you want to read values of selected features from an entity or multiple entities in an entity type."
       ]
     },
     {
@@ -779,15 +713,6 @@
         "print(users_read_df)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "b2cfa09ef11d"
-      },
-      "source": [
-        "### Read feature values for movies"
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -809,13 +734,18 @@
         "id": "AK2Glzkq5OW7"
       },
       "source": [
-        "## Batch serve feature values from Feature Store\n",
+        "## Batch Serve Featurestore's Feature Values from Vertex AI Feature Store\n",
         "\n",
-        "Batch Serving is used to fetch a large batch of feature values for high-throughput, and is typically used for training a model or batch prediction. In this section, you learn how to prepare training examples by using the Feature Store's batch serve function.\n",
-        "\n",
-        "### Read instances from source file\n",
-        "\n",
-        "Define the source file and destination file. "
+        "Batch Serving is used to fetch a large batch of feature values for high-throughput, and is typically used for training a model or batch prediction. In this section, you will learn how to prepare for training examples by using the Featurestore's batch serve function."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "hxsotHUe5OW7"
+      },
+      "source": [
+        "#### Read Instances Source File"
       ]
     },
     {
@@ -826,17 +756,7 @@
       },
       "outputs": [],
       "source": [
-        "GCS_READ_INSTANCES_CSV_URI = \"gs://cloud-samples-data-us-central1/vertex-ai/feature-store/datasets/movie_prediction.csv\"\n",
-        "READ_INSTANCES_CSV_FN = \"data.csv\""
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "3f2c558b649f"
-      },
-      "source": [
-        "Copy the instances from the source file to the destination file locally."
+        "GCS_READ_INSTANCES_CSV_URI = \"gs://cloud-samples-data-us-central1/vertex-ai/feature-store/datasets/movie_prediction.csv\""
       ]
     },
     {
@@ -856,9 +776,7 @@
         "id": "T5DW1MFt5OW7"
       },
       "source": [
-        "### Load the instances\n",
-        "\n",
-        "Load the instances from CSV file into a Pandas dataframe."
+        "#### Load Csv File into a Pandas DataFrame"
       ]
     },
     {
@@ -869,7 +787,7 @@
       },
       "outputs": [],
       "source": [
-        "read_instances_df = pd.read_csv(READ_INSTANCES_CSV_FN)\n",
+        "read_instances_df = pd.read_csv(read_instances_csv_fn)\n",
         "print(read_instances_df)"
       ]
     },
@@ -879,9 +797,7 @@
         "id": "LsgNNH8G5OW8"
       },
       "source": [
-        "### Change the data type\n",
-        "\n",
-        "Change the data type of the timestamp field from `Timestamp` to `Datetime64`."
+        "#### Change the Dtype of `Timestamp` to `Datetime64`"
       ]
     },
     {
@@ -903,9 +819,7 @@
         "id": "ao1dC5Pc5OW8"
       },
       "source": [
-        "### Batch serve feature values from Feature Store\n",
-        "\n",
-        "Serve the batch response to a dataframe and display the data."
+        "#### Batch Serve Feature Values from Movie Predictions Featurestore"
       ]
     },
     {
@@ -926,21 +840,42 @@
         "movie_predictions_df"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "29gLNORP5OW8"
+      },
+      "source": [
+        "## Read the Updated Feature Values"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "XN84znoI5OW8"
       },
       "source": [
-        "## Read the latest feature values\n",
-        "\n",
-        "In Feature Store, you access the latest or the last available feature values unless a specific time is provided. Now, you test this feature by ingesting new data to the entity types and reading it from the Feature Store.\n",
-        "\n",
-        "### Ingest updated feature values\n",
-        "\n",
-        "Now, you update the feature values by running the following cell. \n",
-        "\n",
-        "**Note:** For comparison, you can try printing the feature values read from the entity types earlier (those in `movies_read_df` variable). "
+        "#### Recall Read from the Entity Type Shows Feature Values from the Last Ingestion"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "wtmshq_n5OW9"
+      },
+      "outputs": [],
+      "source": [
+        "print(movies_read_df)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "feTUJjqG5OW9"
+      },
+      "source": [
+        "#### Ingest Updated Feature Values"
       ]
     },
     {
@@ -951,13 +886,21 @@
       },
       "outputs": [],
       "source": [
-        "# Create a dataframe for the new data\n",
         "update_movies_df = pd.DataFrame(\n",
         "    data=[[\"movie_03\", 4.3], [\"movie_04\", 4.8]],\n",
         "    columns=[\"movie_id\", \"average_rating\"],\n",
         ")\n",
-        "\n",
-        "# Ingest the new data from the dataframe\n",
+        "print(update_movies_df)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "aKKhSzUc5OW9"
+      },
+      "outputs": [],
+      "source": [
         "movies_entity_type.ingest_from_df(\n",
         "    feature_ids=[\"average_rating\"],\n",
         "    feature_time=datetime.datetime.now(),\n",
@@ -972,9 +915,7 @@
         "id": "s47WCIvL5OW9"
       },
       "source": [
-        "### Fetch the latest feature values\n",
-        "\n",
-        "Reading from the entity type gives you the updated feature values from the latest ingestion."
+        "#### Read from the Entity Type Shows Updated Feature Values from the Latest Ingestion"
       ]
     },
     {
@@ -992,18 +933,22 @@
         "print(update_movies_read_df)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "wsvCRzn_5OW9"
+      },
+      "source": [
+        "## Point-in-Time Correctness"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "R1YGRNsW5OW9"
       },
       "source": [
-        "## Point-in-time correctness\n",
-        "\n",
-        "Vertex AI Feature Store captures feature values for a feature at a specific point in time. In case there are missing values in your past data, you can backfill them using batch serving.\n",
-        "\n",
-        "### Missing data\n",
-        "Recall that response from the batch serve from last ingestion has some missing data in it."
+        "#### Recall Batch Serve From the Last Ingestion Has Missing Data"
       ]
     },
     {
@@ -1014,7 +959,6 @@
       },
       "outputs": [],
       "source": [
-        "# Print the response\n",
         "print(movie_predictions_df)"
       ]
     },
@@ -1024,9 +968,7 @@
         "id": "abQRF6mx5OW-"
       },
       "source": [
-        "### Backfill/correct point-in-time data\n",
-        "\n",
-        "Impute the missing data based on the timestamps."
+        "#### Backfill/Correct Point-in-Time Data"
       ]
     },
     {
@@ -1037,7 +979,6 @@
       },
       "outputs": [],
       "source": [
-        "# Impute the users data\n",
         "backfill_users_df = pd.DataFrame(\n",
         "    data=[[\"bob\", 34, \"Male\", [\"Drama\"], \"2020-02-13 09:35:15\"]],\n",
         "    columns=[\"user_id\", \"age\", \"gender\", \"liked_genres\", \"update_time\"],\n",
@@ -1054,7 +995,6 @@
       },
       "outputs": [],
       "source": [
-        "# Impute the movies data\n",
         "backfill_movies_df = pd.DataFrame(\n",
         "    data=[[\"movie_04\", 4.2, \"The Dark Knight\", \"Action\", \"2020-02-13 09:35:15\"]],\n",
         "    columns=[\"movie_id\", \"average_rating\", \"title\", \"genres\", \"update_time\"],\n",
@@ -1069,9 +1009,7 @@
         "id": "WXb4JUhu5OW-"
       },
       "source": [
-        "### Ingest the backfilled/corrected data\n",
-        "\n",
-        "Ingest the imputed point-in-time data from dataframe to the entity types in feature store."
+        "#### Ingest Backfill/Correct Point-in-Time Data"
       ]
     },
     {
@@ -1082,7 +1020,6 @@
       },
       "outputs": [],
       "source": [
-        "# Ingest the users data\n",
         "users_entity_type.ingest_from_df(\n",
         "    feature_ids=[\"age\", \"gender\", \"liked_genres\"],\n",
         "    feature_time=\"update_time\",\n",
@@ -1099,7 +1036,6 @@
       },
       "outputs": [],
       "source": [
-        "# Ingest the users data\n",
         "movies_entity_type.ingest_from_df(\n",
         "    feature_ids=[\"average_rating\", \"title\", \"genres\"],\n",
         "    feature_time=\"update_time\",\n",
@@ -1114,8 +1050,7 @@
         "id": "1e62Ku6W5OW_"
       },
       "source": [
-        "### Fetch the latest data\n",
-        "Batch serve the latest ingested data with backfill/correction to a dataframe to ensure the feature store is updated. "
+        "#### Batch Serve From the Latest Ingestion with Backfill/Correction Has Reduced Missing Data"
       ]
     },
     {
@@ -1147,7 +1082,7 @@
         "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
         "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
         "\n",
-        "Otherwise, you can delete the individual resources you created in this tutorial:"
+        "You can also keep the project but delete the featurestore by running the code below:"
       ]
     },
     {
@@ -1158,7 +1093,6 @@
       },
       "outputs": [],
       "source": [
-        "# Delete the feature store\n",
         "movie_predictions_feature_store.delete(force=True)"
       ]
     }

notebooks/community/managed_notebooks/ads_targetting/training-multi-class-classification-model-for-ads-targeting-usecase.ipynb

@@ -1,55 +1,29 @@
 {
   "cells": [
     {
-      "cell_type": "code",
-      "execution_count": null,
+      "cell_type": "markdown",
       "metadata": {
         "id": "503077811e70"
       },
-      "outputs": [],
-      "source": [
-        "# Copyright 2022 Google LLC\n",
-        "#\n",
-        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
-        "# you may not use this file except in compliance with the License.\n",
-        "# You may obtain a copy of the License at\n",
-        "#\n",
-        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
-        "#\n",
-        "# Unless required by applicable law or agreed to in writing, software\n",
-        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
-        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
-        "# See the License for the specific language governing permissions and\n",
-        "# limitations under the License."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "e885ac09bc73"
-      },
       "source": [
         "# Train a multi-class classification model for ads-targeting\n",
-        "<table align=\"left\">\n",
         "\n",
-        "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/ads_targetting/training-multi-class-classification-model-for-ads-targeting-usecase.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
-        "    </a>\n",
-        "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/ads_targetting/training-multi-class-classification-model-for-ads-targeting-usecase.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
-        "      View on GitHub\n",
-        "    </a>\n",
-        "  </td>\n",
-        "  <td>\n",
-        "<a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/workbench/ads_targetting/training-multi-class-classification-model-for-ads-targeting-usecase.ipynb\" target='_blank'>\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
-        "    </a>\n",
-        "  </td>                                                                                               \n",
-        "</table>"
+        "## Table of contents\n",
+        "\n",
+        "* [Overview](#section-1)\n",
+        "* [Dataset](#section-2)\n",
+        "* [Objective](#section-3)\n",
+        "* [Costs](#section-4)\n",
+        "* [Tutorial](#section-5)\n",
+        "\t- [Fetch the data from BigQuery](#section-5)\n",
+        "    - [Preprocess the data](#section-6)\n",
+        "    - [Train a TensorFlow model](#section-7)\n",
+        "    - [Run the model on test data](#section-8)\n",
+        "    - [Automating the execution of the notebook using executor](#section-9)\n",
+        "    - [Scheduled runs on executor](#section-10)\n",
+        "    - [Parameterizing the variables](#section-11)\n",
+        "* [Save the model to a Cloud Storage path](#section-12)\n",
+        "* [Clean up](#section-13)\n"
       ]
     },
     {
@@ -59,27 +33,25 @@
       },
       "source": [
         "## Overview\n",
+        "<a name=\"section-1\"></a>\n",
         "\n",
         "This tutorial demonstrates how to build a machine learning model for an ads-targeting use case. Ads-targeting is an advertisement technique where chosen or tailor-made ads are shown to the customers based on their past behavior and preferences. Targeted ads are meant to reach specific customers based on demographics, psychographics, behavior, and other second-order activities that are learned usually through data collected from the customers.\n",
         "\n",
-        "*Note: If you are using [Vertex AI Workbench managed notebooks](https://cloud.google.com/vertex-ai/docs/workbench/managed/create-instance) instance use the `TensorFlow 2 (Local)` kernel. Some components of this notebook may not work in other notebook environments.*\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "1bea2b6e9b25"
-      },
-      "source": [
-        "### Objective\n",
+        "*Note: This notebook file was designed to run in a [Vertex AI Workbench managed notebooks](https://cloud.google.com/vertex-ai/docs/workbench/managed/create-instance) instance using the `TensorFlow 2 (Local)` kernel. Some components of this notebook may not work in other notebook environments.*\n",
         "\n",
-        "In this tutorial, you learn how to collect data from BigQuery, preprocess it, and train a multi-class classification model on an e-commerce dataset. \n",
+        "## Dataset\n",
+        "<a name=\"section-2\"></a>\n",
         "\n",
-        "This tutorial uses the following Google Cloud ML services and resources:\n",
+        "This tutorial uses the `looker-private-demo.ecomm` dataset in BigQuery. The dataset consists of information about various advertisement campaigns including the demographics of users who have clicked and made some purchases after seeing the ads. For this tutorial, the top three campaigns from the USA are selected from this dataset and user information for those who have made purchases shall be used to train a model with the campaigns as the classes. The idea is to see if the advertisement and the user data can be used to identify which campaign is best-suited for the user.\n",
         "\n",
-        "- BigQuery\n",
+        "The dataset can be accessed by pinning the `looker-private-demo` project in BigQuery. Instead of going to the BigQuery user interface, this process can be performed from the JupyterLab user interface on a Vertex AI Workbench managed notebooks instance. Vertex AI Workbench managed notebooks instances support browsing through the datasets and tables from BigQuery through its BigQuery integration. \n",
         "\n",
-        "The steps performed include:\n",
+        "<img src=\"images/Bigquery_UI_new.PNG\"></img>\n",
+        "\n",
+        "## Objective\n",
+        "<a name=\"section-3\"></a>\n",
+        "\n",
+        "This tutorial demonstrates how to collect data from BigQuery, preprocess it, and train a multi-class classification model on an E-commerce dataset. The steps performed include the following:\n",
         "\n",
         "- Fetch the required data from BigQuery\n",
         "- Preprocess the data\n",
@@ -87,31 +59,10 @@
         "- Evaluate the loss for the trained model\n",
         "- Automate the notebook execution using the executor feature\n",
         "- Save the model to a Cloud Storage path\n",
-        "- Clean up the created resources"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "34d623e6dfa3"
-      },
-      "source": [
-        "## Dataset\n",
+        "- Clean up the created resources\n",
         "\n",
-        "This tutorial uses the `looker-private-demo.ecomm` dataset in BigQuery. The dataset consists of information about various advertisement campaigns including the demographics of users who have clicked and made some purchases after seeing the ads. For this tutorial, the top three campaigns from the USA are selected from this dataset and user information for those who have made purchases shall be used to train a model with the campaigns as the classes. The idea is to see if the advertisement and the user data can be used to identify which campaign is best-suited for the user.\n",
-        "\n",
-        "The dataset can be accessed by pinning the `looker-private-demo` project in BigQuery. If you are using Vertex AI Workbench managed notebooks instance, instead of going to the BigQuery user interface, this process can be performed from the JupyterLab user interface. Vertex AI Workbench managed notebooks instances support browsing through the datasets and tables from BigQuery through its BigQuery integration. \n",
-        "\n",
-        "<img src=\"images/Bigquery_UI_new.PNG\"></img>"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "ee02650bb7fd"
-      },
-      "source": [
         "### Costs \n",
+        "<a name=\"section-4\"></a>\n",
         "\n",
         "This tutorial uses billable components of Google Cloud:\n",
         "\n",
@@ -127,144 +78,6 @@
         "to generate a cost estimate based on your projected usage."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "y320EIk-kXT7"
-      },
-      "source": [
-        "### Set up your local development environment\n",
-        "\n",
-        "**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets\n",
-        "all the requirements to run this notebook. You can skip this step.\n",
-        "\n",
-        "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
-        "You need the following:\n",
-        "\n",
-        "* The Google Cloud SDK\n",
-        "* Git\n",
-        "* Python 3\n",
-        "* virtualenv\n",
-        "* Jupyter notebook running in a virtual environment with Python 3\n",
-        "\n",
-        "The Google Cloud guide to [Setting up a Python development\n",
-        "environment](https://cloud.google.com/python/setup) and the [Jupyter\n",
-        "installation guide](https://jupyter.org/install) provide detailed instructions\n",
-        "for meeting these requirements. The following steps provide a condensed set of\n",
-        "instructions:\n",
-        "\n",
-        "1. [Install and initialize the Cloud SDK.](https://cloud.google.com/sdk/docs/)\n",
-        "\n",
-        "1. [Install Python 3.](https://cloud.google.com/python/setup#installing_python)\n",
-        "\n",
-        "1. [Install\n",
-        "   virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv)\n",
-        "   and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
-        "\n",
-        "1. To install Jupyter, run `pip3 install jupyter` on the\n",
-        "command-line in a terminal shell.\n",
-        "\n",
-        "1. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
-        "\n",
-        "1. Open this notebook in the Jupyter Notebook Dashboard."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "1DouUvNOkXT8"
-      },
-      "source": [
-        "### Install additional packages\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "Ayt1jhFXkXT9"
-      },
-      "outputs": [],
-      "source": [
-        "import os\n",
-        "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
-        "! pip3 install {USER_FLAG} --upgrade pandas-gbq 'google-cloud-bigquery[bqstorage,pandas]' tensorflow scikit-learn protobuf==3.20.3 -q"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "aNeMRbpukXUA"
-      },
-      "source": [
-        "### Restart the kernel\n",
-        "\n",
-        "After you install the additional packages, you need to restart the notebook kernel so it can find the packages."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "dJ_yvi_9kXUB"
-      },
-      "outputs": [],
-      "source": [
-        "# Automatically restart kernel after installs\n",
-        "import os\n",
-        "\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    # Automatically restart kernel after installs\n",
-        "    import IPython\n",
-        "\n",
-        "    app = IPython.Application.instance()\n",
-        "    app.kernel.do_shutdown(True)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "a47846030fef"
-      },
-      "source": [
-        "## Before you begin"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "71b29797836c"
-      },
-      "source": [
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. <a href=\"https://console.cloud.google.com/cloud-resource-manager\" target=\"_blank\">Select or create a Google Cloud project</a>. When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "1. <a href=\"https://cloud.google.com/billing/docs/how-to/modify-project\" target=\"_blank\">Make sure that billing is enabled for your project</a>.\n",
-        "\n",
-        "1. <a href=\"https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com\" target=\"_blank\">Enable the Vertex AI API</a>.\n",
-        "\n",
-        "1. If you are running this notebook locally, you will need to install the <a href=\"https://cloud.google.com/sdk\" target=\"_blank\">Cloud SDK</a>.\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -276,32 +89,6 @@
         "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "3c8049930470"
-      },
-      "outputs": [],
-      "source": [
-        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "a36c4b991a39"
-      },
-      "outputs": [],
-      "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
-        "    # Get your GCP project id from gcloud\n",
-        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
-        "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID:\", PROJECT_ID)"
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -310,41 +97,34 @@
       },
       "outputs": [],
       "source": [
-        "! gcloud config set project $PROJECT_ID"
+        "PROJECT_ID = \"\"\n",
+        "\n",
+        "# Get your Google Cloud project ID from gcloud\n",
+        "if not os.getenv(\"IS_TESTING\"):\n",
+        "    shell_output=!gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "    PROJECT_ID = shell_output[0]\n",
+        "    print(\"Project ID: \", PROJECT_ID)"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "03d8d65b914d"
+        "id": "d0058f55f8cf"
       },
       "source": [
-        "#### Region\n",
-        "\n",
-        "You can also change the `REGION` variable, which is used for operations\n",
-        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
-        "\n",
-        "- Americas: `us-central1`\n",
-        "- Europe: `europe-west4`\n",
-        "- Asia Pacific: `asia-east1`\n",
-        "\n",
-        "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
-        "\n",
-        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
+        "Otherwise, set your project ID here."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "3281bedf6d3c"
+        "id": "19579640c063"
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
+        "    PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -353,9 +133,9 @@
         "id": "b2b04f364669"
       },
       "source": [
-        "#### UUID\n",
+        "#### Timestamp\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a UUID for each instance session, and append it onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append it onto the name of resources you create in this tutorial."
       ]
     },
     {
@@ -366,84 +146,9 @@
       },
       "outputs": [],
       "source": [
-        "import random\n",
-        "import string\n",
+        "from datetime import datetime\n",
         "\n",
-        "\n",
-        "# Generate a uuid of a specifed length(default=8)\n",
-        "def generate_uuid(length: int = 8) -> str:\n",
-        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
-        "\n",
-        "\n",
-        "UUID = generate_uuid()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "OoPGk5KOkXUG"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
-        "authenticated. Skip this step.\n",
-        "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions\n",
-        "when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "1. In the Cloud Console, go to the [**Create service account key**\n",
-        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
-        "\n",
-        "2. Click **Create service account**.\n",
-        "\n",
-        "3. In the **Service account name** field, enter a name, and\n",
-        "   click **Create**.\n",
-        "\n",
-        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
-        "into the filter box, and select\n",
-        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
-        "local environment.\n",
-        "\n",
-        "6. Enter the path to your service account key as the\n",
-        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "Teyy6LGqkXUG"
-      },
-      "outputs": [],
-      "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = \"google.colab\" in sys.modules\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
     {
@@ -456,8 +161,20 @@
         "\n",
         "**The following steps are required, regardless of your notebook environment.**\n",
         "\n",
+        "When you submit a training job using the Cloud SDK, you upload a Python package\n",
+        "containing your training code to a Cloud Storage bucket. Vertex AI runs\n",
+        "the code from this package. In this tutorial, Vertex AI also saves the\n",
+        "trained model that results from your job in the same bucket. Using this model artifact, you can then\n",
+        "create Vertex AI model and endpoint resources in order to serve\n",
+        "online predictions.\n",
+        "\n",
         "Set the name of your Cloud Storage bucket below. It must be unique across all\n",
-        "Cloud Storage buckets.\n"
+        "Cloud Storage buckets.\n",
+        "\n",
+        "You may also change the `REGION` variable, which is used for operations\n",
+        "throughout the rest of this notebook. Make sure to [choose a region where Vertex AI services are\n",
+        "available](https://cloud.google.com/vertex-ai/docs/general/locations#available_regions). You may\n",
+        "not use a Multi-Regional Storage bucket for training with Vertex AI."
       ]
     },
     {
@@ -468,8 +185,8 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "REGION = \"[your-region]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -480,9 +197,8 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
-        "    BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
       ]
     },
     {
@@ -502,7 +218,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_URI"
+        "! gsutil mb -l $REGION $BUCKET_NAME"
       ]
     },
     {
@@ -522,37 +238,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_URI"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "bmnMD2MjkXUJ"
-      },
-      "source": [
-        "### Import libraries and define constants"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "oqtZRqDEkXUJ"
-      },
-      "outputs": [],
-      "source": [
-        "import warnings\n",
-        "\n",
-        "import pandas as pd\n",
-        "from google.cloud.bigquery import Client\n",
-        "from sklearn.model_selection import train_test_split\n",
-        "from sklearn.preprocessing import StandardScaler\n",
-        "from tensorflow.keras import Sequential\n",
-        "from tensorflow.keras.layers import Dense\n",
-        "from tensorflow.keras.utils import to_categorical\n",
-        "\n",
-        "warnings.filterwarnings(\"ignore\")"
+        "! gsutil ls -al $BUCKET_NAME"
       ]
     },
     {
@@ -564,7 +250,7 @@
         "## Tutorial\n",
         "\n",
         "### Fetch the data from BigQuery \n",
-        "If you are using ***Vertex AI Workbench managed notebooks instance***, below cell which starts with \"#@bigquery\" will be a SQL Query. If you are using Vertex AI Workbench user managed notebooks instance or Colab it will be a markdown cell."
+        "<a name=\"section-5\"></a>"
       ]
     },
     {
@@ -645,22 +331,11 @@
         "id": "923fdd823683"
       },
       "source": [
-        "If you are using Vertex AI Workbench managed notebooks instance, once the results from BigQuery are displayed in the above cell, click the **Query and load as DataFrame** button and execute the generated code stub to fetch the data into the current notebook as a dataframe.\n",
+        "Once the results from BigQuery are displayed in the above cell, click the **Query and load as DataFrame** button and execute the generated code stub to fetch the data into the current notebook as a dataframe.\n",
         "\n",
         "*Note: By default the data is loaded into a `df` variable, though this can be changed before executing the cell if required.*"
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "3d44a10b6884"
-      },
-      "outputs": [],
-      "source": [
-        "client = Client(project=PROJECT_ID)"
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -669,6 +344,12 @@
       },
       "outputs": [],
       "source": [
+        "# The following two lines are only necessary to run once.\n",
+        "# Comment out otherwise for speed-up.\n",
+        "from google.cloud.bigquery import Client\n",
+        "\n",
+        "client = Client()\n",
+        "\n",
         "query = \"\"\"WITH traindata AS (\n",
         "SELECT b.* except(ad_event_id, user_id), c.* except(id), d.* except(keyword_id, ad_id), a.amount, a.device_type, e.name\n",
         "FROM `looker-private-demo.ecomm.ad_events` a\n",
@@ -698,7 +379,45 @@
       },
       "source": [
         "### Preprocess the data\n",
-        "Select the necessary columns from the e-commerce data and divide them based on their type (numerical/categorical)."
+        "<a name=\"section-6\"></a>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e8503e799eec"
+      },
+      "source": [
+        "Import the required libraries."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "5b11973ccf76"
+      },
+      "outputs": [],
+      "source": [
+        "import warnings\n",
+        "\n",
+        "import pandas as pd\n",
+        "from sklearn.model_selection import train_test_split\n",
+        "from sklearn.preprocessing import StandardScaler\n",
+        "from tensorflow.keras import Sequential\n",
+        "from tensorflow.keras.layers import Dense\n",
+        "from tensorflow.keras.utils import to_categorical\n",
+        "\n",
+        "warnings.filterwarnings(\"ignore\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e48d156d8bb6"
+      },
+      "source": [
+        "Select the necessary columns from the E-commerce data and divide them based on their type (numerical/categorical)."
       ]
     },
     {
@@ -728,8 +447,7 @@
         "id": "ace612851261"
       },
       "source": [
-        "#### Select top three campaigns\n",
-        "From the current dataset, only the top three campaigns will be chosen to target the users. All the relevant information about the advertisement and the user who purchased an item after seeing the advertisement is available in the dataframe already. "
+        "From the current dataset, only the top three camapigns will be chosen to target the users. All the relevant information about the advertisement and the user who purchased an item after seeing the advertisement is available in the dataframe already. "
       ]
     },
     {
@@ -749,7 +467,7 @@
         "id": "f89106348ffe"
       },
       "source": [
-        "#### Encode the target variable."
+        "Encode the target variable."
       ]
     },
     {
@@ -766,11 +484,10 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "c2d5338b1b95"
+        "id": "8902f763d1ca"
       },
       "source": [
-        "#### One-hot encode the categorical variables\n",
-        "After one-hot encoding, the first level-column is dropped to avoid the [dummy-variable trap](https://en.wikipedia.org/wiki/Dummy_variable_(statistics)) scenario. This process is called *dummy-encoding*."
+        "One-hot encode the categorical variables. After one-hot encoding, the first level-column is dropped to avoid the [dummy-variable trap](https://en.wikipedia.org/wiki/Dummy_variable_(statistics)) scenario. This process is called *dummy-encoding*."
       ]
     },
     {
@@ -804,7 +521,7 @@
         "id": "3abf027eda2d"
       },
       "source": [
-        "#### Split the data into train and test"
+        "Split the data into train and test."
       ]
     },
     {
@@ -829,7 +546,7 @@
         "id": "d1a32b9d9640"
       },
       "source": [
-        "#### Scale the data"
+        "Scale the data."
       ]
     },
     {
@@ -852,6 +569,15 @@
       },
       "source": [
         "### Train a TensorFlow model\n",
+        "<a name=\"section-7\"></a>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3e7656556a48"
+      },
+      "source": [
         "Convert the target column to a categorical encoded colum (one-hot encoded)."
       ]
     },
@@ -873,7 +599,7 @@
         "id": "3dd0014a7e1d"
       },
       "source": [
-        "#### Define hyperparameters for model training\n",
+        "Define hyperparameters for model training. \n",
         "\n",
         "*Note: Comment or remove the parameters from the following cell if they are provided already as an input parameter through the executor feature.*"
       ]
@@ -898,7 +624,7 @@
         "id": "406b731f576b"
       },
       "source": [
-        "#### Define the architecture and compile the model"
+        "Define the architecture and compile the model."
       ]
     },
     {
@@ -938,7 +664,7 @@
         "id": "4ab12c34f258"
       },
       "source": [
-        "#### Train the model"
+        "Fit the model."
       ]
     },
     {
@@ -952,13 +678,23 @@
         "history = model.fit(X_train, y_train_categ, epochs=50, verbose=1)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "51a2d0b52df3"
+      },
+      "source": [
+        "### Run the model on test data\n",
+        "<a name=\"section-8\"></a>"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "f08445f2cd02"
       },
       "source": [
-        "### Evaluate the model on test data."
+        "Evaluate the model on test data."
       ]
     },
     {
@@ -973,24 +709,16 @@
         "print(f\"Test results - Loss: {test_results}\")"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "81ef0e081340"
-      },
-      "source": [
-        "**Note:**  Please note that executor feature is available only in Vertex AI Workbench managed notebooks"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "9769168778e8"
       },
       "source": [
-        "### Automating the execution of the notebook using executor in Vertex AI Workbench managed notebooks instance\n",
+        "### Automating the execution of the notebook using executor\n",
+        "<a name=\"section-9\"></a>\n",
         "\n",
-        "If you are using Vertex AI Workbench managed notebooks instance, the executor can help you run a notebook file from start to end, with your choice of the environment, machine type, input parameters, and other characteristics. After setting up an execution, the notebook is executed as a job in Vertex AI custom training. Your jobs can be monitored from the <b>Notebook Executor</b> pane in the menu on the left.\n",
+        "The executor can help you run a notebook file from start to end, with your choice of the environment, machine type, input parameters, and other characteristics. After setting up an execution, the notebook is executed as a job in Vertex AI custom training. Your jobs can be monitored from the <b>Notebook Executor</b> pane in the menu on the left.\n",
         "\n",
         "<img src=\"images/executor.png\"></img>\n",
         "\n",
@@ -1003,9 +731,10 @@
         "id": "cf486c351581"
       },
       "source": [
-        "### Scheduled runs on executor in Vertex AI Workbench managed notebooks instance\n",
+        "### Scheduled runs on executor\n",
+        "<a name=\"section-10\"></a>\n",
         "\n",
-        "Vertex AI Workbench managed noteboook runs can also be scheduled recurringly with the executor. To do so, select <b>Schedule-based recurring executions</b> as the run type instead of <b>One-time execution</b>. The frequency of the job and the time when it executes is provided when you create the execution.\n",
+        "Notebook runs can also be scheduled recurringly with the executor. To do so, select <b>Schedule-based recurring executions</b> as the run type instead of <b>One-time execution</b>. The frequency of the job and the time when it executes is provided when you create the execution.\n",
         "\n",
         "<img src=\"images/executor_scheduled_runs2.png\"></img>"
       ]
@@ -1017,8 +746,9 @@
       },
       "source": [
         "### Parameterizing the variables\n",
+        "<a name=\"section-11\"></a>\n",
         "\n",
-        "If you are using Vertex AI Workbench managed notebooks instance, executor lets you run a notebook with different sets of input parameters. If required, constants in the notebook can be treated as arguments to a function, and when you submit the execution, you can provide those constants as input parameters.\n",
+        "Executor lets you run a notebook with different sets of input parameters. If required, constants in the notebook can be treated as arguments to a function, and when you submit the execution, you can provide those constants as input parameters.\n",
         "\n",
         "<img src=\"images/executor_input_parameters.png\"></img>\n",
         "\n",
@@ -1032,6 +762,7 @@
       },
       "source": [
         "### Save the model to a Cloud Storage path\n",
+        "<a name=\"section-12\"></a>\n",
         "\n",
         "TensorFlow's `model.save()` method supports Cloud Storage paths as well as the local file paths while writing the model object to a file. It needs to be ensured that the service account being used to run this notebook has `write` permissions to the specified Cloud Storage path."
       ]
@@ -1044,7 +775,7 @@
       },
       "outputs": [],
       "source": [
-        "GCS_PATH = BUCKET_URI + \"/path-to-save/\"\n",
+        "GCS_PATH = \"gs://\" + BUCKET_NAME + \"/[path-to-save]/\"\n",
         "model.save(GCS_PATH)"
       ]
     },
@@ -1054,7 +785,8 @@
         "id": "29c0ca2a517a"
       },
       "source": [
-        "## Cleaning up\n",
+        "## Clean up\n",
+        "<a name=\"section-13\"></a>\n",
         "\n",
         "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
         "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
@@ -1070,11 +802,7 @@
       },
       "outputs": [],
       "source": [
-        "# Delete the Cloud Storage bucket\n",
-        "\n",
-        "delete_bucket = False\n",
-        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
-        "    ! gsutil -m rm -r $BUCKET_URI"
+        "! gsutil -m rm -r [cloud-storage-folder-path-to-delete]"
       ]
     }
   ],