fix: git rendering issue

* Update Keras stable diffusion notebooks with cloud notebook template style and fix bugs.

* fix minor typos

.cloud-build/Dockerfile

@@ -1,194 +0,0 @@
-# Copyright 2020 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# We want to use LTS ubuntu from our mirror because dockerhub has a
-# rate limit.
-# FROM mirror.gcr.io/library/ubuntu:18.04
-# However, now the above image is not working, we're using our own cache
-FROM gcr.io/cloud-devrel-kokoro-resources/ubuntu:20.04
-
-ENV DEBIAN_FRONTEND noninteractive
-
-# Ensure local Python is preferred over distribution Python.
-ENV PATH /usr/local/bin:$PATH
-
-# http://bugs.python.org/issue19846
-# At the moment, setting "LANG=C" on a Linux system fundamentally breaks
-# Python 3.
-ENV LANG C.UTF-8
-
-# Install dependencies.
-RUN apt-get update \
-  && apt-get install -y --no-install-recommends \
-    apt-transport-https \
-    build-essential \
-    ca-certificates \
-    curl \
-    dirmngr \
-    git \
-    gcc \
-    gpg-agent \
-    graphviz \
-    libbz2-dev \
-    libdb5.3-dev \
-    libexpat1-dev \
-    libffi-dev \
-    liblzma-dev \
-    libmagickwand-dev \
-    libmemcached-dev \
-    libpython3-dev \
-    libreadline-dev \
-    libsnappy-dev \
-    libssl-dev \
-    libsqlite3-dev \
-    portaudio19-dev \
-    pkg-config \
-    redis-server \
-    software-properties-common \
-    ssh \
-    sudo \
-    systemd \
-    tcl \
-    tcl-dev \
-    tk \
-    tk-dev \
-    uuid-dev \
-    wget \
-    zlib1g-dev \
-  && apt-get clean autoclean \
-  && apt-get autoremove -y \
-  && rm -rf /var/lib/apt/lists/* \
-  && rm -f /var/cache/apt/archives/*.deb
-
-# Install docker
-RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
-
-RUN add-apt-repository \
-   "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
-   $(lsb_release -cs) \
-   stable"
-
-RUN apt-get update \
-  && apt-get install -y --no-install-recommends \
-    docker-ce \
-  && apt-get clean autoclean \
-  && apt-get autoremove -y \
-  && rm -rf /var/lib/apt/lists/* \
-  && rm -f /var/cache/apt/archives/*.deb
-
-# Install Bazel for compiling Tink in Cloud SQL Client Side Encryption Samples
-# TODO: Delete this section once google/tink#483 is resolved
-RUN apt install -y curl gpgconf gpg \
-  && curl -fsSL https://bazel.build/bazel-release.pub.gpg | gpg --dearmor > bazel.gpg \
-  && mv bazel.gpg /etc/apt/trusted.gpg.d/ \
-  && echo "deb [arch=amd64] https://storage.googleapis.com/bazel-apt stable jdk1.8" | sudo tee /etc/apt/sources.list.d/bazel.list \
-  && apt update &&  apt install -y bazel \
-  && apt-get clean autoclean \
-  && apt-get autoremove -y \
-  && rm -rf /var/lib/apt/lists/* \
-  && rm -f /var/cache/apt/archives/*.deb
-
-# Install Microsoft ODBC 17 Driver and unixodbc for testing SQL Server samples
-RUN curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - \
-  && curl https://packages.microsoft.com/config/ubuntu/20.04/prod.list > /etc/apt/sources.list.d/mssql-release.list \
-  && apt-get update \
-  && ACCEPT_EULA=Y apt-get install -y --no-install-recommends \
-    msodbcsql17 \
-    unixodbc-dev \
-  && apt-get clean autoclean \
-  && apt-get autoremove -y \
-  && rm -rf /var/lib/apt/lists/* \
-  && rm -f /var/cache/apt/archives/*.deb
-
-COPY fetch_gpg_keys.sh /tmp
-# Install the desired versions of Python.
-RUN set -ex \
-    && export GNUPGHOME="$(mktemp -d)" \
-    && echo "disable-ipv6" >> "${GNUPGHOME}/dirmngr.conf" \
-    && /tmp/fetch_gpg_keys.sh \
-    && for PYTHON_VERSION in 2.7.18 3.6.13 3.7.10 3.8.8 3.9.2; do \
-        wget --no-check-certificate -O python-${PYTHON_VERSION}.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz" \
-        && wget --no-check-certificate -O python-${PYTHON_VERSION}.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc" \
-        && gpg --batch --verify python-${PYTHON_VERSION}.tar.xz.asc python-${PYTHON_VERSION}.tar.xz \
-        && rm -r python-${PYTHON_VERSION}.tar.xz.asc \
-        && mkdir -p /usr/src/python-${PYTHON_VERSION} \
-        && tar -xJC /usr/src/python-${PYTHON_VERSION} --strip-components=1 -f python-${PYTHON_VERSION}.tar.xz \
-        && rm python-${PYTHON_VERSION}.tar.xz \
-        && cd /usr/src/python-${PYTHON_VERSION} \
-        && ./configure \
-            --enable-shared \
-            # This works only on Python 2.7 and throws a warning on every other
-            # version, but seems otherwise harmless.
-            --enable-unicode=ucs4 \
-            --with-system-ffi \
-            --without-ensurepip \
-        && make -j$(nproc) \
-        && make install \
-        && ldconfig \
-  ; done \
-  && rm -rf "${GNUPGHOME}" \
-  && rm -rf /usr/src/python* \
-  && rm -rf ~/.cache/
-
-
-# Install pip on Python 3.6 only.
-# If the environment variable is called "PIP_VERSION", pip explodes with
-# "ValueError: invalid truth value '<VERSION>'"
-ENV PYTHON_PIP_VERSION 20.2.4
-RUN wget --no-check-certificate -O /tmp/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' \
-    && python3.6 /tmp/get-pip.py "pip==$PYTHON_PIP_VERSION" \
-  # we use "--force-reinstall" for the case where the version of pip we're trying to install is the same as the version bundled with Python
-  # ("Requirement already up-to-date: pip==8.1.2 in /usr/local/lib/python3.6/site-packages")
-  # https://github.com/docker-library/python/pull/143#issuecomment-241032683
-    && pip3 install --no-cache-dir --upgrade --force-reinstall "pip==$PYTHON_PIP_VERSION" \
-  # then we use "pip list" to ensure we don't have more than one pip version installed
-  # https://github.com/docker-library/python/pull/100
-    && [ "$(pip list |tac|tac| awk -F '[ ()]+' '$1 == "pip" { print $2; exit }')" = "$PYTHON_PIP_VERSION" ]
-
-# Ensure Pip for python3
-RUN python3 /tmp/get-pip.py
-RUN rm /tmp/get-pip.py
-
-# Install "virtualenv", since the vast majority of users of this image
-# will want it.
-RUN pip install --no-cache-dir virtualenv
-
-# Setup Cloud SDK
-ENV CLOUD_SDK_VERSION 339.0.0
-# Use system python for cloud sdk.
-ENV CLOUDSDK_PYTHON python3.6
-RUN wget https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-$CLOUD_SDK_VERSION-linux-x86_64.tar.gz
-RUN tar xzf google-cloud-sdk-$CLOUD_SDK_VERSION-linux-x86_64.tar.gz
-RUN /google-cloud-sdk/install.sh
-ENV PATH /google-cloud-sdk/bin:$PATH
-
-# Enable redis-server on boot.
-RUN sudo systemctl enable redis-server.service
-
-# Create a user and allow sudo
-
-# kbuilder uid on the default Kokoro image
-ARG UID=1000
-ARG USERNAME=kbuilder
-
-# Add a new user to the container image.
-# This is needed for ssh and sudo access.
-
-# Add a new user with the caller's uid and the username.
-RUN useradd -d /h -u ${UID} ${USERNAME}
-
-# Allow nopasswd sudo
-RUN echo "${USERNAME} ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
-
-CMD ["python3.6"]

.cloud-build/ExecuteChangedNotebooks.py

@@ -1,312 +0,0 @@
-#!/usr/bin/env python
-# Copyright 2021 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import argparse
-import dataclasses
-import datetime
-import functools
-import pathlib
-import os
-import subprocess
-from pathlib import Path
-from typing import List, Optional
-import concurrent
-from tabulate import tabulate
-
-import ExecuteNotebook
-
-
-def str2bool(v):
-    if isinstance(v, bool):
-        return v
-    if v.lower() in ("yes", "true", "t", "y", "1"):
-        return True
-    elif v.lower() in ("no", "false", "f", "n", "0"):
-        return False
-    else:
-        raise argparse.ArgumentTypeError("Boolean value expected.")
-
-
-def format_timedelta(delta: datetime.timedelta) -> str:
-    """Formats a timedelta duration to [N days] %H:%M:%S format"""
-    seconds = int(delta.total_seconds())
-
-    secs_in_a_day = 86400
-    secs_in_a_hour = 3600
-    secs_in_a_min = 60
-
-    days, seconds = divmod(seconds, secs_in_a_day)
-    hours, seconds = divmod(seconds, secs_in_a_hour)
-    minutes, seconds = divmod(seconds, secs_in_a_min)
-
-    time_fmt = f"{hours:02d}:{minutes:02d}:{seconds:02d}"
-
-    if days > 0:
-        suffix = "s" if days > 1 else ""
-        return f"{days} day{suffix} {time_fmt}"
-
-    return time_fmt
-
-
-@dataclasses.dataclass
-class NotebookExecutionResult:
-    notebook: str
-    duration: datetime.timedelta
-    is_pass: bool
-    error_message: Optional[str]
-
-
-def execute_notebook(
-    artifacts_path: str,
-    variable_project_id: str,
-    variable_region: str,
-    should_log_output: bool,
-    should_use_new_kernel: bool,
-    notebook: str,
-) -> NotebookExecutionResult:
-    print(f"Running notebook: {notebook}")
-
-    result = NotebookExecutionResult(
-        notebook=notebook,
-        duration=datetime.timedelta(seconds=0),
-        is_pass=False,
-        error_message=None,
-    )
-
-    # TODO: Handle cases where multiple notebooks have the same name
-    time_start = datetime.datetime.now()
-    try:
-        ExecuteNotebook.execute_notebook(
-            notebook_file_path=notebook,
-            output_file_folder=artifacts_path,
-            replacement_map={
-                "PROJECT_ID": variable_project_id,
-                "REGION": variable_region,
-            },
-            should_log_output=should_log_output,
-            should_use_new_kernel=should_use_new_kernel,
-        )
-        result.duration = datetime.datetime.now() - time_start
-        result.is_pass = True
-        print(f"{notebook} PASSED in {format_timedelta(result.duration)}.")
-    except Exception as error:
-        result.duration = datetime.datetime.now() - time_start
-        result.is_pass = False
-        result.error_message = str(error)
-        print(
-            f"{notebook} FAILED in {format_timedelta(result.duration)}: {result.error_message}"
-        )
-
-    return result
-
-
-def run_changed_notebooks(
-    test_paths_file: str,
-    base_branch: Optional[str],
-    output_folder: str,
-    variable_project_id: str,
-    variable_region: str,
-    should_parallelize: bool,
-    should_use_separate_kernels: bool,
-):
-    """
-    Run the notebooks that exist under the folders defined in the test_paths_file.
-    It only runs notebooks that have differences from the Git base_branch.
-
-    The executed notebooks are saved in the output_folder.
-
-    Variables are also injected into the notebooks such as the variable_project_id and variable_region.
-
-    Args:
-        test_paths_file (str):
-            Required. The new-line delimited file to folders and files that need checking.
-            Folders are checked recursively.
-        base_branch (str):
-            Optional. If provided, only the files that have changed from the base_branch will be checked.
-            If not provided, all files will be checked.
-        output_folder (str):
-            Required. The folder to write executed notebooks to.
-        variable_project_id (str):
-            Required. The value for PROJECT_ID to inject into notebooks.
-        variable_region (str):
-            Required. The value for REGION to inject into notebooks.
-        should_parallelize (bool):
-            Required. Should run notebooks in parallel using a thread pool as opposed to in sequence.
-        should_use_separate_kernels (bool):
-            Note: Dependencies don't install correctly when this is set to True
-            See https://github.com/nteract/papermill/issues/625
-
-            Required. Should run each notebook in a separate and independent virtual environment.
-    """
-
-    test_paths = []
-    with open(test_paths_file) as file:
-        lines = [line.strip() for line in file.readlines()]
-        lines = [line for line in lines if len(line) > 0]
-        test_paths = [line for line in lines]
-
-    if len(test_paths) == 0:
-        raise RuntimeError("No test folders found.")
-
-    print(f"Checking folders: {test_paths}")
-
-    # Find notebooks
-    notebooks = []
-    if base_branch:
-        print(f"Looking for notebooks that changed from branch: {base_branch}")
-        notebooks = subprocess.check_output(
-            ["git", "diff", "--name-only", f"origin/{base_branch}..."] + test_paths
-        )
-    else:
-        print(f"Looking for all notebooks.")
-        notebooks = subprocess.check_output(["git", "ls-files"] + test_paths)
-
-    notebooks = notebooks.decode("utf-8").split("\n")
-    notebooks = [notebook for notebook in notebooks if notebook.endswith(".ipynb")]
-    notebooks = [notebook for notebook in notebooks if len(notebook) > 0]
-    notebooks = [notebook for notebook in notebooks if Path(notebook).exists()]
-
-    # Create paths
-    artifacts_path = Path(output_folder)
-    artifacts_path.mkdir(parents=True, exist_ok=True)
-    artifacts_path.joinpath("success").mkdir(parents=True, exist_ok=True)
-    artifacts_path.joinpath("failure").mkdir(parents=True, exist_ok=True)
-
-    notebook_execution_results: List[NotebookExecutionResult] = []
-
-    if len(notebooks) > 0:
-        print(f"Found {len(notebooks)} modified notebooks: {notebooks}")
-
-        if should_parallelize and len(notebooks) > 1:
-            print(
-                "Running notebooks in parallel, so no logs will be displayed. Please wait..."
-            )
-            with concurrent.futures.ThreadPoolExecutor(max_workers=None) as executor:
-                notebook_execution_results = list(
-                    executor.map(
-                        functools.partial(
-                            execute_notebook,
-                            artifacts_path,
-                            variable_project_id,
-                            variable_region,
-                            False,
-                            should_use_separate_kernels,
-                        ),
-                        notebooks,
-                    )
-                )
-        else:
-            notebook_execution_results = [
-                execute_notebook(
-                    artifacts_path=artifacts_path,
-                    variable_project_id=variable_project_id,
-                    variable_region=variable_region,
-                    notebook=notebook,
-                    should_log_output=True,
-                    should_use_new_kernel=should_use_separate_kernels,
-                )
-                for notebook in notebooks
-            ]
-    else:
-        print("No notebooks modified in this pull request.")
-
-    print("\n=== RESULTS ===\n")
-
-    notebooks_sorted = sorted(
-        notebook_execution_results,
-        key=lambda result: result.is_pass,
-        reverse=True,
-    )
-    # Print results
-    print(
-        tabulate(
-            [
-                [
-                    os.path.basename(os.path.normpath(result.notebook)),
-                    "PASSED" if result.is_pass else "FAILED",
-                    format_timedelta(result.duration),
-                    result.error_message or "--",
-                ]
-                for result in notebooks_sorted
-            ],
-            headers=["file", "status", "duration", "error"],
-        )
-    )
-
-    print("\n=== END RESULTS===\n")
-
-
-parser = argparse.ArgumentParser(description="Run changed notebooks.")
-parser.add_argument(
-    "--test_paths_file",
-    type=pathlib.Path,
-    help="The path to the file that has newline-limited folders of notebooks that should be tested.",
-    required=True,
-)
-parser.add_argument(
-    "--base_branch",
-    help="The base git branch to diff against to find changed files.",
-    required=False,
-)
-parser.add_argument(
-    "--output_folder",
-    type=pathlib.Path,
-    help="The path to the folder to store executed notebooks.",
-    required=True,
-)
-parser.add_argument(
-    "--variable_project_id",
-    type=str,
-    help="The GCP project id. This is used to inject a variable value into the notebook before running.",
-    required=True,
-)
-parser.add_argument(
-    "--variable_region",
-    type=str,
-    help="The GCP region. This is used to inject a variable value into the notebook before running.",
-    required=True,
-)
-
-# Note: Dependencies don't install correctly when this is set to True
-parser.add_argument(
-    "--should_parallelize",
-    type=str2bool,
-    nargs="?",
-    const=True,
-    default=False,
-    help="Should run notebooks in parallel.",
-)
-
-# Note: This isn't guaranteed to work correctly due to existing Papermill issue
-# See https://github.com/nteract/papermill/issues/625
-parser.add_argument(
-    "--should_use_separate_kernels",
-    type=str2bool,
-    nargs="?",
-    const=True,
-    default=False,
-    help="(Experimental) Should run each notebook in a separate and independent virtual environment.",
-)
-
-args = parser.parse_args()
-run_changed_notebooks(
-    test_paths_file=args.test_paths_file,
-    base_branch=args.base_branch,
-    output_folder=args.output_folder,
-    variable_project_id=args.variable_project_id,
-    variable_region=args.variable_region,
-    should_parallelize=args.should_parallelize,
-    should_use_separate_kernels=args.should_use_separate_kernels,
-)

.cloud-build/ExecuteNotebook.py

@@ -1,175 +0,0 @@
-#!/usr/bin/env python
-# Copyright 2021 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import json
-import sys
-import nbformat
-import os
-import errno
-from NotebookProcessors import RemoveNoExecuteCells, UpdateVariablesPreprocessor
-from typing import Dict, Tuple
-import papermill as pm
-import shutil
-import virtualenv
-import uuid
-from jupyter_client.kernelspecapp import KernelSpecManager
-
-# This script is used to execute a notebook and write out the output notebook.
-# The replaces calling the nbconvert via command-line, which doesn't write the output notebook correctly when there are errors during execution.
-
-STAGING_FOLDER = "staging"
-ENVIRONMENTS_PATH = "environments"
-KERNELS_SPECS_PATH = "kernel_specs"
-
-
-def create_and_install_kernel() -> Tuple[str, str]:
-    # Create environment
-    kernel_name = str(uuid.uuid4())
-    env_name = f"{ENVIRONMENTS_PATH}/{kernel_name}"
-    # venv.create(env_name, system_site_packages=True, with_pip=True)
-    virtualenv.cli_run([env_name, "--system-site-packages"])
-
-    # Create kernel spec
-    kernel_spec = {
-        "argv": [
-            f"{env_name}/bin/python",
-            "-m",
-            "ipykernel_launcher",
-            "-f",
-            "{connection_file}",
-        ],
-        "display_name": "Python 3",
-        "language": "python",
-    }
-    kernel_spec_folder = os.path.join(KERNELS_SPECS_PATH, kernel_name)
-    kernel_spec_file = os.path.join(kernel_spec_folder, "kernel.json")
-
-    # Create kernel spec folder
-    if not os.path.exists(os.path.dirname(kernel_spec_file)):
-        try:
-            os.makedirs(os.path.dirname(kernel_spec_file))
-        except OSError as exc:  # Guard against race condition
-            if exc.errno != errno.EEXIST:
-                raise
-
-    with open(kernel_spec_file, mode="w", encoding="utf-8") as f:
-        json.dump(kernel_spec, f)
-
-    # Install kernel
-    kernel_spec_manager = KernelSpecManager()
-    kernel_spec_manager.install_kernel_spec(
-        source_dir=kernel_spec_folder, kernel_name=kernel_name
-    )
-
-    return kernel_name, env_name
-
-
-def execute_notebook(
-    notebook_file_path: str,
-    output_file_folder: str,
-    replacement_map: Dict[str, str],
-    should_log_output: bool,
-    should_use_new_kernel: bool,
-):
-    # Create staging directory if it doesn't exist
-    staging_file_path = f"{STAGING_FOLDER}/{notebook_file_path}"
-    if not os.path.exists(os.path.dirname(staging_file_path)):
-        try:
-            os.makedirs(os.path.dirname(staging_file_path))
-        except OSError as exc:  # Guard against race condition
-            if exc.errno != errno.EEXIST:
-                raise
-
-    file_name = os.path.basename(os.path.normpath(notebook_file_path))
-
-    # Create environments folder
-    if not os.path.exists(ENVIRONMENTS_PATH):
-        try:
-            os.makedirs(ENVIRONMENTS_PATH)
-        except OSError as exc:  # Guard against race condition
-            if exc.errno != errno.EEXIST:
-                raise
-
-    # Create and install kernel
-    kernel_name = next(
-        iter(KernelSpecManager().find_kernel_specs().keys()), None
-    )  # Find first existing kernel and use as default
-    env_name = None
-    if should_use_new_kernel:
-        kernel_name, env_name = create_and_install_kernel()
-
-    # Read notebook
-    with open(notebook_file_path) as f:
-        nb = nbformat.read(f, as_version=4)
-
-    has_error = False
-
-    # Execute notebook
-    try:
-        # Create preprocessors
-        remove_no_execute_cells_preprocessor = RemoveNoExecuteCells()
-        update_variables_preprocessor = UpdateVariablesPreprocessor(
-            replacement_map=replacement_map
-        )
-
-        # Use no-execute preprocessor
-        (
-            nb,
-            resources,
-        ) = remove_no_execute_cells_preprocessor.preprocess(nb)
-
-        (nb, resources) = update_variables_preprocessor.preprocess(nb, resources)
-
-        # print(f"Staging modified notebook to: {staging_file_path}")
-        with open(staging_file_path, mode="w", encoding="utf-8") as f:
-            nbformat.write(nb, f)
-
-        # Execute notebook
-        pm.execute_notebook(
-            input_path=staging_file_path,
-            output_path=staging_file_path,
-            kernel_name=kernel_name,
-            progress_bar=should_log_output,
-            request_save_on_cell_execute=should_log_output,
-            log_output=should_log_output,
-            stdout_file=sys.stdout if should_log_output else None,
-            stderr_file=sys.stderr if should_log_output else None,
-        )
-    except Exception:
-        # print(f"Error executing the notebook: {notebook_file_path}.\n\n")
-        has_error = True
-
-        raise
-
-    finally:
-        # Clear env
-        if env_name is not None:
-            shutil.rmtree(path=env_name)
-
-        # Copy execute notebook
-        output_file_path = os.path.join(
-            output_file_folder, "failure" if has_error else "success", file_name
-        )
-
-        # Create directories if they don't exist
-        if not os.path.exists(os.path.dirname(output_file_path)):
-            try:
-                os.makedirs(os.path.dirname(output_file_path))
-            except OSError as exc:  # Guard against race condition
-                if exc.errno != errno.EEXIST:
-                    raise
-
-        # print(f"Writing output to: {output_file_path}")
-        shutil.move(staging_file_path, output_file_path)

.cloud-build/UpdateNotebookVariables.py

@@ -1,81 +0,0 @@
-#!/usr/bin/env python
-# Copyright 2021 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import re
-
-"""
- This script is used to update variables in the notebook via regex
- It requires variables to be defined in particular format
-
-For example, if your variable was PROJECT_ID, use:
-
-    PROJECT_ID = "[your_project_here]"
-
-Single-quotes also work:
-
-    PROJECT_ID = '[your_project_here]'
-
-Variables in conditionals can also be replaced:
-
-    PROJECT_ID == "[your_project_here]"
-"""
-
-
-def get_updated_value(content: str, variable_name: str, variable_value: str) -> str:
-    return re.sub(
-        rf"({variable_name}.*?=.*?[\",\'])\[.+?\]([\",\'].*?)",
-        rf"\1{variable_value}\2",
-        content,
-        flags=re.M,
-    )
-
-
-def test_update_value():
-    new_content = get_updated_value(
-        content='asdf\nPROJECT_ID = "[your-project-id]" #@param {type:"string"} \nasdf',
-        variable_name="PROJECT_ID",
-        variable_value="sample-project",
-    )
-    assert (
-        new_content
-        == 'asdf\nPROJECT_ID = "sample-project" #@param {type:"string"} \nasdf'
-    )
-
-
-def test_update_value_single_quotes():
-    new_content = get_updated_value(
-        content="PROJECT_ID = '[your-project-id]'",
-        variable_name="PROJECT_ID",
-        variable_value="sample-project",
-    )
-    assert new_content == "PROJECT_ID = 'sample-project'"
-
-
-def test_update_value_avoidance():
-    new_content = get_updated_value(
-        content="PROJECT_ID = shell_output[0] ",
-        variable_name="PROJECT_ID",
-        variable_value="sample-project",
-    )
-    assert new_content == "PROJECT_ID = shell_output[0] "
-
-
-def test_region():
-    new_content = get_updated_value(
-        content='REGION = "[your-region]"  # @param {type:"string"}',
-        variable_name="REGION",
-        variable_value="us-central1",
-    )
-    assert new_content == 'REGION = "us-central1"  # @param {type:"string"}'

.cloud-build/cleanup/cleanup-requirements.txt

@@ -1 +1,2 @@
+ratemate
 google-cloud-aiplatform

.cloud-build/cleanup/cleanup.py

@@ -1,11 +1,23 @@
 from typing import List
+from ratemate import RateLimit
 from resource_cleanup_manager import (
-    ResourceCleanupManager,
     DatasetResourceCleanupManager,
-    EndpointResourceCleanupManager,
     ModelResourceCleanupManager,
+    EndpointResourceCleanupManager,
+    ResourceCleanupManager,
+    MatchingEngineIndexEndpointResourceCleanupManager,
+    MatchingEngineIndexResourceCleanupManager,
+    FeatureStoreCleanupManager,
+    PipelineJobCleanupManager,
+    TrainingJobCleanupManager,
+    HyperparameterTuningCleanupManager,
+    BatchPredictionJobCleanupManager,
+    ExperimentCleanupManager,
+    BucketCleanupManager
 )
 
+rate_limit = RateLimit(max_count=25, per=60, greedy=False)
+
 
 def run_cleanup_managers(managers: List[ResourceCleanupManager], is_dry_run: bool):
     for manager in managers:
@@ -15,14 +27,17 @@ def run_cleanup_managers(managers: List[ResourceCleanupManager], is_dry_run: boo
         resources = manager.list()
         print(f"Found {len(resources)} {type_name}'s")
         for resource in resources:
-            if not manager.is_deletable(resource):
-                continue
-
-            if is_dry_run:
-                resource_name = manager.resource_name(resource)
-                print(f"Will delete '{type_name}': {resource_name}")
-            else:
-                manager.delete(resource)
+            try:
+                if not manager.is_deletable(resource):
+                    continue
+                if is_dry_run:
+                    resource_name = manager.resource_name(resource)
+                    print(f"Will delete '{type_name}': {resource_name}")
+                else:
+                    rate_limit.wait()  # wait before deleting
+                    manager.delete(resource)
+            except Exception as exception:
+                print(exception)
 
         print("")
 
@@ -33,10 +48,19 @@ if is_dry_run:
     print("Starting cleanup in dry run mode...")
 
 # List of all cleanup managers
-managers = [
+managers: List[ResourceCleanupManager] = [
     DatasetResourceCleanupManager(),
     EndpointResourceCleanupManager(),
-    ModelResourceCleanupManager(),
+    ModelResourceCleanupManager(),  # ModelResourceCleanupManager must follow EndpointResourceCleanupManager due to deployed models blocking model deletion.
+    MatchingEngineIndexEndpointResourceCleanupManager(),
+    MatchingEngineIndexResourceCleanupManager(),
+    FeatureStoreCleanupManager(),
+    PipelineJobCleanupManager(),
+    TrainingJobCleanupManager(),
+    HyperparameterTuningCleanupManager(),
+    BatchPredictionJobCleanupManager(),
+    ExperimentCleanupManager(), # Experiment missing _resource_noun
+    BucketCleanupManager()
 ]
 
 run_cleanup_managers(managers=managers, is_dry_run=is_dry_run)

.cloud-build/cleanup/resource_cleanup_manager.py

@@ -1,8 +1,18 @@
+'''
+READ FIRST BEFORE MAKING CHANGES
+-  Create a convention for resources created from vertex-ai-samples GH. We already have one IIRC
+- Only delete those objects as part of our clean-up script.
+- Don't run any tests on python-docs-samples-tests project, especially ones that affect resources created outside of our purview
+- Add --dry-run option to the clean-up script. This option will just output the list of resources the script will delete instead of actually deleting the resources.
+- Have a larger conversation in DEE before touching any resources that were not created as part of vertex-ai-samples
+'''
 import abc
+from typing import Any, Type
+
 from google.cloud import aiplatform
-from typing import Any
-from proto.datetime_helpers import DatetimeWithNanoseconds
 from google.cloud.aiplatform import base
+from google.cloud import storage
+from proto.datetime_helpers import DatetimeWithNanoseconds
 
 # If a resource was updated within this number of seconds, do not delete.
 RESOURCE_UPDATE_BUFFER_IN_SECONDS = 60 * 60 * 8
@@ -40,7 +50,7 @@ class ResourceCleanupManager(abc.ABC):
         # Check that it wasn't created too recently, to prevent race conditions
         if time_difference <= RESOURCE_UPDATE_BUFFER_IN_SECONDS:
             print(
-                f"Skipping '{resource}' due update_time being '{time_difference}', which is less than '{RESOURCE_UPDATE_BUFFER_IN_SECONDS}'."
+                f"Skipping '{resource}' due to update_time being '{time_difference}', which is less than '{RESOURCE_UPDATE_BUFFER_IN_SECONDS}'."
             )
             return False
 
@@ -50,7 +60,7 @@ class ResourceCleanupManager(abc.ABC):
 class VertexAIResourceCleanupManager(ResourceCleanupManager):
     @property
     @abc.abstractmethod
-    def vertex_ai_resource(self) -> base.VertexAiResourceNounWithFutureManager:
+    def vertex_ai_resource(self) -> Type[base.VertexAiResourceNounWithFutureManager]:
         pass
 
     @property
@@ -60,13 +70,15 @@ class VertexAIResourceCleanupManager(ResourceCleanupManager):
     def list(self) -> Any:
         return self.vertex_ai_resource.list()
 
-    def resource_name(self, resource: Any) -> str:
+    def resource_name(
+        self, resource: Type[base.VertexAiResourceNounWithFutureManager]
+    ) -> str:
         return resource.display_name
 
     def delete(self, resource):
         resource.delete()
 
-    def get_seconds_since_modification(self, resource: Any) -> bool:
+    def get_seconds_since_modification(self, resource: Any) -> float:
         update_time = resource.update_time
         current_time = DatetimeWithNanoseconds.now(tz=update_time.tzinfo)
         return (current_time - update_time).total_seconds()
@@ -74,14 +86,139 @@ class VertexAIResourceCleanupManager(ResourceCleanupManager):
 
 class DatasetResourceCleanupManager(VertexAIResourceCleanupManager):
     vertex_ai_resource = aiplatform.datasets._Dataset
+    dataset_types = [
+        aiplatform.ImageDataset,
+        aiplatform.TabularDataset,
+        aiplatform.TextDataset,
+        aiplatform.TimeSeriesDataset,
+        aiplatform.VideoDataset,
+    ]
+
+    def list(self) -> Any:
+        return [
+            dataset
+            for dataset_type in self.dataset_types
+            for dataset in dataset_type.list()
+        ]
 
 
 class EndpointResourceCleanupManager(VertexAIResourceCleanupManager):
     vertex_ai_resource = aiplatform.Endpoint
 
     def delete(self, resource):
+        for deployed_model_id in [
+            models.id for models in resource._gca_resource.deployed_models
+        ]:
+            resource._undeploy(deployed_model_id=deployed_model_id)
         resource.delete(force=True)
 
 
 class ModelResourceCleanupManager(VertexAIResourceCleanupManager):
     vertex_ai_resource = aiplatform.Model
+
+
+class MatchingEngineIndexResourceCleanupManager(VertexAIResourceCleanupManager):
+    vertex_ai_resource = aiplatform.MatchingEngineIndex
+
+
+class MatchingEngineIndexEndpointResourceCleanupManager(VertexAIResourceCleanupManager):
+    vertex_ai_resource = aiplatform.MatchingEngineIndexEndpoint
+
+    def delete(self, resource):
+        resource.undeploy_all()
+        resource.delete(force=True)
+
+class FeatureStoreCleanupManager(VertexAIResourceCleanupManager):
+    vertex_ai_resource = aiplatform.Featurestore
+
+    def resource_name(self, resource: Any) -> str:
+        return resource.name
+
+class PipelineJobCleanupManager(VertexAIResourceCleanupManager):
+    vertex_ai_resource = aiplatform.PipelineJob
+
+class TrainingJobCleanupManager(VertexAIResourceCleanupManager):
+    vertex_ai_resource = aiplatform.training_jobs._CustomTrainingJob
+
+    job_types = [
+            aiplatform.AutoMLImageTrainingJob,
+            aiplatform.AutoMLTextTrainingJob,
+            aiplatform.AutoMLTabularTrainingJob,
+            aiplatform.AutoMLVideoTrainingJob,
+            aiplatform.AutoMLForecastingTrainingJob,
+            aiplatform.CustomJob,
+            aiplatform.CustomTrainingJob,
+            aiplatform.CustomContainerTrainingJob,
+            aiplatform.CustomPythonPackageTrainingJob
+    ]
+
+    def list(self) -> Any:
+        return [
+            job
+            for job_type in self.job_types
+            for job in job_type.list()
+        ]
+
+class HyperparameterTuningCleanupManager(VertexAIResourceCleanupManager):
+    vertex_ai_resource = aiplatform.HyperparameterTuningJob
+
+
+class BatchPredictionJobCleanupManager(VertexAIResourceCleanupManager):
+    vertex_ai_resource = aiplatform.BatchPredictionJob
+
+class ExperimentCleanupManager(VertexAIResourceCleanupManager):
+    vertex_ai_resource = aiplatform.Experiment
+
+    @property
+    def type_name(self) -> str:
+        return "Experiment"
+
+    def resource_name(self, resource: Any) -> str:
+        return resource.name
+
+    def get_seconds_since_modification(self, resource: Any) -> float:
+        update_time = resource._metadata_context.update_time
+        current_time = DatetimeWithNanoseconds.now()
+        return float(current_time.timestamp() - update_time.timestamp())
+
+class BucketCleanupManager(ResourceCleanupManager):
+    vertex_ai_resource = storage.bucket.Bucket
+
+    def list(self) -> Any:
+        storage_client = storage.Client()
+        return list(storage_client.list_buckets())
+
+    def delete(self, resource):
+        try:
+            resource.delete(force=True)
+        except Exception as e:
+            print(e)
+
+    @property
+    def type_name(self) -> str:
+        return "Bucket"
+
+    def get_seconds_since_modification(self, resource: Any) -> float:
+        # Bucket has no last_update property, only time created
+        created_time = resource.time_created
+        current_time = DatetimeWithNanoseconds.now()
+        return float(current_time.timestamp() - created_time.timestamp())
+
+    def resource_name(self, resource: Any) -> str:
+        return resource.name
+
+    def is_deletable(self, resource: Any) -> bool:
+        time_difference = self.get_seconds_since_modification(resource)
+
+        if not self.resource_name(resource).startswith('your-bucket-name'):
+            print(f"Skipping '{resource}' not a Vertex AI notebook bucket")
+            return False
+
+        # Check that it wasn't created too recently, to prevent race conditions
+        if time_difference <= RESOURCE_UPDATE_BUFFER_IN_SECONDS:
+            print(
+                f"Skipping '{resource}' due to update_time being '{time_difference}', which is less than '{RESOURCE_UPDATE_BUFFER_IN_SECONDS}'."
+            )
+            return False
+
+        return True

.cloud-build/execute_changed_notebooks_cli.py

@@ -0,0 +1,172 @@
+#!/usr/bin/env python
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""A CLI to process changed notebooks and execute them on Google Cloud Build"""
+
+import argparse
+import pathlib
+import os
+
+import execute_changed_notebooks_helper
+
+
+def str2bool(v):
+    if isinstance(v, bool):
+        return v
+    if v.lower() in ("yes", "true", "t", "y", "1"):
+        return True
+    elif v.lower() in ("no", "false", "f", "n", "0"):
+        return False
+    else:
+        raise argparse.ArgumentTypeError("Boolean value expected.")
+
+
+parser = argparse.ArgumentParser(description="Run changed notebooks.")
+parser.add_argument(
+    "--test_paths_file",
+    type=pathlib.Path,
+    help="The path to the file that has newline-limited folders of notebooks that should be tested.",
+    required=True,
+)
+parser.add_argument(
+    "--test_percent",
+    type=int,
+    help="The percent of notebooks to be tested (between 1 and 100).",
+    required=False,
+    default=100,
+)
+parser.add_argument(
+    "--build_id",
+    type=str,
+    help="The build id (which may be a Cloud Build job specific or user explicit.",
+    required=True
+)
+parser.add_argument(
+    "--base_branch",
+    help="The base git branch to diff against to find changed files.",
+    required=False,
+)
+parser.add_argument(
+    "--container_uri",
+    type=str,
+    help="The container uri to run each notebook in.",
+    required=True,
+)
+parser.add_argument(
+    "--variable_project_id",
+    type=str,
+    help="The GCP project id. This is used to inject a variable value into the notebook before running.",
+    required=True,
+)
+parser.add_argument(
+    "--variable_region",
+    type=str,
+    help="The GCP region. This is used to inject a variable value into the notebook before running.",
+    required=True,
+)
+parser.add_argument(
+    "--variable_service_account",
+    type=str,
+    help="A service account. This is used to inject a variable value into the notebook before running. This is not the account that will run the notebook.",
+    required=True,
+)
+parser.add_argument(
+    "--variable_vpc_network",
+    type=str,
+    help="The full VPC network name. See https://cloud.google.com/compute/docs/networks-and-firewalls#networks. Format is projects/{project}/global/networks/{network}, where {project} is a project number, as in '12345', and {network} is network name. See <https://cloud.google.com/compute/docs/reference/rest/v1/networks/insert> for details. This is used to inject a variable value into the notebook before running.",
+    required=False,
+)
+parser.add_argument(
+    "--staging_bucket",
+    type=str,
+    help="The GCP directory for staging temporary files.",
+    required=True,
+)
+parser.add_argument(
+    "--artifacts_bucket",
+    type=str,
+    help="The GCP directory for storing executed notebooks.",
+    required=True,
+)
+parser.add_argument(
+    "--timeout",
+    type=int,
+    help="Timeout in seconds",
+    default=86400,
+    required=False,
+)
+parser.add_argument(
+    "--private_pool_id",
+    type=str,
+    help="The private pool id.",
+    required=False,
+)
+parser.add_argument(
+    "--should_parallelize",
+    type=str2bool,
+    nargs="?",
+    const=True,
+    default=True,
+    help="Should run notebooks in parallel.",
+)
+parser.add_argument(
+    "--dry_run",
+    type=str2bool,
+    default=False,
+    help="Dry run for testing - no execution",
+)
+
+args = parser.parse_args()
+
+changed_notebooks = execute_changed_notebooks_helper.get_changed_notebooks(
+    test_paths_file=args.test_paths_file,
+    base_branch=args.base_branch,
+)
+
+
+results_bucket = f"{args.artifacts_bucket}"
+# artifacts_bucket may get set by trigger to a full gs:// folder path
+if results_bucket.startswith("gs://"):
+    results_bucket = results_bucket[5:]
+results_bucket = results_bucket.split('/')[0]
+results_file = f"build_results/{args.build_id}.json"
+
+if args.test_percent == 100:
+    notebooks = changed_notebooks
+    accumulative_results = {}
+else:
+    accumulative_results = execute_changed_notebooks_helper.load_results(results_bucket, results_file)
+
+    notebooks = [changed_notebook for changed_notebook in changed_notebooks if execute_changed_notebooks_helper.select_notebook(changed_notebook, accumulative_results, args.test_percent)]
+
+if args.dry_run:
+    print("Dry run ...\n")
+    for notebook in notebooks:
+        print(f"Would execute: {notebook}")
+else:
+    execute_changed_notebooks_helper.process_and_execute_notebooks(
+        notebooks=notebooks,
+        container_uri=args.container_uri,
+        staging_bucket=args.staging_bucket,
+        artifacts_bucket=args.artifacts_bucket,
+        results_file=results_file,
+        should_parallelize=args.should_parallelize,
+        timeout=args.timeout,
+        variable_project_id=args.variable_project_id,
+        variable_region=args.variable_region,
+        variable_service_account=args.variable_service_account,
+        variable_vpc_network=args.variable_vpc_network,
+        private_pool_id=args.private_pool_id
+)

.cloud-build/execute_changed_notebooks_helper.py

@@ -0,0 +1,608 @@
+#!/usr/bin/env python
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import concurrent
+import dataclasses
+import datetime
+import functools
+import json
+import git
+import operator
+import os
+import io
+import json
+import pathlib
+import re
+import subprocess
+import random
+from google.cloud import storage
+import utils
+from typing import List, Optional, Dict, Any
+from utils import util
+
+import execute_notebook_helper
+import execute_notebook_remote
+import nbformat
+from google.cloud.devtools.cloudbuild_v1.types import BuildOperationMetadata
+from ratemate import RateLimit
+from tabulate import tabulate
+from utils import NotebookProcessors, util
+
+# A buffer so that workers finish before the orchestrating job
+WORKER_TIMEOUT_BUFFER_IN_SECONDS: int = 60 * 60
+PYTHON_VERSION = "3.9"  # Set default python version
+
+
+def format_timedelta(delta: datetime.timedelta) -> str:
+    """Formats a timedelta duration to [N days] %H:%M:%S format"""
+    seconds = int(delta.total_seconds())
+
+    secs_in_a_day = 86400
+    secs_in_a_hour = 3600
+    secs_in_a_min = 60
+
+    days, seconds = divmod(seconds, secs_in_a_day)
+    hours, seconds = divmod(seconds, secs_in_a_hour)
+    minutes, seconds = divmod(seconds, secs_in_a_min)
+
+    time_fmt = f"{hours:02d}:{minutes:02d}:{seconds:02d}"
+
+    if days > 0:
+        suffix = "s" if days > 1 else ""
+        return f"{days} day{suffix} {time_fmt}"
+
+    return time_fmt
+
+
+@dataclasses.dataclass
+class NotebookExecutionResult:
+    name: str
+    path: str
+    duration: datetime.timedelta
+    start_time: datetime.datetime
+    is_pass: bool
+    log_url: str
+    output_uri: str
+    build_id: str
+    logs_bucket: str
+    error_message: Optional[str]
+
+    @property
+    def output_uri_web(self) -> Optional[str]:
+        if self.output_uri.startswith("gs://"):
+            return f"https://storage.googleapis.com/{self.output_uri[5:]}"
+        else:
+            return None
+
+def load_results(results_bucket: str,
+        results_file: str) -> Dict[str,Any]:
+    '''
+    Load accumulated notebook test results
+    '''
+
+    print("Loading existing accumulative results ...")
+    accumulative_results = {}
+    try:
+        client = storage.Client()
+        bucket = client.bucket(results_bucket)
+
+        build_results_dir = os.path.dirname(results_file)
+        blobs = client.list_blobs(results_bucket, prefix=build_results_dir)
+        for blob in blobs:
+            content = util.download_blob_into_memory(results_bucket, blob.name, download_as_text=True)
+            build_results = json.loads(content)
+            for notebook in build_results:
+                if notebook in accumulative_results:
+                    accumulative_results[notebook] += build_results[notebook]
+                else:
+                    accumulative_results[notebook] = build_results[notebook]
+
+        print(accumulative_results)
+    except Exception as e:
+        print(e)
+
+    # If there are no accumulative results, an empty dict is returned
+    return accumulative_results
+
+def select_notebook(changed_notebook: str,
+                    accumulative_results: Dict[str, Any],
+                    test_percent: int) -> bool:
+    '''
+    Algorithm to randomly select a notebook, but weight the propbability of selected based on past failures
+    '''
+
+    if changed_notebook in accumulative_results:
+        pass_count = accumulative_results[changed_notebook]['passed']
+        fail_count = accumulative_results[changed_notebook]['failed']
+    else:
+        pass_count = 1
+        fail_count = 0
+
+    inferred_failure_rate = fail_count / (pass_count + fail_count)
+
+    # If failure rate is high, the chance of testing should be higher
+    should_test_due_to_failure = random.uniform(0, 1) < inferred_failure_rate
+
+    # Additionally, only test a percentage of these
+    should_test_due_to_random_subset = random.uniform(0, 1) < test_percent
+
+    return should_test_due_to_failure and should_test_due_to_random_subset
+
+
+def _process_notebook(
+    notebook_path: str,
+    variable_project_id: str,
+    variable_region: str,
+    variable_service_account: str,
+    variable_vpc_network: Optional[str],
+):
+    # Read notebook
+    with open(notebook_path) as f:
+        nb = nbformat.read(f, as_version=4)
+
+    # Create preprocessors
+    remove_no_execute_cells_preprocessor = NotebookProcessors.RemoveNoExecuteCells()
+    update_variables_preprocessor = NotebookProcessors.UpdateVariablesPreprocessor(
+        replacement_map={
+            "PROJECT_ID": variable_project_id,
+            "REGION": variable_region,
+            "SERVICE_ACCOUNT": variable_service_account,
+            "VPC_NETWORK": variable_vpc_network,
+        },
+    )
+    unique_strings_preprocessor = NotebookProcessors.UniqueStringsPreprocessor()
+
+    # Use no-execute preprocessor
+    (
+        nb,
+        resources,
+    ) = remove_no_execute_cells_preprocessor.preprocess(nb)
+
+    (nb, resources) = update_variables_preprocessor.preprocess(nb, resources)
+    (nb, resources) = unique_strings_preprocessor.preprocess(nb, resources)
+
+    with open(notebook_path, mode="w", encoding="utf-8") as new_file:
+        nbformat.write(nb, new_file)
+
+
+def _get_notebook_python_version(notebook_path: str) -> str:
+    """
+    Get the python version for running the notebook if it is specified in
+    the notebook.
+    """
+    python_version = PYTHON_VERSION
+
+    # Load the notebook
+    file = open(notebook_path)
+    src = file.read()
+    nb_json = json.loads(src)
+
+    # Iterate over the cells in the ipynb
+    for cell in nb_json["cells"]:
+        if cell["cell_type"] == "markdown":
+            markdown = str.join("", cell["source"])
+
+            # Look for the python version specification pattern
+            re_match = re.search(
+                "python version = (\d\.\d)", markdown, flags=re.IGNORECASE
+            )
+            if re_match:
+                # get the version number
+                python_version = re_match.group(1)
+                break
+
+    return python_version
+
+
+def _create_tag(filepath: str) -> str:
+    tag = os.path.basename(os.path.normpath(filepath))
+    tag = re.sub("[^0-9a-zA-Z_.-]+", "-", tag)
+
+    if tag.startswith(".") or tag.startswith("-"):
+        tag = tag[1:]
+
+    return tag
+
+
+rate_limit = RateLimit(max_count=10, per=60, greedy=True)
+
+
+def process_and_execute_notebook(
+    container_uri: str,
+    staging_bucket: str,
+    artifacts_bucket: str,
+    variable_project_id: str,
+    variable_region: str,
+    variable_service_account: str,
+    variable_vpc_network: Optional[str],
+    private_pool_id: Optional[str],
+    deadline: datetime.datetime,
+    notebook: str,
+    should_get_tail_logs: bool = False,
+) -> NotebookExecutionResult:
+    rate_limit.wait()  # wait before creating the task
+
+    print(f"Running notebook: {notebook}")
+
+    # Handle empty strings
+    if not variable_vpc_network:
+        variable_vpc_network = None
+
+    if not private_pool_id:
+        private_pool_id = None
+
+    # Create paths
+    notebook_output_uri = "/".join([artifacts_bucket, pathlib.Path(notebook).name])
+
+    # Create tag from notebook
+    tag = _create_tag(filepath=notebook)
+
+    result = NotebookExecutionResult(
+        name=tag,
+        path=notebook,
+        duration=datetime.timedelta(seconds=0),
+        start_time=datetime.datetime.now(),
+        is_pass=False,
+        output_uri=notebook_output_uri,
+        log_url="",
+        build_id="",
+        logs_bucket="",
+        error_message=None,
+    )
+
+    # TODO: Handle cases where multiple notebooks have the same name
+    operation = None
+    try:
+        # Get the python version for running the notebook if specified
+        notebook_exec_python_version = _get_notebook_python_version(
+            notebook_path=notebook
+        )
+        print(f"Running notebook with python {notebook_exec_python_version}")
+
+        # Pre-process notebook by substituting variable names
+        _process_notebook(
+            notebook_path=notebook,
+            variable_project_id=variable_project_id,
+            variable_region=variable_region,
+            variable_service_account=variable_service_account,
+            variable_vpc_network=variable_vpc_network,
+        )
+
+        # Upload the pre-processed code to a GCS bucket
+        code_archive_uri = util.archive_code_and_upload(staging_bucket=staging_bucket)
+
+        # Calculate timeout in seconds
+        timeout_in_seconds = max(
+            int((deadline - datetime.datetime.now()).total_seconds()), 1
+        )
+
+        operation = execute_notebook_remote.execute_notebook_remote(
+            code_archive_uri=code_archive_uri,
+            notebook_uri=notebook,
+            notebook_output_uri=notebook_output_uri,
+            container_uri=container_uri,
+            tag=tag,
+            private_pool_id=private_pool_id,
+            private_pool_region=variable_region,
+            timeout_in_seconds=timeout_in_seconds,
+            python_version=notebook_exec_python_version,
+        )
+
+        operation_metadata = BuildOperationMetadata(mapping=operation.metadata)
+        result.build_id = operation_metadata.build.id
+        result.log_url = operation_metadata.build.log_url
+        result.logs_bucket = operation_metadata.build.logs_bucket
+
+        # Block and wait for the result
+        operation_result = operation.result(timeout=timeout_in_seconds)
+
+        result.duration = datetime.datetime.now() - result.start_time
+        result.is_pass = True
+        print(f"{notebook} PASSED in {format_timedelta(result.duration)}.")
+
+    except Exception as error:
+        result.error_message = str(error)
+
+        if operation and should_get_tail_logs:
+            # Extract the logs
+            logs_bucket = operation_metadata.build.logs_bucket
+
+            # Download tail end of logs file
+            log_file_uri = f"{logs_bucket}/log-{result.build_id}.txt"
+
+            # Use gcloud to get tail
+            try:
+                result.error_message = subprocess.check_output(
+                    ["gsutil", "cat", "-r", "-1000", log_file_uri], encoding="UTF-8"
+                )
+            except Exception as error:
+                result.error_message = str(error)
+
+        result.duration = datetime.datetime.now() - result.start_time
+        result.is_pass = False
+
+        print(
+            f"{notebook} FAILED in {format_timedelta(result.duration)}: {result.error_message}"
+        )
+
+    return result
+
+
+def get_changed_notebooks(
+    test_paths_file: str,
+    base_branch: Optional[str] = None,
+) -> List[str]:
+    """
+    Get the notebooks that exist under the folders defined in the test_paths_file.
+    It only returns notebooks that have differences from the Git base_branch.
+    """
+
+    test_paths = []
+    with open(test_paths_file) as file:
+        lines = [line.strip() for line in file.readlines()]
+        lines = [line for line in lines if len(line) > 0]
+        test_paths = [line for line in lines]
+
+    if len(test_paths) == 0:
+        raise RuntimeError("No test folders found.")
+
+    print(f"Checking folders: {test_paths}")
+
+    # Find notebooks
+    notebooks = []
+
+    # Instantiate GitPython objects
+    repo = git.Repo(os.getcwd())
+    index = repo.index
+
+    if base_branch:
+        # Get the point at which this branch branches off from main
+        branching_commits = repo.merge_base("HEAD", f"origin/{base_branch}")
+
+        if len(branching_commits) > 0:
+            branching_commit = branching_commits[0]
+            print(f"Looking for notebooks that changed from branch: {branching_commit}")
+
+            notebooks = [
+                diff.b_path
+                for diff in index.diff(branching_commit, paths=test_paths)
+                if diff.b_path is not None
+            ]
+        else:
+            notebooks = []
+    else:
+        print(f"Looking for all notebooks.")
+        notebooks_str = subprocess.check_output(["git", "ls-files"] + test_paths)
+        notebooks = notebooks_str.decode("utf-8").split("\n")
+
+    notebooks = [notebook for notebook in notebooks if notebook.endswith(".ipynb")]
+    notebooks = [notebook for notebook in notebooks if len(notebook) > 0]
+    notebooks = [notebook for notebook in notebooks if pathlib.Path(notebook).exists()]
+
+    if len(notebooks) > 0:
+        print(f"Found {len(notebooks)} notebooks:")
+        for notebook in notebooks:
+            print(f"\t{notebook}")
+
+    return notebooks
+
+def _save_results(results: List[NotebookExecutionResult],
+                  artifacts_bucket: str,
+                  results_file: str):
+
+    artifacts_bucket = artifacts_bucket.replace("gs://", "").split('/')[0]
+
+    print("Updating build results ...")
+    build_results = {}
+    for result in results:
+        if result.is_pass:
+            pass_count = 1
+            fail_count = 0
+        else:
+            pass_count = 0
+            fail_count = 1
+        build_results[result.path] = {
+                'duration': result.duration.total_seconds(),
+                'start_time': str(result.start_time),
+                'passed': pass_count,
+                'failed': fail_count
+        }
+        print(f"adding {result.path}")
+
+    print("Saving accumulative results ...")
+    content = json.dumps(build_results)
+
+    client = storage.Client()
+    bucket = client.get_bucket(artifacts_bucket)
+    bucket.blob(str(results_file)).upload_from_string(content, 'text/json')
+
+
+
+def process_and_execute_notebooks(
+    notebooks: List[str],
+    container_uri: str,
+    staging_bucket: str,
+    artifacts_bucket: str,
+    results_file: str,
+    should_parallelize: bool,
+    timeout: int,
+    variable_project_id: str,
+    variable_region: str,
+    variable_service_account: str,
+    variable_vpc_network: Optional[str] = None,
+    private_pool_id: Optional[str] = None,
+):
+    """
+    Run the notebooks that exist under the folders defined in the test_paths_file.
+    It only runs notebooks that have differences from the Git base_branch.
+
+    The executed notebooks are saved in the artifacts_bucket.
+
+    Variables are also injected into the notebooks such as the variable_project_id and variable_region.
+
+    Args:
+        test_paths_file (str):
+            Required. The new-line delimited file to folders and files that need checking.
+            Folders are checked recursively.
+        base_branch (str):
+            Optional. If provided, only the files that have changed from the base_branch will be checked.
+            If not provided, all files will be checked.
+        staging_bucket (str):
+            Required. The GCS staging bucket to write source code to.
+        artifacts_bucket (str):
+            Required. The GCS staging bucket to write executed notebooks to.
+        results_file (str):
+            Required: The path to the artifacts bucket to save results 
+        variable_project_id (str):
+            Required. The value for PROJECT_ID to inject into notebooks.
+        variable_region (str):
+            Required. The value for REGION to inject into notebooks.
+        should_parallelize (bool):
+            Required. Should run notebooks in parallel using a thread pool as opposed to in sequence.
+        timeout (str):
+            Required. Timeout string according to https://cloud.google.com/build/docs/build-config-file-schema#timeout.
+    """
+
+    # Calculate deadline
+    deadline = datetime.datetime.now() + datetime.timedelta(
+        seconds=max(timeout - WORKER_TIMEOUT_BUFFER_IN_SECONDS, 0)
+    )
+
+    if len(notebooks) >= 1:
+        notebook_execution_results: List[NotebookExecutionResult] = []
+
+        print(f"Found {len(notebooks)} modified notebooks: {notebooks}")
+
+        if should_parallelize and len(notebooks) > 1:
+            print(
+                "Running notebooks in parallel, so no logs will be displayed. Please wait..."
+            )
+            with concurrent.futures.ThreadPoolExecutor(max_workers=100) as executor:
+                print(f"Max workers: {executor._max_workers}")
+
+                notebook_execution_results = list(
+                    executor.map(
+                        functools.partial(
+                            process_and_execute_notebook,
+                            container_uri,
+                            staging_bucket,
+                            artifacts_bucket,
+                            variable_project_id,
+                            variable_region,
+                            variable_service_account,
+                            variable_vpc_network,
+                            private_pool_id,
+                            deadline,
+                        ),
+                        notebooks,
+                    )
+                )
+        else:
+            notebook_execution_results = [
+                process_and_execute_notebook(
+                    container_uri=container_uri,
+                    staging_bucket=staging_bucket,
+                    artifacts_bucket=artifacts_bucket,
+                    variable_project_id=variable_project_id,
+                    variable_region=variable_region,
+                    variable_service_account=variable_service_account,
+                    variable_vpc_network=variable_vpc_network,
+                    private_pool_id=private_pool_id,
+                    deadline=deadline,
+                    notebook=notebook,
+                )
+                for notebook in notebooks
+            ]
+
+        print("\n=== RESULTS ===\n")
+
+        results_sorted = sorted(
+            notebook_execution_results,
+            key=lambda result: result.is_pass,
+            reverse=True,
+        )
+
+        # Print results
+        print(
+            tabulate(
+                [
+                    [
+                        result.name,
+                        "PASSED" if result.is_pass else "FAILED",
+                        format_timedelta(result.duration),
+                        result.log_url,
+                        result.output_uri,
+                        result.output_uri_web,
+                        result.logs_bucket,
+                    ]
+                    for result in results_sorted
+                ],
+                headers=[
+                    "build_tag",
+                    "status",
+                    "duration",
+                    "log_url",
+                    "output_uri",
+                    "output_uri_web",
+                    "logs_bucket",
+                ],
+            )
+        )
+
+        if len(notebooks) == 1:
+            print("=" * 100)
+            print("The notebook execution build log:\n")
+            print("=" * 100)
+
+            build_id = results_sorted[0].build_id
+            logs_bucket_name = (results_sorted[0].logs_bucket).replace("gs://", "")
+            log_file_name = f"log-{build_id}.txt"
+
+            log_contents = util.download_blob_into_memory(
+                bucket_name=logs_bucket_name,
+                blob_name=log_file_name,
+                download_as_text=True,
+            )
+
+            # Remove extra steps from the log
+            match = re.search("starting Step #4", log_contents, flags=re.IGNORECASE)
+
+            if match is not None:
+                match_index = match.span()[0]
+                print(log_contents[match_index:])
+            else:
+                print(log_contents)
+
+        _save_results(results_sorted, 
+                      artifacts_bucket, 
+                      results_file)
+
+        print("\n=== END RESULTS===\n")
+
+        total_notebook_duration = functools.reduce(
+            operator.add,
+            [datetime.timedelta(seconds=0)]
+            + [result.duration for result in results_sorted],
+        )
+
+        print(
+            f"Cumulative notebook duration: {format_timedelta(total_notebook_duration)}"
+        )
+
+        # Raise error if any notebooks failed
+        if not all([result.is_pass for result in results_sorted]):
+            raise RuntimeError("Notebook failures detected. See logs for details")
+    else:
+        print("No notebooks modified in this pull request.")

.cloud-build/execute_notebook_cli.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""A CLI to download (optional) and run a single notebook locally"""
+
+import argparse
+
+import execute_notebook_helper
+
+parser = argparse.ArgumentParser(description="Run a single notebook locally.")
+parser.add_argument(
+    "--notebook_source",
+    type=str,
+    help="Local filepath or GCS URI to notebook.",
+    required=True,
+)
+parser.add_argument(
+    "--output_file_or_uri",
+    type=str,
+    help="Local file or GCS URI to save executed notebook to.",
+    required=True,
+)
+
+args = parser.parse_args()
+execute_notebook_helper.execute_notebook(
+    notebook_source=args.notebook_source,
+    output_file_or_uri=args.output_file_or_uri,
+    should_log_output=True,
+)

.cloud-build/execute_notebook_helper.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Methods to run a notebook locally"""
+
+import errno
+import os
+import shutil
+import sys
+
+import papermill as pm
+from google.cloud.aiplatform import utils
+from utils import util
+
+# This script is used to execute a notebook and write out the output notebook.
+
+# This is used to force papermill to use this kernel to run the notebook instead of any defined inside the notebook itself
+DEFAULT_KERNEL_NAME = "python3"
+
+
+def execute_notebook(
+    notebook_source: str,
+    output_file_or_uri: str,
+    should_log_output: bool,
+):
+    """Execute a single notebook using Papermill"""
+    file_name = os.path.basename(os.path.normpath(notebook_source))
+
+    # Download notebook if it's a GCS URI
+    if notebook_source.startswith("gs://"):
+        # Extract uri components
+        bucket_name, prefix = utils.extract_bucket_and_prefix_from_gcs_path(
+            notebook_source
+        )
+
+        # Download remote notebook to local file system
+        notebook_source = file_name
+        util.download_file(
+            bucket_name=bucket_name, blob_name=prefix, destination_file=notebook_source
+        )
+
+    execution_exception = None
+
+    print("\n=== DOWNLOAD EXECUTED NOTEBOOK ===\n")
+    print(f"Please debug the executed notebook by downloading the executed notebook:")
+
+    print("Option 1. Using gsutil. Run the following command in your terminal.")
+    print(f'\tgsutil cp "{output_file_or_uri}" .')
+
+    print("Option 2. Using this link.")
+    print(f"\thttps://storage.googleapis.com/{output_file_or_uri[5:]}")
+
+    print("\n======\n")
+
+    # Execute notebook
+    try:
+        # Execute notebook
+        pm.execute_notebook(
+            input_path=notebook_source,
+            output_path=notebook_source,
+            progress_bar=should_log_output,
+            request_save_on_cell_execute=should_log_output,
+            kernel_name=DEFAULT_KERNEL_NAME,
+            log_output=should_log_output,
+            stdout_file=sys.stdout if should_log_output else None,
+            stderr_file=sys.stderr if should_log_output else None,
+        )
+    except Exception as exception:
+        execution_exception = exception
+    finally:
+        # Copy executed notebook
+        if output_file_or_uri.startswith("gs://"):
+            # Upload to GCS path
+            util.upload_file(notebook_source, remote_file_path=output_file_or_uri)
+
+            print("\n=== EXECUTION FINISHED ===\n")
+        else:
+            # Create directories if they don't exist
+            if not os.path.exists(os.path.dirname(output_file_or_uri)):
+                try:
+                    os.makedirs(os.path.dirname(output_file_or_uri))
+                except OSError as exc:  # Guard against race condition
+                    if exc.errno != errno.EEXIST:
+                        raise
+
+            print(f"Writing output to: {output_file_or_uri}")
+            shutil.move(notebook_source, output_file_or_uri)
+
+        if execution_exception:
+            raise execution_exception

.cloud-build/execute_notebook_remote.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Methods to run a notebook on Google Cloud Build"""
+
+from re import sub
+from typing import Optional
+
+import google.auth
+import yaml
+from google.api_core import client_options, operation
+from google.cloud.aiplatform import utils
+from google.cloud.devtools import cloudbuild_v1
+from google.cloud.devtools.cloudbuild_v1.types import Source, StorageSource
+from google.protobuf import duration_pb2
+from yaml.loader import FullLoader
+
+CLOUD_BUILD_FILEPATH = ".cloud-build/notebook-execution-test-cloudbuild-single.yaml"
+SERVICE_BASE_PATH = "cloudbuild.googleapis.com"
+
+
+def execute_notebook_remote(
+    code_archive_uri: str,
+    notebook_uri: str,
+    notebook_output_uri: str,
+    container_uri: str,
+    private_pool_id: Optional[str],
+    private_pool_region: Optional[str],
+    tag: Optional[str],
+    timeout_in_seconds: Optional[int] = None,
+    python_version: Optional[str] = None
+) -> operation.Operation:
+    """Create and execute a single notebook on Google Cloud Build"""
+    # Load build steps from YAML
+
+    cloudbuild_config = yaml.load(open(CLOUD_BUILD_FILEPATH), Loader=FullLoader)
+
+    substitutions = {
+        "_PYTHON_IMAGE": container_uri,
+        "_NOTEBOOK_GCS_URI": notebook_uri,
+        "_NOTEBOOK_OUTPUT_GCS_URI": notebook_output_uri,
+        "_PYTHON_VERSION" : f"python{python_version}"
+    }
+
+    if python_version is not None:
+      substitutions["_PYTHON_VERSION"] = "python" + python_version
+
+    build = cloudbuild_v1.Build()
+
+    options: Optional[client_options.ClientOptions] = None
+    if private_pool_id and private_pool_region:
+        # substitutions["_PRIVATE_POOL_NAME"] = private_pool_id
+        build.options = cloudbuild_config.get("options")
+        build.options.pool = {"name": private_pool_id}
+
+        # Switch to the regional endpoint of the pool
+        options = client_options.ClientOptions(
+            api_endpoint=f"{private_pool_region}-{SERVICE_BASE_PATH}"
+        )
+
+    # Authorize the client with Google defaults
+    credentials, project_id = google.auth.default()
+
+    client = cloudbuild_v1.services.cloud_build.CloudBuildClient(client_options=options)
+
+    (
+        source_archived_file_gcs_bucket,
+        source_archived_file_gcs_object,
+    ) = utils.extract_bucket_and_prefix_from_gcs_path(code_archive_uri)
+
+    build.source = Source(
+        storage_source=StorageSource(
+            bucket=source_archived_file_gcs_bucket,
+            object_=source_archived_file_gcs_object,
+        )
+    )
+
+    build.steps = cloudbuild_config["steps"]
+    build.substitutions = substitutions
+    build.timeout = duration_pb2.Duration(seconds=timeout_in_seconds)
+    build.queue_ttl = duration_pb2.Duration(seconds=timeout_in_seconds)
+
+    if tag:
+        build.tags = [tag]
+
+    operation = client.create_build(project_id=project_id, build=build)
+    # Print the in-progress operation
+    # print("IN PROGRESS:")
+    # print(operation.metadata)
+
+    # Print the completed status
+    # print("RESULT:", result.status)
+    return operation

.cloud-build/notebook-execution-test-cloudbuild-single.yaml

@@ -0,0 +1,38 @@
+steps:
+  # Show the gcloud info and check if gcloud exists
+  - name: ${_PYTHON_IMAGE}
+    entrypoint: /bin/sh
+    args:
+    - -c
+    - 'gcloud config list --quiet'
+  # Check the Python version
+  - name: ${_PYTHON_IMAGE}
+    entrypoint: /bin/sh
+    args:
+    - -c
+    - ${_PYTHON_VERSION} .cloud-build/CheckPythonVersion.py -q
+  # Create a virtual environment
+  - name: ${_PYTHON_IMAGE}
+    entrypoint: /bin/sh
+    args:
+    - -c
+    - ${_PYTHON_VERSION} -m venv workspace/env
+  # Install Python dependencies
+  - name: ${_PYTHON_IMAGE}
+    entrypoint: /bin/sh
+    args:
+    - -c
+    - . workspace/env/bin/activate &&
+      python -m pip -q install -U pip &&
+      python -m pip -q install -U -r .cloud-build/requirements.txt
+  # Install Python dependencies and run testing script
+  - name: ${_PYTHON_IMAGE}
+    entrypoint: /bin/sh
+    args:
+    - -c
+    - |
+      . workspace/env/bin/activate &&
+      python .cloud-build/execute_notebook_cli.py --notebook_source "${_NOTEBOOK_GCS_URI}" --output_file_or_uri "${_NOTEBOOK_OUTPUT_GCS_URI}"
+    env:
+    - 'IS_TESTING=1'
+timeout: 86400s

.cloud-build/notebook-execution-test-cloudbuild.yaml

@@ -4,41 +4,42 @@ steps:
     entrypoint: /bin/sh
     args:
     - -c
-    - 'gcloud config list'
+    - gcloud config list --quiet
   # Check the Python version
   - name: ${_PYTHON_IMAGE}
     entrypoint: /bin/sh
     args:
     - -c
-    - 'python3 .cloud-build/CheckPythonVersion.py'
-  # Fetch base branch if required
+    - python3 .cloud-build/CheckPythonVersion.py -q
+  # Fetch full repo for diff purposes
+  - name: gcr.io/cloud-builders/git
+    args: [fetch, --unshallow, --quiet]
+  # Create a virtual environment
   - name: ${_PYTHON_IMAGE}
     entrypoint: /bin/sh
     args:
     - -c
-    - 'if [ -n "${_BASE_BRANCH}" ]; then git fetch origin "${_BASE_BRANCH}":refs/remotes/origin/"${_BASE_BRANCH}"; else echo "Skipping fetch."; fi'
+    - python3 -m venv workspace/env
   # Install Python dependencies
   - name: ${_PYTHON_IMAGE}
-    entrypoint: pip
-    args: ['install', '--upgrade',  '--user', '--requirement', '.cloud-build/requirements.txt']
+    entrypoint: /bin/sh
+    args:
+    - -c
+    - . workspace/env/bin/activate &&
+      python3 -m pip -q install -U pip &&
+      python3 -m pip -q install -U -r .cloud-build/requirements.txt
   # Install Python dependencies and run testing script
+  # TODO: Only pass in private_pool_id if it is set
   - name: ${_PYTHON_IMAGE}
     entrypoint: /bin/sh
     args:
     - -c
-    - 'python3 -m pip freeze && python3 .cloud-build/ExecuteChangedNotebooks.py --test_paths_file "${_TEST_PATHS_FILE}" --base_branch "${_FORCED_BASE_BRANCH}" --output_folder ${BUILD_ID} --variable_project_id ${PROJECT_ID} --variable_region ${_GCP_REGION}'
+    - |
+      . workspace/env/bin/activate &&
+      python3 .cloud-build/execute_changed_notebooks_cli.py --test_paths_file "${_TEST_PATHS_FILE}" --base_branch "${_FORCED_BASE_BRANCH}" --container_uri ${_PYTHON_IMAGE} --staging_bucket ${_GCS_STAGING_BUCKET}  --artifacts_bucket ${_GCS_STAGING_BUCKET}/executed_notebooks/PR_${_PR_NUMBER}/BUILD_${BUILD_ID} --variable_project_id ${PROJECT_ID} --variable_region ${_GCP_REGION} --variable_service_account ${_GCP_SERVICE_ACCOUNT} --variable_vpc_network "${_GPC_VPC_NETWORK_NAME}" `if [ ! -z "${_PRIVATE_POOL_NAME}" ]; then echo "--private_pool_id ${_PRIVATE_POOL_NAME}"; fi` --build_id ${BUILD_ID} --test_percent=${_TEST_PERCENT}
     env:
     - 'IS_TESTING=1'
-  # Manually copy artifacts to GCS
-  - name: gcr.io/cloud-builders/gsutil
-    entrypoint: /bin/sh
-    args: 
-    - -c
-    - 'if [ $(ls -pR "/workspace/${BUILD_ID}"  | grep -v / | grep -v ^$ | wc -l) -ne 0 ]; then gsutil -m -q rsync -r "/workspace/${BUILD_ID}" "gs://${_GCS_ARTIFACTS_BUCKET}/test-artifacts/PR_${_PR_NUMBER}/BUILD_${BUILD_ID}/"; else echo "No artifacts to copy."; fi'
-  # Fail if there is anything in the failure folder
-  - name: ${_PYTHON_IMAGE}
-    entrypoint: /bin/sh
-    args: 
-    - -c
-    - 'echo "Download executed notebooks with this command: \"mkdir -p artifacts && gsutil rsync -r gs://${_GCS_ARTIFACTS_BUCKET}/test-artifacts/PR_${_PR_NUMBER}/BUILD_${BUILD_ID} artifacts/\"" && if [ "$(ls -A /workspace/${BUILD_ID}/failure | wc -l)" -ne 0 ]; then exit 1; else exit 0; fi' 
 timeout: 86400s
+options:
+  pool:
+    name: ${_PRIVATE_POOL_NAME}

.cloud-build/requirements.txt

@@ -1,8 +1,17 @@
-ipython>=7.0
-jupyter>=1.0
-nbconvert>=6.0
-papermill>=2.3
-numpy>=1.19
-pandas>=1.2
-matplotlib>=3.4
+ipython
+numpy
+jupyter
+nbconvert
+papermill
+matplotlib
 tabulate
+google-cloud-aiplatform
+google-cloud-storage
+google-cloud-build
+google-cloud-storage
+ratemate
+GitPython
+tqdm
+fsspec
+pandas
+

.cloud-build/test_failing.txt

@@ -0,0 +1,40 @@
+notebooks/official/training/pytorch_gcs_data_training.ipynb
+notebooks/official/custom/custom_training_tensorboard_profiler.ipynb
+notebooks/official/custom/custom-tabular-bq-managed-dataset.ipynb
+notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-locally-trained-models.ipynb
+notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb
+notebooks/official/tabnet/tabnet_vertex_tutorial.ipynb
+notebooks/official/tabnet/get_started_with_tabnet.ipynb
+notebooks/official/pipelines/google_cloud_pipeline_components_automl_text.ipynb
+notebooks/official/pipelines/multicontender_vs_champion_deployment_method.ipynb
+notebooks/official/pipelines/google_cloud_pipeline_components_automl_images.ipynb
+notebooks/official/pipelines/rapid_prototyping_bqml_automl.ipynb
+notebooks/official/pipelines/challenger_vs_blessed_deployment_method.ipynb
+notebooks/official/matching_engine/sdk_matching_engine_create_stack_overflow_embeddings.ipynb
+notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb
+notebooks/official/matching_engine/sdk_matching_engine_create_text_to_image_embeddings.ipynb
+notebooks/official/explainable_ai/sdk_custom_image_classification_online_explain.ipynb
+notebooks/official/explainable_ai/xai_image_classification_feature_attributions.ipynb
+notebooks/official/explainable_ai/sdk_custom_image_classification_batch_explain.ipynb
+notebooks/official/tabular_workflows/tabnet_on_vertex_pipelines.ipynb
+notebooks/official/model_registry/get_started_with_model_registry.ipynb
+notebooks/official/model_registry/bqml_vertexai_model_registry.ipynb
+notebooks/official/sdk/SDK_Custom_Training_Python_Package_Managed_Text_Dataset_Tensorflow_Serving_Container.ipynb
+notebooks/official/model_monitoring/batch_prediction_model_monitoring.ipynb
+notebooks/official/model_monitoring/get_started_with_model_monitoring_setup.ipynb
+notebooks/official/model_monitoring/get_started_with_model_monitoring_custom.ipynb
+notebooks/official/model_monitoring/get_started_with_model_monitoring_custom_tf_serving.ipynb
+notebooks/official/model_monitoring/model_monitoring.ipynb
+notebooks/official/tensorboard/tensorboard_profiler_custom_training_with_prebuilt_container.ipynb
+notebooks/official/tensorboard/tensorboard_hyperparameter_tuning_with_hparams.ipynb
+notebooks/official/tensorboard/tensorboard_profiler_custom_training.ipynb
+notebooks/official/model_evaluation/custom_tabular_regression_model_evaluation.ipynb
+notebooks/official/model_evaluation/custom_tabular_classification_model_evaluation.ipynb
+notebooks/official/model_evaluation/automl_video_classification_model_evaluation.ipynb
+notebooks/official/experiments/comparing_local_trained_models.ipynb
+notebooks/official/automl/automl_image_classification_online_online_prediction.ipynb
+notebooks/official/automl/automl-text-classification.ipynb
+notebooks/official/automl/sdk_automl_video_object_tracking_batch.ipynb
+notebooks/official/feature_store/sdk-feature-store-pandas.ipynb
+notebooks/official/prediction/custom_batch_prediction_feature_filter.ipynb
+notebooks/official/prediction/pytorch_image_classification_with_prebuilt_serving_containers.ipynb

.cloud-build/test_notebook_vm.txt

@@ -0,0 +1,5 @@
+notebooks/official/vizier/gapic-vizier-multi-objective-optimization.ipynb
+notebooks/official/pipelines/lightweight_functions_component_io_kfp.ipynb
+notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-locally-trained-models.ipynb
+notebooks/official/custom/custom-tabular-bq-managed-dataset.ipynb
+.cloud-build/tests/python_version_test.ipynb

.cloud-build/test_passing.txt

@@ -0,0 +1,80 @@
+notebooks/official/training/hyperparameter_tuning_tensorflow.ipynb
+notebooks/official/training/get_started_with_vertex_distributed_training.ipynb
+notebooks/official/training/hyperparameter_tuning_xgboost.ipynb
+notebooks/official/training/multi_node_ddp_gloo_vertex_training_with_custom_container.ipynb
+notebooks/official/training/distributed_hyperparameter_tuning.ipynb
+notebooks/official/training/pytorch-text-sentiment-classification-custom-train-deploy.ipynb
+notebooks/official/training/xgboost_data_parallel_training_on_cpu_using_dask.ipynb
+notebooks/official/training/multi_node_ddp_nccl_vertex_training_with_custom_container.ipynb
+notebooks/official/bigquery_ml/get_started_with_bqml_training.ipynb
+notebooks/official/bigquery_ml/bqml-online-prediction.ipynb
+notebooks/official/custom/custom_training_container_and_model_registry.ipynb
+notebooks/official/custom/sdk-custom-image-classification-online.ipynb
+notebooks/official/custom/sdk-custom-image-classification-batch.ipynb
+notebooks/official/custom/SDK_FBProphet_Forecasting_Online.ipynb
+notebooks/official/custom/get_started_vertex_training_xgboost.ipynb
+notebooks/official/custom/get_started_with_vertex_endpoint_and_shared_vm.ipynb
+notebooks/official/custom/SDK_Custom_Container_Prediction.ipynb
+notebooks/official/reduction_server/pytorch_distributed_training_reduction_server.ipynb
+notebooks/official/tabnet/ai-explanations-tabnet-algorithm.ipynb
+notebooks/official/vizier/get_started_vertex_vizier.ipynb
+notebooks/official/vizier/gapic-vizier-multi-objective-optimization.ipynb
+notebooks/official/pipelines/get_started_with_hpt_pipeline_components.ipynb
+notebooks/official/pipelines/google_cloud_pipeline_components_automl_tabular.ipynb
+notebooks/official/pipelines/custom_tabular_train_batch_pred_bq_pipeline.ipynb
+notebooks/official/pipelines/metrics_viz_run_compare_kfp.ipynb
+notebooks/official/pipelines/google_cloud_pipeline_components_model_upload_predict_evaluate.ipynb
+notebooks/official/pipelines/google_cloud_pipeline_components_model_train_upload_deploy.ipynb
+notebooks/official/pipelines/get_started_with_machine_management.ipynb
+notebooks/official/pipelines/custom_model_training_and_batch_prediction.ipynb
+notebooks/official/pipelines/control_flow_kfp.ipynb
+notebooks/official/pipelines/lightweight_functions_component_io_kfp.ipynb
+notebooks/official/pipelines/google_cloud_pipeline_components_bqml_text.ipynb
+notebooks/official/pipelines/pipelines_intro_kfp.ipynb
+notebooks/official/pipelines/automl_tabular_classification_beans.ipynb
+notebooks/official/pipelines/google_cloud_pipeline_components_dataproc_tabular.ipynb
+notebooks/official/explainable_ai/sdk_automl_tabular_classification_online_explain.ipynb
+notebooks/official/explainable_ai/sdk_custom_tabular_regression_online_explain.ipynb
+notebooks/official/explainable_ai/sdk_automl_tabular_binary_classification_batch_explain.ipynb
+notebooks/official/explainable_ai/sdk_custom_tabular_regression_online_explain_get_metadata.ipynb
+notebooks/official/explainable_ai/sdk_custom_tabular_regression_batch_explain.ipynb
+notebooks/official/tabular_workflows/prophet_on_vertex_pipelines.ipynb
+notebooks/official/tabular_workflows/wide_and_deep_on_vertex_pipelines.ipynb
+notebooks/official/sdk/SDK_AutoML_Video_Classification.ipynb
+notebooks/official/model_monitoring/get_started_with_model_monitoring_automl.ipynb
+notebooks/official/model_monitoring/get_started_with_model_monitoring_automl_image_batch.ipynb
+notebooks/official/model_monitoring/get_started_with_model_monitoring_automl_image_online.ipynb
+notebooks/official/model_monitoring/get_started_with_model_monitoring_xgboost.ipynb
+notebooks/official/tensorboard/tensorboard_custom_training_with_custom_container.ipynb
+notebooks/official/tensorboard/tensorboard_custom_training_with_prebuilt_container.ipynb
+notebooks/official/tensorboard/tensorboard_vertex_ai_pipelines_integration.ipynb
+notebooks/official/model_evaluation/automl_text_classification_model_evaluation.ipynb
+notebooks/official/model_evaluation/get_started_with_custom_model_evaluation_import.ipynb
+notebooks/official/model_evaluation/automl_tabular_classification_model_evaluation.ipynb
+notebooks/official/model_evaluation/automl_tabular_regression_model_evaluation.ipynb
+notebooks/official/experiments/get_started_with_vertex_experiments.ipynb
+notebooks/official/experiments/comparing_pipeline_runs.ipynb
+notebooks/official/experiments/get_started_with_vertex_experiments_autologging.ipynb
+notebooks/official/experiments/build_model_experimentation_lineage_with_prebuild_code.ipynb
+notebooks/official/experiments/delete_outdated_tensorboard_experiments.ipynb
+notebooks/official/automl/sdk_automl_tabular_regression_batch_bq.ipynb
+notebooks/official/automl/sdk_automl_text_sentiment_analysis_online.ipynb
+notebooks/official/automl/sdk_automl_text_entity_extraction_online.ipynb
+notebooks/official/automl/sdk_automl_forecasting_hierarchical_batch.ipynb
+notebooks/official/automl/automl_text_entity_extraction_batch_prediction.ipynb
+notebooks/official/automl/automl_image_classification_batch_prediction.ipynb
+notebooks/official/automl/automl_text_sentiment_analysis_batch_prediction.ipynb
+notebooks/official/automl/sdk_automl_tabular_regression_online_bq.ipynb
+notebooks/official/automl/get_started_automl_training.ipynb
+notebooks/official/automl/automl-tabular-classification.ipynb
+notebooks/official/automl/automl_image_object_detection_export_edge.ipynb
+notebooks/official/automl/sdk_automl_image_object_detection_batch.ipynb
+notebooks/official/automl/automl_tabular_on_vertex_pipelines.ipynb
+notebooks/official/automl/sdk_automl_video_classification_batch.ipynb
+notebooks/official/automl/sdk_automl_video_action_recognition_batch.ipynb
+notebooks/official/automl/sdk_automl_tabular_forecasting_batch.ipynb
+notebooks/official/automl/automl_image_object_detection_online_prediction.ipynb
+notebooks/official/automl/automl_forecasting_bqml_arima_plus_comparison.ipynb
+notebooks/official/datasets/get_started_bq_datasets.ipynb
+notebooks/official/datasets/get_started_with_data_labeling.ipynb
+notebooks/official/feature_store/feature_store_streaming_ingestion_sdk.ipynb

.cloud-build/test_single_notebook_vm.txt

@@ -0,0 +1 @@
+notebooks/official/custom/custom-tabular-bq-managed-dataset.ipynb

.cloud-build/tests/flaky.py

@@ -0,0 +1,46 @@
+# grep PASSED tests.txt | cut -c 10-100 >passed.txt
+
+import os
+
+repo_dir = '/home/jupyter/vertex-ai-samples/'
+repo_dir_len = len(repo_dir)
+official_dir = repo_dir + 'notebooks/official'
+
+entries = os.scandir(official_dir)
+folders = []
+for entry in entries:
+    if entry.is_dir():
+        folders.append(entry.path)
+
+# Passing
+with open('passed.txt', 'r') as pass_file:
+    notebook_names = pass_file.readlines()
+        
+notebooks = []
+for folder in folders:
+    entries = os.scandir(folder)
+    for entry in entries:
+        for notebook in notebook_names:
+            if entry.name == notebook.rstrip():
+                notebooks.append(entry.path[repo_dir_len:])
+                
+with open('passing_tests.txt', 'w') as f:
+    for notebook in notebooks:
+        f.write(notebook + '\n')
+        
+        
+# Failing
+with open('failed.txt', 'r') as fail_file:
+    notebook_names = fail_file.readlines()
+        
+notebooks = []
+for folder in folders:
+    entries = os.scandir(folder)
+    for entry in entries:
+        for notebook in notebook_names:
+            if entry.name == notebook.rstrip():
+                notebooks.append(entry.path[repo_dir_len:])
+                
+with open('failing_tests.txt', 'w') as f:
+    for notebook in notebooks:
+        f.write(notebook + '\n')

.cloud-build/tests/python_version_test.ipynb

@@ -0,0 +1,61 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "57a3d44ed8a8"
+      },
+      "source": [
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**_NOTE_**: This notebook has been tested in the following environment:\n",
+        "\n",
+        "* Python version = 3.7\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI API and Compute Engine API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component).\n",
+        "\n",
+        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "c6516f90311b"
+      },
+      "outputs": [],
+      "source": [
+        "# test if the right python version is being used\n",
+        "import sys\n",
+        "\n",
+        "actual_python_version = f\"{sys.version_info.major}.{sys.version_info.minor}\"\n",
+        "print(f\"Runtime python version: {actual_python_version}\")\n",
+        "\n",
+        "assert actual_python_version == \"3.7\", \"Wrong python version!\""
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "python_version_test.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

.cloud-build/utils/NotebookProcessors.py

@@ -13,9 +13,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from nbconvert.preprocessors import Preprocessor
 from typing import Dict
-import UpdateNotebookVariables
+import random
+import string
+
+from nbconvert.preprocessors import Preprocessor
+
+from . import UpdateNotebookVariables as update_notebook_variables
 
 
 class RemoveNoExecuteCells(Preprocessor):
@@ -41,7 +45,7 @@ class UpdateVariablesPreprocessor(Preprocessor):
         # VARIABLE_NAME = '[description]'
 
         for variable_name, variable_value in replacement_map.items():
-            content = UpdateNotebookVariables.get_updated_value(
+            content = update_notebook_variables.get_updated_value(
                 content=content,
                 variable_name=variable_name,
                 variable_value=variable_value,
@@ -60,4 +64,37 @@ class UpdateVariablesPreprocessor(Preprocessor):
 
             executable_cells.append(cell)
         notebook.cells = executable_cells
-        return notebook, resources
+        return notebook, resources
+
+
+# Generate a uuid of a specifed length
+def generate_uuid(length: int = 8) -> str:
+    return "".join(random.choices(string.ascii_lowercase + string.digits, k=length))
+
+
+class UniqueStringsPreprocessor(Preprocessor):
+    # A preprocessor that replaces strings that end with "-unique" or "_unique" with a uuid.
+
+    @staticmethod
+    def update_unique_strings(content: str):
+        # Replace strings that end with "-unique" or "_unique" with a uuid.
+
+        unique_id = generate_uuid()
+        return (
+            content.replace('-unique"', f'-{unique_id}"')
+            .replace("-unique'", f'-{unique_id}"')
+            .replace('_unique"', f'_{unique_id}"')
+            .replace("_unique'", f'_{unique_id}"')
+        )
+
+    def preprocess(self, notebook, resources=None):
+        executable_cells = []
+        for cell in notebook.cells:
+            if cell.cell_type == "code":
+                cell.source = self.update_unique_strings(
+                    content=cell.source,
+                )
+
+            executable_cells.append(cell)
+        notebook.cells = executable_cells
+        return notebook, resources

.cloud-build/utils/UpdateNotebookVariables.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+
+"""
+ This script is used to update variables in the notebook via regex
+ It requires variables to be defined in particular format
+
+For example, if your variable was PROJECT_ID, use:
+
+    PROJECT_ID = "[your_project_here]"
+
+Single-quotes also work:
+
+    PROJECT_ID = '[your_project_here]'
+
+Variables in conditionals can also be replaced:
+
+    PROJECT_ID == "[your_project_here]"
+"""
+
+
+def get_updated_value(content: str, variable_name: str, variable_value: str) -> str:
+    return re.sub(
+        rf"({variable_name}.*? = .*?[\",\'])\[.+?\]([\",\'].*?)",
+        rf"\g<1>{variable_value}\g<2>",
+        content,
+        flags=re.M,
+    )

.cloud-build/utils/tests/test_preprocessors.py

@@ -0,0 +1,14 @@
+from utils import NotebookProcessors
+
+
+def test_update_value():
+    # Test that the content was updated
+    preprocessor = NotebookProcessors.UniqueStringsPreprocessor()
+
+    content = 'PROJECT_ID = "your-project-id-unique"'
+
+    new_content = preprocessor.update_unique_strings(content)
+
+    assert new_content != content
+    assert new_content.startswith('PROJECT_ID = "your-project-id-')
+    assert new_content.endswith('"')

.cloud-build/utils/tests/test_update_notebook_variables.py

@@ -0,0 +1,63 @@
+from utils import UpdateNotebookVariables
+
+
+def test_update_value():
+    new_content = UpdateNotebookVariables.get_updated_value(
+        content='asdf\nPROJECT_ID = "[your-project-id]" #@param {type:"string"} \nasdf',
+        variable_name="PROJECT_ID",
+        variable_value="sample-project",
+    )
+    assert (
+        new_content
+        == 'asdf\nPROJECT_ID = "sample-project" #@param {type:"string"} \nasdf'
+    )
+
+
+def test_update_value_single_quotes():
+    new_content = UpdateNotebookVariables.get_updated_value(
+        content="PROJECT_ID = '[your-project-id]'",
+        variable_name="PROJECT_ID",
+        variable_value="sample-project",
+    )
+    assert new_content == "PROJECT_ID = 'sample-project'"
+
+
+def test_update_value_avoidance():
+    new_content = UpdateNotebookVariables.get_updated_value(
+        content="PROJECT_ID = shell_output[0] ",
+        variable_name="PROJECT_ID",
+        variable_value="sample-project",
+    )
+    assert new_content == "PROJECT_ID = shell_output[0] "
+
+
+def test_region():
+    new_content = UpdateNotebookVariables.get_updated_value(
+        content='REGION = "[your-region]"  # @param {type:"string"}',
+        variable_name="REGION",
+        variable_value="us-central1",
+    )
+    assert new_content == 'REGION = "us-central1"  # @param {type:"string"}'
+
+
+def test_region_equal_equals_ignore():
+    # Tests that == is ignored
+    new_content = UpdateNotebookVariables.get_updated_value(
+        content='REGION == "[your-region]"  # @param {type:"string"}',
+        variable_name="REGION",
+        variable_value="us-central1",
+    )
+    assert new_content == 'REGION == "[your-region]"  # @param {type:"string"}'
+
+
+def test_service_account():
+    # Tests that == is ignored
+    new_content = UpdateNotebookVariables.get_updated_value(
+        content='SERVICE_ACCOUNT = "[your-service-account]"  # @param {type:"string"}',
+        variable_name="SERVICE_ACCOUNT",
+        variable_value="12345-compute@developer.gserviceaccount.com",
+    )
+    assert (
+        new_content
+        == 'SERVICE_ACCOUNT = "12345-compute@developer.gserviceaccount.com"  # @param {type:"string"}'
+    )

.cloud-build/utils/util.py

@@ -0,0 +1,86 @@
+import os
+import subprocess
+import tarfile
+import uuid
+from datetime import datetime
+from typing import Optional, Union
+
+from google.auth import credentials as auth_credentials
+from google.cloud import storage
+from google.cloud.aiplatform import utils
+
+
+def download_file(bucket_name: str, blob_name: str, destination_file: str) -> str:
+    """Copies a remote GCS file to a local path"""
+    remote_file_path = "".join(["gs://", "/".join([bucket_name, blob_name])])
+
+    subprocess.check_output(
+        ["gsutil", "cp", remote_file_path, destination_file], encoding="UTF-8"
+    )
+
+    return destination_file
+
+
+def upload_file(
+    local_file_path: str,
+    remote_file_path: str,
+) -> str:
+    """Copies a local file to a GCS path"""
+    subprocess.check_output(
+        ["gsutil", "cp", local_file_path, remote_file_path], encoding="UTF-8"
+    )
+
+    return remote_file_path
+
+
+def archive_code_and_upload(staging_bucket: str):
+    # Archive all source in current directory
+    unique_id = uuid.uuid4()
+    source_archived_file = f"source_archived_{unique_id}.tar.gz"
+
+    git_files = subprocess.check_output(
+        ["git", "ls-tree", "-r", "HEAD", "--name-only"], encoding="UTF-8"
+    ).split("\n")
+
+    with tarfile.open(source_archived_file, "w:gz") as tar:
+        for file in git_files:
+            if len(file) > 0 and os.path.exists(file):
+                tar.add(file)
+
+    # Upload archive to GCS bucket
+    source_archived_file_gcs = upload_file(
+        local_file_path=f"{source_archived_file}",
+        remote_file_path="/".join(
+            [staging_bucket, "code_archives", source_archived_file]
+        ),
+    )
+
+    print(f"Uploaded source code archive to {source_archived_file_gcs}")
+
+    return source_archived_file_gcs
+
+
+def download_blob_into_memory(
+    bucket_name: str, blob_name: str, download_as_text: Optional[bool] = False
+) -> Union[bytes, str]:
+    """
+    Downloads a blob into memory as byte or as text if
+    download_as_text is set to True.
+    """
+
+    storage_client = storage.Client()
+
+    bucket = storage_client.bucket(bucket_name)
+
+    # Construct a client side representation of a blob.
+    blob = bucket.blob(blob_name)
+
+    # Download the blob content
+    if download_as_text:
+        contents = blob.download_as_text()
+    else:
+        contents = blob.download_as_bytes()
+
+    print(f"Downloaded storage object {blob_name} from bucket {bucket_name}.")
+
+    return contents

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,17 +1,28 @@
-If you are opening a PR for `Official Notebooks` under the [notebooks/official](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/master/notebooks/official) folder, follow this mandatory checklist:
-- [ ] Use the [notebook template](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/notebook_template.ipynb) as a starting point.
+**REQUIRED:** Add a summary of your PR here, typically including why the change is needed and what was changed. Include any design alternatives for discussion purposes.
+
+<br>
+--- YOUR PR SUMMARY GOES HERE ---
+<br><br><br>
+
+**REQUIRED:** Fill out the below checklists or remove if irrelevant
+1. If you are opening a PR for `Official Notebooks` under the [notebooks/official](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/notebooks/official) folder, follow this mandatory checklist:
+- [ ] Use the [notebook template](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/notebook_template.ipynb) as a starting point.
 - [ ] Follow the style and grammar rules outlined in the above notebook template.
 - [ ] Verify the notebook runs successfully in Colab since the automated tests cannot guarantee this even when it passes.
-- [ ] Passes all the required automated checks
+- [ ] Passes all the required automated checks. You can locally test for formatting and linting with these [instructions](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/CONTRIBUTING.md#code-quality-checks).
 - [ ] You have consulted with a tech writer to see if tech writer review is necessary. If so, the notebook has been reviewed by a tech writer, and they have approved it.
-- [ ] This notebook has been added to the [CODEOWNERS](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/docs/CODEOWNERS) file under `# Official Notebooks` section, pointing to the author or the author's team.
+- [ ] This notebook has been added to the [CODEOWNERS](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/CODEOWNERS) file under the `Official Notebooks` section, pointing to the author or the author's team.
 - [ ] The Jupyter notebook cleans up any artifacts it has created (datasets, ML models, endpoints, etc) so as not to eat up unnecessary resources.
 
+<br>
 
-If you are opening a PR for `Community Notebooks` under the [notebooks/community](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/master/notebooks/community) folder:
-- [ ] This notebook has been added to the [CODEOWNERS](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/docs/CODEOWNERS) file under the `# Community Notebooks` section, pointing to the author or the author's team.
+2. If you are opening a PR for `Community Notebooks` under the [notebooks/community](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/notebooks/community) folder:
+- [ ] This notebook has been added to the [CODEOWNERS](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/CODEOWNERS) file under the `Community Notebooks` section, pointing to the author or the author's team.
+- [ ] Passes all the required formatting and linting checks. You can locally test with these [instructions](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/CONTRIBUTING.md#code-quality-checks).
 
+<br>
 
-If you are opening a PR for `Community Content` under the [community-content](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/master/community-content) folder:
+3. If you are opening a PR for `Community Content` under the [community-content](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/community-content) folder:
 - [ ] Make sure your main `Content Directory Name` is descriptive, informative, and includes some of the key products and attributes of your content, so that it is differentiable from other content
-- [ ] The main content directory has been added to the [CODEOWNERS](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/docs/CODEOWNERS) file under the `# Community Content` section, pointing to the author or the author's team.
+- [ ] The main content directory has been added to the [CODEOWNERS](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/community-content/CODEOWNERS) file under the `Community Content` section, pointing to the author or the author's team.
+- [ ] Passes all the required formatting and linting checks. You can locally test with these [instructions](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/CONTRIBUTING.md#code-quality-checks).

.github/workflows/ci.yaml

@@ -7,13 +7,15 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.x'      
     - name: Fetch pull request branch
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
       with:
         fetch-depth: 0      
-    - name: Fetch base master branch
-      run: git fetch -u "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY" master:master
+    - name: Fetch base main branch
+      run: git fetch -u "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY" main:main
     - name: Install requirements
       run: python3 -m pip install -U -r .github/workflows/linter/requirements.txt
     - name: Format and lint notebooks

.github/workflows/linter/Dockerfile

@@ -0,0 +1,20 @@
+# To use this image, run this command with the desired notebook args from the top-level vertex-ai-samples directory:
+# 1. To lint all changed notebooks:
+# docker run -v ${PWD}:/setup/app gcr.io/python-docs-samples-tests/notebook_linter:latest
+# 2. To lint specific notebooks:
+# docker run -v ${PWD}:/setup/app gcr.io/python-docs-samples-tests/notebook_linter:latest notebooks/1.ipynb notebooks/2.ipynb
+
+FROM python:3.10
+
+WORKDIR setup
+
+COPY ./requirements.txt .
+COPY ./run_linter.sh .
+
+# Install dependencies.
+RUN pip install --upgrade pip
+RUN pip install -r requirements.txt
+
+WORKDIR app
+
+ENTRYPOINT ["/setup/run_linter.sh"]

.github/workflows/linter/requirements.txt

@@ -2,8 +2,9 @@ git+https://github.com/tensorflow/docs
 ipython
 jupyter
 nbconvert
-black==20.8b1
-pyupgrade==2.7.3
-isort==5.6.4
-flake8==3.9.0
-nbqa==0.6.0
+black==22.12.0
+pyupgrade==3.4.0
+isort==5.12.0
+flake8==6.0.0
+nbqa==1.5.3
+

.github/workflows/linter/run_linter.sh

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# This script automatically formats and lints all notebooks that have changed from the head of the master branch.
+# This script automatically formats and lints all notebooks that have changed from the head of the main branch.
 #
 # Options:
 # -t: Test-mode. Only test if format and linting are required but make no changes to files.
@@ -47,12 +47,22 @@ done
 
 echo "Test mode: $is_test"
 
+# Read in user-provided notebooks
+notebooks=()
+for arg in "$@"; do
+    if [[ $arg == *.ipynb ]]; then
+        notebooks+=("$arg")
+    fi
+done
+
 # Only check notebooks in test folders modified in this pull request.
 # Note: Use process substitution to persist the data in the array
-notebooks=()
-while read -r file || [ -n "$line" ]; do
-    notebooks+=("$file")
-done < <(git diff --name-only master... | grep '\.ipynb$')
+if [ ${#notebooks[@]} -eq 0 ]; then
+    echo "Checking for changed notebooked using git"
+    while read -r file || [ -n "$line" ]; do
+        notebooks+=("$file")
+    done < <(git diff --name-only main... | grep '\.ipynb$')
+fi
 
 problematic_notebooks=()
 if [ ${#notebooks[@]} -gt 0 ]; then
@@ -68,7 +78,7 @@ if [ ${#notebooks[@]} -gt 0 ]; then
 
             if [ "$is_test" = true ]; then
                 echo "Running nbfmt..."
-                python3 -m tensorflow_docs.tools.nbfmt --remove_outputs --test "$notebook"
+                python3 -m tensorflow_docs.tools.nbfmt --test "$notebook"
                 NBFMT_RTN=$?
                 # echo "Running black..."
                 # python3 -m nbqa black "$notebook" --check
@@ -84,19 +94,19 @@ if [ ${#notebooks[@]} -gt 0 ]; then
                 FLAKE8_RTN=$?
             else
                 echo "Running black..."
-                python3 -m nbqa black "$notebook" --nbqa-mutate
+                python3 -m nbqa black "$notebook"
                 BLACK_RTN=$?
                 echo "Running pyupgrade..."
-                python3 -m nbqa pyupgrade "$notebook" --nbqa-mutate
+                python3 -m nbqa pyupgrade "$notebook"
                 PYUPGRADE_RTN=$?
                 echo "Running isort..."
-                python3 -m nbqa isort "$notebook" --nbqa-mutate
+                python3 -m nbqa isort "$notebook"
                 ISORT_RTN=$?
                 echo "Running nbfmt..."
-                python3 -m tensorflow_docs.tools.nbfmt --remove_outputs "$notebook"
+                python3 -m tensorflow_docs.tools.nbfmt "$notebook"
                 NBFMT_RTN=$?
                 echo "Running flake8..."
-                python3 -m nbqa flake8 "$notebook" --show-source --extend-ignore=W391,E501,F821,E402,F404,W503,E203,E722,W293,W291 --nbqa-mutate
+                python3 -m nbqa flake8 "$notebook" --show-source --extend-ignore=W391,E501,F821,E402,F404,W503,E203,E722,W293,W291
                 FLAKE8_RTN=$?
             fi
 

CODEOWNERS

@@ -0,0 +1,6 @@
+# See https://help.github.com/en/articles/about-code-owners
+# for more info about CODEOWNERS file.
+
+# These owners will be the default owners for everything in
+# the repo. Unless a later match takes precedence.
+*  @GoogleCloudPlatform/vertex-ai-samples-owners

CODE_OF_CONDUCT.md

@@ -0,0 +1,43 @@
+# Contributor Code of Conduct
+
+As contributors and maintainers of this project,
+and in the interest of fostering an open and welcoming community,
+we pledge to respect all people who contribute through reporting issues,
+posting feature requests, updating documentation,
+submitting pull requests or patches, and other activities.
+
+We are committed to making participation in this project
+a harassment-free experience for everyone,
+regardless of level of experience, gender, gender identity and expression,
+sexual orientation, disability, personal appearance,
+body size, race, ethnicity, age, religion, or nationality.
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery
+* Personal attacks
+* Trolling or insulting/derogatory comments
+* Public or private harassment
+* Publishing other's private information,
+such as physical or electronic
+addresses, without explicit permission
+* Other unethical or unprofessional conduct.
+
+Project maintainers have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct.
+By adopting this Code of Conduct,
+project maintainers commit themselves to fairly and consistently
+applying these principles to every aspect of managing this project.
+Project maintainers who do not follow or enforce the Code of Conduct
+may be permanently removed from the project team.
+
+This code of conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community.
+
+Instances of abusive, harassing, or otherwise unacceptable behavior
+may be reported by opening an issue
+or contacting one or more of the project maintainers.
+
+This Code of Conduct is adapted from the [Contributor Covenant](http://contributor-covenant.org), version 1.2.0,
+available at [http://contributor-covenant.org/version/1/2/0/](http://contributor-covenant.org/version/1/2/0/)

CONTRIBUTING.md

@@ -15,6 +15,41 @@ You generally only need to submit a CLA once, so if you've already submitted one
 (even if it was for a different project), you probably don't need to do it
 again.
 
+## Code Quality Checks
+
+All notebooks in this project are checked for formatting and style, to ensure a
+consistent experience. To test notebooks prior to submitting a pull request,
+you can follow these steps.
+
+From a command-line terminal (e.g. from Vertex Workbench or locally), install
+the code analysis tools:
+
+```shell
+pip3 install --user -U nbqa black flake8 isort pyupgrade git+https://github.com/tensorflow/docs
+```
+
+You'll likely need to add the directory where these were installed to your PATH:
+
+```shell
+export PATH="$HOME/.local/bin:$PATH"
+```
+
+Then, set an environment variable for your notebook (or directory):
+
+```shell
+export notebook="your-notebook.ipynb"
+```
+
+Finally, run this code block to check for errors. Each step will attempt to
+automatically fix any issues. If the fixes can't be performed automatically,
+then you will need to manually address them before submitting your PR.
+
+Note: For official, only submit one notebook per PR.
+
+```shell
+docker run -v ${PWD}:/setup/app gcr.io/cloud-devrel-public-resources/notebook_linter:latest your_notebook
+```
+
 ## Code Reviews
 
 All submissions, including submissions by project members, require review. We

README.md

@@ -6,15 +6,32 @@ Welcome to the Google Cloud [Vertex AI](https://cloud.google.com/vertex-ai/docs/
 
 ## Overview
 
-The repository contains [Notebooks](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/notebooks) and [Community Content](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/master/community-content) that demonstrate how to develop and manage ML workflows using Google Cloud Vertex AI. 
+The repository contains [notebooks](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/master/notebooks) and [community content](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/master/community-content) that demonstrate how to develop and manage ML workflows using Google Cloud Vertex AI.
+
+## Repository structure
+
+```bash
+├── community-content - Sample code and tutorials contributed by the community
+├── notebooks
+│   ├── community - Notebooks contributed by the community
+│   ├── official - Notebooks demonstrating use of each Vertex AI service
+│   │   ├── automl
+│   │   ├── custom
+│   │   ├── ...
+```
 
 ## Contributing
 
-Contributions welcome! See the [Contributing Guide](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/docs/contributing.md).
+Contributions welcome! See the [Contributing Guide](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/CONTRIBUTING.md).
 
 ## Getting help
 
 Please use the [issues page](https://github.com/GoogleCloudPlatform/vertex-ai-samples/issues) to provide feedback or submit a bug report.
 
 ## Disclaimer
+
 This is not an officially supported Google product. The code in this repository is for demonstrative purposes only.
+
+## Feedback
+
+Please feel free to fill out our [survey](https://bit.ly/vertex-ai-samples-survey) to give us feedback on the repo and its content.

SECURITY.md

@@ -0,0 +1,7 @@
+# Security Policy
+
+To report a security issue, please use [g.co/vulnz](https://g.co/vulnz).
+
+The Google Security Team will respond within 5 working days of your report on g.co/vulnz.
+
+We use g.co/vulnz for our intake, and do coordination and disclosure here using GitHub Security Advisory to privately discuss and fix the issue.

community-content/CODEOWNERS

@@ -0,0 +1,12 @@
+*  @vertex-ai-samples-contributors  @GoogleCloudPlatform/cloudml-samples-owners
+/tf_agents_bandits_movie_recommendation_with_kfp_and_vertex_sdk	@yinghsienwu
+/pytorch_pre_built_images_deployment @googleapis/vertex-prediction-team
+/pytorch_text_classification_using_vertex_sdk_and_gcloud  @RajeshThallam
+/pytorch_text_classification_using_vertex_sdk_and_gcloud		@RajeshThallam @ultrons
+/sklearn_text_classification_from_script_using_vertex_sdk       @maxhardt
+/pluto_on_workbench @wkharold
+/cpr-examples @samthrasher
+/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines @Ark-kun
+/pipeline_components @Ark-kun
+/pipeline_components/image_ml_model_training @lakeyk
+/prediction_featurestore_integration @googleapis/vertex-prediction-team

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_logistic_regression_model_using_Scikit_learn_and_import_to_Vertex_AI/pipeline.component.yaml

@@ -0,0 +1,83 @@
+name: Train tabular classification logistic regression model using Scikit learn pipeline
+metadata:
+  annotations:
+    author: Alexey Volkov <alexey.volkov@ark-kun.com>
+    canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_classification_logistic_regression_model_using_Scikit_learn_and_import_to_Vertex_AI/pipeline.component.yaml
+    sdk: https://cloud-pipelines.net/pipeline-editor/
+implementation:
+  graph:
+    tasks:
+      Download from GCS:
+        componentRef:
+          digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
+        arguments:
+          GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
+        annotations:
+          editor.position: '{"x":40,"y":40,"width":180,"height":40}'
+      Select columns using Pandas on CSV data:
+        componentRef:
+          digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: Data
+              taskId: Download from GCS
+          column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
+        annotations:
+          editor.position: '{"x":40,"y":140,"width":180,"height":54}'
+      Fill all missing values using Pandas on CSV data:
+        componentRef:
+          digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: transformed_table
+              taskId: Select columns using Pandas on CSV data
+              type: CSV
+          replacement_value: '0'
+        annotations:
+          editor.position: '{"x":40,"y":250,"width":180,"height":54}'
+      Binarize column using Pandas on CSV data:
+        componentRef:
+          digest: d699afd4d7cae862708717cc160f4394ed0c04e536e9515923ef1e8865f01d44
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: transformed_table
+              taskId: Fill all missing values using Pandas on CSV data
+              type: CSV
+          column_name: tips
+          predicate: '> 0'
+          new_column_name: class
+        annotations:
+          editor.position: '{"x":40,"y":380,"width":180,"height":54}'
+      Train logistic regression model using scikit learn from CSV:
+        componentRef:
+          digest: a864625a822e4b1c8ef6fe4ae1454fd90f15438f70a6712bb4c30e0dda4d35b7
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/cb44b75c9c062fcc40c2b905b2024b4493dbc62b/components/ML_frameworks/Scikit_learn/Train_logistic_regression_model/from_CSV/component.yaml
+        arguments:
+          dataset:
+            taskOutput:
+              outputName: transformed_table
+              taskId: Binarize column using Pandas on CSV data
+              type: CSV
+          label_column_name: class
+        annotations:
+          editor.position: '{"x":40,"y":510,"width":180,"height":70}'
+      Upload Scikit learn pickle model to Google Cloud Vertex AI:
+        componentRef:
+          digest: 81c91c8d7d21ec97e0872f669d68bd89edea87279d703685db54aa94743bebcd
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/workaround_for_buggy_KFPv2_compiler/component.yaml
+        arguments:
+          model:
+            taskOutput:
+              outputName: model
+              taskId: Train logistic regression model using scikit learn from CSV
+              type: ScikitLearnPickleModel
+        annotations:
+          editor.position: '{"x":40,"y":660,"width":180,"height":70}'
+    outputValues: {}

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_logistic_regression_model_using_Scikit_learn_and_import_to_Vertex_AI/pipeline.py

@@ -0,0 +1,73 @@
+# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
+from kfp import components
+
+# %% Loading components
+download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/storage/download/component.yaml")
+select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml")
+fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
+binarize_column_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Binarize_column/in_CSV_format/component.yaml")
+train_logistic_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/1f5cf6e06409b704064b2086c0a705e4e6b4fcde/community-content/pipeline_components/ML_frameworks/Scikit_learn/Train_logistic_regression_model/from_CSV/component.yaml")
+upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/component.yaml")
+deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
+
+# %% Pipeline definition
+def train_tabular_classification_logistic_regression_model_using_Scikit_learn_pipeline():
+    dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
+    feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]  # Excluded "trip_total"
+    label_column = "tips"
+    # Deploying the model might incur additional costs over time
+    deploy_model = False
+
+    classification_label_column = "class"
+    all_columns = [label_column] + feature_columns
+
+    training_data = download_from_gcs_op(
+        gcs_path=dataset_gcs_uri
+    ).outputs["Data"]
+
+    training_data = select_columns_using_Pandas_on_CSV_data_op(
+        table=training_data,
+        column_names=all_columns,
+    ).outputs["transformed_table"]
+
+    # Cleaning the NaN values.
+    training_data = fill_all_missing_values_using_Pandas_on_CSV_data_op(
+        table=training_data,
+        replacement_value="0",
+        #replacement_type_name="float",
+    ).outputs["transformed_table"]
+
+    classification_training_data = binarize_column_using_Pandas_on_CSV_data_op(
+        table=training_data,
+        column_name=label_column,
+        predicate="> 0",
+        new_column_name=classification_label_column,
+    ).outputs["transformed_table"]
+
+    model = train_logistic_regression_model_using_scikit_learn_from_CSV_op(
+        dataset=classification_training_data,
+        label_column_name=classification_label_column,
+        # Optional:
+        #penalty="l2",
+        #solver="lbfgs",
+        #max_iterations=100,
+        #multi_class_mode="auto",
+        #random_seed=0,
+    ).outputs["model"]
+
+    vertex_model_name = upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op(
+        model=model,
+    ).outputs["model_name"]
+
+    # Deploying the model might incur additional costs over time
+    if deploy_model:
+        sklearn_vertex_endpoint_name = deploy_model_to_endpoint_op(
+            model_name=vertex_model_name,
+        ).outputs["endpoint_name"]
+
+pipeline_func = train_tabular_classification_logistic_regression_model_using_Scikit_learn_pipeline
+
+# %% Pipeline submission
+if __name__ == '__main__':
+    from google.cloud import aiplatform
+    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_model_using_PyTorch_and_import_to_Vertex_AI/pipeline.component.yaml

@@ -0,0 +1,114 @@
+name: Train tabular classification model using PyTorch pipeline
+metadata:
+  annotations:
+    author: Alexey Volkov <alexey.volkov@ark-kun.com>
+    canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_classification_model_using_PyTorch_and_import_to_Vertex_AI/pipeline.component.yaml
+    sdk: https://cloud-pipelines.net/pipeline-editor/
+implementation:
+  graph:
+    tasks:
+      Download from GCS:
+        componentRef:
+          digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
+        arguments:
+          GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
+        annotations:
+          editor.position: '{"x":240,"y":40,"width":180,"height":40}'
+      Select columns using Pandas on CSV data:
+        componentRef:
+          digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: Data
+              taskId: Download from GCS
+          column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
+        annotations:
+          editor.position: '{"x":240,"y":140,"width":180,"height":54}'
+      Fill all missing values using Pandas on CSV data:
+        componentRef:
+          digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: transformed_table
+              taskId: Select columns using Pandas on CSV data
+              type: CSV
+          replacement_value: '0'
+        annotations:
+          editor.position: '{"x":240,"y":250,"width":180,"height":54}'
+      Create fully connected pytorch network:
+        componentRef:
+          digest: d03d8248fd358a0275ec33568ee7dd7dce576cc112b09dfafe2651e4d97e04a9
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1a2ef3eeb77bc278f33cad0dd29008ea2431e191/components/PyTorch/Create_fully_connected_network/component.yaml
+        arguments:
+          input_size: '7'
+          hidden_layer_sizes: '[10]'
+          activation_name: elu
+          output_activation_name: sigmoid
+        annotations:
+          editor.position: '{"x":40,"y":360,"width":180,"height":54}'
+      Binarize column using Pandas on CSV data:
+        componentRef:
+          digest: d699afd4d7cae862708717cc160f4394ed0c04e536e9515923ef1e8865f01d44
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: transformed_table
+              taskId: Fill all missing values using Pandas on CSV data
+              type: CSV
+          column_name: tips
+          predicate: ' > 0'
+          new_column_name: class
+        annotations:
+          editor.position: '{"x":240,"y":360,"width":180,"height":54}'
+      Train pytorch model from csv:
+        componentRef:
+          digest: 40f3185eb61e9727f41a4e0c05dd3d3b44bd802aa0f378cfc31756560033949a
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml
+        arguments:
+          model:
+            taskOutput:
+              outputName: model
+              taskId: Create fully connected pytorch network
+              type: PyTorchScriptModule
+          training_data:
+            taskOutput:
+              outputName: transformed_table
+              taskId: Binarize column using Pandas on CSV data
+              type: CSV
+          label_column_name: class
+          loss_function_name: binary_cross_entropy
+        annotations:
+          editor.position: '{"x":240,"y":490,"width":180,"height":40}'
+      Create PyTorch Model Archive with base handler:
+        componentRef:
+          digest: 8298b5ee1b0f0879f893add4cf352c8dec7cf9e21bb9db134c91a2d046cdb0ec
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/46d51383e6554b7f3ab4fd8cf614d8c2b422fb22/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml
+        arguments:
+          Model:
+            taskOutput:
+              outputName: trained_model
+              taskId: Train pytorch model from csv
+              type: PyTorchScriptModule
+          Model name: model
+          Model version: '1.0'
+        annotations:
+          editor.position: '{"x":240,"y":590,"width":180,"height":54}'
+      Upload PyTorch model archive to Google Cloud Vertex AI:
+        componentRef:
+          digest: 4450212fae7b9001482aca7eb78b28413c205506eccf08a04e7754a8dfa99004
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/workaround_for_buggy_KFPv2_compiler/component.yaml
+        arguments:
+          model_archive:
+            taskOutput:
+              outputName: Model archive
+              taskId: Create PyTorch Model Archive with base handler
+              type: PyTorchModelArchive
+        annotations:
+          editor.position: '{"x":240,"y":720,"width":180,"height":70}'
+    outputValues: {}

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_model_using_PyTorch_and_import_to_Vertex_AI/pipeline.py

@@ -0,0 +1,95 @@
+# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
+from kfp import components
+
+# %% Loading components
+download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/storage/download/component.yaml")
+select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml")
+fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
+binarize_column_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Binarize_column/in_CSV_format/component.yaml")
+create_fully_connected_pytorch_network_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Create_fully_connected_network/component.yaml")
+train_pytorch_model_from_csv_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml")
+create_pytorch_model_archive_with_base_handler_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml")
+upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/component.yaml")
+deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
+
+# %% Pipeline definition
+def train_tabular_classification_model_using_PyTorch_pipeline():
+    dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
+    feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]  # Excluded "trip_total"
+    label_column = "tips"
+    # Deploying the model might incur additional costs over time
+    deploy_model = False
+
+    classification_label_column = "class"
+    all_columns = [label_column] + feature_columns
+
+    training_data = download_from_gcs_op(
+        gcs_path=dataset_gcs_uri
+    ).outputs["Data"]
+
+    training_data = select_columns_using_Pandas_on_CSV_data_op(
+        table=training_data,
+        column_names=all_columns,
+    ).outputs["transformed_table"]
+
+    # Cleaning the NaN values.
+    training_data = fill_all_missing_values_using_Pandas_on_CSV_data_op(
+        table=training_data,
+        replacement_value="0",
+        #replacement_type_name="float",
+    ).outputs["transformed_table"]
+
+    classification_training_data = binarize_column_using_Pandas_on_CSV_data_op(
+        table=training_data,
+        column_name=label_column,
+        predicate=" > 0",
+        new_column_name=classification_label_column,
+    ).outputs["transformed_table"]
+
+    network = create_fully_connected_pytorch_network_op(
+        input_size=len(feature_columns),
+        # Optional:
+        hidden_layer_sizes=[10],
+        activation_name="elu",
+        output_activation_name="sigmoid",
+        # output_size=1,
+    ).outputs["model"]
+
+    model = train_pytorch_model_from_csv_op(
+        model=network,
+        training_data=classification_training_data,
+        label_column_name=classification_label_column,
+        loss_function_name="binary_cross_entropy",
+        # Optional:
+        #number_of_epochs=1,
+        #learning_rate=0.1,
+        #optimizer_name="Adadelta",
+        #optimizer_parameters={},
+        #batch_size=32,
+        #batch_log_interval=100,
+        #random_seed=0,
+    ).outputs["trained_model"]
+
+    model_archive = create_pytorch_model_archive_with_base_handler_op(
+        model=model,
+        # Optional:
+        # model_name="model",
+        # model_version="1.0",
+    ).outputs["Model archive"]
+
+    vertex_model_name = upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op(
+        model_archive=model_archive,
+    ).outputs["model_name"]
+
+    # Deploying the model might incur additional costs over time
+    if deploy_model:
+        vertex_endpoint_name = deploy_model_to_endpoint_op(
+            model_name=vertex_model_name,
+        ).outputs["endpoint_name"]
+
+pipeline_func=train_tabular_classification_model_using_PyTorch_pipeline
+
+# %% Pipeline submission
+if __name__ == '__main__':
+    from google.cloud import aiplatform
+    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_model_using_TensorFlow_and_import_to_Vertex_AI/pipeline.component.yaml

@@ -0,0 +1,132 @@
+name: Train tabular classification model using TensorFlow pipeline
+metadata:
+  annotations:
+    author: Alexey Volkov <alexey.volkov@ark-kun.com>
+    canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_classification_model_using_TensorFlow_and_import_to_Vertex_AI/pipeline.component.yaml
+    sdk: https://cloud-pipelines.net/pipeline-editor/
+implementation:
+  graph:
+    tasks:
+      Download from GCS:
+        componentRef:
+          digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
+        arguments:
+          GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
+        annotations:
+          editor.position: '{"x":40,"y":40,"width":180,"height":40}'
+      Select columns using Pandas on CSV data:
+        componentRef:
+          digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: Data
+              taskId: Download from GCS
+          column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
+        annotations:
+          editor.position: '{"x":40,"y":140,"width":180,"height":54}'
+      Fill all missing values using Pandas on CSV data:
+        componentRef:
+          digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: transformed_table
+              taskId: Select columns using Pandas on CSV data
+              type: CSV
+          replacement_value: '0'
+        annotations:
+          editor.position: '{"x":40,"y":250,"width":180,"height":54}'
+      Binarize column using Pandas on CSV data:
+        componentRef:
+          digest: d699afd4d7cae862708717cc160f4394ed0c04e536e9515923ef1e8865f01d44
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: transformed_table
+              taskId: Fill all missing values using Pandas on CSV data
+              type: CSV
+          column_name: tips
+          predicate: ' > 0'
+          new_column_name: class
+        annotations:
+          editor.position: '{"x":40,"y":370,"width":180,"height":54}'
+      Split rows into subsets:
+        componentRef:
+          digest: a609c3c9196484290f24a1174955f95b27f07a7b458aa5cb8cde28866cb2cb46
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: transformed_table
+              taskId: Binarize column using Pandas on CSV data
+              type: CSV
+          fraction_1: '0.8'
+        annotations:
+          editor.position: '{"x":170,"y":500,"width":180,"height":40}'
+      Create fully connected tensorflow network:
+        componentRef:
+          digest: bfcafbc5ce711b1f69cabf1338212d10d50136a73db9f9f7c984de7b80b4bfb0
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/9ca0f9eecf5f896f65b8538bbd809747052617d1/components/tensorflow/Create_fully_connected_network/component.yaml
+        arguments:
+          input_size: '7'
+          hidden_layer_sizes: '[10]'
+          activation_name: elu
+          output_activation_name: sigmoid
+        annotations:
+          editor.position: '{"x":370,"y":500,"width":180,"height":54}'
+      Train model using Keras on CSV:
+        componentRef:
+          digest: 42ae60c889034dbad74815653e95b4f7d576b5f47f803173e8679c7b54984609
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c504a4010348c50eaaf6d4337586ccc008f4dcef/components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml
+        arguments:
+          training_data:
+            taskOutput:
+              outputName: split_1
+              taskId: Split rows into subsets
+              type: CSV
+          model:
+            taskOutput:
+              outputName: model
+              taskId: Create fully connected tensorflow network
+              type: TensorflowSavedModel
+          label_column_name: class
+          loss_function_name: binary_crossentropy
+          number_of_epochs: '10'
+        annotations:
+          editor.position: '{"x":40,"y":620,"width":180,"height":54}'
+      Upload Tensorflow model to Google Cloud Vertex AI:
+        componentRef:
+          digest: 2e45263ff640b1a688e359b6936e27a81b2407749a84f340af2aa5547e0cb92c
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/workaround_for_buggy_KFPv2_compiler/component.yaml
+        arguments:
+          model:
+            taskOutput:
+              outputName: trained_model
+              taskId: Train model using Keras on CSV
+              type: TensorflowSavedModel
+        annotations:
+          editor.position: '{"x":40,"y":750,"width":180,"height":54}'
+      Predict with TensorFlow model on CSV data:
+        componentRef:
+          digest: 921bb1563e93a78233b8acceab87055b9154ccf5595d056028cf0396ca224cd4
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/59c759ce6f543184e30db6817d2a703879bc0f39/components/tensorflow/Predict/on_CSV/component.yaml
+        arguments:
+          dataset:
+            taskOutput:
+              outputName: split_2
+              taskId: Split rows into subsets
+              type: CSV
+          model:
+            taskOutput:
+              outputName: trained_model
+              taskId: Train model using Keras on CSV
+              type: TensorflowSavedModel
+          label_column_name: class
+        annotations:
+          editor.position: '{"x":240,"y":750,"width":180,"height":54}'
+    outputValues: {}

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_model_using_TensorFlow_and_import_to_Vertex_AI/pipeline.py

@@ -0,0 +1,106 @@
+# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
+from kfp import components
+
+# %% Loading components
+download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/storage/download/component.yaml")
+select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml")
+fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
+binarize_column_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Binarize_column/in_CSV_format/component.yaml")
+split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
+create_fully_connected_tensorflow_network_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Create_fully_connected_network/component.yaml")
+train_model_using_Keras_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml")
+predict_with_TensorFlow_model_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Predict/on_CSV/component.yaml")
+upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/component.yaml")
+deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
+
+# %% Pipeline definition
+def train_tabular_classification_model_using_TensorFlow_pipeline():
+    dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
+    feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]  # Excluded "trip_total"
+    label_column = "tips"
+    training_set_fraction = 0.8
+    # Deploying the model might incur additional costs over time
+    deploy_model = False
+
+    classification_label_column = "class"
+    all_columns = [label_column] + feature_columns
+
+    dataset = download_from_gcs_op(
+        gcs_path=dataset_gcs_uri
+    ).outputs["Data"]
+
+    dataset = select_columns_using_Pandas_on_CSV_data_op(
+        table=dataset,
+        column_names=all_columns,
+    ).outputs["transformed_table"]
+
+    dataset = fill_all_missing_values_using_Pandas_on_CSV_data_op(
+        table=dataset,
+        replacement_value="0",
+        # # Optional:
+        # column_names=None,  # =[...]
+    ).outputs["transformed_table"]
+
+    classification_dataset = binarize_column_using_Pandas_on_CSV_data_op(
+        table=dataset,
+        column_name=label_column,
+        predicate=" > 0",
+        new_column_name=classification_label_column,
+    ).outputs["transformed_table"]
+
+    split_task = split_rows_into_subsets_op(
+        table=classification_dataset,
+        fraction_1=training_set_fraction,
+    )
+    classification_training_data = split_task.outputs["split_1"]
+    classification_testing_data = split_task.outputs["split_2"]
+
+    network = create_fully_connected_tensorflow_network_op(
+        input_size=len(feature_columns),
+        # Optional:
+        hidden_layer_sizes=[10],
+        activation_name="elu",
+        output_activation_name="sigmoid",
+        # output_size=1,
+    ).outputs["model"]
+
+    model = train_model_using_Keras_on_CSV_op(
+        training_data=classification_training_data,
+        model=network,
+        label_column_name=classification_label_column,
+        # Optional:
+        loss_function_name="binary_crossentropy",
+        number_of_epochs=10,
+        #learning_rate=0.1,
+        #optimizer_name="Adadelta",
+        #optimizer_parameters={},
+        #batch_size=32,
+        #metric_names=["mean_absolute_error"],
+        #random_seed=0,
+    ).outputs["trained_model"]
+
+    predictions = predict_with_TensorFlow_model_on_CSV_data_op(
+        dataset=classification_testing_data,
+        model=model,
+        # label_column_name needs to be set when doing prediction on a dataset that has labels
+        label_column_name=classification_label_column,
+        # Optional:
+        # batch_size=1000,
+    ).outputs["predictions"]
+
+    vertex_model_name = upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op(
+        model=model,
+    ).outputs["model_name"]
+
+    # Deploying the model might incur additional costs over time
+    if deploy_model:
+        vertex_endpoint_name = deploy_model_to_endpoint_op(
+            model_name=vertex_model_name,
+        ).outputs["endpoint_name"]
+
+pipeline_func = train_tabular_classification_model_using_TensorFlow_pipeline
+
+# %% Pipeline submission
+if __name__ == '__main__':
+    from google.cloud import aiplatform
+    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_model_using_XGBoost_and_import_to_Vertex_AI/pipeline.component.yaml

@@ -0,0 +1,115 @@
+name: Train tabular classification model using XGBoost pipeline
+metadata:
+  annotations:
+    author: Alexey Volkov <alexey.volkov@ark-kun.com>
+    canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_classification_model_using_XGBoost_and_import_to_Vertex_AI/pipeline.component.yaml
+    sdk: https://cloud-pipelines.net/pipeline-editor/
+implementation:
+  graph:
+    tasks:
+      Download from GCS:
+        componentRef:
+          digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
+        arguments:
+          GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
+        annotations:
+          editor.position: '{"x":40,"y":40,"width":180,"height":40}'
+      Select columns using Pandas on CSV data:
+        componentRef:
+          digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: Data
+              taskId: Download from GCS
+          column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
+        annotations:
+          editor.position: '{"x":40,"y":140,"width":180,"height":54}'
+      Fill all missing values using Pandas on CSV data:
+        componentRef:
+          digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: transformed_table
+              taskId: Select columns using Pandas on CSV data
+              type: CSV
+          replacement_value: '0'
+        annotations:
+          editor.position: '{"x":40,"y":250,"width":180,"height":54}'
+      Binarize column using Pandas on CSV data:
+        componentRef:
+          digest: d699afd4d7cae862708717cc160f4394ed0c04e536e9515923ef1e8865f01d44
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: transformed_table
+              taskId: Fill all missing values using Pandas on CSV data
+              type: CSV
+          column_name: tips
+          predicate: '> 0'
+          new_column_name: class
+        annotations:
+          editor.position: '{"x":40,"y":380,"width":180,"height":54}'
+      Split rows into subsets:
+        componentRef:
+          digest: a609c3c9196484290f24a1174955f95b27f07a7b458aa5cb8cde28866cb2cb46
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: transformed_table
+              taskId: Binarize column using Pandas on CSV data
+              type: CSV
+          fraction_1: '0.8'
+        annotations:
+          editor.position: '{"x":170,"y":510,"width":180,"height":40}'
+      Train XGBoost model on CSV:
+        componentRef:
+          digest: 538c5a01eb38deaf532d619f0bbeaff4efc550fe1f0f776fc06791097b68ceac
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/58d3a47f904f32a64af8403330ba7e2134cae46d/components/XGBoost/Train/component.yaml
+        arguments:
+          training_data:
+            taskOutput:
+              outputName: split_1
+              taskId: Split rows into subsets
+              type: CSV
+          label_column_name: class
+          objective: binary:logistic
+        annotations:
+          editor.position: '{"x":40,"y":630,"width":180,"height":40}'
+      Upload XGBoost model to Google Cloud Vertex AI:
+        componentRef:
+          digest: 5a5a273c403670743820986c03a4175b7cb4595a556524fefcce403656286977
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/workaround_for_buggy_KFPv2_compiler/component.yaml
+        arguments:
+          model:
+            taskOutput:
+              outputName: model
+              taskId: Train XGBoost model on CSV
+              type: XGBoostModel
+        annotations:
+          editor.position: '{"x":40,"y":750,"width":180,"height":54}'
+      Xgboost predict on CSV:
+        componentRef:
+          digest: 0876233a0c7306fefec188bd70f059b46d1fb5aa57be231799570e3bbbdd0d95
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/4694ec97baccf59284c2a1db4aa2250c22291eab/components/XGBoost/Predict/component.yaml
+        arguments:
+          data:
+            taskOutput:
+              outputName: split_2
+              taskId: Split rows into subsets
+              type: CSV
+          model:
+            taskOutput:
+              outputName: model
+              taskId: Train XGBoost model on CSV
+              type: XGBoostModel
+          label_column_name: class
+        annotations:
+          editor.position: '{"x":240,"y":750,"width":180,"height":40}'
+    outputValues: {}

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_model_using_XGBoost_and_import_to_Vertex_AI/pipeline.py

@@ -0,0 +1,94 @@
+# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
+from kfp import components
+
+# %% Loading components
+download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/storage/download/component.yaml")
+select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml")
+fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
+binarize_column_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Binarize_column/in_CSV_format/component.yaml")
+split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
+train_XGBoost_model_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Train/component.yaml")
+xgboost_predict_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Predict/component.yaml")
+upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/component.yaml")
+deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
+
+# %% Pipeline definition
+def train_tabular_classification_model_using_XGBoost_pipeline():
+    dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
+    feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]  # Excluded "trip_total"
+    label_column = "tips"
+    training_set_fraction = 0.8
+    # Deploying the model might incur additional costs over time
+    deploy_model = False
+
+    classification_label_column = "class"
+    all_columns = [label_column] + feature_columns
+
+    dataset = download_from_gcs_op(
+        gcs_path=dataset_gcs_uri
+    ).outputs["Data"]
+
+    dataset = select_columns_using_Pandas_on_CSV_data_op(
+        table=dataset,
+        column_names=all_columns,
+    ).outputs["transformed_table"]
+
+    dataset = fill_all_missing_values_using_Pandas_on_CSV_data_op(
+        table=dataset,
+        replacement_value="0",
+        # # Optional:
+        # column_names=None,  # =[...]
+    ).outputs["transformed_table"]
+
+    classification_dataset = binarize_column_using_Pandas_on_CSV_data_op(
+        table=dataset,
+        column_name=label_column,
+        predicate="> 0",
+        new_column_name=classification_label_column,
+    ).outputs["transformed_table"]
+
+    split_task = split_rows_into_subsets_op(
+        table=classification_dataset,
+        fraction_1=training_set_fraction,
+    )
+    classification_training_data = split_task.outputs["split_1"]
+    classification_testing_data = split_task.outputs["split_2"]
+
+    model = train_XGBoost_model_on_CSV_op(
+        training_data=classification_training_data,
+        label_column_name=classification_label_column,
+        objective="binary:logistic",
+        # Optional:
+        #starting_model=None,
+        #num_iterations=10,
+        #booster_params={},
+        #booster="gbtree",
+        #learning_rate=0.3,
+        #min_split_loss=0,
+        #max_depth=6,
+    ).outputs["model"]
+
+    # Predicting on the testing data
+    predictions = xgboost_predict_on_CSV_op(
+        data=classification_testing_data,
+        model=model,
+        # label_column needs to be set when doing prediction on a dataset that has labels
+        label_column_name=classification_label_column,
+    ).outputs["predictions"]
+
+    vertex_model_name = upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op(
+        model=model,
+    ).outputs["model_name"]
+
+    # Deploying the model might incur additional costs over time
+    if deploy_model:
+        vertex_endpoint_name = deploy_model_to_endpoint_op(
+            model_name=vertex_model_name,
+        ).outputs["endpoint_name"]
+
+pipeline_func = train_tabular_classification_model_using_XGBoost_pipeline
+
+# %% Pipeline submission
+if __name__ == '__main__':
+    from google.cloud import aiplatform
+    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_model_using_all_frameworks_and_import_to_Vertex_AI/pipeline.component.yaml

@@ -0,0 +1,257 @@
+name: Train tabular classification model using all frameworks pipeline
+metadata:
+  annotations:
+    author: Alexey Volkov <alexey.volkov@ark-kun.com>
+    canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_classification_model_using_all_frameworks_and_import_to_Vertex_AI/pipeline.component.yaml
+    sdk: https://cloud-pipelines.net/pipeline-editor/
+implementation:
+  graph:
+    tasks:
+      Download from GCS:
+        componentRef:
+          digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
+        arguments:
+          GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
+        annotations:
+          editor.position: '{"x":550,"y":40,"width":180,"height":40}'
+      Select columns using Pandas on CSV data:
+        componentRef:
+          digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: Data
+              taskId: Download from GCS
+          column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
+        annotations:
+          editor.position: '{"x":550,"y":140,"width":180,"height":54}'
+      Fill all missing values using Pandas on CSV data:
+        componentRef:
+          digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: transformed_table
+              taskId: Select columns using Pandas on CSV data
+              type: CSV
+          replacement_value: '0'
+        annotations:
+          editor.position: '{"x":550,"y":250,"width":180,"height":54}'
+      Binarize column using Pandas on CSV data:
+        componentRef:
+          digest: d699afd4d7cae862708717cc160f4394ed0c04e536e9515923ef1e8865f01d44
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: transformed_table
+              taskId: Fill all missing values using Pandas on CSV data
+              type: CSV
+          column_name: tips
+          predicate: ' > 0'
+          new_column_name: class
+        annotations:
+          editor.position: '{"x":550,"y":380,"width":180,"height":54}'
+      Split rows into subsets:
+        componentRef:
+          digest: a609c3c9196484290f24a1174955f95b27f07a7b458aa5cb8cde28866cb2cb46
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: transformed_table
+              taskId: Binarize column using Pandas on CSV data
+              type: CSV
+          fraction_1: '0.8'
+        annotations:
+          editor.position: '{"x":550,"y":490,"width":180,"height":40}'
+      Create fully connected pytorch network:
+        componentRef:
+          digest: d03d8248fd358a0275ec33568ee7dd7dce576cc112b09dfafe2651e4d97e04a9
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1a2ef3eeb77bc278f33cad0dd29008ea2431e191/components/PyTorch/Create_fully_connected_network/component.yaml
+        arguments:
+          input_size: '7'
+          hidden_layer_sizes: '[10]'
+          activation_name: elu
+          output_activation_name: sigmoid
+        annotations:
+          editor.position: '{"x":380,"y":620,"width":180,"height":54}'
+      Create fully connected tensorflow network:
+        componentRef:
+          digest: bfcafbc5ce711b1f69cabf1338212d10d50136a73db9f9f7c984de7b80b4bfb0
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/9ca0f9eecf5f896f65b8538bbd809747052617d1/components/tensorflow/Create_fully_connected_network/component.yaml
+        arguments:
+          input_size: '7'
+          hidden_layer_sizes: '[10]'
+          activation_name: elu
+          output_activation_name: sigmoid
+        annotations:
+          editor.position: '{"x":40,"y":630,"width":180,"height":54}'
+      Train model using Keras on CSV:
+        componentRef:
+          digest: 42ae60c889034dbad74815653e95b4f7d576b5f47f803173e8679c7b54984609
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c504a4010348c50eaaf6d4337586ccc008f4dcef/components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml
+        arguments:
+          training_data:
+            taskOutput:
+              outputName: split_1
+              taskId: Split rows into subsets
+              type: CSV
+          model:
+            taskOutput:
+              outputName: model
+              taskId: Create fully connected tensorflow network
+              type: TensorflowSavedModel
+          label_column_name: class
+          loss_function_name: binary_crossentropy
+          number_of_epochs: '10'
+        annotations:
+          editor.position: '{"x":40,"y":750,"width":180,"height":54}'
+      Train pytorch model from csv:
+        componentRef:
+          digest: 40f3185eb61e9727f41a4e0c05dd3d3b44bd802aa0f378cfc31756560033949a
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml
+        arguments:
+          model:
+            taskOutput:
+              outputName: model
+              taskId: Create fully connected pytorch network
+              type: PyTorchScriptModule
+          training_data:
+            taskOutput:
+              outputName: split_1
+              taskId: Split rows into subsets
+              type: CSV
+          label_column_name: class
+          loss_function_name: binary_cross_entropy
+        annotations:
+          editor.position: '{"x":380,"y":750,"width":180,"height":40}'
+      Train XGBoost model on CSV:
+        componentRef:
+          digest: 538c5a01eb38deaf532d619f0bbeaff4efc550fe1f0f776fc06791097b68ceac
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/58d3a47f904f32a64af8403330ba7e2134cae46d/components/XGBoost/Train/component.yaml
+        arguments:
+          training_data:
+            taskOutput:
+              outputName: split_1
+              taskId: Split rows into subsets
+              type: CSV
+          label_column_name: class
+          objective: binary:logistic
+        annotations:
+          editor.position: '{"x":720,"y":750,"width":180,"height":40}'
+      Train logistic regression model using scikit learn from CSV:
+        componentRef:
+          digest: a864625a822e4b1c8ef6fe4ae1454fd90f15438f70a6712bb4c30e0dda4d35b7
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/cb44b75c9c062fcc40c2b905b2024b4493dbc62b/components/ML_frameworks/Scikit_learn/Train_logistic_regression_model/from_CSV/component.yaml
+        arguments:
+          dataset:
+            taskOutput:
+              outputName: split_1
+              taskId: Split rows into subsets
+              type: CSV
+          label_column_name: class
+        annotations:
+          editor.position: '{"x":1030,"y":750,"width":180,"height":70}'
+      Predict with TensorFlow model on CSV data:
+        componentRef:
+          digest: 921bb1563e93a78233b8acceab87055b9154ccf5595d056028cf0396ca224cd4
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/59c759ce6f543184e30db6817d2a703879bc0f39/components/tensorflow/Predict/on_CSV/component.yaml
+        arguments:
+          dataset:
+            taskOutput:
+              outputName: split_2
+              taskId: Split rows into subsets
+              type: CSV
+          model:
+            taskOutput:
+              outputName: trained_model
+              taskId: Train model using Keras on CSV
+              type: TensorflowSavedModel
+          label_column_name: class
+        annotations:
+          editor.position: '{"x":160,"y":880,"width":180,"height":54}'
+      Create PyTorch Model Archive with base handler:
+        componentRef:
+          digest: 8298b5ee1b0f0879f893add4cf352c8dec7cf9e21bb9db134c91a2d046cdb0ec
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/46d51383e6554b7f3ab4fd8cf614d8c2b422fb22/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml
+        arguments:
+          Model:
+            taskOutput:
+              outputName: trained_model
+              taskId: Train pytorch model from csv
+              type: PyTorchScriptModule
+          Model name: model
+          Model version: '1.0'
+        annotations:
+          editor.position: '{"x":380,"y":880,"width":180,"height":54}'
+      Xgboost predict on CSV:
+        componentRef:
+          digest: 0876233a0c7306fefec188bd70f059b46d1fb5aa57be231799570e3bbbdd0d95
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/4694ec97baccf59284c2a1db4aa2250c22291eab/components/XGBoost/Predict/component.yaml
+        arguments:
+          data:
+            taskOutput:
+              outputName: split_2
+              taskId: Split rows into subsets
+              type: CSV
+          model:
+            taskOutput:
+              outputName: model
+              taskId: Train XGBoost model on CSV
+              type: XGBoostModel
+          label_column_name: class
+        annotations:
+          editor.position: '{"x":810,"y":880,"width":180,"height":40}'
+      Upload Scikit learn pickle model to Google Cloud Vertex AI:
+        componentRef:
+          digest: 81c91c8d7d21ec97e0872f669d68bd89edea87279d703685db54aa94743bebcd
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/workaround_for_buggy_KFPv2_compiler/component.yaml
+        arguments:
+          model:
+            taskOutput:
+              outputName: model
+              taskId: Train logistic regression model using scikit learn from CSV
+              type: ScikitLearnPickleModel
+        annotations:
+          editor.position: '{"x":1030,"y":880,"width":180,"height":70}'
+      Upload Tensorflow model to Google Cloud Vertex AI:
+        componentRef:
+          digest: 2e45263ff640b1a688e359b6936e27a81b2407749a84f340af2aa5547e0cb92c
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/workaround_for_buggy_KFPv2_compiler/component.yaml
+        arguments:
+          model:
+            taskOutput:
+              outputName: trained_model
+              taskId: Train model using Keras on CSV
+              type: TensorflowSavedModel
+        annotations:
+          editor.position: '{"x":40,"y":1010,"width":180,"height":54}'
+      Upload PyTorch model archive to Google Cloud Vertex AI:
+        componentRef:
+          digest: 4450212fae7b9001482aca7eb78b28413c205506eccf08a04e7754a8dfa99004
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/workaround_for_buggy_KFPv2_compiler/component.yaml
+        arguments:
+          model_archive:
+            taskOutput:
+              outputName: Model archive
+              taskId: Create PyTorch Model Archive with base handler
+              type: PyTorchModelArchive
+        annotations:
+          editor.position: '{"x":380,"y":1010,"width":180,"height":70}'
+      Upload XGBoost model to Google Cloud Vertex AI:
+        componentRef:
+          digest: 5a5a273c403670743820986c03a4175b7cb4595a556524fefcce403656286977
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/workaround_for_buggy_KFPv2_compiler/component.yaml
+        arguments:
+          model:
+            taskOutput:
+              outputName: model
+              taskId: Train XGBoost model on CSV
+              type: XGBoostModel
+        annotations:
+          editor.position: '{"x":720,"y":1010,"width":180,"height":54}'
+    outputValues: {}

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_model_using_all_frameworks_and_import_to_Vertex_AI/pipeline.py

@@ -0,0 +1,224 @@
+# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
+from kfp import components
+
+# %% Loading components
+download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/storage/download/component.yaml")
+select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml")
+fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
+binarize_column_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Binarize_column/in_CSV_format/component.yaml")
+split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
+
+# TensorFlow
+create_fully_connected_tensorflow_network_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Create_fully_connected_network/component.yaml")
+train_model_using_Keras_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml")
+predict_with_TensorFlow_model_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Predict/on_CSV/component.yaml")
+upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/component.yaml")
+
+# PyTorch
+create_fully_connected_pytorch_network_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Create_fully_connected_network/component.yaml")
+train_pytorch_model_from_csv_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml")
+create_pytorch_model_archive_with_base_handler_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml")
+upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/component.yaml")
+
+# XGBoost
+train_XGBoost_model_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Train/component.yaml")
+xgboost_predict_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Predict/component.yaml")
+upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/component.yaml")
+
+# Scikit-learn
+#train_linear_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/1f5cf6e06409b704064b2086c0a705e4e6b4fcde/community-content/pipeline_components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml")
+train_logistic_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/1f5cf6e06409b704064b2086c0a705e4e6b4fcde/community-content/pipeline_components/ML_frameworks/Scikit_learn/Train_logistic_regression_model/from_CSV/component.yaml")
+upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/component.yaml")
+
+# Vertex AI
+deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
+
+# %% Pipeline definition
+def train_tabular_classification_model_using_all_frameworks_pipeline():
+    dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
+    feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]  # Excluded "trip_total"
+    label_column = "tips"
+    training_set_fraction = 0.8
+    # Deploying the model might incur additional costs over time
+    deploy_model = False
+
+    classification_label_column = "class"
+    all_columns = [label_column] + feature_columns
+
+    dataset = download_from_gcs_op(
+        gcs_path=dataset_gcs_uri
+    ).outputs["Data"]
+
+    dataset = select_columns_using_Pandas_on_CSV_data_op(
+        table=dataset,
+        column_names=all_columns,
+    ).outputs["transformed_table"]
+
+    dataset = fill_all_missing_values_using_Pandas_on_CSV_data_op(
+        table=dataset,
+        replacement_value="0",
+        # # Optional:
+        # column_names=None,  # =[...]
+    ).outputs["transformed_table"]
+
+    classification_dataset = binarize_column_using_Pandas_on_CSV_data_op(
+        table=dataset,
+        column_name=label_column,
+        predicate=" > 0",
+        new_column_name=classification_label_column,
+    ).outputs["transformed_table"]
+
+    split_task = split_rows_into_subsets_op(
+        table=classification_dataset,
+        fraction_1=training_set_fraction,
+    )
+    classification_training_data = split_task.outputs["split_1"]
+    classification_testing_data = split_task.outputs["split_2"]
+
+    # TensorFlow
+    tensorflow_network = create_fully_connected_tensorflow_network_op(
+        input_size=len(feature_columns),
+        # Optional:
+        hidden_layer_sizes=[10],
+        activation_name="elu",
+        output_activation_name="sigmoid",
+        # output_size=1,
+    ).outputs["model"]
+
+    tensorflow_model = train_model_using_Keras_on_CSV_op(
+        training_data=classification_training_data,
+        model=tensorflow_network,
+        label_column_name=classification_label_column,
+        # Optional:
+        loss_function_name="binary_crossentropy",
+        number_of_epochs=10,
+        #learning_rate=0.1,
+        #optimizer_name="Adadelta",
+        #optimizer_parameters={},
+        #batch_size=32,
+        #metric_names=["mean_absolute_error"],
+        #random_seed=0,
+    ).outputs["trained_model"]
+
+    tensorflow_predictions = predict_with_TensorFlow_model_on_CSV_data_op(
+        dataset=classification_testing_data,
+        model=tensorflow_model,
+        # label_column_name needs to be set when doing prediction on a dataset that has labels
+        label_column_name=classification_label_column,
+        # Optional:
+        # batch_size=1000,
+    ).outputs["predictions"]
+
+    tensorflow_vertex_model_name = upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op(
+        model=tensorflow_model,
+    ).outputs["model_name"]
+
+    # Deploying the model might incur additional costs over time
+    if deploy_model:
+        tensorflow_vertex_endpoint_name = deploy_model_to_endpoint_op(
+            model_name=tensorflow_vertex_model_name,
+        ).outputs["endpoint_name"]
+
+    # PyTorch
+    pytorch_network = create_fully_connected_pytorch_network_op(
+        input_size=len(feature_columns),
+        # Optional:
+        hidden_layer_sizes=[10],
+        activation_name="elu",
+        output_activation_name="sigmoid",
+        # output_size=1,
+    ).outputs["model"]
+
+    pytorch_model = train_pytorch_model_from_csv_op(
+        model=pytorch_network,
+        training_data=classification_training_data,
+        label_column_name=classification_label_column,
+        loss_function_name="binary_cross_entropy",
+        # Optional:
+        #number_of_epochs=1,
+        #learning_rate=0.1,
+        #optimizer_name="Adadelta",
+        #optimizer_parameters={},
+        #batch_size=32,
+        #batch_log_interval=100,
+        #random_seed=0,
+    ).outputs["trained_model"]
+
+    pytorch_model_archive = create_pytorch_model_archive_with_base_handler_op(
+        model=pytorch_model,
+        # Optional:
+        # model_name="model",
+        # model_version="1.0",
+    ).outputs["Model archive"]
+
+    pytorch_vertex_model_name = upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op(
+        model_archive=pytorch_model_archive,
+    ).outputs["model_name"]
+
+    # Deploying the model might incur additional costs over time
+    if deploy_model:
+        pytorch_vertex_endpoint_name = deploy_model_to_endpoint_op(
+            model_name=pytorch_vertex_model_name,
+        ).outputs["endpoint_name"]
+
+    # XGBoost
+    xgboost_model = train_XGBoost_model_on_CSV_op(
+        training_data=classification_training_data,
+        label_column_name=classification_label_column,
+        objective="binary:logistic",
+        # Optional:
+        #starting_model=None,
+        #num_iterations=10,
+        #booster_params={},
+        #booster="gbtree",
+        #learning_rate=0.3,
+        #min_split_loss=0,
+        #max_depth=6,
+    ).outputs["model"]
+
+    # Predicting on the testing data
+    xgboost_predictions = xgboost_predict_on_CSV_op(
+        data=classification_testing_data,
+        model=xgboost_model,
+        # label_column needs to be set when doing prediction on a dataset that has labels
+        label_column_name=classification_label_column,
+    ).outputs["predictions"]
+
+    xgboost_vertex_model_name = upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op(
+        model=xgboost_model,
+    ).outputs["model_name"]
+
+    # Deploying the model might incur additional costs over time
+    if deploy_model:
+        xgboost_vertex_endpoint_name = deploy_model_to_endpoint_op(
+            model_name=xgboost_vertex_model_name,
+        ).outputs["endpoint_name"]
+
+    # Scikit-learn
+    sklearn_model = train_logistic_regression_model_using_scikit_learn_from_CSV_op(
+        dataset=classification_training_data,
+        label_column_name=classification_label_column,
+        # Optional:
+        #penalty="l2",
+        #solver="lbfgs",
+        #max_iterations=100,
+        #multi_class_mode="auto",
+        #random_seed=0,
+    ).outputs["model"]
+
+    sklearn_vertex_model_name = upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op(
+        model=sklearn_model,
+    ).outputs["model_name"]
+
+    # Deploying the model might incur additional costs over time
+    if deploy_model:
+        sklearn_vertex_endpoint_name = deploy_model_to_endpoint_op(
+            model_name=sklearn_vertex_model_name,
+        ).outputs["endpoint_name"]
+
+pipeline_func=train_tabular_classification_model_using_all_frameworks_pipeline
+
+# %% Pipeline submission
+if __name__ == '__main__':
+    from google.cloud import aiplatform
+    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_regression_linear_model_using_Scikit_learn_and_import_to_Vertex_AI/pipeline.component.yaml

@@ -0,0 +1,68 @@
+name: Train tabular regression linear model using Scikit learn pipeline
+metadata:
+  annotations:
+    author: Alexey Volkov <alexey.volkov@ark-kun.com>
+    canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_regression_linear_model_using_Scikit_learn_and_import_to_Vertex_AI/pipeline.component.yaml
+    sdk: https://cloud-pipelines.net/pipeline-editor/
+implementation:
+  graph:
+    tasks:
+      Download from GCS:
+        componentRef:
+          digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
+        arguments:
+          GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
+        annotations:
+          editor.position: '{"x":40,"y":40,"width":180,"height":40}'
+      Select columns using Pandas on CSV data:
+        componentRef:
+          digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: Data
+              taskId: Download from GCS
+          column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
+        annotations:
+          editor.position: '{"x":40,"y":140,"width":180,"height":54}'
+      Fill all missing values using Pandas on CSV data:
+        componentRef:
+          digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: transformed_table
+              taskId: Select columns using Pandas on CSV data
+              type: CSV
+          replacement_value: '0'
+        annotations:
+          editor.position: '{"x":40,"y":250,"width":180,"height":54}'
+      Train linear regression model using scikit learn from CSV:
+        componentRef:
+          digest: c7fe7912ab0d1fb45d201d452e9ce6be5544e7d8c6d229db7a4b931ff58560f3
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/f807e02b54d4886c65a05f40848fd51c72407f40/components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml
+        arguments:
+          dataset:
+            taskOutput:
+              outputName: transformed_table
+              taskId: Fill all missing values using Pandas on CSV data
+              type: CSV
+          label_column_name: tips
+        annotations:
+          editor.position: '{"x":40,"y":360,"width":180,"height":54}'
+      Upload Scikit learn pickle model to Google Cloud Vertex AI:
+        componentRef:
+          digest: 81c91c8d7d21ec97e0872f669d68bd89edea87279d703685db54aa94743bebcd
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/workaround_for_buggy_KFPv2_compiler/component.yaml
+        arguments:
+          model:
+            taskOutput:
+              outputName: model
+              taskId: Train linear regression model using scikit learn from CSV
+              type: ScikitLearnPickleModel
+        annotations:
+          editor.position: '{"x":40,"y":490,"width":180,"height":70}'
+    outputValues: {}

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_regression_linear_model_using_Scikit_learn_and_import_to_Vertex_AI/pipeline.py

@@ -0,0 +1,57 @@
+# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
+from kfp import components
+
+# %% Loading components
+download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/storage/download/component.yaml")
+select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml")
+fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
+train_linear_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/1f5cf6e06409b704064b2086c0a705e4e6b4fcde/community-content/pipeline_components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml")
+upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/component.yaml")
+deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
+
+# %% Pipeline definition
+def train_tabular_regression_linear_model_using_Scikit_learn_pipeline():
+    dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
+    feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]  # Excluded "trip_total"
+    label_column = "tips"
+    all_columns = [label_column] + feature_columns
+    # Deploying the model might incur additional costs over time
+    deploy_model = False
+
+    training_data = download_from_gcs_op(
+        gcs_path=dataset_gcs_uri
+    ).outputs["Data"]
+
+    training_data = select_columns_using_Pandas_on_CSV_data_op(
+        table=training_data,
+        column_names=all_columns,
+    ).outputs["transformed_table"]
+
+    # Cleaning the NaN values.
+    training_data = fill_all_missing_values_using_Pandas_on_CSV_data_op(
+        table=training_data,
+        replacement_value="0",
+        #replacement_type_name="float",
+    ).outputs["transformed_table"]
+
+    model = train_linear_regression_model_using_scikit_learn_from_CSV_op(
+        dataset=training_data,
+        label_column_name=label_column,
+    ).outputs["model"]
+
+    vertex_model_name = upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op(
+        model=model,
+    ).outputs["model_name"]
+
+    # Deploying the model might incur additional costs over time
+    if deploy_model:
+        sklearn_vertex_endpoint_name = deploy_model_to_endpoint_op(
+            model_name=vertex_model_name,
+        ).outputs["endpoint_name"]
+
+pipeline_func = train_tabular_regression_linear_model_using_Scikit_learn_pipeline
+
+# %% Pipeline submission
+if __name__ == '__main__':
+    from google.cloud import aiplatform
+    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_regression_model_using_PyTorch_and_import_to_Vertex_AI/pipeline.component.yaml

@@ -0,0 +1,97 @@
+name: Train tabular regression model using PyTorch pipeline
+metadata:
+  annotations:
+    author: Alexey Volkov <alexey.volkov@ark-kun.com>
+    canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_regression_model_using_PyTorch_and_import_to_Vertex_AI/pipeline.component.yaml
+    sdk: https://cloud-pipelines.net/pipeline-editor/
+implementation:
+  graph:
+    tasks:
+      Download from GCS:
+        componentRef:
+          digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
+        arguments:
+          GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
+        annotations:
+          editor.position: '{"x":240,"y":40,"width":180,"height":40}'
+      Select columns using Pandas on CSV data:
+        componentRef:
+          digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: Data
+              taskId: Download from GCS
+          column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
+        annotations:
+          editor.position: '{"x":240,"y":130,"width":180,"height":54}'
+      Create fully connected pytorch network:
+        componentRef:
+          digest: d03d8248fd358a0275ec33568ee7dd7dce576cc112b09dfafe2651e4d97e04a9
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1a2ef3eeb77bc278f33cad0dd29008ea2431e191/components/PyTorch/Create_fully_connected_network/component.yaml
+        arguments:
+          input_size: '7'
+          hidden_layer_sizes: '[10]'
+          activation_name: elu
+        annotations:
+          editor.position: '{"x":40,"y":240,"width":180,"height":54}'
+      Fill all missing values using Pandas on CSV data:
+        componentRef:
+          digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: transformed_table
+              taskId: Select columns using Pandas on CSV data
+              type: CSV
+          replacement_value: '0'
+        annotations:
+          editor.position: '{"x":240,"y":240,"width":180,"height":54}'
+      Train pytorch model from csv:
+        componentRef:
+          digest: 40f3185eb61e9727f41a4e0c05dd3d3b44bd802aa0f378cfc31756560033949a
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml
+        arguments:
+          model:
+            taskOutput:
+              outputName: model
+              taskId: Create fully connected pytorch network
+              type: PyTorchScriptModule
+          training_data:
+            taskOutput:
+              outputName: transformed_table
+              taskId: Fill all missing values using Pandas on CSV data
+              type: CSV
+          label_column_name: tips
+        annotations:
+          editor.position: '{"x":240,"y":380,"width":180,"height":40}'
+      Create PyTorch Model Archive with base handler:
+        componentRef:
+          digest: 8298b5ee1b0f0879f893add4cf352c8dec7cf9e21bb9db134c91a2d046cdb0ec
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/46d51383e6554b7f3ab4fd8cf614d8c2b422fb22/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml
+        arguments:
+          Model:
+            taskOutput:
+              outputName: trained_model
+              taskId: Train pytorch model from csv
+              type: PyTorchScriptModule
+          Model name: model
+          Model version: '1.0'
+        annotations:
+          editor.position: '{"x":240,"y":500,"width":180,"height":54}'
+      Upload PyTorch model archive to Google Cloud Vertex AI:
+        componentRef:
+          digest: 4450212fae7b9001482aca7eb78b28413c205506eccf08a04e7754a8dfa99004
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/workaround_for_buggy_KFPv2_compiler/component.yaml
+        arguments:
+          model_archive:
+            taskOutput:
+              outputName: Model archive
+              taskId: Create PyTorch Model Archive with base handler
+              type: PyTorchModelArchive
+        annotations:
+          editor.position: '{"x":240,"y":630,"width":180,"height":70}'
+    outputValues: {}

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_regression_model_using_PyTorch_and_import_to_Vertex_AI/pipeline.py

@@ -0,0 +1,85 @@
+# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
+from kfp import components
+
+# %% Loading components
+download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/storage/download/component.yaml")
+select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml")
+fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
+create_fully_connected_pytorch_network_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Create_fully_connected_network/component.yaml")
+train_pytorch_model_from_csv_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml")
+create_pytorch_model_archive_with_base_handler_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml")
+upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/component.yaml")
+deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
+
+# %% Pipeline definition
+def train_tabular_regression_model_using_PyTorch_pipeline():
+    dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
+    feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]  # Excluded "trip_total"
+    label_column = "tips"
+    all_columns = [label_column] + feature_columns
+    # Deploying the model might incur additional costs over time
+    deploy_model = False
+
+    training_data = download_from_gcs_op(
+        gcs_path=dataset_gcs_uri
+    ).outputs["Data"]
+
+    training_data = select_columns_using_Pandas_on_CSV_data_op(
+        table=training_data,
+        column_names=all_columns,
+    ).outputs["transformed_table"]
+
+    # Cleaning the NaN values.
+    training_data = fill_all_missing_values_using_Pandas_on_CSV_data_op(
+        table=training_data,
+        replacement_value="0",
+        #replacement_type_name="float",
+    ).outputs["transformed_table"]
+
+    network = create_fully_connected_pytorch_network_op(
+        input_size=len(feature_columns),
+        # Optional:
+        hidden_layer_sizes=[10],
+        activation_name="elu",
+        # output_activation_name=None,
+        # output_size=1,
+    ).outputs["model"]
+
+    model = train_pytorch_model_from_csv_op(
+        model=network,
+        training_data=training_data,
+        label_column_name=label_column,
+        # Optional:
+        #loss_function_name="mse_loss",
+        #number_of_epochs=1,
+        #learning_rate=0.1,
+        #optimizer_name="Adadelta",
+        #optimizer_parameters={},
+        #batch_size=32,
+        #batch_log_interval=100,
+        #random_seed=0,
+    ).outputs["trained_model"]
+
+    model_archive = create_pytorch_model_archive_with_base_handler_op(
+        model=model,
+        # Optional:
+        # model_name="model",
+        # model_version="1.0",
+    ).outputs["Model archive"]
+
+    vertex_model_name = upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op(
+        model_archive=model_archive,
+    ).outputs["model_name"]
+
+    # Deploying the model might incur additional costs over time
+    if deploy_model:
+        vertex_endpoint_name = deploy_model_to_endpoint_op(
+            model_name=vertex_model_name,
+        ).outputs["endpoint_name"]
+
+pipeline_func=train_tabular_regression_model_using_PyTorch_pipeline
+
+# %% Pipeline submission
+if __name__ == '__main__':
+    from google.cloud import aiplatform
+    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_regression_model_using_TensorFlow_and_import_to_Vertex_AI/pipeline.component.yaml

@@ -0,0 +1,116 @@
+name: Train tabular regression model using Tensorflow pipeline
+metadata:
+  annotations:
+    author: Alexey Volkov <alexey.volkov@ark-kun.com>
+    canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_regression_model_using_TensorFlow_and_import_to_Vertex_AI/pipeline.component.yaml
+    sdk: https://cloud-pipelines.net/pipeline-editor/
+implementation:
+  graph:
+    tasks:
+      Download from GCS:
+        componentRef:
+          digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
+        arguments:
+          GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
+        annotations:
+          editor.position: '{"x":40,"y":40,"width":180,"height":40}'
+      Select columns using Pandas on CSV data:
+        componentRef:
+          digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: Data
+              taskId: Download from GCS
+          column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
+        annotations:
+          editor.position: '{"x":40,"y":140,"width":180,"height":54}'
+      Fill all missing values using Pandas on CSV data:
+        componentRef:
+          digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: transformed_table
+              taskId: Select columns using Pandas on CSV data
+              type: CSV
+          replacement_value: '0'
+        annotations:
+          editor.position: '{"x":40,"y":250,"width":180,"height":54}'
+      Split rows into subsets:
+        componentRef:
+          digest: a609c3c9196484290f24a1174955f95b27f07a7b458aa5cb8cde28866cb2cb46
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: transformed_table
+              taskId: Fill all missing values using Pandas on CSV data
+              type: CSV
+          fraction_1: '0.8'
+        annotations:
+          editor.position: '{"x":170,"y":380,"width":180,"height":40}'
+      Create fully connected tensorflow network:
+        componentRef:
+          digest: bfcafbc5ce711b1f69cabf1338212d10d50136a73db9f9f7c984de7b80b4bfb0
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/9ca0f9eecf5f896f65b8538bbd809747052617d1/components/tensorflow/Create_fully_connected_network/component.yaml
+        arguments:
+          input_size: '7'
+          hidden_layer_sizes: '[10]'
+          activation_name: elu
+        annotations:
+          editor.position: '{"x":370,"y":380,"width":180,"height":54}'
+      Train model using Keras on CSV:
+        componentRef:
+          digest: 42ae60c889034dbad74815653e95b4f7d576b5f47f803173e8679c7b54984609
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c504a4010348c50eaaf6d4337586ccc008f4dcef/components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml
+        arguments:
+          training_data:
+            taskOutput:
+              outputName: split_1
+              taskId: Split rows into subsets
+              type: CSV
+          model:
+            taskOutput:
+              outputName: model
+              taskId: Create fully connected tensorflow network
+              type: TensorflowSavedModel
+          label_column_name: tips
+          number_of_epochs: '10'
+          metric_names: '["mean_absolute_error"]'
+        annotations:
+          editor.position: '{"x":40,"y":500,"width":180,"height":54}'
+      Upload Tensorflow model to Google Cloud Vertex AI:
+        componentRef:
+          digest: 2e45263ff640b1a688e359b6936e27a81b2407749a84f340af2aa5547e0cb92c
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/workaround_for_buggy_KFPv2_compiler/component.yaml
+        arguments:
+          model:
+            taskOutput:
+              outputName: trained_model
+              taskId: Train model using Keras on CSV
+              type: TensorflowSavedModel
+        annotations:
+          editor.position: '{"x":40,"y":630,"width":180,"height":54}'
+      Predict with TensorFlow model on CSV data:
+        componentRef:
+          digest: 921bb1563e93a78233b8acceab87055b9154ccf5595d056028cf0396ca224cd4
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/59c759ce6f543184e30db6817d2a703879bc0f39/components/tensorflow/Predict/on_CSV/component.yaml
+        arguments:
+          dataset:
+            taskOutput:
+              outputName: split_2
+              taskId: Split rows into subsets
+              type: CSV
+          model:
+            taskOutput:
+              outputName: trained_model
+              taskId: Train model using Keras on CSV
+              type: TensorflowSavedModel
+          label_column_name: tips
+        annotations:
+          editor.position: '{"x":240,"y":630,"width":180,"height":54}'
+    outputValues: {}

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_regression_model_using_TensorFlow_and_import_to_Vertex_AI/pipeline.py

@@ -0,0 +1,97 @@
+# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
+from kfp import components
+
+# %% Loading components
+download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/storage/download/component.yaml")
+select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml")
+fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
+split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
+create_fully_connected_tensorflow_network_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Create_fully_connected_network/component.yaml")
+train_model_using_Keras_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml")
+predict_with_TensorFlow_model_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Predict/on_CSV/component.yaml")
+upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/component.yaml")
+deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
+
+# %% Pipeline definition
+def train_tabular_regression_model_using_Tensorflow_pipeline():
+    dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
+    feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]  # Excluded "trip_total"
+    label_column = "tips"
+    training_set_fraction = 0.8
+    # Deploying the model might incur additional costs over time
+    deploy_model = False
+
+    all_columns = [label_column] + feature_columns
+
+    dataset = download_from_gcs_op(
+        gcs_path=dataset_gcs_uri
+    ).outputs["Data"]
+
+    dataset = select_columns_using_Pandas_on_CSV_data_op(
+        table=dataset,
+        column_names=all_columns,
+    ).outputs["transformed_table"]
+
+    dataset = fill_all_missing_values_using_Pandas_on_CSV_data_op(
+        table=dataset,
+        replacement_value="0",
+        # # Optional:
+        # column_names=None,  # =[...]
+    ).outputs["transformed_table"]
+
+    split_task = split_rows_into_subsets_op(
+        table=dataset,
+        fraction_1=training_set_fraction,
+    )
+    training_data = split_task.outputs["split_1"]
+    testing_data = split_task.outputs["split_2"]
+
+    network = create_fully_connected_tensorflow_network_op(
+        input_size=len(feature_columns),
+        # Optional:
+        hidden_layer_sizes=[10],
+        activation_name="elu",
+        # output_activation_name=None,
+        # output_size=1,
+    ).outputs["model"]
+
+    model = train_model_using_Keras_on_CSV_op(
+        training_data=training_data,
+        model=network,
+        label_column_name=label_column,
+        # Optional:
+        #loss_function_name="mean_squared_error",
+        number_of_epochs=10,
+        #learning_rate=0.1,
+        #optimizer_name="Adadelta",
+        #optimizer_parameters={},
+        #batch_size=32,
+        metric_names=["mean_absolute_error"],
+        #random_seed=0,
+    ).outputs["trained_model"]
+
+    predictions = predict_with_TensorFlow_model_on_CSV_data_op(
+        dataset=testing_data,
+        model=model,
+        # label_column_name needs to be set when doing prediction on a dataset that has labels
+        label_column_name=label_column,
+        # Optional:
+        # batch_size=1000,
+    ).outputs["predictions"]
+
+    vertex_model_name = upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op(
+        model=model,
+    ).outputs["model_name"]
+
+    # Deploying the model might incur additional costs over time
+    if deploy_model:
+        vertex_endpoint_name = deploy_model_to_endpoint_op(
+            model_name=vertex_model_name,
+        ).outputs["endpoint_name"]
+
+pipeline_func=train_tabular_regression_model_using_Tensorflow_pipeline
+
+# %% Pipeline submission
+if __name__ == '__main__':
+    from google.cloud import aiplatform
+    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_regression_model_using_XGBoost_and_import_to_Vertex_AI/pipeline.component.yaml

@@ -0,0 +1,99 @@
+name: Train tabular regression model using XGBoost pipeline
+metadata:
+  annotations:
+    author: Alexey Volkov <alexey.volkov@ark-kun.com>
+    canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_regression_model_using_XGBoost_and_import_to_Vertex_AI/pipeline.component.yaml
+    sdk: https://cloud-pipelines.net/pipeline-editor/
+implementation:
+  graph:
+    tasks:
+      Download from GCS:
+        componentRef:
+          digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
+        arguments:
+          GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
+        annotations:
+          editor.position: '{"x":40,"y":40,"width":180,"height":40}'
+      Select columns using Pandas on CSV data:
+        componentRef:
+          digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: Data
+              taskId: Download from GCS
+          column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
+        annotations:
+          editor.position: '{"x":40,"y":140,"width":180,"height":54}'
+      Fill all missing values using Pandas on CSV data:
+        componentRef:
+          digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: transformed_table
+              taskId: Select columns using Pandas on CSV data
+              type: CSV
+          replacement_value: '0'
+        annotations:
+          editor.position: '{"x":40,"y":250,"width":180,"height":54}'
+      Split rows into subsets:
+        componentRef:
+          digest: a609c3c9196484290f24a1174955f95b27f07a7b458aa5cb8cde28866cb2cb46
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: transformed_table
+              taskId: Fill all missing values using Pandas on CSV data
+              type: CSV
+          fraction_1: '0.8'
+        annotations:
+          editor.position: '{"x":170,"y":360,"width":180,"height":40}'
+      Train XGBoost model on CSV:
+        componentRef:
+          digest: 538c5a01eb38deaf532d619f0bbeaff4efc550fe1f0f776fc06791097b68ceac
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/58d3a47f904f32a64af8403330ba7e2134cae46d/components/XGBoost/Train/component.yaml
+        arguments:
+          training_data:
+            taskOutput:
+              outputName: split_1
+              taskId: Split rows into subsets
+              type: CSV
+          label_column_name: tips
+        annotations:
+          editor.position: '{"x":40,"y":480,"width":180,"height":40}'
+      Upload XGBoost model to Google Cloud Vertex AI:
+        componentRef:
+          digest: 5a5a273c403670743820986c03a4175b7cb4595a556524fefcce403656286977
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/workaround_for_buggy_KFPv2_compiler/component.yaml
+        arguments:
+          model:
+            taskOutput:
+              outputName: model
+              taskId: Train XGBoost model on CSV
+              type: XGBoostModel
+        annotations:
+          editor.position: '{"x":40,"y":600,"width":180,"height":54}'
+      Xgboost predict on CSV:
+        componentRef:
+          digest: 0876233a0c7306fefec188bd70f059b46d1fb5aa57be231799570e3bbbdd0d95
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/4694ec97baccf59284c2a1db4aa2250c22291eab/components/XGBoost/Predict/component.yaml
+        arguments:
+          data:
+            taskOutput:
+              outputName: split_2
+              taskId: Split rows into subsets
+              type: CSV
+          model:
+            taskOutput:
+              outputName: model
+              taskId: Train XGBoost model on CSV
+              type: XGBoostModel
+          label_column_name: tips
+        annotations:
+          editor.position: '{"x":240,"y":600,"width":180,"height":40}'
+    outputValues: {}

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_regression_model_using_XGBoost_and_import_to_Vertex_AI/pipeline.py

@@ -0,0 +1,85 @@
+# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
+from kfp import components
+
+# %% Loading components
+download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/storage/download/component.yaml")
+select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml")
+fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
+split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
+train_XGBoost_model_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Train/component.yaml")
+xgboost_predict_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Predict/component.yaml")
+upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/component.yaml")
+deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
+
+# %% Pipeline definition
+def train_tabular_regression_model_using_XGBoost_pipeline():
+    dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
+    feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]  # Excluded "trip_total"
+    label_column = "tips"
+    training_set_fraction = 0.8
+    # Deploying the model might incur additional costs over time
+    deploy_model = False
+
+    all_columns = [label_column] + feature_columns
+
+    dataset = download_from_gcs_op(
+        gcs_path=dataset_gcs_uri
+    ).outputs["Data"]
+
+    dataset = select_columns_using_Pandas_on_CSV_data_op(
+        table=dataset,
+        column_names=all_columns,
+    ).outputs["transformed_table"]
+
+    dataset = fill_all_missing_values_using_Pandas_on_CSV_data_op(
+        table=dataset,
+        replacement_value="0",
+        # # Optional:
+        # column_names=None,  # =[...]
+    ).outputs["transformed_table"]
+
+    split_task = split_rows_into_subsets_op(
+        table=dataset,
+        fraction_1=training_set_fraction,
+    )
+    training_data = split_task.outputs["split_1"]
+    testing_data = split_task.outputs["split_2"]
+
+    model = train_XGBoost_model_on_CSV_op(
+        training_data=training_data,
+        label_column_name=label_column,
+        # Optional:
+        #starting_model=None,
+        #num_iterations=10,
+        #booster_params={},
+        #objective="reg:squarederror",
+        #booster="gbtree",
+        #learning_rate=0.3,
+        #min_split_loss=0,
+        #max_depth=6,
+    ).outputs["model"]
+
+    # Predicting on the testing data
+    predictions = xgboost_predict_on_CSV_op(
+        data=testing_data,
+        model=model,
+        # label_column needs to be set when doing prediction on a dataset that has labels
+        label_column_name=label_column,
+    ).outputs["predictions"]
+
+    vertex_model_name = upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op(
+        model=model,
+    ).outputs["model_name"]
+
+    # Deploying the model might incur additional costs over time
+    if deploy_model:
+        vertex_endpoint_name = deploy_model_to_endpoint_op(
+            model_name=vertex_model_name,
+        ).outputs["endpoint_name"]
+
+pipeline_func = train_tabular_regression_model_using_XGBoost_pipeline
+
+# %% Pipeline submission
+if __name__ == '__main__':
+    from google.cloud import aiplatform
+    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_regression_model_using_all_frameworks_and_import_to_Vertex_AI/pipeline.component.yaml

@@ -0,0 +1,238 @@
+name: Train tabular regression model using all frameworks pipeline
+metadata:
+  annotations:
+    author: Alexey Volkov <alexey.volkov@ark-kun.com>
+    canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_regression_model_using_all_frameworks_and_import_to_Vertex_AI/pipeline.component.yaml
+    sdk: https://cloud-pipelines.net/pipeline-editor/
+implementation:
+  graph:
+    tasks:
+      Download from GCS:
+        componentRef:
+          digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
+        arguments:
+          GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
+        annotations:
+          editor.position: '{"x":550,"y":40,"width":180,"height":40}'
+      Select columns using Pandas on CSV data:
+        componentRef:
+          digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: Data
+              taskId: Download from GCS
+          column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
+        annotations:
+          editor.position: '{"x":550,"y":140,"width":180,"height":54}'
+      Fill all missing values using Pandas on CSV data:
+        componentRef:
+          digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: transformed_table
+              taskId: Select columns using Pandas on CSV data
+              type: CSV
+          replacement_value: '0'
+        annotations:
+          editor.position: '{"x":550,"y":250,"width":180,"height":54}'
+      Split rows into subsets:
+        componentRef:
+          digest: a609c3c9196484290f24a1174955f95b27f07a7b458aa5cb8cde28866cb2cb46
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml
+        arguments:
+          table:
+            taskOutput:
+              outputName: transformed_table
+              taskId: Fill all missing values using Pandas on CSV data
+              type: CSV
+          fraction_1: '0.8'
+        annotations:
+          editor.position: '{"x":550,"y":360,"width":180,"height":40}'
+      Create fully connected pytorch network:
+        componentRef:
+          digest: d03d8248fd358a0275ec33568ee7dd7dce576cc112b09dfafe2651e4d97e04a9
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1a2ef3eeb77bc278f33cad0dd29008ea2431e191/components/PyTorch/Create_fully_connected_network/component.yaml
+        arguments:
+          input_size: '7'
+          hidden_layer_sizes: '[10]'
+          activation_name: elu
+        annotations:
+          editor.position: '{"x":380,"y":490,"width":180,"height":54}'
+      Create fully connected tensorflow network:
+        componentRef:
+          digest: bfcafbc5ce711b1f69cabf1338212d10d50136a73db9f9f7c984de7b80b4bfb0
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/9ca0f9eecf5f896f65b8538bbd809747052617d1/components/tensorflow/Create_fully_connected_network/component.yaml
+        arguments:
+          input_size: '7'
+          hidden_layer_sizes: '[10]'
+          activation_name: elu
+        annotations:
+          editor.position: '{"x":40,"y":500,"width":180,"height":54}'
+      Train model using Keras on CSV:
+        componentRef:
+          digest: 42ae60c889034dbad74815653e95b4f7d576b5f47f803173e8679c7b54984609
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c504a4010348c50eaaf6d4337586ccc008f4dcef/components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml
+        arguments:
+          training_data:
+            taskOutput:
+              outputName: split_1
+              taskId: Split rows into subsets
+              type: CSV
+          model:
+            taskOutput:
+              outputName: model
+              taskId: Create fully connected tensorflow network
+              type: TensorflowSavedModel
+          label_column_name: tips
+          number_of_epochs: '10'
+          metric_names: '["mean_absolute_error"]'
+        annotations:
+          editor.position: '{"x":40,"y":620,"width":180,"height":54}'
+      Train pytorch model from csv:
+        componentRef:
+          digest: 40f3185eb61e9727f41a4e0c05dd3d3b44bd802aa0f378cfc31756560033949a
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml
+        arguments:
+          model:
+            taskOutput:
+              outputName: model
+              taskId: Create fully connected pytorch network
+              type: PyTorchScriptModule
+          training_data:
+            taskOutput:
+              outputName: split_1
+              taskId: Split rows into subsets
+              type: CSV
+          label_column_name: tips
+        annotations:
+          editor.position: '{"x":380,"y":620,"width":180,"height":40}'
+      Train XGBoost model on CSV:
+        componentRef:
+          digest: 538c5a01eb38deaf532d619f0bbeaff4efc550fe1f0f776fc06791097b68ceac
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/58d3a47f904f32a64af8403330ba7e2134cae46d/components/XGBoost/Train/component.yaml
+        arguments:
+          training_data:
+            taskOutput:
+              outputName: split_1
+              taskId: Split rows into subsets
+              type: CSV
+          label_column_name: tips
+        annotations:
+          editor.position: '{"x":720,"y":620,"width":180,"height":40}'
+      Train linear regression model using scikit learn from CSV:
+        componentRef:
+          digest: c7fe7912ab0d1fb45d201d452e9ce6be5544e7d8c6d229db7a4b931ff58560f3
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/f807e02b54d4886c65a05f40848fd51c72407f40/components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml
+        arguments:
+          dataset:
+            taskOutput:
+              outputName: split_1
+              taskId: Split rows into subsets
+              type: CSV
+          label_column_name: tips
+        annotations:
+          editor.position: '{"x":1030,"y":620,"width":180,"height":54}'
+      Predict with TensorFlow model on CSV data:
+        componentRef:
+          digest: 921bb1563e93a78233b8acceab87055b9154ccf5595d056028cf0396ca224cd4
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/59c759ce6f543184e30db6817d2a703879bc0f39/components/tensorflow/Predict/on_CSV/component.yaml
+        arguments:
+          dataset:
+            taskOutput:
+              outputName: split_2
+              taskId: Split rows into subsets
+              type: CSV
+          model:
+            taskOutput:
+              outputName: trained_model
+              taskId: Train model using Keras on CSV
+              type: TensorflowSavedModel
+          label_column_name: tips
+        annotations:
+          editor.position: '{"x":160,"y":750,"width":180,"height":54}'
+      Create PyTorch Model Archive with base handler:
+        componentRef:
+          digest: 8298b5ee1b0f0879f893add4cf352c8dec7cf9e21bb9db134c91a2d046cdb0ec
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/46d51383e6554b7f3ab4fd8cf614d8c2b422fb22/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml
+        arguments:
+          Model:
+            taskOutput:
+              outputName: trained_model
+              taskId: Train pytorch model from csv
+              type: PyTorchScriptModule
+          Model name: model
+          Model version: '1.0'
+        annotations:
+          editor.position: '{"x":380,"y":750,"width":180,"height":54}'
+      Xgboost predict on CSV:
+        componentRef:
+          digest: 0876233a0c7306fefec188bd70f059b46d1fb5aa57be231799570e3bbbdd0d95
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/4694ec97baccf59284c2a1db4aa2250c22291eab/components/XGBoost/Predict/component.yaml
+        arguments:
+          data:
+            taskOutput:
+              outputName: split_2
+              taskId: Split rows into subsets
+              type: CSV
+          model:
+            taskOutput:
+              outputName: model
+              taskId: Train XGBoost model on CSV
+              type: XGBoostModel
+          label_column_name: tips
+        annotations:
+          editor.position: '{"x":810,"y":750,"width":180,"height":40}'
+      Upload Scikit learn pickle model to Google Cloud Vertex AI:
+        componentRef:
+          digest: 81c91c8d7d21ec97e0872f669d68bd89edea87279d703685db54aa94743bebcd
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/workaround_for_buggy_KFPv2_compiler/component.yaml
+        arguments:
+          model:
+            taskOutput:
+              outputName: model
+              taskId: Train linear regression model using scikit learn from CSV
+              type: ScikitLearnPickleModel
+        annotations:
+          editor.position: '{"x":1030,"y":750,"width":180,"height":70}'
+      Upload Tensorflow model to Google Cloud Vertex AI:
+        componentRef:
+          digest: 2e45263ff640b1a688e359b6936e27a81b2407749a84f340af2aa5547e0cb92c
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/workaround_for_buggy_KFPv2_compiler/component.yaml
+        arguments:
+          model:
+            taskOutput:
+              outputName: trained_model
+              taskId: Train model using Keras on CSV
+              type: TensorflowSavedModel
+        annotations:
+          editor.position: '{"x":40,"y":880,"width":180,"height":54}'
+      Upload PyTorch model archive to Google Cloud Vertex AI:
+        componentRef:
+          digest: 4450212fae7b9001482aca7eb78b28413c205506eccf08a04e7754a8dfa99004
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/workaround_for_buggy_KFPv2_compiler/component.yaml
+        arguments:
+          model_archive:
+            taskOutput:
+              outputName: Model archive
+              taskId: Create PyTorch Model Archive with base handler
+              type: PyTorchModelArchive
+        annotations:
+          editor.position: '{"x":380,"y":880,"width":180,"height":70}'
+      Upload XGBoost model to Google Cloud Vertex AI:
+        componentRef:
+          digest: 5a5a273c403670743820986c03a4175b7cb4595a556524fefcce403656286977
+          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/workaround_for_buggy_KFPv2_compiler/component.yaml
+        arguments:
+          model:
+            taskOutput:
+              outputName: model
+              taskId: Train XGBoost model on CSV
+              type: XGBoostModel
+        annotations:
+          editor.position: '{"x":720,"y":880,"width":180,"height":54}'
+    outputValues: {}

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_regression_model_using_all_frameworks_and_import_to_Vertex_AI/pipeline.py

@@ -0,0 +1,208 @@
+# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
+from kfp import components
+
+# %% Loading components
+download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/storage/download/component.yaml")
+select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml")
+fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
+split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
+
+# TensorFlow
+create_fully_connected_tensorflow_network_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Create_fully_connected_network/component.yaml")
+train_model_using_Keras_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml")
+predict_with_TensorFlow_model_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Predict/on_CSV/component.yaml")
+upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/component.yaml")
+
+# PyTorch
+create_fully_connected_pytorch_network_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Create_fully_connected_network/component.yaml")
+train_pytorch_model_from_csv_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml")
+create_pytorch_model_archive_with_base_handler_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml")
+upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/component.yaml")
+
+# XGBoost
+train_XGBoost_model_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Train/component.yaml")
+xgboost_predict_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Predict/component.yaml")
+upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/component.yaml")
+
+# Scikit-learn
+train_linear_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/1f5cf6e06409b704064b2086c0a705e4e6b4fcde/community-content/pipeline_components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml")
+upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/component.yaml")
+
+# Vertex AI
+deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
+
+# %% Pipeline definition
+def train_tabular_regression_model_using_all_frameworks_pipeline():
+    dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
+    feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]  # Excluded "trip_total"
+    label_column = "tips"
+    training_set_fraction = 0.8
+    # Deploying the model might incur additional costs over time
+    deploy_model = False
+
+    all_columns = [label_column] + feature_columns
+
+    dataset = download_from_gcs_op(
+        gcs_path=dataset_gcs_uri
+    ).outputs["Data"]
+
+    dataset = select_columns_using_Pandas_on_CSV_data_op(
+        table=dataset,
+        column_names=all_columns,
+    ).outputs["transformed_table"]
+
+    dataset = fill_all_missing_values_using_Pandas_on_CSV_data_op(
+        table=dataset,
+        replacement_value="0",
+        # # Optional:
+        # column_names=None,  # =[...]
+    ).outputs["transformed_table"]
+
+    split_task = split_rows_into_subsets_op(
+        table=dataset,
+        fraction_1=training_set_fraction,
+    )
+    training_data = split_task.outputs["split_1"]
+    testing_data = split_task.outputs["split_2"]
+
+    # TensorFlow
+    tensorflow_network = create_fully_connected_tensorflow_network_op(
+        input_size=len(feature_columns),
+        # Optional:
+        hidden_layer_sizes=[10],
+        activation_name="elu",
+        # output_activation_name=None,
+        # output_size=1,
+    ).outputs["model"]
+
+    tensorflow_model = train_model_using_Keras_on_CSV_op(
+        training_data=training_data,
+        model=tensorflow_network,
+        label_column_name=label_column,
+        # Optional:
+        #loss_function_name="mean_squared_error",
+        number_of_epochs=10,
+        #learning_rate=0.1,
+        #optimizer_name="Adadelta",
+        #optimizer_parameters={},
+        #batch_size=32,
+        metric_names=["mean_absolute_error"],
+        #random_seed=0,
+    ).outputs["trained_model"]
+
+    tensorflow_predictions = predict_with_TensorFlow_model_on_CSV_data_op(
+        dataset=testing_data,
+        model=tensorflow_model,
+        # label_column_name needs to be set when doing prediction on a dataset that has labels
+        label_column_name=label_column,
+        # Optional:
+        # batch_size=1000,
+    ).outputs["predictions"]
+
+    tensorflow_vertex_model_name = upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op(
+        model=tensorflow_model,
+    ).outputs["model_name"]
+
+    # Deploying the model might incur additional costs over time
+    if deploy_model:
+        tensorflow_vertex_endpoint_name = deploy_model_to_endpoint_op(
+            model_name=tensorflow_vertex_model_name,
+        ).outputs["endpoint_name"]
+
+    # PyTorch
+    pytorch_network = create_fully_connected_pytorch_network_op(
+        input_size=len(feature_columns),
+        # Optional:
+        hidden_layer_sizes=[10],
+        activation_name="elu",
+        # output_activation_name=None,
+        # output_size=1,
+    ).outputs["model"]
+
+    pytorch_model = train_pytorch_model_from_csv_op(
+        model=pytorch_network,
+        training_data=training_data,
+        label_column_name=label_column,
+        # Optional:
+        #loss_function_name="mse_loss",
+        #number_of_epochs=1,
+        #learning_rate=0.1,
+        #optimizer_name="Adadelta",
+        #optimizer_parameters={},
+        #batch_size=32,
+        #batch_log_interval=100,
+        #random_seed=0,
+    ).outputs["trained_model"]
+
+    pytorch_model_archive = create_pytorch_model_archive_with_base_handler_op(
+        model=pytorch_model,
+        # Optional:
+        # model_name="model",
+        # model_version="1.0",
+    ).outputs["Model archive"]
+
+    pytorch_vertex_model_name = upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op(
+        model_archive=pytorch_model_archive,
+    ).outputs["model_name"]
+
+    # Deploying the model might incur additional costs over time
+    if deploy_model:
+        pytorch_vertex_endpoint_name = deploy_model_to_endpoint_op(
+            model_name=pytorch_vertex_model_name,
+        ).outputs["endpoint_name"]
+
+    # XGBoost
+    xgboost_model = train_XGBoost_model_on_CSV_op(
+        training_data=training_data,
+        label_column_name=label_column,
+        # Optional:
+        #starting_model=None,
+        #num_iterations=10,
+        #booster_params={},
+        #objective="reg:squarederror",
+        #booster="gbtree",
+        #learning_rate=0.3,
+        #min_split_loss=0,
+        #max_depth=6,
+    ).outputs["model"]
+
+    # Predicting on the testing data
+    xgboost_predictions = xgboost_predict_on_CSV_op(
+        data=testing_data,
+        model=xgboost_model,
+        # label_column needs to be set when doing prediction on a dataset that has labels
+        label_column_name=label_column,
+    ).outputs["predictions"]
+
+    xgboost_vertex_model_name = upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op(
+        model=xgboost_model,
+    ).outputs["model_name"]
+
+    # Deploying the model might incur additional costs over time
+    if deploy_model:
+        xgboost_vertex_endpoint_name = deploy_model_to_endpoint_op(
+            model_name=xgboost_vertex_model_name,
+        ).outputs["endpoint_name"]
+
+    # Scikit-learn
+    sklearn_model = train_linear_regression_model_using_scikit_learn_from_CSV_op(
+        dataset=training_data,
+        label_column_name=label_column,
+    ).outputs["model"]
+
+    sklearn_vertex_model_name = upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op(
+        model=sklearn_model,
+    ).outputs["model_name"]
+
+    # Deploying the model might incur additional costs over time
+    if deploy_model:
+        sklearn_vertex_endpoint_name = deploy_model_to_endpoint_op(
+            model_name=sklearn_vertex_model_name,
+        ).outputs["endpoint_name"]
+
+pipeline_func=train_tabular_regression_model_using_all_frameworks_pipeline
+
+# %% Pipeline submission
+if __name__ == '__main__':
+    from google.cloud import aiplatform
+    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

community-content/alphafold_on_workbench/AlphaFold.ipynb

@@ -0,0 +1,824 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "pc5-mbsX9PZC"
+      },
+      "source": [
+        "# AlphaFold On Vertex AI Workbench\n",
+        "\n",
+        "[Vertex AI Workbench](https://cloud.google.com/vertex-ai/docs/workbench) offers an end-to-end notebook-based production environment that can be preconfigured with the runtime dependencies necessary to run AlphaFold on Vertex AI. With [User-Managed Notebooks](https://cloud.google.com/vertex-ai/docs/workbench/user-managed/introduction), you can configure a GPU accelerator to run AlphaFold using Tensorflow, without having to install and manage drivers or JupyterLab instances. This notebook allows you to easily predict the structure of a protein using a slightly simplified version of [AlphaFold v2.1.0](https://doi.org/10.1038/s41586-021-03819-2). \n",
+        "\n",
+        "## ![](https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/community-content/alphafold_on_workbench/vertexai_40.png) [Launch this Notebook in Vertex AI Workbench](https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/raw/main/community-content/alphafold_on_workbench/AlphaFold.ipynb)\n",
+        "\n",
+        "**Differences to AlphaFold v2.1.0**\n",
+        "\n",
+        "In comparison to AlphaFold v2.1.0, this notebook notebook uses **no templates (homologous structures)** and a selected portion of the [BFD database](https://bfd.mmseqs.com/). We have validated these changes on several thousand recent PDB structures. While accuracy will be near-identical to the full AlphaFold system on many targets, a small fraction have a large drop in accuracy due to the smaller MSA and lack of templates. For best reliability, we recommend instead using the [full open source AlphaFold](https://github.com/deepmind/alphafold/), or the [AlphaFold Protein Structure Database](https://alphafold.ebi.ac.uk/).\n",
+        "\n",
+        "**This notebook has an small drop in average accuracy for multimers compared to local AlphaFold installation, for full multimer accuracy it is highly recommended to run [AlphaFold locally](https://github.com/deepmind/alphafold#running-alphafold).** Moreover, the AlphaFold-Multimer requires searching for MSA for every unique sequence in the complex, hence it is substantially slower. If your notebook times-out due to slow multimer MSA search, we recommend running AlphaFold locally.\n",
+        "\n",
+        "Please note that this notebook is provided as an early-access prototype and is not a finished product. It is provided for theoretical modelling only and caution should be exercised in its use. \n",
+        "\n",
+        "**Citing this work**\n",
+        "\n",
+        "Any publication that discloses findings arising from using this notebook should [cite](https://github.com/deepmind/alphafold/#citing-this-work) the [AlphaFold paper](https://doi.org/10.1038/s41586-021-03819-2).\n",
+        "\n",
+        "**Licenses**\n",
+        "\n",
+        "This Colab uses the [AlphaFold model parameters](https://github.com/deepmind/alphafold/#model-parameters-license) which are subject to the Creative Commons Attribution 4.0 International ([CC BY 4.0](https://creativecommons.org/licenses/by/4.0/legalcode)) license. The Colab itself is provided under the [Apache 2.0 license](https://www.apache.org/licenses/LICENSE-2.0). See the full license statement below.\n",
+        "\n",
+        "\n",
+        "**More information**\n",
+        "\n",
+        "You can find more information about how AlphaFold works in the following papers:\n",
+        "\n",
+        "*   [AlphaFold methods paper](https://www.nature.com/articles/s41586-021-03819-2)\n",
+        "*   [AlphaFold predictions of the human proteome paper](https://www.nature.com/articles/s41586-021-03828-1)\n",
+        "*   [AlphaFold-Multimer paper](https://www.biorxiv.org/content/10.1101/2021.10.04.463034v1)\n",
+        "\n",
+        "FAQ on how to interpret AlphaFold predictions are [here](https://alphafold.ebi.ac.uk/faq)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "b7a02613eb1a"
+      },
+      "source": [
+        "## Download AlphaFold Data"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "collapsed": true,
+        "jupyter": {
+          "source_hidden": true
+        },
+        "cellView": "form",
+        "id": "woIxeCPygt7K"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "import subprocess\n",
+        "import sys\n",
+        "\n",
+        "import alphafold.common\n",
+        "import tqdm.notebook\n",
+        "from IPython.utils import io\n",
+        "\n",
+        "TQDM_BAR_FORMAT = (\n",
+        "    \"{l_bar}{bar}| {n_fmt}/{total_fmt} [elapsed: {elapsed} remaining: {remaining}]\"\n",
+        ")\n",
+        "\n",
+        "SOURCE_URL = (\n",
+        "    \"https://storage.googleapis.com/alphafold/alphafold_params_colab_2022-01-19.tar\"\n",
+        ")\n",
+        "PARAMS_DIR = \"alphafold/data/params\"\n",
+        "PARAMS_PATH = os.path.join(PARAMS_DIR, os.path.basename(SOURCE_URL))\n",
+        "ALPHAFOLD_COMMON_DIR = os.path.dirname(alphafold.common.__file__)\n",
+        "\n",
+        "try:\n",
+        "    with tqdm.notebook.tqdm(total=100, bar_format=TQDM_BAR_FORMAT) as pbar:\n",
+        "        with io.capture_output() as captured:\n",
+        "\n",
+        "            # Download and store stereo_chemical_props.txt\n",
+        "            !mkdir -p ~/content/alphafold/alphafold/common\n",
+        "            !mkdir -p /opt/conda/lib/python3.7/site-packages/alphafold/common/\n",
+        "            !wget -q -P ~/content/alphafold/alphafold/common https://git.scicore.unibas.ch/schwede/openstructure/-/raw/7102c63615b64735c4941278d92b554ec94415f8/modules/mol/alg/src/stereo_chemical_props.txt\n",
+        "            pbar.update(18)\n",
+        "            !cp -f ~/content/alphafold/alphafold/common/stereo_chemical_props.txt \"{ALPHAFOLD_COMMON_DIR}\"\n",
+        "\n",
+        "            # Download alphafold_params_colab_2021-10-27.tar\n",
+        "            !mkdir --parents \"{PARAMS_DIR}\"\n",
+        "            !wget -O \"{PARAMS_PATH}\" \"{SOURCE_URL}\"\n",
+        "            pbar.update(27)\n",
+        "\n",
+        "            # Un-tar alphafold_params_colab_2021-10-27.tar\n",
+        "            !tar --extract --verbose --file=\"{PARAMS_PATH}\" --directory=\"{PARAMS_DIR}\" --preserve-permissions\n",
+        "            # !rm \"{PARAMS_PATH}\"\n",
+        "            pbar.update(55)\n",
+        "\n",
+        "except subprocess.CalledProcessError:\n",
+        "    print(captured)\n",
+        "    raise"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d8926b7d5529"
+      },
+      "source": [
+        "## Configure GPU Acceleration"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "collapsed": true,
+        "jupyter": {
+          "source_hidden": true
+        },
+        "cellView": "form",
+        "id": "VzJ5iMjTtoZw"
+      },
+      "outputs": [],
+      "source": [
+        "# Confirm accelerator configuration\n",
+        "import jax\n",
+        "\n",
+        "if jax.local_devices()[0].platform == \"tpu\":\n",
+        "    raise RuntimeError(\n",
+        "        \"TPU runtime not supported. Please configure GPU acceleration on the VM.\"\n",
+        "    )\n",
+        "elif jax.local_devices()[0].platform == \"cpu\":\n",
+        "    print(\n",
+        "        \"CPU-only runtime is not recommended, because prediction execution will be slow. For better performance, consider GPU acceleration on the VM.\"\n",
+        "    )\n",
+        "else:\n",
+        "    print(f\"Running with {jax.local_devices()[0].device_kind} GPU\")\n",
+        "\n",
+        "# Make sure all necessary environment variables are set.\n",
+        "import os\n",
+        "\n",
+        "os.environ[\"TF_FORCE_UNIFIED_MEMORY\"] = \"1\"\n",
+        "os.environ[\"XLA_PYTHON_CLIENT_MEM_FRACTION\"] = \"2.0\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "W4JpOs6oA-QS"
+      },
+      "source": [
+        "## Making a prediction\n",
+        "\n",
+        "Please paste the sequence of your protein in the text box below, then run the remaining cells via _Run_ > _Run Selected Cell and All Below_. You can also run the cells individually by pressing the _Play_ button on the left.\n",
+        "\n",
+        "Note that the search against databases and the actual prediction can take some time, from minutes to hours, depending on the length of the protein and what type of GPU you allocate (see FAQ below).\n",
+        "\n",
+        "To start, enter the amino acid sequence(s) to fold ⬇️\n",
+        "\n",
+        "If you enter only a single sequence, the monomer model will be used. If you enter multiple sequences, the multimer model will be used."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b310d44229d0"
+      },
+      "outputs": [],
+      "source": [
+        "# Input sequences (type: str)\n",
+        "sequence_1 = \"MAAHKGAEHHHKAAEHHEQAAKHHHAAAEHHEKGEHEQAAHHADTAYAHHKHAEEHAAQAAKHDAEHHAPKPH\"\n",
+        "sequence_2 = \"\"\n",
+        "sequence_3 = \"\"\n",
+        "sequence_4 = \"\"\n",
+        "sequence_5 = \"\"\n",
+        "sequence_6 = \"\"\n",
+        "sequence_7 = \"\"\n",
+        "sequence_8 = \"\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "collapsed": true,
+        "jupyter": {
+          "source_hidden": true
+        },
+        "cellView": "form",
+        "id": "rowN0bVYLe9n"
+      },
+      "outputs": [],
+      "source": [
+        "from alphafold.notebooks import notebook_utils\n",
+        "\n",
+        "input_sequences = (\n",
+        "    sequence_1,\n",
+        "    sequence_2,\n",
+        "    sequence_3,\n",
+        "    sequence_4,\n",
+        "    sequence_5,\n",
+        "    sequence_6,\n",
+        "    sequence_7,\n",
+        "    sequence_8,\n",
+        ")\n",
+        "\n",
+        "# If folding a complex target and all the input sequences are\n",
+        "# prokaryotic then set `is_prokaryotic` to `True`. Set to `False`\n",
+        "# otherwise or if the origin is unknown.\n",
+        "\n",
+        "is_prokaryote = False  # @param {type:\"boolean\"}\n",
+        "\n",
+        "MIN_SINGLE_SEQUENCE_LENGTH = 16\n",
+        "MAX_SINGLE_SEQUENCE_LENGTH = 2500\n",
+        "MAX_MULTIMER_LENGTH = 2500\n",
+        "\n",
+        "# Validate the input.\n",
+        "sequences, model_type_to_use = notebook_utils.validate_input(\n",
+        "    input_sequences=input_sequences,\n",
+        "    min_length=MIN_SINGLE_SEQUENCE_LENGTH,\n",
+        "    max_length=MAX_SINGLE_SEQUENCE_LENGTH,\n",
+        "    max_multimer_length=MAX_MULTIMER_LENGTH,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "db551d4877ea"
+      },
+      "source": [
+        "## Search against genetic databases\n",
+        "\n",
+        "Once this cell has been executed, you will see statistics about the multiple sequence alignment (MSA) that will be used by AlphaFold. In particular, you’ll see how well each residue is covered by similar  sequences in the MSA."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "collapsed": true,
+        "jupyter": {
+          "source_hidden": true
+        },
+        "cellView": "form",
+        "id": "2tTeTTsLKPjB"
+      },
+      "outputs": [],
+      "source": [
+        "import collections\n",
+        "import copy\n",
+        "import random\n",
+        "from concurrent import futures\n",
+        "from urllib import request\n",
+        "\n",
+        "import matplotlib.pyplot as plt\n",
+        "import numpy as np\n",
+        "import py3Dmol\n",
+        "from alphafold.common import protein\n",
+        "from alphafold.data import (feature_processing, msa_pairing, pipeline,\n",
+        "                            pipeline_multimer)\n",
+        "from alphafold.data.tools import jackhmmer\n",
+        "from alphafold.model import config, data, model\n",
+        "from alphafold.relax import relax, utils\n",
+        "from IPython import display\n",
+        "from ipywidgets import GridspecLayout, Output\n",
+        "\n",
+        "# Color bands for visualizing plddt\n",
+        "PLDDT_BANDS = [\n",
+        "    (0, 50, \"#FF7D45\"),\n",
+        "    (50, 70, \"#FFDB13\"),\n",
+        "    (70, 90, \"#65CBF3\"),\n",
+        "    (90, 100, \"#0053D6\"),\n",
+        "]\n",
+        "\n",
+        "# --- Find the closest source ---\n",
+        "test_url_pattern = (\n",
+        "    \"https://storage.googleapis.com/alphafold-colab{:s}/latest/uniref90_2021_03.fasta.1\"\n",
+        ")\n",
+        "ex = futures.ThreadPoolExecutor(3)\n",
+        "\n",
+        "\n",
+        "def fetch(source):\n",
+        "    request.urlretrieve(test_url_pattern.format(source))\n",
+        "    return source\n",
+        "\n",
+        "\n",
+        "fs = [ex.submit(fetch, source) for source in [\"\", \"-europe\", \"-asia\"]]\n",
+        "source = None\n",
+        "for f in futures.as_completed(fs):\n",
+        "    source = f.result()\n",
+        "    ex.shutdown()\n",
+        "    break\n",
+        "\n",
+        "JACKHMMER_BINARY_PATH = \"/usr/bin/jackhmmer\"\n",
+        "DB_ROOT_PATH = f\"https://storage.googleapis.com/alphafold-colab{source}/latest/\"\n",
+        "# The z_value is the number of sequences in a database.\n",
+        "MSA_DATABASES = [\n",
+        "    {\n",
+        "        \"db_name\": \"uniref90\",\n",
+        "        \"db_path\": f\"{DB_ROOT_PATH}uniref90_2021_03.fasta\",\n",
+        "        \"num_streamed_chunks\": 59,\n",
+        "        \"z_value\": 135_301_051,\n",
+        "    },\n",
+        "    {\n",
+        "        \"db_name\": \"smallbfd\",\n",
+        "        \"db_path\": f\"{DB_ROOT_PATH}bfd-first_non_consensus_sequences.fasta\",\n",
+        "        \"num_streamed_chunks\": 17,\n",
+        "        \"z_value\": 65_984_053,\n",
+        "    },\n",
+        "    {\n",
+        "        \"db_name\": \"mgnify\",\n",
+        "        \"db_path\": f\"{DB_ROOT_PATH}mgy_clusters_2019_05.fasta\",\n",
+        "        \"num_streamed_chunks\": 71,\n",
+        "        \"z_value\": 304_820_129,\n",
+        "    },\n",
+        "]\n",
+        "\n",
+        "# Search UniProt and construct the all_seq features only for heteromers, not homomers.\n",
+        "if model_type_to_use == notebook_utils.ModelType.MULTIMER and len(set(sequences)) > 1:\n",
+        "    MSA_DATABASES.extend(\n",
+        "        [\n",
+        "            # Swiss-Prot and TrEMBL are concatenated together as UniProt.\n",
+        "            {\n",
+        "                \"db_name\": \"uniprot\",\n",
+        "                \"db_path\": f\"{DB_ROOT_PATH}uniprot_2021_03.fasta\",\n",
+        "                \"num_streamed_chunks\": 98,\n",
+        "                \"z_value\": 219_174_961 + 565_254,\n",
+        "            },\n",
+        "        ]\n",
+        "    )\n",
+        "\n",
+        "TOTAL_JACKHMMER_CHUNKS = sum(cfg[\"num_streamed_chunks\"] for cfg in MSA_DATABASES)\n",
+        "\n",
+        "MAX_HITS = {\n",
+        "    \"uniref90\": 10_000,\n",
+        "    \"smallbfd\": 5_000,\n",
+        "    \"mgnify\": 501,\n",
+        "    \"uniprot\": 50_000,\n",
+        "}\n",
+        "\n",
+        "\n",
+        "def get_msa(fasta_path):\n",
+        "    \"\"\"Searches for MSA for the given sequence using chunked Jackhmmer search.\"\"\"\n",
+        "\n",
+        "    # Run the search against chunks of genetic databases.\n",
+        "    raw_msa_results = collections.defaultdict(list)\n",
+        "    with tqdm.notebook.tqdm(\n",
+        "        total=TOTAL_JACKHMMER_CHUNKS, bar_format=TQDM_BAR_FORMAT\n",
+        "    ) as pbar:\n",
+        "\n",
+        "        def jackhmmer_chunk_callback(i):\n",
+        "            pbar.update(n=1)\n",
+        "\n",
+        "        for db_config in MSA_DATABASES:\n",
+        "            db_name = db_config[\"db_name\"]\n",
+        "            pbar.set_description(f\"Searching {db_name}\")\n",
+        "            jackhmmer_runner = jackhmmer.Jackhmmer(\n",
+        "                binary_path=JACKHMMER_BINARY_PATH,\n",
+        "                database_path=db_config[\"db_path\"],\n",
+        "                get_tblout=True,\n",
+        "                num_streamed_chunks=db_config[\"num_streamed_chunks\"],\n",
+        "                streaming_callback=jackhmmer_chunk_callback,\n",
+        "                z_value=db_config[\"z_value\"],\n",
+        "            )\n",
+        "            # Group the results by database name.\n",
+        "            raw_msa_results[db_name].extend(jackhmmer_runner.query(fasta_path))\n",
+        "\n",
+        "    return raw_msa_results\n",
+        "\n",
+        "\n",
+        "features_for_chain = {}\n",
+        "raw_msa_results_for_sequence = {}\n",
+        "for sequence_index, sequence in enumerate(sequences, start=1):\n",
+        "    print(f\"\\nGetting MSA for sequence {sequence_index}\")\n",
+        "\n",
+        "    fasta_path = f\"target_{sequence_index}.fasta\"\n",
+        "    with open(fasta_path, \"wt\") as f:\n",
+        "        f.write(f\">query\\n{sequence}\")\n",
+        "\n",
+        "    # Don't do redundant work for multiple copies of the same chain in the multimer.\n",
+        "    if sequence not in raw_msa_results_for_sequence:\n",
+        "        raw_msa_results = get_msa(fasta_path=fasta_path)\n",
+        "        raw_msa_results_for_sequence[sequence] = raw_msa_results\n",
+        "    else:\n",
+        "        raw_msa_results = copy.deepcopy(raw_msa_results_for_sequence[sequence])\n",
+        "\n",
+        "    # Extract the MSAs from the Stockholm files.\n",
+        "    # NB: deduplication happens later in pipeline.make_msa_features.\n",
+        "    single_chain_msas = []\n",
+        "    uniprot_msa = None\n",
+        "    for db_name, db_results in raw_msa_results.items():\n",
+        "        merged_msa = notebook_utils.merge_chunked_msa(\n",
+        "            results=db_results, max_hits=MAX_HITS.get(db_name)\n",
+        "        )\n",
+        "        if merged_msa.sequences and db_name != \"uniprot\":\n",
+        "            single_chain_msas.append(merged_msa)\n",
+        "            msa_size = len(set(merged_msa.sequences))\n",
+        "            print(\n",
+        "                f\"{msa_size} unique sequences found in {db_name} for sequence {sequence_index}\"\n",
+        "            )\n",
+        "        elif merged_msa.sequences and db_name == \"uniprot\":\n",
+        "            uniprot_msa = merged_msa\n",
+        "\n",
+        "    notebook_utils.show_msa_info(\n",
+        "        single_chain_msas=single_chain_msas, sequence_index=sequence_index\n",
+        "    )\n",
+        "\n",
+        "    # Turn the raw data into model features.\n",
+        "    feature_dict = {}\n",
+        "    feature_dict.update(\n",
+        "        pipeline.make_sequence_features(\n",
+        "            sequence=sequence, description=\"query\", num_res=len(sequence)\n",
+        "        )\n",
+        "    )\n",
+        "    feature_dict.update(pipeline.make_msa_features(msas=single_chain_msas))\n",
+        "    # We don't use templates in AlphaFold notebook, add only empty placeholder features.\n",
+        "    feature_dict.update(\n",
+        "        notebook_utils.empty_placeholder_template_features(\n",
+        "            num_templates=0, num_res=len(sequence)\n",
+        "        )\n",
+        "    )\n",
+        "\n",
+        "    # Construct the all_seq features only for heteromers, not homomers.\n",
+        "    if (\n",
+        "        model_type_to_use == notebook_utils.ModelType.MULTIMER\n",
+        "        and len(set(sequences)) > 1\n",
+        "    ):\n",
+        "        valid_feats = msa_pairing.MSA_FEATURES + (\n",
+        "            \"msa_uniprot_accession_identifiers\",\n",
+        "            \"msa_species_identifiers\",\n",
+        "        )\n",
+        "        all_seq_features = {\n",
+        "            f\"{k}_all_seq\": v\n",
+        "            for k, v in pipeline.make_msa_features([uniprot_msa]).items()\n",
+        "            if k in valid_feats\n",
+        "        }\n",
+        "        feature_dict.update(all_seq_features)\n",
+        "\n",
+        "    features_for_chain[protein.PDB_CHAIN_IDS[sequence_index - 1]] = feature_dict\n",
+        "\n",
+        "\n",
+        "# Do further feature post-processing depending on the model type.\n",
+        "if model_type_to_use == notebook_utils.ModelType.MONOMER:\n",
+        "    np_example = features_for_chain[protein.PDB_CHAIN_IDS[0]]\n",
+        "\n",
+        "elif model_type_to_use == notebook_utils.ModelType.MULTIMER:\n",
+        "    all_chain_features = {}\n",
+        "    for chain_id, chain_features in features_for_chain.items():\n",
+        "        all_chain_features[chain_id] = pipeline_multimer.convert_monomer_features(\n",
+        "            chain_features, chain_id\n",
+        "        )\n",
+        "\n",
+        "    all_chain_features = pipeline_multimer.add_assembly_features(all_chain_features)\n",
+        "\n",
+        "    np_example = feature_processing.pair_and_merge(\n",
+        "        all_chain_features=all_chain_features, is_prokaryote=is_prokaryote\n",
+        "    )\n",
+        "\n",
+        "    # Pad MSA to avoid zero-sized extra_msa.\n",
+        "    np_example = pipeline_multimer.pad_msa(np_example, min_num_seq=512)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9640643486bd"
+      },
+      "source": [
+        "## Run AlphaFold\n",
+        "\n",
+        "Once this cell has been executed, a zip-archive \"prediction.zip\" with the obtained prediction will be saved on the VM, and available for download to your computer in the sidebar. In case you are having issues with the relaxation stage, you can disable it below. Warning: This means that the prediction might have distracting small stereochemical violations."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "collapsed": true,
+        "jupyter": {
+          "source_hidden": true
+        },
+        "cellView": "form",
+        "id": "XUo6foMQxwS2"
+      },
+      "outputs": [],
+      "source": [
+        "run_relax = True\n",
+        "\n",
+        "# --- Run the model ---\n",
+        "if model_type_to_use == notebook_utils.ModelType.MONOMER:\n",
+        "    model_names = config.MODEL_PRESETS[\"monomer\"] + (\"model_2_ptm\",)\n",
+        "elif model_type_to_use == notebook_utils.ModelType.MULTIMER:\n",
+        "    model_names = config.MODEL_PRESETS[\"multimer\"]\n",
+        "\n",
+        "output_dir = \"prediction\"\n",
+        "os.makedirs(output_dir, exist_ok=True)\n",
+        "\n",
+        "plddts = {}\n",
+        "ranking_confidences = {}\n",
+        "pae_outputs = {}\n",
+        "unrelaxed_proteins = {}\n",
+        "\n",
+        "with tqdm.notebook.tqdm(total=len(model_names) + 1, bar_format=TQDM_BAR_FORMAT) as pbar:\n",
+        "    for model_name in model_names:\n",
+        "        pbar.set_description(f\"Running {model_name}\")\n",
+        "\n",
+        "        cfg = config.model_config(model_name)\n",
+        "        if model_type_to_use == notebook_utils.ModelType.MONOMER:\n",
+        "            cfg.data.eval.num_ensemble = 1\n",
+        "        elif model_type_to_use == notebook_utils.ModelType.MULTIMER:\n",
+        "            cfg.model.num_ensemble_eval = 1\n",
+        "        params = data.get_model_haiku_params(model_name, \"./alphafold/data\")\n",
+        "        model_runner = model.RunModel(cfg, params)\n",
+        "        processed_feature_dict = model_runner.process_features(\n",
+        "            np_example, random_seed=0\n",
+        "        )\n",
+        "        prediction = model_runner.predict(\n",
+        "            processed_feature_dict, random_seed=random.randrange(sys.maxsize)\n",
+        "        )\n",
+        "\n",
+        "        mean_plddt = prediction[\"plddt\"].mean()\n",
+        "\n",
+        "        if model_type_to_use == notebook_utils.ModelType.MONOMER:\n",
+        "            if \"predicted_aligned_error\" in prediction:\n",
+        "                pae_outputs[model_name] = (\n",
+        "                    prediction[\"predicted_aligned_error\"],\n",
+        "                    prediction[\"max_predicted_aligned_error\"],\n",
+        "                )\n",
+        "            else:\n",
+        "                # Monomer models are sorted by mean pLDDT. Do not put monomer pTM models here as they\n",
+        "                # should never get selected.\n",
+        "                ranking_confidences[model_name] = prediction[\"ranking_confidence\"]\n",
+        "                plddts[model_name] = prediction[\"plddt\"]\n",
+        "        elif model_type_to_use == notebook_utils.ModelType.MULTIMER:\n",
+        "            # Multimer models are sorted by pTM+ipTM.\n",
+        "            ranking_confidences[model_name] = prediction[\"ranking_confidence\"]\n",
+        "            plddts[model_name] = prediction[\"plddt\"]\n",
+        "            pae_outputs[model_name] = (\n",
+        "                prediction[\"predicted_aligned_error\"],\n",
+        "                prediction[\"max_predicted_aligned_error\"],\n",
+        "            )\n",
+        "\n",
+        "        # Set the b-factors to the per-residue plddt.\n",
+        "        final_atom_mask = prediction[\"structure_module\"][\"final_atom_mask\"]\n",
+        "        b_factors = prediction[\"plddt\"][:, None] * final_atom_mask\n",
+        "        unrelaxed_protein = protein.from_prediction(\n",
+        "            processed_feature_dict,\n",
+        "            prediction,\n",
+        "            b_factors=b_factors,\n",
+        "            remove_leading_feature_dimension=(\n",
+        "                model_type_to_use == notebook_utils.ModelType.MONOMER\n",
+        "            ),\n",
+        "        )\n",
+        "        unrelaxed_proteins[model_name] = unrelaxed_protein\n",
+        "\n",
+        "        # Delete unused outputs to save memory.\n",
+        "        del model_runner\n",
+        "        del params\n",
+        "        del prediction\n",
+        "        pbar.update(n=1)\n",
+        "\n",
+        "    # --- AMBER relax the best model ---\n",
+        "\n",
+        "    # Find the best model according to the mean pLDDT.\n",
+        "    best_model_name = max(\n",
+        "        ranking_confidences.keys(), key=lambda x: ranking_confidences[x]\n",
+        "    )\n",
+        "\n",
+        "    if run_relax:\n",
+        "        pbar.set_description(\"AMBER relaxation\")\n",
+        "        amber_relaxer = relax.AmberRelaxation(\n",
+        "            max_iterations=0,\n",
+        "            tolerance=2.39,\n",
+        "            stiffness=10.0,\n",
+        "            exclude_residues=[],\n",
+        "            max_outer_iterations=3,\n",
+        "        )\n",
+        "        relaxed_pdb, _, _ = amber_relaxer.process(\n",
+        "            prot=unrelaxed_proteins[best_model_name]\n",
+        "        )\n",
+        "    else:\n",
+        "        print(\"Warning: Running without the relaxation stage.\")\n",
+        "        relaxed_pdb = protein.to_pdb(unrelaxed_proteins[best_model_name])\n",
+        "    pbar.update(n=1)  # Finished AMBER relax.\n",
+        "\n",
+        "# Construct multiclass b-factors to indicate confidence bands\n",
+        "# 0=very low, 1=low, 2=confident, 3=very high\n",
+        "banded_b_factors = []\n",
+        "for plddt in plddts[best_model_name]:\n",
+        "    for idx, (min_val, max_val, _) in enumerate(PLDDT_BANDS):\n",
+        "        if plddt >= min_val and plddt <= max_val:\n",
+        "            banded_b_factors.append(idx)\n",
+        "            break\n",
+        "banded_b_factors = np.array(banded_b_factors)[:, None] * final_atom_mask\n",
+        "to_visualize_pdb = utils.overwrite_b_factors(relaxed_pdb, banded_b_factors)\n",
+        "\n",
+        "\n",
+        "# Write out the prediction\n",
+        "pred_output_path = os.path.join(output_dir, \"selected_prediction.pdb\")\n",
+        "with open(pred_output_path, \"w\") as f:\n",
+        "    f.write(relaxed_pdb)\n",
+        "\n",
+        "\n",
+        "# --- Visualise the prediction & confidence ---\n",
+        "show_sidechains = True\n",
+        "\n",
+        "\n",
+        "def plot_plddt_legend():\n",
+        "    \"\"\"Plots the legend for pLDDT.\"\"\"\n",
+        "    thresh = [\n",
+        "        \"Very low (pLDDT < 50)\",\n",
+        "        \"Low (70 > pLDDT > 50)\",\n",
+        "        \"Confident (90 > pLDDT > 70)\",\n",
+        "        \"Very high (pLDDT > 90)\",\n",
+        "    ]\n",
+        "\n",
+        "    colors = [x[2] for x in PLDDT_BANDS]\n",
+        "\n",
+        "    plt.figure(figsize=(2, 2))\n",
+        "    for c in colors:\n",
+        "        plt.bar(0, 0, color=c)\n",
+        "    plt.legend(thresh, frameon=False, loc=\"center\", fontsize=20)\n",
+        "    plt.xticks([])\n",
+        "    plt.yticks([])\n",
+        "    ax = plt.gca()\n",
+        "    ax.spines[\"right\"].set_visible(False)\n",
+        "    ax.spines[\"top\"].set_visible(False)\n",
+        "    ax.spines[\"left\"].set_visible(False)\n",
+        "    ax.spines[\"bottom\"].set_visible(False)\n",
+        "    plt.title(\"Model Confidence\", fontsize=20, pad=20)\n",
+        "    return plt\n",
+        "\n",
+        "\n",
+        "# Show the structure coloured by chain if the multimer model has been used.\n",
+        "if model_type_to_use == notebook_utils.ModelType.MULTIMER:\n",
+        "    multichain_view = py3Dmol.view(width=800, height=600)\n",
+        "    multichain_view.addModelsAsFrames(to_visualize_pdb)\n",
+        "    multichain_style = {\"cartoon\": {\"colorscheme\": \"chain\"}}\n",
+        "    multichain_view.setStyle({\"model\": -1}, multichain_style)\n",
+        "    multichain_view.zoomTo()\n",
+        "    multichain_view.show()\n",
+        "\n",
+        "# Color the structure by per-residue pLDDT\n",
+        "color_map = {i: bands[2] for i, bands in enumerate(PLDDT_BANDS)}\n",
+        "view = py3Dmol.view(width=800, height=600)\n",
+        "view.addModelsAsFrames(to_visualize_pdb)\n",
+        "style = {\"cartoon\": {\"colorscheme\": {\"prop\": \"b\", \"map\": color_map}}}\n",
+        "if show_sidechains:\n",
+        "    style[\"stick\"] = {}\n",
+        "view.setStyle({\"model\": -1}, style)\n",
+        "view.zoomTo()\n",
+        "\n",
+        "grid = GridspecLayout(1, 2)\n",
+        "out = Output()\n",
+        "with out:\n",
+        "    view.show()\n",
+        "grid[0, 0] = out\n",
+        "\n",
+        "out = Output()\n",
+        "with out:\n",
+        "    plot_plddt_legend().show()\n",
+        "grid[0, 1] = out\n",
+        "\n",
+        "display.display(grid)\n",
+        "\n",
+        "# Display pLDDT and predicted aligned error (if output by the model).\n",
+        "if pae_outputs:\n",
+        "    num_plots = 2\n",
+        "else:\n",
+        "    num_plots = 1\n",
+        "\n",
+        "plt.figure(figsize=[8 * num_plots, 6])\n",
+        "plt.subplot(1, num_plots, 1)\n",
+        "plt.plot(plddts[best_model_name])\n",
+        "plt.title(\"Predicted LDDT\")\n",
+        "plt.xlabel(\"Residue\")\n",
+        "plt.ylabel(\"pLDDT\")\n",
+        "\n",
+        "if num_plots == 2:\n",
+        "    plt.subplot(1, 2, 2)\n",
+        "    pae, max_pae = list(pae_outputs.values())[0]\n",
+        "    plt.imshow(pae, vmin=0.0, vmax=max_pae, cmap=\"Greens_r\")\n",
+        "    plt.colorbar(fraction=0.046, pad=0.04)\n",
+        "\n",
+        "    # Display lines at chain boundaries.\n",
+        "    best_unrelaxed_prot = unrelaxed_proteins[best_model_name]\n",
+        "    total_num_res = best_unrelaxed_prot.residue_index.shape[-1]\n",
+        "    chain_ids = best_unrelaxed_prot.chain_index\n",
+        "    for chain_boundary in np.nonzero(chain_ids[:-1] - chain_ids[1:]):\n",
+        "        if chain_boundary.size:\n",
+        "            plt.plot([0, total_num_res], [chain_boundary, chain_boundary], color=\"red\")\n",
+        "            plt.plot([chain_boundary, chain_boundary], [0, total_num_res], color=\"red\")\n",
+        "\n",
+        "    plt.title(\"Predicted Aligned Error\")\n",
+        "    plt.xlabel(\"Scored residue\")\n",
+        "    plt.ylabel(\"Aligned residue\")\n",
+        "\n",
+        "# Save the predicted aligned error (if it exists).\n",
+        "pae_output_path = os.path.join(output_dir, \"predicted_aligned_error.json\")\n",
+        "if pae_outputs:\n",
+        "    # Save predicted aligned error in the same format as the AF EMBL DB.\n",
+        "    pae_data = notebook_utils.get_pae_json(pae=pae, max_pae=max_pae.item())\n",
+        "    with open(pae_output_path, \"w\") as f:\n",
+        "        f.write(pae_data)\n",
+        "\n",
+        "!zip -q -r {output_dir}.zip {output_dir}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "lUQAn5LYC5n4"
+      },
+      "source": [
+        "### Interpreting the prediction\n",
+        "\n",
+        "In general predicted LDDT (pLDDT) is best used for intra-domain confidence, whereas Predicted Aligned Error (PAE) is best used for determining between domain or between chain confidence.\n",
+        "\n",
+        "Please see the [AlphaFold methods paper](https://www.nature.com/articles/s41586-021-03819-2), the [AlphaFold predictions of the human proteome paper](https://www.nature.com/articles/s41586-021-03828-1), and the [AlphaFold-Multimer paper](https://www.biorxiv.org/content/10.1101/2021.10.04.463034v1) as well as [our FAQ](https://alphafold.ebi.ac.uk/faq) on how to interpret AlphaFold predictions."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "jeb2z8DIA4om"
+      },
+      "source": [
+        "## FAQ & Troubleshooting\n",
+        "\n",
+        "\n",
+        "*   How do I get a predicted protein structure for my protein?\n",
+        "    *   Connect the notebook to the Jupyter kernel \"Python 3 (ipykernel)\".\n",
+        "    *   Paste the amino acid sequence of your protein (without any headers) into the variable sequence_1 in \"Making a Prediction\".\n",
+        "    *   Run all cells in the notebook, either by running them individually or via \"Kernel\"/\"Restart Kernel and Run All Cells...\"\n",
+        "    *   The predicted protein structure will be downloaded once all cells have been executed. Note: This can take minutes to hours - see below.\n",
+        "*   How long will this take?\n",
+        "    *   The search against genetic databases can take minutes to hours.\n",
+        "    *   Running AlphaFold and generating the prediction can take minutes to hours, depending on the length of your protein and on which GPU-type your VM has access to.\n",
+        "*   My notebook no longer seems to be doing anything, what should I do?\n",
+        "    *   Some steps may take minutes to hours to complete.\n",
+        "    *   If nothing happens or if you receive an error message, try restarting your notebook runtime via \"Kernel\"/\"Restart Kernel and Run All Cells...\".\n",
+        "    *   If this doesn’t help, try resetting restarting your VM inside the GCloud Console (\"Compute Engine\"/\"VM Instances\").\n",
+        "*   How does this compare to the open-source version of AlphaFold?\n",
+        "    *   This notebook version of AlphaFold searches a selected portion of the BFD dataset and currently doesn’t use templates, so its accuracy is reduced in comparison to the full version of AlphaFold that is described in the [AlphaFold paper](https://doi.org/10.1038/s41586-021-03819-2) and [Github repo](https://github.com/deepmind/alphafold/) (the full version is available via the inference script).\n",
+        "*   I received a warning “Notebook requires high RAM”, what do I do?\n",
+        "    *   In the \"Compute Engine\"/\"VM Instances\" Console menu, you can reconfigure the host VM settings. See [Changing the machine type of a VM instance](https://cloud.google.com/compute/docs/instances/changing-machine-type-of-stopped-instance) for instructions.\n",
+        "*   Does this tool install anything on my computer?\n",
+        "    *   No, everything happens in the VM instance within your Google Cloud project.\n",
+        "*   How should I share feedback and bug reports?\n",
+        "    *   Please share any feedback and bug reports as an [issue](https://github.com/GoogleCloudPlatform/vertex-ai-samples/issues) on Github.\n",
+        "\n",
+        "\n",
+        "## Related work\n",
+        "\n",
+        "Take a look at these Colab notebooks provided by the community (please note that these notebooks may vary from our validated AlphaFold system and we cannot guarantee their accuracy):\n",
+        "\n",
+        "*   The [ColabFold AlphaFold2 notebook](https://colab.research.google.com/github/sokrypton/ColabFold/blob/main/AlphaFold2.ipynb) by Sergey Ovchinnikov, Milot Mirdita and Martin Steinegger, which uses an API hosted at the Södinglab based on the MMseqs2 server ([Mirdita et al. 2019, Bioinformatics](https://academic.oup.com/bioinformatics/article/35/16/2856/5280135)) for the multiple sequence alignment creation.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "YfPhvYgKC81B"
+      },
+      "source": [
+        "# License and Disclaimer\n",
+        "\n",
+        "This is not an officially-supported Google product.\n",
+        "\n",
+        "This notebook and other information provided is for theoretical modelling only, caution should be exercised in its use. It is provided ‘as-is’ without any warranty of any kind, whether expressed or implied. Information is not intended to be a substitute for professional medical advice, diagnosis, or treatment, and does not constitute medical or other professional advice.\n",
+        "\n",
+        "Copyright 2021 DeepMind Technologies Limited.\n",
+        "\n",
+        "\n",
+        "## AlphaFold Code License\n",
+        "\n",
+        "Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0.\n",
+        "\n",
+        "Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.\n",
+        "\n",
+        "## Model Parameters License\n",
+        "\n",
+        "The AlphaFold parameters are made available under the terms of the Creative Commons Attribution 4.0 International (CC BY 4.0) license. You can find details at: https://creativecommons.org/licenses/by/4.0/legalcode\n",
+        "\n",
+        "\n",
+        "## Third-party software\n",
+        "\n",
+        "Use of the third-party software, libraries or code referred to in the [Acknowledgements section](https://github.com/deepmind/alphafold/#acknowledgements) in the AlphaFold README may be governed by separate terms and conditions or license provisions. Your use of the third-party software, libraries or code is subject to any such terms and you should check that you can comply with any applicable restrictions or terms and conditions before use.\n",
+        "\n",
+        "\n",
+        "## Mirrored Databases\n",
+        "\n",
+        "The following databases have been mirrored by DeepMind, and are available with reference to the following:\n",
+        "* UniProt: v2021\\_03 (unmodified), by The UniProt Consortium, available under a [Creative Commons Attribution-NoDerivatives 4.0 International License](http://creativecommons.org/licenses/by-nd/4.0/).\n",
+        "* UniRef90: v2021\\_03 (unmodified), by The UniProt Consortium, available under a [Creative Commons Attribution-NoDerivatives 4.0 International License](http://creativecommons.org/licenses/by-nd/4.0/).\n",
+        "* MGnify: v2019\\_05 (unmodified), by Mitchell AL et al., available free of all copyright restrictions and made fully and freely available for both non-commercial and commercial use under [CC0 1.0 Universal (CC0 1.0) Public Domain Dedication](https://creativecommons.org/publicdomain/zero/1.0/).\n",
+        "* BFD: (modified), by Steinegger M. and Söding J., modified by DeepMind, available under a [Creative Commons Attribution-ShareAlike 4.0 International License](https://creativecommons.org/licenses/by/4.0/). See the Methods section of the [AlphaFold proteome paper](https://www.nature.com/articles/s41586-021-03828-1) for details."
+      ]
+    }
+  ],
+  "metadata": {
+    "accelerator": "GPU",
+    "colab": {
+      "collapsed_sections": [],
+      "name": "AlphaFold.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

community-content/alphafold_on_workbench/Dockerfile

@@ -0,0 +1,82 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG CUDA_MAJOR=11
+ARG CUDA_MINOR=0
+
+FROM gcr.io/deeplearning-platform-release/base-cu110
+
+ARG CUDA_MAJOR
+ARG CUDA_MINOR
+
+SHELL ["/bin/bash", "-c"]
+
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+      build-essential \
+      cmake \
+      cuda-command-line-tools-${CUDA_MAJOR}-${CUDA_MINOR} \
+      git \
+      hmmer \
+      kalign \
+      tzdata \
+      wget \
+    && rm -rf /var/lib/apt/lists/*
+
+# Compile HHsuite from source.
+RUN git clone --branch v3.3.0 https://github.com/soedinglab/hh-suite.git /tmp/hh-suite \
+    && mkdir /tmp/hh-suite/build \
+    && pushd /tmp/hh-suite/build \
+    && cmake -DCMAKE_INSTALL_PREFIX=/opt/hhsuite .. \
+    && make -j 4 && make install \
+    && ln -s /opt/hhsuite/bin/* /usr/bin \
+    && popd \
+    && rm -rf /tmp/hh-suite
+
+ENV PATH="/opt/conda/bin:$PATH"
+RUN conda update -qy conda \
+    && conda install -y -c conda-forge \
+      openmm=7.5.1 \
+      cudatoolkit==${CUDA_VERSION} \
+      pdbfixer \
+      pip \
+      python=3.7
+
+COPY . /app/alphafold
+
+# Install pip packages.
+RUN pip3 install --upgrade pip \
+    && pip3 install -r /app/alphafold/requirements.txt \
+    && pip3 install py3Dmol tqdm \
+    && pip3 install --upgrade jax==0.2.14 jaxlib==0.1.69+cuda${CUDA_MAJOR}${CUDA_MINOR} -f \
+      https://storage.googleapis.com/jax-releases/jax_releases.html
+
+# Install alphafold.
+WORKDIR /app/alphafold
+RUN python setup.py install
+
+# Apply OpenMM patch.
+WORKDIR /opt/conda/lib/python3.7/site-packages
+RUN patch -p0 < /app/alphafold/docker/openmm.patch
+
+# Creating a tmp location for jackhmmr; not mounting through to host though.
+RUN sudo mkdir -m 777 --parents /tmp/ramdisk
+
+# We need to run `ldconfig` first to ensure GPUs are visible, due to some quirk
+# with Debian. See https://github.com/NVIDIA/nvidia-docker/issues/1399 for
+# details.
+# ENTRYPOINT does not support easily running multiple commands, so instead we
+# write a shell script to wrap them up.
+WORKDIR /home/jupyter
+RUN echo '#!/bin/bash\nldconfig\n\'
+

community-content/alphafold_on_workbench/build_docker.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+set -e
+
+# Prod (Publicly viewable)
+PROJECT=cloud-devrel-public-resources
+REPOSITORY=alphafold
+LOCAL_IMAGE=alphafold-on-gcp
+REMOTE_IMAGE=${LOCAL_IMAGE?}
+TAG=latest
+REGISTRY="us-west1-docker.pkg.dev/${PROJECT?}/${REPOSITORY?}/${REMOTE_IMAGE?}:${TAG?}"
+
+git clone https://github.com/deepmind/alphafold.git
+
+cp Dockerfile alphafold/docker/Dockerfile
+cp AlphaFold.ipynb alphafold/notebooks/AlphaFold.ipynb
+
+cd alphafold && sudo docker build --tag ${LOCAL_IMAGE?}:${TAG?} -f docker/Dockerfile .
+
+sudo docker tag ${LOCAL_IMAGE?}:${TAG?} ${REGISTRY?}
+sudo docker push ${REGISTRY?}

community-content/cpr-examples/timm_serving/.dockerignore

@@ -0,0 +1,5 @@
+testdata/*
+build.py
+test.py
+state_dict.pth
+config.json

community-content/cpr-examples/timm_serving/.gitignore

@@ -0,0 +1,6 @@
+cpr_model_server.py
+entrypoint.py
+state_dict.pth
+config.json
+**/__pycache__
+!testdata/**

community-content/cpr-examples/timm_serving/README.md

@@ -0,0 +1,110 @@
+# CPR Example: PyTorch Image Models (timm)
+
+## About CPR
+
+CPR ([custom prediction routines](https://github.com/googleapis/python-aiplatform/blob/main/google/cloud/aiplatform/prediction/README.md)) is a framework designed by Google Cloud developers to make it easier to combine machine learning models with custom preprocessing and postprocessing logic in a real-time serving application. 
+
+## Using this example
+
+This code is a self-contained example of a custom model server project built using CPR.
+
+As is, you can use it to serve the ViT-Small image classification model from Ross Wightman's [`timm`](https://github.com/rwightman/pytorch-image-models) library of image model implementations in PyTorch. Both CPU and GPU are supported.
+
+You can also consider using the code here as a template for your own CPR project if you want to use a different model from `timm`, a different PyTorch model, or an entirely different framework.
+
+### Requirements
+
+In order to use this example, you'll need Docker and Python 3 installed on your system.
+
+To get started, first create a virtual environment in an empty directory:
+```sh
+mkdir cpr-example
+python3 -m venv cpr-example
+cd cpr-example && source bin/activate
+``` 
+
+Then, clone the [vertex-ai-samples repo](https://github.com/GoogleCloudPlatform/vertex-ai-samples) in that directory:
+```sh
+git clone https://github.com/GoogleCloudPlatform/vertex-ai-samples.git
+cd vertex-ai-samples/community-content/cpr-examples/timm_serving
+```
+
+Finally, install the Python modules required to build and run the model server:
+```sh
+pip install -r requirements.txt
+```
+
+### Auth
+
+This example uses Google Cloud Storage for hosting model artifacts and Artifact Registry to store the container image. 
+You'll need to authorize yourself before you can interact with these.
+
+First, log in to GCP with application default credentials:
+```sh
+gcloud auth application-default login
+```
+
+Next, if you haven't done so already, set up the [gcloud credential helper](https://cloud.google.com/artifact-registry/docs/docker/authentication)
+for the Artifact Registry region where you intend to host the image.  
+```
+gcloud auth configure-docker <region>-docker.pkg.dev
+```
+
+
+### Predictor
+
+The `TimmPredictor` class in `timm_serving/predictor.py` implements most of the important logic for the server.
+
+- `load(artifacts_dir)`: The predictor's `load` method is called when the server starts up in order to set up the predictor, usually by loading model weights and any artifacts needed for preprocessing and postprocessing. In this example, we initialize the saved model from the `state_dict.pth` file located inside the `artifacts_dir` folder and create the preprocessing transform from the model config.
+
+- `preprocess`, `predict`, `postprocess`: These methods are applied in sequence to the deserialized JSON data from each request. 
+    - `preprocess` decodes images from base64 and apply cropping, scaling and normalizing transforms. 
+    - `predict` runs the ViT-Small model on the preprocessed images and returns class scores.
+    - `postprocess` finds the top five classes and packs the class names, probabilities, and indices in a serializable result.
+
+### Building the container
+
+To build the model server locally, run the build command:
+```sh
+python build.py build
+```
+
+You can edit configuration values such as the model server's base image, the name and tag assigned to the image, and the path where model weights are stored locally.
+
+When you run the build command, model weights are downloaded and the model server container is built.
+
+### Running local tests
+
+`test.py` contains a suite of unit tests for the predictor as well as end-to-end tests for the model server. 
+
+To run the tests:
+```sh
+python test.py
+```
+
+All of the test images are public domain.
+- [Cat](https://commons.wikimedia.org/wiki/File:Stray_cat_on_wall.jpg)
+- [Airplane](https://commons.wikimedia.org/wiki/File:Airplanes_jets.jpg)
+- The infamous [mandrill](https://commons.wikimedia.org/wiki/File:Wikipedia-sipi-image-db-mandrill-4.2.03.png)
+
+### Deploying to Vertex AI
+
+Before uploading or deploying the container, you'll need to modify `config.py` to set appropriate values for: 
+- `project_id`: Your GCP project id.
+- `region`: Region where the model will be uploaded and deployed.
+- `repository`: [Artifact Registry repository](https://cloud.google.com/artifact-registry/docs/repositories/create-repos) in your project where the container image will be uploaded. 
+- `artifacts_gcs_dir`: Folder in a [Google Cloud Storage bucket](https://cloud.google.com/storage/docs/creating-buckets) where the model weights will be uploaded.
+
+Once this is done, first upload the model:
+```sh
+python build.py upload
+```
+
+Then deploy it:
+```sh
+python build.py deploy
+```
+
+If you run the deploy command again, it will create a new endpoint. If you want to undeploy the model, you can do so using the Vertex AI dashboard on the Google Cloud console, or use `gcloud ai endpoints undeploy` from the command line.
+
+After deploying successfully, you can run `python build.py probe` to send a sample request to the deployed model. 

community-content/cpr-examples/timm_serving/build.py

@@ -0,0 +1,117 @@
+# Copyright 2022 Google LLC
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     https://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Build the model server container."""
+import json
+import logging
+import os
+import pathlib
+from typing import Sequence
+
+from absl import app
+from absl import logging
+from config import CPRConfig
+from google.cloud import aiplatform
+from google.cloud.aiplatform import prediction as cpr
+import smart_open
+import timm
+from timm_serving import predictor
+import torch
+
+
+def build_container(config: CPRConfig, tag: str) -> cpr.LocalModel:
+    """Build the model server container.
+
+    Args:
+      tag: Output image tag.
+
+    Returns:
+      LocalModel exposing the built model server.
+    """
+    return cpr.LocalModel.build_cpr_model(
+        src_dir=os.path.join(os.getcwd()),
+        output_image_uri=tag,
+        base_image=config.base_image,
+        predictor=predictor.TimmPredictor,
+        requirements_path=os.path.join(os.getcwd(), "requirements.txt"),
+    )
+
+
+def save_model_artifact(destination: str) -> None:
+    """Save a copy of the model state dict."""
+    model = timm.create_model(predictor.TimmPredictor.TIMM_MODEL_NAME, pretrained=True)
+    dest_file = os.path.join(destination, predictor.TimmPredictor.WEIGHTS_FILE)
+    with smart_open.open(dest_file, "wb") as f:
+        torch.save(model, f)
+    logging.info("Saved model to %s", dest_file)
+    logging.info("%s parameters", sum(p.numel() for p in model.parameters()))
+
+
+def upload_model(config: CPRConfig) -> aiplatform.Model:
+    """Tag and upload the model server."""
+    ar_tag = (
+        f"{config.region}-docker.pkg.dev/{config.project_id}"
+        f"/{config.repository}/{config.image}"
+    )
+    local_model = build_container(config, tag=ar_tag)
+    aiplatform.init(project=config.project_id, location=config.region)
+    local_model.push_image()
+    aip_model = aiplatform.Model.upload(
+        local_model=local_model,
+        display_name=predictor.TimmPredictor.TIMM_MODEL_NAME,
+        artifact_uri=config.artifact_gcs_dir,
+    )
+    config.model_name = aip_model.resource_name
+    config.save()
+    return aip_model
+
+
+def deploy_model(config: CPRConfig) -> aiplatform.Endpoint:
+    """Deploy the model server to a Vertex Prediction endpoint."""
+    aiplatform.init(project=config.project_id, location=config.region)
+    aip_model = aiplatform.Model(model_name=config.model_name)
+    endpoint = aip_model.deploy(machine_type=config.machine_type)
+    config.endpoint_name = endpoint.resource_name
+    config.save()
+    return endpoint
+
+
+def probe_prediction(config: CPRConfig, request_path: str) -> None:
+    """Send a sample prediction request to the Vertex Prediction endpoint."""
+    aiplatform.init(project=config.project_id, location=config.region)
+    aip_endpoint = aiplatform.Endpoint(endpoint_name=config.endpoint_name)
+    with open(request_path) as f:
+        logging.info(aip_endpoint.predict(**json.load(f)))
+
+
+def main(argv: Sequence[str]):
+    config = CPRConfig()
+    if pathlib.Path(config.config_file).exists():
+        config.load()
+
+    actions = set(argv[1:])
+    if "build" in actions:
+        build_container(config, config.image)
+        save_model_artifact(config.artifact_local_dir)
+    if "upload" in actions:
+        save_model_artifact(config.artifact_gcs_dir)
+        upload_model(config)
+    if "deploy" in actions:
+        deploy_model(config)
+    if "probe" in actions:
+        probe_prediction(config, request_path="sample_request.json")
+
+
+if __name__ == "__main__":
+    app.run(main)

community-content/cpr-examples/timm_serving/config.py

@@ -0,0 +1,76 @@
+# Copyright 2022 Google LLC
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     https://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import dataclasses
+import json
+
+
+@dataclasses.dataclass
+class CPRConfig(object):
+    """Configure the build process by editing the default values here.
+
+    config_file: File path used to save values in this config. (Some
+      values, such as the model name, are generated at build time and
+      depended on by future steps, so saving it allows this script to
+      deploy the model without re-uploading it, for example.)
+
+    base_image: Base Docker image on top of which the model server will
+      be built. By default, a Debian-based Python 3 image without GPU
+      support will be used.
+
+    image: Name and tag assigned to the built model server image.
+
+    artifact_local_dir: Local directory where a copy of the pretrained model weights
+      will be saved.
+
+    region: Google Cloud Region where the model will be uploaded during the
+      build process.
+
+    project_id: Google Cloud project ID.
+
+    repository: Name of the Artifact Registry repository where the container
+      will be uploaded.
+
+    artifact_gcs_dir: Location on GCS where a copy of the pretrained model
+      weights will be uploaded.
+
+    model_name: Full resource path of the uploaded model. This is a write-only
+      field, the value is generated by Vertex AI when the model is uploaded.
+
+    endpoint_name: Full resource path of the created endpoint. This is a
+      write-only field, the value is generated by Vertex AI when the model is
+      deployed to an endpoint.
+
+    machine_type: Machine type to use when deploying the model.
+    """
+
+    config_file: str = "config.json"
+    base_image: str = "python:3.10-bullseye"
+    image: str = "timm_predictor:latest"
+    artifact_local_dir: str = ""
+    region: str = "us-central1"
+    project_id: str = "<your project ID here>"
+    repository: str = "cpr-images"
+    artifact_gcs_dir: str = "gs://<your bucket ID here>/timm-vit224/"
+    model_name: str = ""
+    endpoint_name: str = ""
+    machine_type: str = "n1-standard-2"
+
+    def save(self):
+        with open(self.config_file, "w") as f:
+            json.dump(dataclasses.asdict(self), f, indent=2)
+
+    def load(self):
+        with open(self.config_file) as f:
+            self.__init__(**json.load(f))

community-content/cpr-examples/timm_serving/requirements.txt

@@ -0,0 +1,8 @@
+absl-py==1.1.0
+fastapi==0.75.2
+uvicorn==0.18.2
+timm==0.5.4
+smart_open==6.0.0
+
+google-cloud-storage>=1.26.0,<2.0.0dev
+google-cloud-aiplatform[prediction]>=1.16.0

community-content/cpr-examples/timm_serving/test.py

@@ -0,0 +1,255 @@
+"""Test the timm_serving predictor."""
+import base64
+import json
+import logging
+import os
+import pickle
+from typing import List, Dict
+
+from absl import flags
+from absl import logging
+from absl.testing import absltest
+from config import CPRConfig
+import fastapi
+from google.cloud import aiplatform
+from google.cloud.aiplatform import prediction as cpr
+import PIL
+from timm_serving import predictor
+import torch
+
+VIT_SMALL_PARAMS = 22878952
+
+
+def b64_encode_file(path: str) -> str:
+    """Encode a file's contents as base64.
+
+    Args:
+      path: Path to the file.
+
+    Returns:
+      Base64-encoded contents of the file.
+    """
+    with open(path, "rb") as f:
+        return str(base64.b64encode(f.read()), encoding="utf-8")
+
+
+def make_instance_dict(
+    image_paths: List[str], base64_encodings: List[str]
+) -> Dict[str, List[str]]:
+    """Generate a dictionary similar to a parsed prediction server request.
+
+    Args:
+      image_paths: Paths to image files to include.
+      base64_encodings: Pre-encoded base64 strings.
+
+    Returns:
+      Dictionary of instances in the format accepted by the preprocessor.
+    """
+    instances = [s for s in base64_encodings]
+    for path in image_paths:
+        instances.append(b64_encode_file(path))
+    return {"instances": instances}
+
+
+def count_parameters(model: torch.nn.Module):
+    """Count the parameters in a Pytorch model.
+
+    Args:
+      model: Pytorch model (nn.Module).
+
+    Returns:
+      Number of parameters in the model.
+
+    """
+    return sum(p.numel() for p in model.parameters())
+
+
+class PredictorUnitTests(absltest.TestCase):
+    """Unit tests for timm_serving.predictor."""
+
+    def setUp(self):
+        super().setUp()
+        self.config = CPRConfig()
+        try:
+            self.config.load()
+        except FileNotFoundError:
+            logging.info("No saved config file found, using default values.")
+        self.predictor = predictor.TimmPredictor()
+
+    def test_load_from_saved_state_dict_ok(self):
+        self.predictor.load(self.config.artifact_local_dir)
+        self.assertEqual(count_parameters(self.predictor._model), VIT_SMALL_PARAMS)
+
+    def test_load_bad_path(self):
+        with self.assertRaises(FileNotFoundError):
+            self.predictor.load("testdata/")
+        with self.assertRaisesRegex(ValueError, "not a directory"):
+            self.predictor.load("blah")
+
+    def test_load_bad_data(self):
+        with self.assertRaises(pickle.UnpicklingError):
+            self.predictor.load("testdata/bad_model_1")
+        with self.assertRaisesRegex(RuntimeError, "Invalid magic number"):
+            self.predictor.load("testdata/bad_model_2")
+
+    def test_preprocess_ok(self):
+        self.predictor.load(self.config.artifact_local_dir)
+        instance_dict = make_instance_dict(
+            base64_encodings=[],
+            image_paths=[
+                "testdata/airplane.jpg",
+                "testdata/mandrill.tiff",
+                "testdata/mandrill.tiff",
+                "testdata/cat_alpha.png",
+            ],
+        )
+        result = self.predictor.preprocess(instance_dict)
+        self.assertEqual(result.size(), torch.Size([4, 3, 224, 224]))
+        self.assertEqual(result.dtype, torch.float32)
+
+    def test_preprocess_no_instances(self):
+        self.predictor.load(self.config.artifact_local_dir)
+        with self.assertRaises(fastapi.HTTPException) as ctx:
+            self.predictor.preprocess({})
+        self.assertEqual(ctx.exception.status_code, 400)
+        self.assertRegex(ctx.exception.detail, 'must contain "instances"')
+
+    def test_preprocess_wrong_shape_instances(self):
+        self.predictor.load(self.config.artifact_local_dir)
+        instance_dict = {"instances": [[b64_encode_file("testdata/mandrill.tiff")]]}
+        with self.assertRaises(fastapi.HTTPException) as ctx:
+            self.predictor.preprocess(instance_dict)
+        self.assertEqual(ctx.exception.status_code, 400)
+        self.assertRegex(ctx.exception.detail, "not 'list'")
+
+    def test_preprocess_bad_base64(self):
+        self.predictor.load(self.config.artifact_local_dir)
+        instance_dict = make_instance_dict(base64_encodings=["!@#$"], image_paths=[])
+        with self.assertRaises(fastapi.HTTPException) as ctx:
+            self.predictor.preprocess(instance_dict)
+        self.assertEqual(ctx.exception.status_code, 400)
+        self.assertRegex(ctx.exception.detail, "[Bb]ase64")
+
+    def test_preprocess_not_image_data(self):
+        self.predictor.load(self.config.artifact_local_dir)
+        instance_dict = make_instance_dict(
+            base64_encodings=[], image_paths=["testdata/bad.jpg"]
+        )
+        with self.assertRaises(fastapi.HTTPException) as ctx:
+            self.predictor.preprocess(instance_dict)
+        self.assertEqual(ctx.exception.status_code, 400)
+        self.assertRegex(ctx.exception.detail, "image file")
+
+    def test_predict_ok(self):
+        self.predictor.load(self.config.artifact_local_dir)
+        inputs = torch.zeros(size=[2, 3, 224, 224], dtype=torch.float32)
+        if torch.cuda.device_count() > 0:
+            inputs = inputs.cuda()
+        result = self.predictor.predict(inputs)
+        self.assertEqual(result.size(), torch.Size([2, 1000]))
+        self.assertEqual(result.dtype, torch.float32)
+
+    def test_postprocess_ok(self):
+        class_probs = torch.zeros(size=[2, 1000])
+        class_probs[0, 0] = 1
+        class_probs[1, 123] = 1
+        result = self.predictor.postprocess(class_probs)
+        predictions = result["predictions"]
+        self.assertLen(predictions[0]["class_names"], 5)
+        self.assertLen(predictions[0]["indices"], 5)
+        self.assertLen(predictions[0]["probabilities"], 5)
+        self.assertLen(predictions[1]["class_names"], 5)
+        self.assertLen(predictions[1]["indices"], 5)
+        self.assertLen(predictions[1]["probabilities"], 5)
+        self.assertContainsSubsequence(predictions[0]["class_names"][0], "tench")
+        self.assertContainsSubsequence(
+            predictions[1]["class_names"][0], "spiny lobster"
+        )
+
+
+class ServerEndToEndTests(absltest.TestCase):
+    """End-to-end tests for the model server, using LocalEndpoint."""
+
+    def setUp(self):
+        super().setUp()
+        self.config = CPRConfig()
+        try:
+            self.config.load()
+        except FileNotFoundError:
+            logging.info("No saved config file found, using default values.")
+        self.local_model = cpr.LocalModel(
+            serving_container_spec=aiplatform.gapic.ModelContainerSpec(
+                image_uri=self.config.image
+            )
+        )
+
+        self.local_endpoint = self.local_model.deploy_to_local_endpoint(
+            artifact_uri=self.config.artifact_local_dir or os.getcwd()
+        )
+        self.local_endpoint.serve()
+
+    def tearDown(self):
+        self.local_endpoint.stop()
+        super().tearDown()
+
+    def test_e2e_healthcheck_ok(self):
+        health_check_response = self.local_endpoint.run_health_check()
+        self.assertEqual(health_check_response.status_code, 200)
+        self.assertEqual(health_check_response.content, b"{}")
+
+    def test_e2e_predict_ok(self):
+        predict_request = json.dumps(
+            make_instance_dict(
+                base64_encodings=[],
+                image_paths=[
+                    "testdata/mandrill.tiff",
+                ],
+            )
+        )
+        response = self.local_endpoint.predict(
+            request=predict_request, headers={"Content-Type": "application/json"}
+        )
+        logging.info(response.content)
+        self.assertEqual(response.status_code, 200)
+        predictions = response.json()["predictions"]
+        self.assertContainsSubsequence(predictions[0]["class_names"][0], "baboon")
+
+    def test_e2e_predict_bad_json_returns_400(self):
+        predict_request = "blah"
+        response = self.local_endpoint.predict(
+            request=predict_request, headers={"Content-Type": "application/json"}
+        )
+        logging.info(response.content)
+        self.assertEqual(response.status_code, 400)
+
+    def test_e2e_predict_no_instances_returns_400(self):
+        predict_request = json.dumps({})
+        response = self.local_endpoint.predict(
+            request=predict_request, headers={"Content-Type": "application/json"}
+        )
+        logging.info(response.content)
+        self.assertEqual(response.status_code, 400)
+
+    def test_e2e_predict_bad_base64_returns_400(self):
+        predict_request = json.dumps(
+            make_instance_dict(base64_encodings=["blah"], image_paths=[])
+        )
+        response = self.local_endpoint.predict(
+            request=predict_request, headers={"Content-Type": "application/json"}
+        )
+        logging.info(response.content)
+        self.assertEqual(response.status_code, 400)
+
+    def test_e2e_predict_bad_image_returns_400(self):
+        predict_request = json.dumps(
+            make_instance_dict(base64_encodings=[], image_paths=["testdata/bad.jpg"])
+        )
+        response = self.local_endpoint.predict(
+            request=predict_request, headers={"Content-Type": "application/json"}
+        )
+        logging.info(response.content)
+        self.assertEqual(response.status_code, 400)
+
+
+if __name__ == "__main__":
+    absltest.main()

community-content/cpr-examples/timm_serving/testdata/bad.jpg

@@ -0,0 +1 @@
+some non-image data

community-content/cpr-examples/timm_serving/testdata/bad.txt

@@ -0,0 +1 @@
+some non-image data

community-content/cpr-examples/timm_serving/testdata/bad_model_1/state_dict.pth

@@ -0,0 +1 @@
+blah

community-content/cpr-examples/timm_serving/timm_serving/predictor.py

@@ -0,0 +1,178 @@
+# Copyright 2022 Google LLC
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     https://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Adapts a pretrained TIMM image classification model to the CPR framework.
+
+Documentation for the TIMM (Torch IMage Models) library is here:
+  https://rwightman.github.io/pytorch-image-models/
+
+Its source can also be found here:
+  https://github.com/rwightman/pytorch-image-models
+"""
+
+import base64
+import binascii
+import io
+import os
+from typing import Dict, List, Union
+
+from fastapi import HTTPException
+from google.cloud.aiplatform import prediction as cpr
+from pathlib import Path
+import PIL
+import smart_open
+import timm
+import torch
+import torch.nn.functional as F
+
+with open(Path(__file__).parent.absolute().joinpath("imagenet.txt")) as f:
+    IMAGENET_CLASSES = f.read().splitlines()
+
+
+class TimmPredictor(cpr.predictor.Predictor):
+    """Predictor class for image models based on TIMM."""
+
+    TIMM_MODEL_NAME = os.getenv("TIMM_MODEL_NAME", default="vit_small_patch32_224")
+    WEIGHTS_FILE = "state_dict.pth"
+    NUM_TOP_CLASSES_TO_RETURN = 5
+
+    def __init__(self):
+        self._cuda = torch.cuda.device_count() > 0
+
+    def load(self, artifacts_uri: str = ""):
+        """Initializes the model and preprocessing transforms.
+
+        Args:
+          artifacts_uri: Directory where state dict is stored. Can be a
+          GCS URI or local path.
+        """
+        if artifacts_uri:
+            artifact_path = os.path.join(artifacts_uri)
+            if not (os.path.isdir(artifact_path) or artifact_path.startswith("gs://")):
+                raise ValueError("Provided artifact_uri is not a directory.")
+        else:
+            artifact_path = os.getcwd()
+
+        artifact_path = os.path.join(artifact_path, self.WEIGHTS_FILE)
+        with smart_open.open(artifact_path, "rb") as f:
+            self._model = torch.load(f)
+
+        if self._cuda:
+            self._model.cuda()
+
+        config = timm.data.resolve_data_config(model=self.TIMM_MODEL_NAME, args=[])
+        self._transform = timm.data.create_transform(
+            is_training=False, use_prefetcher=False, **config
+        )
+
+    def preprocess(self, request_dict: Dict[str, List[str]]) -> torch.Tensor:
+        """Performs preprocessing.
+
+        By default, the server expects a request body consisting of a valid JSON
+        object. This will be parsed by the handler before it's evaluated by the
+        preprocess method.
+
+        Args:
+          request_dict: Parsed request body. We expect that the input consists of
+            a list of base64-encoded image files under the "instances" key. (Any
+            image format that PIL.image.open can handle is okay.)
+
+        Returns:
+          torch.Tensor containing the preprocessed images as a batch. If GPU is
+          available, the result tensor will be stored on GPU.
+        """
+
+        if "instances" not in request_dict:
+            raise HTTPException(
+                status_code=400,
+                detail='Request must contain "instances" as a top-level key.',
+            )
+
+        tensors = []
+
+        for (i, image) in enumerate(request_dict["instances"]):
+            # We use Base64 encoding to handle image data.
+            # This is probably the best we can do while still using JSON input.
+            # Overriding the input format requires building a custom Handler.
+            try:
+                image_bytes = base64.b64decode(image, validate=True)
+            except (binascii.Error, TypeError) as e:
+                raise HTTPException(
+                    status_code=400,
+                    detail=f"Base64 decoding of the input image at index {i} failed:"
+                    f" {str(e)}",
+                )
+
+            try:
+                pil_image = PIL.Image.open(io.BytesIO(image_bytes)).convert("RGB")
+            except PIL.UnidentifiedImageError:
+                raise HTTPException(
+                    status_code=400,
+                    detail=f"The input image at index {i} could not be identified as an"
+                    " image file.",
+                )
+
+            tensors.append(self._transform(pil_image))
+
+        with torch.inference_mode():
+            result = torch.stack(tensors)
+            if self._cuda:
+                result = result.cuda()
+        return result
+
+    def predict(self, instances: torch.Tensor) -> torch.Tensor:
+        """Performs prediction.
+
+        Args:
+          instances: torch.Tensor with type torch.float32 and shape
+            [?, 3, 224, 224], containing the pre-processed input images.
+
+        Returns:
+          Vector of scores with type torch.float32 and shape [?, 1000],
+            representing the model's estimate of the likelihood that the
+            input belongs to the Imagenet class with that index.
+        """
+        with torch.inference_mode():
+            class_scores = self._model(instances)
+        return class_scores
+
+    def postprocess(
+        self, class_scores: torch.Tensor
+    ) -> Dict[str, List[Dict[str, Union[str, int, float]]]]:
+        """Translate the model output into a classification result.
+
+        Args:
+          class_scores: torch.Tensor with type torch.float32 and shape
+            [?, 1000], containing the scores assigned to each class by
+            the model.
+
+        Returns:
+          Dictionary containing the list of classification results. Each
+            classification result contains the probabilities, class names, and
+            class indices of the classes with the top class scores as reported by
+            the model.
+        """
+        class_probs = F.softmax(class_scores, dim=1)
+        top_k = class_probs.topk(self.NUM_TOP_CLASSES_TO_RETURN)
+        top_k_values = top_k.values.numpy().tolist()
+        top_k_indices = top_k.indices.numpy().tolist()
+        predictions = [
+            dict(
+                probabilities=values,
+                indices=indices,
+                class_names=[IMAGENET_CLASSES[int(class_num)] for class_num in indices],
+            )
+            for (values, indices) in zip(top_k_values, top_k_indices)
+        ]
+        return {"predictions": predictions}

community-content/pipeline_components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml

@@ -0,0 +1,64 @@
+name: Train linear regression model using scikit learn from CSV
+metadata:
+  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml'}
+inputs:
+- {name: dataset, type: CSV}
+- {name: label_column_name, type: String}
+outputs:
+- {name: model, type: ScikitLearnPickleModel}
+implementation:
+  container:
+    image: python:3.9
+    command:
+    - sh
+    - -c
+    - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
+      'scikit-learn==1.0.2' 'pandas==1.4.3' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3
+      -m pip install --quiet --no-warn-script-location 'scikit-learn==1.0.2' 'pandas==1.4.3'
+      --user) && "$0" "$@"
+    - sh
+    - -ec
+    - |
+      program_path=$(mktemp)
+      printf "%s" "$0" > "$program_path"
+      python3 -u "$program_path" "$@"
+    - |
+      def _make_parent_dirs_and_return_path(file_path: str):
+          import os
+          os.makedirs(os.path.dirname(file_path), exist_ok=True)
+          return file_path
+
+      def train_linear_regression_model_using_scikit_learn_from_CSV(
+          dataset_path,
+          model_path,
+          label_column_name,
+      ):
+          import pandas
+          import pickle
+          from sklearn import linear_model
+
+          df = pandas.read_csv(dataset_path)
+          model = linear_model.LinearRegression()
+          model.fit(
+              X=df.drop(columns=label_column_name),
+              y=df[label_column_name],
+          )
+
+          with open(model_path, "wb") as f:
+              pickle.dump(model, f)
+
+      import argparse
+      _parser = argparse.ArgumentParser(prog='Train linear regression model using scikit learn from CSV', description='')
+      _parser.add_argument("--dataset", dest="dataset_path", type=str, required=True, default=argparse.SUPPRESS)
+      _parser.add_argument("--label-column-name", dest="label_column_name", type=str, required=True, default=argparse.SUPPRESS)
+      _parser.add_argument("--model", dest="model_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
+      _parsed_args = vars(_parser.parse_args())
+
+      _outputs = train_linear_regression_model_using_scikit_learn_from_CSV(**_parsed_args)
+    args:
+    - --dataset
+    - {inputPath: dataset}
+    - --label-column-name
+    - {inputValue: label_column_name}
+    - --model
+    - {outputPath: model}

community-content/pipeline_components/ML_frameworks/Scikit_learn/Train_logistic_regression_model/from_CSV/component.yaml

@@ -0,0 +1,163 @@
+name: Train logistic regression model using scikit learn from CSV
+description: Train logistic regression model using Scikit-learn
+metadata:
+  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/ML_frameworks/Scikit_learn/Train_logistic_regression_model/from_CSV/component.yaml'}
+inputs:
+- {name: dataset, type: CSV}
+- {name: label_column_name, type: String}
+- {name: penalty, type: String, default: l2, optional: true}
+- {name: solver, type: String, default: lbfgs, optional: true}
+- {name: max_iterations, type: Integer, default: '100', optional: true}
+- {name: multi_class_mode, type: String, default: auto, optional: true}
+- {name: random_seed, type: Integer, default: '0', optional: true}
+outputs:
+- {name: model, type: ScikitLearnPickleModel}
+- {name: model_parameters, type: JsonObject}
+implementation:
+  container:
+    image: python:3.9
+    command:
+    - sh
+    - -c
+    - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
+      'scikit-learn==1.0.2' 'pandas==1.4.3' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3
+      -m pip install --quiet --no-warn-script-location 'scikit-learn==1.0.2' 'pandas==1.4.3'
+      --user) && "$0" "$@"
+    - sh
+    - -ec
+    - |
+      program_path=$(mktemp)
+      printf "%s" "$0" > "$program_path"
+      python3 -u "$program_path" "$@"
+    - |
+      def _make_parent_dirs_and_return_path(file_path: str):
+          import os
+          os.makedirs(os.path.dirname(file_path), exist_ok=True)
+          return file_path
+
+      def train_logistic_regression_model_using_scikit_learn_from_CSV(
+          dataset_path,
+          model_path,
+          label_column_name,
+          penalty = "l2", # l1, l2, elasticnet, none
+          solver = "lbfgs", # newton-cg, lbfgs, liblinear, sag, saga
+          max_iterations = 100,
+          multi_class_mode = "auto", # auto, ovr, multinomial
+          random_seed = 0,
+      ):
+          """Train logistic regression model using Scikit-learn
+
+          See https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html
+          """
+          import json
+          import pandas
+          import pickle
+          from sklearn import linear_model
+
+          df = pandas.read_csv(dataset_path)
+          model = linear_model.LogisticRegression(
+              penalty=penalty,
+              #dual=False,
+              #tol=1e-4,
+              #C=1.0,
+              #fit_intercept=True,
+              #intercept_scaling=1,
+              #class_weight=None,
+              random_state=random_seed,
+              solver=solver,
+              max_iter=max_iterations,
+              multi_class=multi_class_mode,
+              #l1_ratio=None,
+              verbose=1,
+          )
+
+          model_parameters = model.get_params()
+          model_parameters_json = json.dumps(model_parameters, indent=2)
+          print("Model parameters:")
+          print(model_parameters_json)
+          print()
+
+          model.fit(
+              X=df.drop(columns=label_column_name),
+              y=df[label_column_name],
+          )
+
+          with open(model_path, "wb") as f:
+              pickle.dump(model, f)
+
+          return (model_parameters_json,)
+
+      def _serialize_json(obj) -> str:
+          if isinstance(obj, str):
+              return obj
+          import json
+          def default_serializer(obj):
+              if hasattr(obj, 'to_struct'):
+                  return obj.to_struct()
+              else:
+                  raise TypeError("Object of type '%s' is not JSON serializable and does not have .to_struct() method." % obj.__class__.__name__)
+          return json.dumps(obj, default=default_serializer, sort_keys=True)
+
+      import argparse
+      _parser = argparse.ArgumentParser(prog='Train logistic regression model using scikit learn from CSV', description='Train logistic regression model using Scikit-learn')
+      _parser.add_argument("--dataset", dest="dataset_path", type=str, required=True, default=argparse.SUPPRESS)
+      _parser.add_argument("--label-column-name", dest="label_column_name", type=str, required=True, default=argparse.SUPPRESS)
+      _parser.add_argument("--penalty", dest="penalty", type=str, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--solver", dest="solver", type=str, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--max-iterations", dest="max_iterations", type=int, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--multi-class-mode", dest="multi_class_mode", type=str, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--random-seed", dest="random_seed", type=int, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--model", dest="model_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
+      _parser.add_argument("----output-paths", dest="_output_paths", type=str, nargs=1)
+      _parsed_args = vars(_parser.parse_args())
+      _output_files = _parsed_args.pop("_output_paths", [])
+
+      _outputs = train_logistic_regression_model_using_scikit_learn_from_CSV(**_parsed_args)
+
+      _output_serializers = [
+          _serialize_json,
+
+      ]
+
+      import os
+      for idx, output_file in enumerate(_output_files):
+          try:
+              os.makedirs(os.path.dirname(output_file))
+          except OSError:
+              pass
+          with open(output_file, 'w') as f:
+              f.write(_output_serializers[idx](_outputs[idx]))
+    args:
+    - --dataset
+    - {inputPath: dataset}
+    - --label-column-name
+    - {inputValue: label_column_name}
+    - if:
+        cond: {isPresent: penalty}
+        then:
+        - --penalty
+        - {inputValue: penalty}
+    - if:
+        cond: {isPresent: solver}
+        then:
+        - --solver
+        - {inputValue: solver}
+    - if:
+        cond: {isPresent: max_iterations}
+        then:
+        - --max-iterations
+        - {inputValue: max_iterations}
+    - if:
+        cond: {isPresent: multi_class_mode}
+        then:
+        - --multi-class-mode
+        - {inputValue: multi_class_mode}
+    - if:
+        cond: {isPresent: random_seed}
+        then:
+        - --random-seed
+        - {inputValue: random_seed}
+    - --model
+    - {outputPath: model}
+    - '----output-paths'
+    - {outputPath: model_parameters}

community-content/pipeline_components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml

@@ -0,0 +1,41 @@
+name: Create PyTorch Model Archive with base handler
+inputs:
+- {name: Model, type: PyTorchScriptModule}
+- {name: Model name, type: String, default: model}
+- {name: Model version, type: String, default: "1.0"}
+outputs:
+- {name: Model archive, type: PyTorchModelArchive}
+metadata:
+  annotations:
+    author: Alexey Volkov <alexey.volkov@ark-kun.com>
+    canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml'
+implementation:
+  container:
+    image: pytorch/torchserve:0.6.0-cpu
+    command:
+    - bash
+    - -exc
+    - |
+      model_path=$0
+      model_name=$1
+      model_version=$2
+      output_model_archive_path=$3
+
+      mkdir -p "$(dirname "$output_model_archive_path")"
+
+      # TODO: Use the built-in base_handler once my fix is merged: https://github.com/pytorch/serve/pull/1682
+      echo '
+      from ts.torch_handler import base_handler
+      class BaseHandler(base_handler.BaseHandler):
+          pass
+      ' > base_handler.py  # torch-model-archiver needs the handler to have .py extension
+      torch-model-archiver --model-name "$model_name" --version "$model_version" --serialized-file "$model_path" --handler base_handler.py
+
+      # torch-model-archiver does not allow specifying the output path, but always writes to "${model_name}.<format>"
+      expected_model_archive_path="${model_name}.mar"
+      mv "$expected_model_archive_path" "$output_model_archive_path"
+
+    - {inputPath: Model}
+    - {inputValue: Model name}
+    - {inputValue: Model version}
+    - {outputPath: Model archive}

community-content/pipeline_components/PyTorch/Create_fully_connected_network/component.yaml

@@ -0,0 +1,117 @@
+name: Create fully connected pytorch network
+description: Creates fully-connected network in PyTorch ScriptModule format
+metadata:
+  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/PyTorch/Create_fully_connected_network/component.yaml'}
+inputs:
+- {name: input_size, type: Integer}
+- {name: hidden_layer_sizes, type: JsonArray, default: '[]', optional: true}
+- {name: output_size, type: Integer, default: '1', optional: true}
+- {name: activation_name, type: String, default: relu, optional: true}
+- {name: output_activation_name, type: String, optional: true}
+- {name: random_seed, type: Integer, default: '0', optional: true}
+outputs:
+- {name: model, type: PyTorchScriptModule}
+implementation:
+  container:
+    image: pytorch/pytorch:1.7.1-cuda11.0-cudnn8-runtime
+    command:
+    - sh
+    - -ec
+    - |
+      program_path=$(mktemp)
+      printf "%s" "$0" > "$program_path"
+      python3 -u "$program_path" "$@"
+    - |
+      def _make_parent_dirs_and_return_path(file_path: str):
+          import os
+          os.makedirs(os.path.dirname(file_path), exist_ok=True)
+          return file_path
+
+      def create_fully_connected_pytorch_network(
+          input_size,
+          model_path,
+          hidden_layer_sizes = [],
+          output_size = 1,
+          activation_name = 'relu',
+          output_activation_name = None,
+          random_seed = 0,
+      ):
+          '''Creates fully-connected network in PyTorch ScriptModule format'''
+          import torch
+          torch.manual_seed(random_seed)
+
+          activation = getattr(torch, activation_name, None) or getattr(torch.nn.functional, activation_name, None)
+          if not activation:
+              raise ValueError(f'Activation "{activation_name}" was not found.')
+
+          class ActivationLayer(torch.nn.Module):
+              def forward(self, input):
+                  return activation(input)
+
+          layers = []
+          prev_layer_size = input_size
+          for layer_size in hidden_layer_sizes:
+              layer = torch.nn.Linear(prev_layer_size, layer_size)
+              prev_layer_size = layer_size
+              layers.append(layer)
+              layers.append(ActivationLayer())
+
+          # Adding the output layer
+          layers.append(torch.nn.Linear(prev_layer_size, output_size))
+
+          # Adding the optional activation after the output layer
+          if output_activation_name:
+              output_activation = getattr(torch, output_activation_name, None) or getattr(torch.nn.functional, output_activation_name, None)
+              class OutputActivationLayer(torch.nn.Module):
+                  def forward(self, input):
+                      return output_activation(input)
+              layers.append(OutputActivationLayer())
+
+          network = torch.nn.Sequential(*layers)
+          script_module = torch.jit.script(network)
+          print(script_module)
+          script_module.save(model_path)
+
+      import json
+      import argparse
+      _parser = argparse.ArgumentParser(prog='Create fully connected pytorch network', description='Creates fully-connected network in PyTorch ScriptModule format')
+      _parser.add_argument("--input-size", dest="input_size", type=int, required=True, default=argparse.SUPPRESS)
+      _parser.add_argument("--hidden-layer-sizes", dest="hidden_layer_sizes", type=json.loads, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--output-size", dest="output_size", type=int, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--activation-name", dest="activation_name", type=str, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--output-activation-name", dest="output_activation_name", type=str, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--random-seed", dest="random_seed", type=int, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--model", dest="model_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
+      _parsed_args = vars(_parser.parse_args())
+
+      _outputs = create_fully_connected_pytorch_network(**_parsed_args)
+    args:
+    - --input-size
+    - {inputValue: input_size}
+    - if:
+        cond: {isPresent: hidden_layer_sizes}
+        then:
+        - --hidden-layer-sizes
+        - {inputValue: hidden_layer_sizes}
+    - if:
+        cond: {isPresent: output_size}
+        then:
+        - --output-size
+        - {inputValue: output_size}
+    - if:
+        cond: {isPresent: activation_name}
+        then:
+        - --activation-name
+        - {inputValue: activation_name}
+    - if:
+        cond: {isPresent: output_activation_name}
+        then:
+        - --output-activation-name
+        - {inputValue: output_activation_name}
+    - if:
+        cond: {isPresent: random_seed}
+        then:
+        - --random-seed
+        - {inputValue: random_seed}
+    - --model
+    - {outputPath: model}

community-content/pipeline_components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml

@@ -0,0 +1,209 @@
+name: Train pytorch model from csv
+description: Trains PyTorch model
+metadata:
+  annotations:
+    author: Alexey Volkov <alexey.volkov@ark-kun.com>
+    canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml'
+inputs:
+- {name: model, type: PyTorchScriptModule}
+- {name: training_data, type: CSV}
+- {name: label_column_name, type: String}
+- {name: loss_function_name, type: String, default: mse_loss, optional: true}
+- {name: number_of_epochs, type: Integer, default: '1', optional: true}
+- {name: learning_rate, type: Float, default: '0.1', optional: true}
+- {name: optimizer_name, type: String, default: Adadelta, optional: true}
+- {name: optimizer_parameters, type: JsonObject, optional: true}
+- {name: batch_size, type: Integer, default: '32', optional: true}
+- {name: batch_log_interval, type: Integer, default: '100', optional: true}
+- {name: random_seed, type: Integer, default: '0', optional: true}
+outputs:
+- {name: trained_model, type: PyTorchScriptModule}
+implementation:
+  container:
+    image: pytorch/pytorch:1.7.1-cuda11.0-cudnn8-runtime
+    command:
+    - sh
+    - -c
+    - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
+      'pandas==1.4.3' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet
+      --no-warn-script-location 'pandas==1.4.3' --user) && "$0" "$@"
+    - sh
+    - -ec
+    - |
+      program_path=$(mktemp)
+      printf "%s" "$0" > "$program_path"
+      python3 -u "$program_path" "$@"
+    - |
+      def _make_parent_dirs_and_return_path(file_path: str):
+          import os
+          os.makedirs(os.path.dirname(file_path), exist_ok=True)
+          return file_path
+
+      def train_pytorch_model_from_csv(
+          model_path,
+          training_data_path,
+          trained_model_path,
+          label_column_name,
+          loss_function_name = 'mse_loss',
+          number_of_epochs = 1,
+          learning_rate = 0.1,
+          optimizer_name = 'Adadelta',
+          optimizer_parameters = None,
+          batch_size = 32,
+          batch_log_interval = 100,
+          random_seed = 0,
+      ):
+          '''Trains PyTorch model'''
+          import pandas
+          import torch
+
+          torch.manual_seed(random_seed)
+
+          use_cuda = torch.cuda.is_available()
+          device = torch.device("cuda" if use_cuda else "cpu")
+
+          model = torch.jit.load(model_path)
+          model.to(device)
+          model.train()
+
+          optimizer_class = getattr(torch.optim, optimizer_name, None)
+          if not optimizer_class:
+              raise ValueError(f'Optimizer "{optimizer_name}" was not found.')
+
+          optimizer_parameters = optimizer_parameters or {}
+          optimizer_parameters['lr'] = learning_rate
+          optimizer = optimizer_class(model.parameters(), **optimizer_parameters)
+
+          loss_function = getattr(torch, loss_function_name, None) or getattr(torch.nn, loss_function_name, None) or getattr(torch.nn.functional, loss_function_name, None)
+          if not loss_function:
+              raise ValueError(f'Loss function "{loss_function_name}" was not found.')
+
+          class CsvDataset(torch.utils.data.Dataset):
+
+              def __init__(self, file_path, label_column_name, drop_nan_columns_or_rows = 'columns'):
+                  dataframe = pandas.read_csv(file_path).convert_dtypes()
+                  # Preventing error: default_collate: batch must contain tensors, numpy arrays, numbers, dicts or lists; found object
+                  if drop_nan_columns_or_rows == 'columns':
+                      non_nan_data = dataframe.dropna(axis='columns')
+                      removed_columns = set(dataframe.columns) - set(non_nan_data.columns)
+                      if removed_columns:
+                          print('Skipping columns with NaNs: ' + str(removed_columns))
+                      dataframe = non_nan_data
+                  if drop_nan_columns_or_rows == 'rows':
+                      non_nan_data = dataframe.dropna(axis='index')
+                      number_of_removed_rows = len(dataframe) - len(non_nan_data)
+                      if number_of_removed_rows:
+                          print(f'Skipped {number_of_removed_rows} rows with NaNs.')
+                      dataframe = non_nan_data
+                  numerical_data = dataframe.select_dtypes(include='number')
+                  non_numerical_data = dataframe.select_dtypes(exclude='number')
+                  if not non_numerical_data.empty:
+                      print('Skipping non-number columns:')
+                      print(non_numerical_data.dtypes)
+                  self._dataframe = dataframe
+                  self.labels = numerical_data[[label_column_name]]
+                  self.features = numerical_data.drop(columns=[label_column_name])
+
+              def __len__(self):
+                  return len(self._dataframe)
+
+              def __getitem__(self, index):
+                  return [self.features.loc[index].to_numpy(dtype='float32'), self.labels.loc[index].to_numpy(dtype='float32')]
+
+          dataset = CsvDataset(
+              file_path=training_data_path,
+              label_column_name=label_column_name,
+          )
+          train_loader = torch.utils.data.DataLoader(
+              dataset=dataset,
+              batch_size=batch_size,
+              shuffle=True,
+          )
+
+          last_full_batch_loss = None
+          for epoch in range(1, number_of_epochs + 1):
+              for batch_idx, (data, target) in enumerate(train_loader):
+                  data, target = data.to(device), target.to(device)
+                  optimizer.zero_grad()
+                  output = model(data)
+                  loss = loss_function(output, target)
+                  loss.backward()
+                  optimizer.step()
+                  if len(data) == batch_size:
+                      last_full_batch_loss = loss.item()
+                  if batch_idx % batch_log_interval == 0:
+                      print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
+                          epoch, batch_idx * len(data), len(train_loader.dataset),
+                          100. * batch_idx / len(train_loader), loss.item()))
+              print(f'Training epoch {epoch} completed. Last full batch loss: {last_full_batch_loss:.6f}')
+
+          # print(optimizer.state_dict())
+          model.save(trained_model_path)
+
+      import json
+      import argparse
+      _parser = argparse.ArgumentParser(prog='Train pytorch model from csv', description='Trains PyTorch model')
+      _parser.add_argument("--model", dest="model_path", type=str, required=True, default=argparse.SUPPRESS)
+      _parser.add_argument("--training-data", dest="training_data_path", type=str, required=True, default=argparse.SUPPRESS)
+      _parser.add_argument("--label-column-name", dest="label_column_name", type=str, required=True, default=argparse.SUPPRESS)
+      _parser.add_argument("--loss-function-name", dest="loss_function_name", type=str, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--number-of-epochs", dest="number_of_epochs", type=int, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--learning-rate", dest="learning_rate", type=float, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--optimizer-name", dest="optimizer_name", type=str, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--optimizer-parameters", dest="optimizer_parameters", type=json.loads, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--batch-size", dest="batch_size", type=int, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--batch-log-interval", dest="batch_log_interval", type=int, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--random-seed", dest="random_seed", type=int, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--trained-model", dest="trained_model_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
+      _parsed_args = vars(_parser.parse_args())
+
+      _outputs = train_pytorch_model_from_csv(**_parsed_args)
+    args:
+    - --model
+    - {inputPath: model}
+    - --training-data
+    - {inputPath: training_data}
+    - --label-column-name
+    - {inputValue: label_column_name}
+    - if:
+        cond: {isPresent: loss_function_name}
+        then:
+        - --loss-function-name
+        - {inputValue: loss_function_name}
+    - if:
+        cond: {isPresent: number_of_epochs}
+        then:
+        - --number-of-epochs
+        - {inputValue: number_of_epochs}
+    - if:
+        cond: {isPresent: learning_rate}
+        then:
+        - --learning-rate
+        - {inputValue: learning_rate}
+    - if:
+        cond: {isPresent: optimizer_name}
+        then:
+        - --optimizer-name
+        - {inputValue: optimizer_name}
+    - if:
+        cond: {isPresent: optimizer_parameters}
+        then:
+        - --optimizer-parameters
+        - {inputValue: optimizer_parameters}
+    - if:
+        cond: {isPresent: batch_size}
+        then:
+        - --batch-size
+        - {inputValue: batch_size}
+    - if:
+        cond: {isPresent: batch_log_interval}
+        then:
+        - --batch-log-interval
+        - {inputValue: batch_log_interval}
+    - if:
+        cond: {isPresent: random_seed}
+        then:
+        - --random-seed
+        - {inputValue: random_seed}
+    - --trained-model
+    - {outputPath: trained_model}

community-content/pipeline_components/XGBoost/Predict/component.yaml

@@ -0,0 +1,110 @@
+name: Xgboost predict on CSV
+description: Makes predictions using a trained XGBoost model.
+metadata:
+  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/XGBoost/Predict/component.yaml'}
+inputs:
+- {name: data, type: CSV, description: Feature data in Apache Parquet format.}
+- {name: model, type: XGBoostModel, description: Trained model in binary XGBoost format.}
+- {name: label_column_name, type: String, description: Optional. Name of the column
+    containing the label data that is excluded during the prediction., optional: true}
+outputs:
+- {name: predictions, description: Model predictions.}
+implementation:
+  container:
+    image: python:3.10
+    command:
+    - sh
+    - -c
+    - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
+      'xgboost==1.6.1' 'pandas==1.4.3' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3
+      -m pip install --quiet --no-warn-script-location 'xgboost==1.6.1' 'pandas==1.4.3'
+      --user) && "$0" "$@"
+    - sh
+    - -ec
+    - |
+      program_path=$(mktemp)
+      printf "%s" "$0" > "$program_path"
+      python3 -u "$program_path" "$@"
+    - |
+      def _make_parent_dirs_and_return_path(file_path: str):
+          import os
+          os.makedirs(os.path.dirname(file_path), exist_ok=True)
+          return file_path
+
+      def xgboost_predict_on_CSV(
+          data_path,
+          model_path,
+          predictions_path,
+          label_column_name = None,
+      ):
+          """Makes predictions using a trained XGBoost model.
+
+          Args:
+              data_path: Feature data in Apache Parquet format.
+              model_path: Trained model in binary XGBoost format.
+              predictions_path: Model predictions.
+              label_column_name: Optional. Name of the column containing the label data that is excluded during the prediction.
+
+          Annotations:
+              author: Alexey Volkov <alexey.volkov@ark-kun.com>
+          """
+          from pathlib import Path
+
+          import numpy
+          import pandas
+          import xgboost
+
+          df = pandas.read_csv(
+              data_path,
+          ).convert_dtypes()
+          print("Evaluation data information:")
+          df.info(verbose=True)
+          # Converting column types that XGBoost does not support
+          for column_name, dtype in df.dtypes.items():
+              if dtype in ["string", "object"]:
+                  print(f"Treating the {dtype.name} column '{column_name}' as categorical.")
+                  df[column_name] = df[column_name].astype("category")
+                  print(f"Inferred {len(df[column_name].cat.categories)} categories for the '{column_name}' column.")
+              # Working around the XGBoost issue with nullable floats: https://github.com/dmlc/xgboost/issues/8213
+              if pandas.api.types.is_float_dtype(dtype):
+                  # Converting from "Float64" to "float64"
+                  df[column_name] = df[column_name].astype(dtype.name.lower())
+          print("Final evaluation data information:")
+          df.info(verbose=True)
+
+          if label_column_name is not None:
+              df = df.drop(columns=[label_column_name])
+
+          testing_data = xgboost.DMatrix(
+              data=df,
+              enable_categorical=True,
+          )
+
+          model = xgboost.Booster(model_file=model_path)
+
+          predictions = model.predict(testing_data)
+
+          Path(predictions_path).parent.mkdir(parents=True, exist_ok=True)
+          numpy.savetxt(predictions_path, predictions)
+
+      import argparse
+      _parser = argparse.ArgumentParser(prog='Xgboost predict on CSV', description='Makes predictions using a trained XGBoost model.')
+      _parser.add_argument("--data", dest="data_path", type=str, required=True, default=argparse.SUPPRESS)
+      _parser.add_argument("--model", dest="model_path", type=str, required=True, default=argparse.SUPPRESS)
+      _parser.add_argument("--label-column-name", dest="label_column_name", type=str, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--predictions", dest="predictions_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
+      _parsed_args = vars(_parser.parse_args())
+
+      _outputs = xgboost_predict_on_CSV(**_parsed_args)
+    args:
+    - --data
+    - {inputPath: data}
+    - --model
+    - {inputPath: model}
+    - if:
+        cond: {isPresent: label_column_name}
+        then:
+        - --label-column-name
+        - {inputValue: label_column_name}
+    - --predictions
+    - {outputPath: predictions}

community-content/pipeline_components/XGBoost/Train/component.yaml

@@ -0,0 +1,241 @@
+name: Train XGBoost model on CSV
+description: Trains an XGBoost model.
+metadata:
+  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/XGBoost/Train/component.yaml'}
+inputs:
+- {name: training_data, type: CSV, description: Training data in CSV format.}
+- {name: label_column_name, type: String, description: Name of the column containing
+    the label data.}
+- {name: starting_model, type: XGBoostModel, description: Existing trained model to
+    start from (in the binary XGBoost format)., optional: true}
+- {name: num_iterations, type: Integer, description: Number of boosting iterations.,
+  default: '10', optional: true}
+- name: objective
+  type: String
+  description: |-
+    The learning task and the corresponding learning objective.
+    See https://xgboost.readthedocs.io/en/latest/parameter.html#learning-task-parameters
+    The most common values are:
+    "reg:squarederror" - Regression with squared loss (default).
+    "reg:logistic" - Logistic regression.
+    "binary:logistic" - Logistic regression for binary classification, output probability.
+    "binary:logitraw" - Logistic regression for binary classification, output score before logistic transformation
+    "rank:pairwise" - Use LambdaMART to perform pairwise ranking where the pairwise loss is minimized
+    "rank:ndcg" - Use LambdaMART to perform list-wise ranking where Normalized Discounted Cumulative Gain (NDCG) is maximized
+  default: reg:squarederror
+  optional: true
+- {name: booster, type: String, description: 'The booster to use. Can be `gbtree`,
+    `gblinear` or `dart`; `gbtree` and `dart` use tree based models while `gblinear`
+    uses linear functions.', default: gbtree, optional: true}
+- {name: learning_rate, type: Float, description: 'Step size shrinkage used in update
+    to prevents overfitting. Range: [0,1].', default: '0.3', optional: true}
+- name: min_split_loss
+  type: Float
+  description: |-
+    Minimum loss reduction required to make a further partition on a leaf node of the tree.
+    The larger `min_split_loss` is, the more conservative the algorithm will be. Range: [0,Inf].
+  default: '0'
+  optional: true
+- name: max_depth
+  type: Integer
+  description: |-
+    Maximum depth of a tree. Increasing this value will make the model more complex and more likely to overfit.
+    0 indicates no limit on depth. Range: [0,Inf].
+  default: '6'
+  optional: true
+- {name: booster_params, type: JsonObject, description: 'Parameters for the booster.
+    See https://xgboost.readthedocs.io/en/latest/parameter.html', optional: true}
+outputs:
+- {name: model, type: XGBoostModel, description: Trained model in the binary XGBoost
+    format.}
+- {name: model_config, type: XGBoostModelConfig, description: The internal parameter
+    configuration of Booster as a JSON string.}
+implementation:
+  container:
+    image: python:3.10
+    command:
+    - sh
+    - -c
+    - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
+      'xgboost==1.6.1' 'pandas==1.4.3' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3
+      -m pip install --quiet --no-warn-script-location 'xgboost==1.6.1' 'pandas==1.4.3'
+      --user) && "$0" "$@"
+    - sh
+    - -ec
+    - |
+      program_path=$(mktemp)
+      printf "%s" "$0" > "$program_path"
+      python3 -u "$program_path" "$@"
+    - |
+      def _make_parent_dirs_and_return_path(file_path: str):
+          import os
+          os.makedirs(os.path.dirname(file_path), exist_ok=True)
+          return file_path
+
+      def train_XGBoost_model_on_CSV(
+          training_data_path,
+          model_path,
+          model_config_path,
+          label_column_name,
+          starting_model_path = None,
+          num_iterations = 10,
+          # Booster parameters
+          objective = "reg:squarederror",
+          booster = "gbtree",
+          learning_rate = 0.3,
+          min_split_loss = 0,
+          max_depth = 6,
+          booster_params = None,
+      ):
+          """Trains an XGBoost model.
+
+          Args:
+              training_data_path: Training data in CSV format.
+              model_path: Trained model in the binary XGBoost format.
+              model_config_path: The internal parameter configuration of Booster as a JSON string.
+              starting_model_path: Existing trained model to start from (in the binary XGBoost format).
+              label_column_name: Name of the column containing the label data.
+              num_iterations: Number of boosting iterations.
+              booster_params: Parameters for the booster. See https://xgboost.readthedocs.io/en/latest/parameter.html
+              objective: The learning task and the corresponding learning objective.
+                  See https://xgboost.readthedocs.io/en/latest/parameter.html#learning-task-parameters
+                  The most common values are:
+                  "reg:squarederror" - Regression with squared loss (default).
+                  "reg:logistic" - Logistic regression.
+                  "binary:logistic" - Logistic regression for binary classification, output probability.
+                  "binary:logitraw" - Logistic regression for binary classification, output score before logistic transformation
+                  "rank:pairwise" - Use LambdaMART to perform pairwise ranking where the pairwise loss is minimized
+                  "rank:ndcg" - Use LambdaMART to perform list-wise ranking where Normalized Discounted Cumulative Gain (NDCG) is maximized
+              booster: The booster to use. Can be `gbtree`, `gblinear` or `dart`; `gbtree` and `dart` use tree based models while `gblinear` uses linear functions.
+              learning_rate: Step size shrinkage used in update to prevents overfitting. Range: [0,1].
+              min_split_loss: Minimum loss reduction required to make a further partition on a leaf node of the tree.
+                  The larger `min_split_loss` is, the more conservative the algorithm will be. Range: [0,Inf].
+              max_depth: Maximum depth of a tree. Increasing this value will make the model more complex and more likely to overfit.
+                  0 indicates no limit on depth. Range: [0,Inf].
+
+          Annotations:
+              author: Alexey Volkov <alexey.volkov@ark-kun.com>
+          """
+          import pandas
+          import xgboost
+
+          df = pandas.read_csv(
+              training_data_path,
+          ).convert_dtypes()
+          print("Training data information:")
+          df.info(verbose=True)
+          # Converting column types that XGBoost does not support
+          for column_name, dtype in df.dtypes.items():
+              if dtype in ["string", "object"]:
+                  print(f"Treating the {dtype.name} column '{column_name}' as categorical.")
+                  df[column_name] = df[column_name].astype("category")
+                  print(f"Inferred {len(df[column_name].cat.categories)} categories for the '{column_name}' column.")
+              # Working around the XGBoost issue with nullable floats: https://github.com/dmlc/xgboost/issues/8213
+              if pandas.api.types.is_float_dtype(dtype):
+                  # Converting from "Float64" to "float64"
+                  df[column_name] = df[column_name].astype(dtype.name.lower())
+          print()
+          print("Final training data information:")
+          df.info(verbose=True)
+
+          training_data = xgboost.DMatrix(
+              data=df.drop(columns=[label_column_name]),
+              label=df[[label_column_name]],
+              enable_categorical=True,
+          )
+
+          booster_params = booster_params or {}
+          booster_params.setdefault("objective", objective)
+          booster_params.setdefault("booster", booster)
+          booster_params.setdefault("learning_rate", learning_rate)
+          booster_params.setdefault("min_split_loss", min_split_loss)
+          booster_params.setdefault("max_depth", max_depth)
+
+          starting_model = None
+          if starting_model_path:
+              starting_model = xgboost.Booster(model_file=starting_model_path)
+
+          print()
+          print("Training the model:")
+          model = xgboost.train(
+              params=booster_params,
+              dtrain=training_data,
+              num_boost_round=num_iterations,
+              xgb_model=starting_model,
+              evals=[(training_data, "training_data")],
+          )
+
+          # Saving the model in binary format
+          model.save_model(model_path)
+
+          model_config_str = model.save_config()
+          with open(model_config_path, "w") as model_config_file:
+              model_config_file.write(model_config_str)
+
+      import json
+      import argparse
+      _parser = argparse.ArgumentParser(prog='Train XGBoost model on CSV', description='Trains an XGBoost model.')
+      _parser.add_argument("--training-data", dest="training_data_path", type=str, required=True, default=argparse.SUPPRESS)
+      _parser.add_argument("--label-column-name", dest="label_column_name", type=str, required=True, default=argparse.SUPPRESS)
+      _parser.add_argument("--starting-model", dest="starting_model_path", type=str, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--num-iterations", dest="num_iterations", type=int, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--objective", dest="objective", type=str, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--booster", dest="booster", type=str, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--learning-rate", dest="learning_rate", type=float, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--min-split-loss", dest="min_split_loss", type=float, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--max-depth", dest="max_depth", type=int, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--booster-params", dest="booster_params", type=json.loads, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--model", dest="model_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
+      _parser.add_argument("--model-config", dest="model_config_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
+      _parsed_args = vars(_parser.parse_args())
+
+      _outputs = train_XGBoost_model_on_CSV(**_parsed_args)
+    args:
+    - --training-data
+    - {inputPath: training_data}
+    - --label-column-name
+    - {inputValue: label_column_name}
+    - if:
+        cond: {isPresent: starting_model}
+        then:
+        - --starting-model
+        - {inputPath: starting_model}
+    - if:
+        cond: {isPresent: num_iterations}
+        then:
+        - --num-iterations
+        - {inputValue: num_iterations}
+    - if:
+        cond: {isPresent: objective}
+        then:
+        - --objective
+        - {inputValue: objective}
+    - if:
+        cond: {isPresent: booster}
+        then:
+        - --booster
+        - {inputValue: booster}
+    - if:
+        cond: {isPresent: learning_rate}
+        then:
+        - --learning-rate
+        - {inputValue: learning_rate}
+    - if:
+        cond: {isPresent: min_split_loss}
+        then:
+        - --min-split-loss
+        - {inputValue: min_split_loss}
+    - if:
+        cond: {isPresent: max_depth}
+        then:
+        - --max-depth
+        - {inputValue: max_depth}
+    - if:
+        cond: {isPresent: booster_params}
+        then:
+        - --booster-params
+        - {inputValue: booster_params}
+    - --model
+    - {outputPath: model}
+    - --model-config
+    - {outputPath: model_config}

community-content/pipeline_components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml

@@ -0,0 +1,204 @@
+name: Split rows into subsets
+description: Splits the data table according to the split fractions.
+metadata:
+  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml'}
+inputs:
+- {name: table, type: CSV}
+- {name: fraction_1, type: Float, description: 'The proportion of the lines to put
+    into the 1st split. Range: [0, 1]'}
+- name: fraction_2
+  type: Float
+  description: |-
+    The proportion of the lines to put into the 2nd split. Range: [0, 1]
+    If fraction_2 is not specified, then fraction_2 = 1 - fraction_1.
+    The remaining lines go to the 3rd split (if any).
+  optional: true
+- {name: random_seed, type: Integer, default: '0', optional: true}
+outputs:
+- {name: split_1, type: CSV}
+- {name: split_2, type: CSV}
+- {name: split_3, type: CSV}
+- {name: split_1_count, type: Integer}
+- {name: split_2_count, type: Integer}
+- {name: split_3_count, type: Integer}
+implementation:
+  container:
+    image: python:3.9
+    command:
+    - sh
+    - -ec
+    - |
+      program_path=$(mktemp)
+      printf "%s" "$0" > "$program_path"
+      python3 -u "$program_path" "$@"
+    - |
+      def _make_parent_dirs_and_return_path(file_path: str):
+          import os
+          os.makedirs(os.path.dirname(file_path), exist_ok=True)
+          return file_path
+
+      def split_rows_into_subsets(
+          table_path,
+          split_1_path,
+          split_2_path,
+          split_3_path,
+          fraction_1,
+          fraction_2 = None,
+          random_seed = 0,
+      ):
+          """Splits the data table according to the split fractions.
+
+          Args:
+              fraction_1: The proportion of the lines to put into the 1st split. Range: [0, 1]
+              fraction_2: The proportion of the lines to put into the 2nd split. Range: [0, 1]
+                  If fraction_2 is not specified, then fraction_2 = 1 - fraction_1.
+                  The remaining lines go to the 3rd split (if any).
+          """
+          import random
+
+          random.seed(random_seed)
+
+          SHUFFLE_BUFFER_SIZE = 10000
+
+          num_splits = 3
+
+          if fraction_1 < 0 or fraction_1 > 1:
+              raise ValueError("fraction_1 must be in between 0 and 1.")
+
+          if fraction_2 is None:
+              fraction_2 = 1 - fraction_1
+          if fraction_2 < 0 or fraction_2 > 1:
+              raise ValueError("fraction_2 must be in between 0 and 1.")
+
+          fraction_3 = 1 - fraction_1 - fraction_2
+
+          fractions = [
+              fraction_1,
+              fraction_2,
+              fraction_3,
+          ]
+
+          assert sum(fractions) == 1
+
+          written_line_counts = [0] * num_splits
+
+          output_files = [
+              open(split_1_path, "wb"),
+              open(split_2_path, "wb"),
+              open(split_3_path, "wb"),
+          ]
+
+          with open(table_path, "rb") as input_file:
+              # Writing the headers
+              header_line = input_file.readline()
+              for output_file in output_files:
+                  output_file.write(header_line)
+
+              while True:
+                  line_buffer = []
+                  for i in range(SHUFFLE_BUFFER_SIZE):
+                      line = input_file.readline()
+                      if not line:
+                          break
+                      line_buffer.append(line)
+
+                  # We need to exactly partition the lines between the output files
+                  # To overcome possible systematic bias, we could calculate the total numbers
+                  # of lines written to each file and take that into account.
+                  num_read_lines = len(line_buffer)
+                  number_of_lines_for_files = [0] * num_splits
+                  # List that will have the index of the destination file for each line
+                  file_index_for_line = []
+                  remaining_lines = num_read_lines
+                  remaining_fraction = 1
+                  for i in range(num_splits):
+                      number_of_lines_for_file = (
+                          round(remaining_lines * (fractions[i] / remaining_fraction))
+                          if remaining_fraction > 0
+                          else 0
+                      )
+                      number_of_lines_for_files[i] = number_of_lines_for_file
+                      remaining_lines -= number_of_lines_for_file
+                      remaining_fraction -= fractions[i]
+                      file_index_for_line.extend([i] * number_of_lines_for_file)
+
+                  assert remaining_lines == 0, f"{remaining_lines}"
+                  assert len(file_index_for_line) == num_read_lines
+
+                  random.shuffle(file_index_for_line)
+
+                  for i in range(num_read_lines):
+                      output_files[file_index_for_line[i]].write(line_buffer[i])
+                      written_line_counts[file_index_for_line[i]] += 1
+
+                  # Exit if the file ended before we were able to fully fill the buffer
+                  if len(line_buffer) != SHUFFLE_BUFFER_SIZE:
+                      break
+
+          for output_file in output_files:
+              output_file.close()
+
+          return written_line_counts
+
+      def _serialize_int(int_value: int) -> str:
+          if isinstance(int_value, str):
+              return int_value
+          if not isinstance(int_value, int):
+              raise TypeError('Value "{}" has type "{}" instead of int.'.format(str(int_value), str(type(int_value))))
+          return str(int_value)
+
+      import argparse
+      _parser = argparse.ArgumentParser(prog='Split rows into subsets', description='Splits the data table according to the split fractions.')
+      _parser.add_argument("--table", dest="table_path", type=str, required=True, default=argparse.SUPPRESS)
+      _parser.add_argument("--fraction-1", dest="fraction_1", type=float, required=True, default=argparse.SUPPRESS)
+      _parser.add_argument("--fraction-2", dest="fraction_2", type=float, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--random-seed", dest="random_seed", type=int, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--split-1", dest="split_1_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
+      _parser.add_argument("--split-2", dest="split_2_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
+      _parser.add_argument("--split-3", dest="split_3_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
+      _parser.add_argument("----output-paths", dest="_output_paths", type=str, nargs=3)
+      _parsed_args = vars(_parser.parse_args())
+      _output_files = _parsed_args.pop("_output_paths", [])
+
+      _outputs = split_rows_into_subsets(**_parsed_args)
+
+      _output_serializers = [
+          _serialize_int,
+          _serialize_int,
+          _serialize_int,
+
+      ]
+
+      import os
+      for idx, output_file in enumerate(_output_files):
+          try:
+              os.makedirs(os.path.dirname(output_file))
+          except OSError:
+              pass
+          with open(output_file, 'w') as f:
+              f.write(_output_serializers[idx](_outputs[idx]))
+    args:
+    - --table
+    - {inputPath: table}
+    - --fraction-1
+    - {inputValue: fraction_1}
+    - if:
+        cond: {isPresent: fraction_2}
+        then:
+        - --fraction-2
+        - {inputValue: fraction_2}
+    - if:
+        cond: {isPresent: random_seed}
+        then:
+        - --random-seed
+        - {inputValue: random_seed}
+    - --split-1
+    - {outputPath: split_1}
+    - --split-2
+    - {outputPath: split_2}
+    - --split-3
+    - {outputPath: split_3}
+    - '----output-paths'
+    - {outputPath: split_1_count}
+    - {outputPath: split_2_count}
+    - {outputPath: split_3_count}

community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml

@@ -0,0 +1,241 @@
+name: Deploy model to endpoint for Google Cloud Vertex AI Model
+description: Deploys Google Cloud Vertex AI Model to a Google Cloud Vertex AI Endpoint.
+metadata:
+  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/KFPv2_hell/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/workaround_for_buggy_KFPv2_compiler/component.yaml'}
+inputs:
+- {name: model_name, type: String, description: Full resource name of a Google Cloud
+    Vertex AI Model}
+- name: endpoint_name
+  type: String
+  description: |-
+    Optional. Full name of Google Cloud Vertex Endpoint. A new
+    endpoint is created if the name is not passed.
+  optional: true
+- name: machine_type
+  type: String
+  description: |-
+    The type of the machine. See the [list of machine types
+    supported for prediction
+    ](https://cloud.google.com/vertex-ai/docs/predictions/configure-compute#machine-types).
+    Defaults to "n1-standard-2"
+  default: n1-standard-2
+  optional: true
+- name: min_replica_count
+  type: Integer
+  description: |-
+    Optional. The minimum number of machine replicas this deployed
+    model will be always deployed on. If traffic against it increases,
+    it may dynamically be deployed onto more replicas, and as traffic
+    decreases, some of these extra replicas may be freed.
+  default: '1'
+  optional: true
+- name: max_replica_count
+  type: Integer
+  description: |-
+    Optional. The maximum number of replicas this deployed model may
+    be deployed on when the traffic against it increases. If requested
+    value is too large, the deployment will error, but if deployment
+    succeeds then the ability to scale the model to that many replicas
+    is guaranteed (barring service outages). If traffic against the
+    deployed model increases beyond what its replicas at maximum may
+    handle, a portion of the traffic will be dropped. If this value
+    is not provided, the smaller value of min_replica_count or 1 will
+    be used.
+  default: '1'
+  optional: true
+- name: accelerator_type
+  type: String
+  description: |-
+    Optional. Hardware accelerator type. Must also set accelerator_count if used.
+    One of ACCELERATOR_TYPE_UNSPECIFIED, NVIDIA_TESLA_K80, NVIDIA_TESLA_P100,
+    NVIDIA_TESLA_V100, NVIDIA_TESLA_P4, NVIDIA_TESLA_T4
+  optional: true
+- {name: accelerator_count, type: Integer, description: Optional. The number of accelerators
+    to attach to a worker replica., optional: true}
+outputs:
+- {name: endpoint_name, type: String}
+- {name: endpoint_dict, type: JsonObject}
+implementation:
+  container:
+    image: python:3.9
+    command:
+    - sh
+    - -c
+    - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
+      'google-cloud-aiplatform==1.7.0' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3
+      -m pip install --quiet --no-warn-script-location 'google-cloud-aiplatform==1.7.0'
+      --user) && "$0" "$@"
+    - sh
+    - -ec
+    - |
+      program_path=$(mktemp)
+      printf "%s" "$0" > "$program_path"
+      python3 -u "$program_path" "$@"
+    - |
+      def deploy_model_to_endpoint_for_Google_Cloud_Vertex_AI_Model(
+          model_name,
+          endpoint_name = None,
+          machine_type = "n1-standard-2",
+          min_replica_count = 1,
+          max_replica_count = 1,
+          accelerator_type = None,
+          accelerator_count = None,
+          #
+          # Uncomment when anyone requests these:
+          # deployed_model_display_name: str = None,
+          # traffic_percentage: int = 0,
+          # traffic_split: dict = None,
+          # service_account: str = None,
+          # explanation_metadata: "google.cloud.aiplatform_v1.types.explanation_metadata.ExplanationMetadata" = None,
+          # explanation_parameters: "google.cloud.aiplatform_v1.types.explanation.ExplanationParameters" = None,
+          #
+          # encryption_spec_key_name: str = None,
+      ):
+          """Deploys Google Cloud Vertex AI Model to a Google Cloud Vertex AI Endpoint.
+
+          Args:
+              model_name: Full resource name of a Google Cloud Vertex AI Model
+              endpoint_name: Optional. Full name of Google Cloud Vertex Endpoint. A new
+                  endpoint is created if the name is not passed.
+              machine_type: The type of the machine. See the [list of machine types
+                  supported for prediction
+                  ](https://cloud.google.com/vertex-ai/docs/predictions/configure-compute#machine-types).
+                  Defaults to "n1-standard-2"
+              min_replica_count (int):
+                  Optional. The minimum number of machine replicas this deployed
+                  model will be always deployed on. If traffic against it increases,
+                  it may dynamically be deployed onto more replicas, and as traffic
+                  decreases, some of these extra replicas may be freed.
+              max_replica_count (int):
+                  Optional. The maximum number of replicas this deployed model may
+                  be deployed on when the traffic against it increases. If requested
+                  value is too large, the deployment will error, but if deployment
+                  succeeds then the ability to scale the model to that many replicas
+                  is guaranteed (barring service outages). If traffic against the
+                  deployed model increases beyond what its replicas at maximum may
+                  handle, a portion of the traffic will be dropped. If this value
+                  is not provided, the smaller value of min_replica_count or 1 will
+                  be used.
+              accelerator_type (str):
+                  Optional. Hardware accelerator type. Must also set accelerator_count if used.
+                  One of ACCELERATOR_TYPE_UNSPECIFIED, NVIDIA_TESLA_K80, NVIDIA_TESLA_P100,
+                  NVIDIA_TESLA_V100, NVIDIA_TESLA_P4, NVIDIA_TESLA_T4
+              accelerator_count (int):
+                  Optional. The number of accelerators to attach to a worker replica.
+          """
+          import json
+          from google.cloud import aiplatform
+
+          model = aiplatform.Model(model_name=model_name)
+
+          if endpoint_name:
+              endpoint = aiplatform.Endpoint(endpoint_name=endpoint_name)
+          else:
+              endpoint_display_name = model.display_name[:118] + "_endpoint"
+              endpoint = aiplatform.Endpoint.create(
+                  display_name=endpoint_display_name,
+                  project=model.project,
+                  location=model.location,
+                  # encryption_spec_key_name=encryption_spec_key_name,
+                  labels={"component-source": "github-com-ark-kun-pipeline-components"},
+              )
+
+          endpoint = model.deploy(
+              endpoint=endpoint,
+              # deployed_model_display_name=deployed_model_display_name,
+              machine_type=machine_type,
+              min_replica_count=min_replica_count,
+              max_replica_count=max_replica_count,
+              accelerator_type=accelerator_type,
+              accelerator_count=accelerator_count,
+              # service_account=service_account,
+              # explanation_metadata=explanation_metadata,
+              # explanation_parameters=explanation_parameters,
+              # encryption_spec_key_name=encryption_spec_key_name,
+          )
+
+          endpoint_json = json.dumps(endpoint.to_dict(), indent=2)
+          print(endpoint_json)
+          return (endpoint.resource_name, endpoint_json)
+
+      def _serialize_json(obj) -> str:
+          if isinstance(obj, str):
+              return obj
+          import json
+          def default_serializer(obj):
+              if hasattr(obj, 'to_struct'):
+                  return obj.to_struct()
+              else:
+                  raise TypeError("Object of type '%s' is not JSON serializable and does not have .to_struct() method." % obj.__class__.__name__)
+          return json.dumps(obj, default=default_serializer, sort_keys=True)
+
+      def _serialize_str(str_value: str) -> str:
+          if not isinstance(str_value, str):
+              raise TypeError('Value "{}" has type "{}" instead of str.'.format(str(str_value), str(type(str_value))))
+          return str_value
+
+      import argparse
+      _parser = argparse.ArgumentParser(prog='Deploy model to endpoint for Google Cloud Vertex AI Model', description='Deploys Google Cloud Vertex AI Model to a Google Cloud Vertex AI Endpoint.')
+      _parser.add_argument("--model-name", dest="model_name", type=str, required=True, default=argparse.SUPPRESS)
+      _parser.add_argument("--endpoint-name", dest="endpoint_name", type=str, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--machine-type", dest="machine_type", type=str, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--min-replica-count", dest="min_replica_count", type=int, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--max-replica-count", dest="max_replica_count", type=int, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--accelerator-type", dest="accelerator_type", type=str, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--accelerator-count", dest="accelerator_count", type=int, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("----output-paths", dest="_output_paths", type=str, nargs=2)
+      _parsed_args = vars(_parser.parse_args())
+      _output_files = _parsed_args.pop("_output_paths", [])
+
+      _outputs = deploy_model_to_endpoint_for_Google_Cloud_Vertex_AI_Model(**_parsed_args)
+
+      _output_serializers = [
+          _serialize_str,
+          _serialize_json,
+
+      ]
+
+      import os
+      for idx, output_file in enumerate(_output_files):
+          try:
+              os.makedirs(os.path.dirname(output_file))
+          except OSError:
+              pass
+          with open(output_file, 'w') as f:
+              f.write(_output_serializers[idx](_outputs[idx]))
+    args:
+    - --model-name
+    - {inputValue: model_name}
+    - if:
+        cond: {isPresent: endpoint_name}
+        then:
+        - --endpoint-name
+        - {inputValue: endpoint_name}
+    - if:
+        cond: {isPresent: machine_type}
+        then:
+        - --machine-type
+        - {inputValue: machine_type}
+    - if:
+        cond: {isPresent: min_replica_count}
+        then:
+        - --min-replica-count
+        - {inputValue: min_replica_count}
+    - if:
+        cond: {isPresent: max_replica_count}
+        then:
+        - --max-replica-count
+        - {inputValue: max_replica_count}
+    - if:
+        cond: {isPresent: accelerator_type}
+        then:
+        - --accelerator-type
+        - {inputValue: accelerator_type}
+    - if:
+        cond: {isPresent: accelerator_count}
+        then:
+        - --accelerator-count
+        - {inputValue: accelerator_count}
+    - '----output-paths'
+    - {outputPath: endpoint_name}
+    - {outputPath: endpoint_dict}

community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/component.yaml

@@ -0,0 +1,297 @@
+name: Upload PyTorch model archive to Google Cloud Vertex AI
+metadata:
+  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/KFPv2_hell/components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/workaround_for_buggy_KFPv2_compiler/component.yaml'}
+inputs:
+- {name: model_archive, type: PyTorchModelArchive}
+- {name: torchserve_version, type: String, default: 0.6.0, optional: true}
+- name: use_gpu
+  type: Boolean
+  default: "False"
+  optional: true
+- {name: display_name, type: String, optional: true}
+- {name: description, type: String, optional: true}
+- {name: project, type: String, optional: true}
+- {name: location, type: String, optional: true}
+- {name: labels, type: JsonObject, optional: true}
+- {name: staging_bucket, type: String, optional: true}
+outputs:
+- {name: model_name, type: String}
+- {name: model_dict, type: JsonObject}
+implementation:
+  container:
+    image: python:3.9
+    command:
+    - sh
+    - -c
+    - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
+      'google-cloud-aiplatform==1.13.1' 'google-cloud-build==3.8.3' || PIP_DISABLE_PIP_VERSION_CHECK=1
+      python3 -m pip install --quiet --no-warn-script-location 'google-cloud-aiplatform==1.13.1'
+      'google-cloud-build==3.8.3' --user) && "$0" "$@"
+    - sh
+    - -ec
+    - |
+      program_path=$(mktemp)
+      printf "%s" "$0" > "$program_path"
+      python3 -u "$program_path" "$@"
+    - |
+      def upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI(
+          model_archive_path,
+          torchserve_version = "0.6.0",
+          use_gpu = False,
+
+          display_name = None,
+          description = None,
+
+          # Uncomment when anyone requests these:
+          # instance_schema_uri: str = None,
+          # parameters_schema_uri: str = None,
+          # prediction_schema_uri: str = None,
+          # explanation_metadata: "google.cloud.aiplatform_v1.types.explanation_metadata.ExplanationMetadata" = None,
+          # explanation_parameters: "google.cloud.aiplatform_v1.types.explanation.ExplanationParameters" = None,
+
+          project = None,
+          location = None,
+          labels = None,
+          # encryption_spec_key_name: str = None,
+          staging_bucket = None,
+      ):
+          import json
+          import os
+          from google.cloud import aiplatform
+
+          if not location:
+              location = os.environ.get("CLOUD_ML_REGION")
+
+          if not labels:
+              labels = {}
+          labels["component-source"] = "github-com-ark-kun-pipeline-components"
+
+          container_image_tag = torchserve_version + "-" + ("gpu" if use_gpu else "cpu")
+          container_image_uri = f"pytorch/torchserve:{container_image_tag}"
+
+          # Vertex Endpoints refuse to support non-Google container registries.
+          # We have to work around this to reduce user frustration
+          # TODO: Remove this code when Vertex Endpoints service starts supporting other container registries.
+          def copy_container_image(
+              src_container_image_uri,
+              dst_container_image_uri,
+              project_id,
+          ):
+              from google.cloud.devtools import cloudbuild
+              from google import protobuf
+              build_client = cloudbuild.CloudBuildClient()
+              build_config = cloudbuild.Build(
+                  images=[dst_container_image_uri],
+                  steps=[
+                      cloudbuild.BuildStep(
+                          name="gcr.io/cloud-builders/docker",
+                          entrypoint="bash",
+                          args=[
+                              "-exc",
+                              'docker pull --quiet "$0" && docker tag "$0" "$1"',
+                              src_container_image_uri,
+                              dst_container_image_uri,
+                          ],
+                      ),
+                  ],
+                  timeout=protobuf.duration_pb2.Duration(
+                      seconds=1800,
+                  ),
+              )
+              build_operation = build_client.create_build(
+                  project_id=project_id,
+                  build=build_config,
+              )
+              try:
+                  result = build_operation.result()
+              except:
+                  print(f"Logs are available at [{build_operation.metadata.build.log_url}].")
+                  raise
+              return result
+
+          project_id = aiplatform.initializer.global_config.project
+          mirrored_container_uri = f"gcr.io/{project_id}/container_mirror/{container_image_uri}"
+          # FIX: Only mirror when image does not exist
+          # docker does is unable to get the registry data from inside container (it cannot connecto to docker socket):
+          # docker.errors.DockerException: Error while fetching server API version: ('Connection aborted.', FileNotFoundError(2, 'No such file or directory'))
+          # import docker
+          # try:
+          #     docker_client = docker.from_env()
+          #     docker_client.images.get_registry_data(mirrored_container_uri)
+          # except docker.errors.NotFound:
+          if True:
+              print(f"Mirroring {container_image_uri} to {mirrored_container_uri}")
+              copy_container_image(
+                  src_container_image_uri=container_image_uri,
+                  dst_container_image_uri=mirrored_container_uri,
+                  project_id=project_id,
+              )
+          container_image_uri = mirrored_container_uri
+          # End of container image mirroring code
+
+          model_archive_file_name = os.path.basename(model_archive_path)
+          model_archive_dir = os.path.dirname(model_archive_path)
+
+          model = aiplatform.Model.upload(
+              # FIX: Use public image or mirror the official image
+              #serving_container_image_uri="gcr.io/avolkov-31337/mirror/pytorch/torchserve",
+              serving_container_image_uri=container_image_uri,
+              artifact_uri=model_archive_dir,
+              serving_container_command=[
+                  "bash",
+                  "-exc",
+                  '''
+      model_archive_uri="$0"
+      #model_archive_local_path=$(mktemp --suffix ".mar")
+      # For some reason the model must already be inside the model-store directory.
+      model_archive_local_path=./model-store/model.mar
+
+      # Downloading the model archive from GCS
+      # TODO: Fix gsutil bugs (requires project ID, has auth issues) and use gsutil instead.
+      # gsutil cp "$model_archive_uri" "$model_archive_local_path"
+      pip install google-cloud-storage
+      python -c '
+      import sys
+      from google.cloud import storage
+
+      model_archive_uri = sys.argv[1]
+      model_archive_local_path = sys.argv[2]
+
+      storage_client = storage.Client()
+      blob = storage.Blob.from_string(uri=model_archive_uri, client=storage_client)
+      blob.download_to_filename(filename=model_archive_local_path)
+      ' "$model_archive_uri" "$model_archive_local_path"
+
+      #Note: config.properties is owned by root. Our user is not root.
+      echo "
+      service_envelope=json
+      # Needed for external access
+      inference_address=http://0.0.0.0:8080
+      management_address=http://0.0.0.0:8081
+      " > config2.properties
+      torchserve --start --foreground --no-config-snapshots --models main-model="$model_archive_local_path" --model-store ./model-store/ --ts-config config2.properties
+                  ''',
+                  "$(AIP_STORAGE_URI)/" + model_archive_file_name,
+              ],
+              serving_container_predict_route="/predictions/main-model",
+              #serving_container_predict_route="/v1/models/main-model:predict",
+              serving_container_health_route="/ping",
+              serving_container_ports=[8080],
+
+              display_name=display_name,
+              description=description,
+
+              # instance_schema_uri=instance_schema_uri,
+              # parameters_schema_uri=parameters_schema_uri,
+              # prediction_schema_uri=prediction_schema_uri,
+              # explanation_metadata=explanation_metadata,
+              # explanation_parameters=explanation_parameters,
+
+              project=project,
+              location=location,
+              labels=labels,
+              # encryption_spec_key_name=encryption_spec_key_name,
+              staging_bucket=staging_bucket,
+          )
+          model_json = json.dumps(model.to_dict(), indent=2)
+          print(model_json)
+          return (model.resource_name, model_json)
+
+      def _deserialize_bool(s) -> bool:
+          from distutils.util import strtobool
+          return strtobool(s) == 1
+
+      def _serialize_json(obj) -> str:
+          if isinstance(obj, str):
+              return obj
+          import json
+          def default_serializer(obj):
+              if hasattr(obj, 'to_struct'):
+                  return obj.to_struct()
+              else:
+                  raise TypeError("Object of type '%s' is not JSON serializable and does not have .to_struct() method." % obj.__class__.__name__)
+          return json.dumps(obj, default=default_serializer, sort_keys=True)
+
+      def _serialize_str(str_value: str) -> str:
+          if not isinstance(str_value, str):
+              raise TypeError('Value "{}" has type "{}" instead of str.'.format(str(str_value), str(type(str_value))))
+          return str_value
+
+      import json
+      import argparse
+      _parser = argparse.ArgumentParser(prog='Upload PyTorch model archive to Google Cloud Vertex AI', description='')
+      _parser.add_argument("--model-archive", dest="model_archive_path", type=str, required=True, default=argparse.SUPPRESS)
+      _parser.add_argument("--torchserve-version", dest="torchserve_version", type=str, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--use-gpu", dest="use_gpu", type=_deserialize_bool, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--display-name", dest="display_name", type=str, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--description", dest="description", type=str, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--project", dest="project", type=str, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--location", dest="location", type=str, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--labels", dest="labels", type=json.loads, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--staging-bucket", dest="staging_bucket", type=str, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("----output-paths", dest="_output_paths", type=str, nargs=2)
+      _parsed_args = vars(_parser.parse_args())
+      _output_files = _parsed_args.pop("_output_paths", [])
+
+      _outputs = upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI(**_parsed_args)
+
+      _output_serializers = [
+          _serialize_str,
+          _serialize_json,
+
+      ]
+
+      import os
+      for idx, output_file in enumerate(_output_files):
+          try:
+              os.makedirs(os.path.dirname(output_file))
+          except OSError:
+              pass
+          with open(output_file, 'w') as f:
+              f.write(_output_serializers[idx](_outputs[idx]))
+    args:
+    - --model-archive
+    - {inputPath: model_archive}
+    - if:
+        cond: {isPresent: torchserve_version}
+        then:
+        - --torchserve-version
+        - {inputValue: torchserve_version}
+    - if:
+        cond: {isPresent: use_gpu}
+        then:
+        - --use-gpu
+        - {inputValue: use_gpu}
+    - if:
+        cond: {isPresent: display_name}
+        then:
+        - --display-name
+        - {inputValue: display_name}
+    - if:
+        cond: {isPresent: description}
+        then:
+        - --description
+        - {inputValue: description}
+    - if:
+        cond: {isPresent: project}
+        then:
+        - --project
+        - {inputValue: project}
+    - if:
+        cond: {isPresent: location}
+        then:
+        - --location
+        - {inputValue: location}
+    - if:
+        cond: {isPresent: labels}
+        then:
+        - --labels
+        - {inputValue: labels}
+    - if:
+        cond: {isPresent: staging_bucket}
+        then:
+        - --staging-bucket
+        - {inputValue: staging_bucket}
+    - '----output-paths'
+    - {outputPath: model_name}
+    - {outputPath: model_dict}

community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/component.yaml

@@ -0,0 +1,181 @@
+name: Upload Scikit learn pickle model to Google Cloud Vertex AI
+metadata:
+  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/KFPv2_hell/components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/workaround_for_buggy_KFPv2_compiler/component.yaml'}
+inputs:
+- {name: model, type: ScikitLearnPickleModel}
+- {name: sklearn_version, type: String, optional: true}
+- {name: display_name, type: String, optional: true}
+- {name: description, type: String, optional: true}
+- {name: project, type: String, optional: true}
+- {name: location, type: String, optional: true}
+- {name: labels, type: JsonObject, optional: true}
+- {name: staging_bucket, type: String, optional: true}
+outputs:
+- {name: model_name, type: String}
+- {name: model_dict, type: JsonObject}
+implementation:
+  container:
+    image: python:3.9
+    command:
+    - sh
+    - -c
+    - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
+      'google-cloud-aiplatform==1.16.0' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3
+      -m pip install --quiet --no-warn-script-location 'google-cloud-aiplatform==1.16.0'
+      --user) && "$0" "$@"
+    - sh
+    - -ec
+    - |
+      program_path=$(mktemp)
+      printf "%s" "$0" > "$program_path"
+      python3 -u "$program_path" "$@"
+    - |
+      def upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI(
+          model_path,
+          sklearn_version = None,
+
+          display_name = None,
+          description = None,
+
+          # Uncomment when anyone requests these:
+          # instance_schema_uri: str = None,
+          # parameters_schema_uri: str = None,
+          # prediction_schema_uri: str = None,
+          # explanation_metadata: "google.cloud.aiplatform_v1.types.explanation_metadata.ExplanationMetadata" = None,
+          # explanation_parameters: "google.cloud.aiplatform_v1.types.explanation.ExplanationParameters" = None,
+
+          project = None,
+          location = None,
+          labels = None,
+          # encryption_spec_key_name: str = None,
+          staging_bucket = None,
+      ):
+          import json
+          import os
+          import shutil
+          import tempfile
+          from google.cloud import aiplatform
+
+          if not location:
+              location = os.environ.get("CLOUD_ML_REGION")
+
+          if not labels:
+              labels = {}
+          labels["component-source"] = "github-com-ark-kun-pipeline-components"
+
+          # The serving container decides the model type based on the model file extension.
+          # So we need to rename the mode file (e.g. /tmp/inputs/model/data) to *.pkl
+          _, renamed_model_path = tempfile.mkstemp(suffix=".pkl")
+          shutil.copyfile(src=model_path, dst=renamed_model_path)
+
+          model = aiplatform.Model.upload_scikit_learn_model_file(
+              model_file_path=renamed_model_path,
+              sklearn_version=sklearn_version,
+
+              display_name=display_name,
+              description=description,
+
+              # instance_schema_uri=instance_schema_uri,
+              # parameters_schema_uri=parameters_schema_uri,
+              # prediction_schema_uri=prediction_schema_uri,
+              # explanation_metadata=explanation_metadata,
+              # explanation_parameters=explanation_parameters,
+
+              project=project,
+              location=location,
+              labels=labels,
+              # encryption_spec_key_name=encryption_spec_key_name,
+              staging_bucket=staging_bucket,
+          )
+          model_json = json.dumps(model.to_dict(), indent=2)
+          print(model_json)
+          return (model.resource_name, model_json)
+
+      def _serialize_json(obj) -> str:
+          if isinstance(obj, str):
+              return obj
+          import json
+          def default_serializer(obj):
+              if hasattr(obj, 'to_struct'):
+                  return obj.to_struct()
+              else:
+                  raise TypeError("Object of type '%s' is not JSON serializable and does not have .to_struct() method." % obj.__class__.__name__)
+          return json.dumps(obj, default=default_serializer, sort_keys=True)
+
+      def _serialize_str(str_value: str) -> str:
+          if not isinstance(str_value, str):
+              raise TypeError('Value "{}" has type "{}" instead of str.'.format(str(str_value), str(type(str_value))))
+          return str_value
+
+      import json
+      import argparse
+      _parser = argparse.ArgumentParser(prog='Upload Scikit learn pickle model to Google Cloud Vertex AI', description='')
+      _parser.add_argument("--model", dest="model_path", type=str, required=True, default=argparse.SUPPRESS)
+      _parser.add_argument("--sklearn-version", dest="sklearn_version", type=str, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--display-name", dest="display_name", type=str, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--description", dest="description", type=str, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--project", dest="project", type=str, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--location", dest="location", type=str, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--labels", dest="labels", type=json.loads, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--staging-bucket", dest="staging_bucket", type=str, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("----output-paths", dest="_output_paths", type=str, nargs=2)
+      _parsed_args = vars(_parser.parse_args())
+      _output_files = _parsed_args.pop("_output_paths", [])
+
+      _outputs = upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI(**_parsed_args)
+
+      _output_serializers = [
+          _serialize_str,
+          _serialize_json,
+
+      ]
+
+      import os
+      for idx, output_file in enumerate(_output_files):
+          try:
+              os.makedirs(os.path.dirname(output_file))
+          except OSError:
+              pass
+          with open(output_file, 'w') as f:
+              f.write(_output_serializers[idx](_outputs[idx]))
+    args:
+    - --model
+    - {inputPath: model}
+    - if:
+        cond: {isPresent: sklearn_version}
+        then:
+        - --sklearn-version
+        - {inputValue: sklearn_version}
+    - if:
+        cond: {isPresent: display_name}
+        then:
+        - --display-name
+        - {inputValue: display_name}
+    - if:
+        cond: {isPresent: description}
+        then:
+        - --description
+        - {inputValue: description}
+    - if:
+        cond: {isPresent: project}
+        then:
+        - --project
+        - {inputValue: project}
+    - if:
+        cond: {isPresent: location}
+        then:
+        - --location
+        - {inputValue: location}
+    - if:
+        cond: {isPresent: labels}
+        then:
+        - --labels
+        - {inputValue: labels}
+    - if:
+        cond: {isPresent: staging_bucket}
+        then:
+        - --staging-bucket
+        - {inputValue: staging_bucket}
+    - '----output-paths'
+    - {outputPath: model_name}
+    - {outputPath: model_dict}

community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/component.yaml

@@ -0,0 +1,190 @@
+name: Upload Tensorflow model to Google Cloud Vertex AI
+metadata:
+  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/KFPv2_hell/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/workaround_for_buggy_KFPv2_compiler/component.yaml'}
+inputs:
+- {name: model, type: TensorflowSavedModel}
+- {name: tensorflow_version, type: String, optional: true}
+- name: use_gpu
+  type: Boolean
+  default: "False"
+  optional: true
+- {name: display_name, type: String, optional: true}
+- {name: description, type: String, optional: true}
+- {name: project, type: String, optional: true}
+- {name: location, type: String, optional: true}
+- {name: labels, type: JsonObject, optional: true}
+- {name: staging_bucket, type: String, optional: true}
+outputs:
+- {name: model_name, type: String}
+- {name: model_dict, type: JsonObject}
+implementation:
+  container:
+    image: python:3.9
+    command:
+    - sh
+    - -c
+    - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
+      'google-cloud-aiplatform==1.16.0' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3
+      -m pip install --quiet --no-warn-script-location 'google-cloud-aiplatform==1.16.0'
+      --user) && "$0" "$@"
+    - sh
+    - -ec
+    - |
+      program_path=$(mktemp)
+      printf "%s" "$0" > "$program_path"
+      python3 -u "$program_path" "$@"
+    - |
+      def upload_Tensorflow_model_to_Google_Cloud_Vertex_AI(
+          model_path,
+          tensorflow_version = None,
+          use_gpu = False,
+
+          display_name = None,
+          description = None,
+
+          # Uncomment when anyone requests these:
+          # instance_schema_uri: str = None,
+          # parameters_schema_uri: str = None,
+          # prediction_schema_uri: str = None,
+          # explanation_metadata: "google.cloud.aiplatform_v1.types.explanation_metadata.ExplanationMetadata" = None,
+          # explanation_parameters: "google.cloud.aiplatform_v1.types.explanation.ExplanationParameters" = None,
+
+          project = None,
+          location = None,
+          labels = None,
+          # encryption_spec_key_name: str = None,
+          staging_bucket = None,
+      ):
+          import json
+          import os
+          from google.cloud import aiplatform
+
+          if not location:
+              location = os.environ.get("CLOUD_ML_REGION")
+
+          if not labels:
+              labels = {}
+          labels["component-source"] = "github-com-ark-kun-pipeline-components"
+
+          model = aiplatform.Model.upload_tensorflow_saved_model(
+              saved_model_dir=model_path,
+              tensorflow_version=tensorflow_version,
+              use_gpu=use_gpu,
+
+              display_name=display_name,
+              description=description,
+
+              # instance_schema_uri=instance_schema_uri,
+              # parameters_schema_uri=parameters_schema_uri,
+              # prediction_schema_uri=prediction_schema_uri,
+              # explanation_metadata=explanation_metadata,
+              # explanation_parameters=explanation_parameters,
+
+              project=project,
+              location=location,
+              labels=labels,
+              # encryption_spec_key_name=encryption_spec_key_name,
+              staging_bucket=staging_bucket,
+          )
+          model_json = json.dumps(model.to_dict(), indent=2)
+          print(model_json)
+          return (model.resource_name, model_json)
+
+      def _deserialize_bool(s) -> bool:
+          from distutils.util import strtobool
+          return strtobool(s) == 1
+
+      def _serialize_json(obj) -> str:
+          if isinstance(obj, str):
+              return obj
+          import json
+          def default_serializer(obj):
+              if hasattr(obj, 'to_struct'):
+                  return obj.to_struct()
+              else:
+                  raise TypeError("Object of type '%s' is not JSON serializable and does not have .to_struct() method." % obj.__class__.__name__)
+          return json.dumps(obj, default=default_serializer, sort_keys=True)
+
+      def _serialize_str(str_value: str) -> str:
+          if not isinstance(str_value, str):
+              raise TypeError('Value "{}" has type "{}" instead of str.'.format(str(str_value), str(type(str_value))))
+          return str_value
+
+      import json
+      import argparse
+      _parser = argparse.ArgumentParser(prog='Upload Tensorflow model to Google Cloud Vertex AI', description='')
+      _parser.add_argument("--model", dest="model_path", type=str, required=True, default=argparse.SUPPRESS)
+      _parser.add_argument("--tensorflow-version", dest="tensorflow_version", type=str, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--use-gpu", dest="use_gpu", type=_deserialize_bool, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--display-name", dest="display_name", type=str, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--description", dest="description", type=str, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--project", dest="project", type=str, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--location", dest="location", type=str, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--labels", dest="labels", type=json.loads, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--staging-bucket", dest="staging_bucket", type=str, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("----output-paths", dest="_output_paths", type=str, nargs=2)
+      _parsed_args = vars(_parser.parse_args())
+      _output_files = _parsed_args.pop("_output_paths", [])
+
+      _outputs = upload_Tensorflow_model_to_Google_Cloud_Vertex_AI(**_parsed_args)
+
+      _output_serializers = [
+          _serialize_str,
+          _serialize_json,
+
+      ]
+
+      import os
+      for idx, output_file in enumerate(_output_files):
+          try:
+              os.makedirs(os.path.dirname(output_file))
+          except OSError:
+              pass
+          with open(output_file, 'w') as f:
+              f.write(_output_serializers[idx](_outputs[idx]))
+    args:
+    - --model
+    - {inputPath: model}
+    - if:
+        cond: {isPresent: tensorflow_version}
+        then:
+        - --tensorflow-version
+        - {inputValue: tensorflow_version}
+    - if:
+        cond: {isPresent: use_gpu}
+        then:
+        - --use-gpu
+        - {inputValue: use_gpu}
+    - if:
+        cond: {isPresent: display_name}
+        then:
+        - --display-name
+        - {inputValue: display_name}
+    - if:
+        cond: {isPresent: description}
+        then:
+        - --description
+        - {inputValue: description}
+    - if:
+        cond: {isPresent: project}
+        then:
+        - --project
+        - {inputValue: project}
+    - if:
+        cond: {isPresent: location}
+        then:
+        - --location
+        - {inputValue: location}
+    - if:
+        cond: {isPresent: labels}
+        then:
+        - --labels
+        - {inputValue: labels}
+    - if:
+        cond: {isPresent: staging_bucket}
+        then:
+        - --staging-bucket
+        - {inputValue: staging_bucket}
+    - '----output-paths'
+    - {outputPath: model_name}
+    - {outputPath: model_dict}

community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/component.yaml

@@ -0,0 +1,181 @@
+name: Upload XGBoost model to Google Cloud Vertex AI
+metadata:
+  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/workaround_for_buggy_KFPv2_compiler/component.yaml'}
+inputs:
+- {name: model, type: XGBoostModel}
+- {name: xgboost_version, type: String, optional: true}
+- {name: display_name, type: String, optional: true}
+- {name: description, type: String, optional: true}
+- {name: project, type: String, optional: true}
+- {name: location, type: String, optional: true}
+- {name: labels, type: JsonObject, optional: true}
+- {name: staging_bucket, type: String, optional: true}
+outputs:
+- {name: model_name, type: String}
+- {name: model_dict, type: JsonObject}
+implementation:
+  container:
+    image: python:3.9
+    command:
+    - sh
+    - -c
+    - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
+      'google-cloud-aiplatform==1.16.0' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3
+      -m pip install --quiet --no-warn-script-location 'google-cloud-aiplatform==1.16.0'
+      --user) && "$0" "$@"
+    - sh
+    - -ec
+    - |
+      program_path=$(mktemp)
+      printf "%s" "$0" > "$program_path"
+      python3 -u "$program_path" "$@"
+    - |
+      def upload_XGBoost_model_to_Google_Cloud_Vertex_AI(
+          model_path,
+          xgboost_version = None,
+
+          display_name = None,
+          description = None,
+
+          # Uncomment when anyone requests these:
+          # instance_schema_uri: str = None,
+          # parameters_schema_uri: str = None,
+          # prediction_schema_uri: str = None,
+          # explanation_metadata: "google.cloud.aiplatform_v1.types.explanation_metadata.ExplanationMetadata" = None,
+          # explanation_parameters: "google.cloud.aiplatform_v1.types.explanation.ExplanationParameters" = None,
+
+          project = None,
+          location = None,
+          labels = None,
+          # encryption_spec_key_name: str = None,
+          staging_bucket = None,
+      ):
+          import json
+          import os
+          import shutil
+          import tempfile
+          from google.cloud import aiplatform
+
+          if not location:
+              location = os.environ.get("CLOUD_ML_REGION")
+
+          if not labels:
+              labels = {}
+          labels["component-source"] = "github-com-ark-kun-pipeline-components"
+
+          # The serving container decides the model type based on the model file extension.
+          # So we need to rename the mode file (e.g. /tmp/inputs/model/data) to *.pkl
+          _, renamed_model_path = tempfile.mkstemp(suffix=".pkl")
+          shutil.copyfile(src=model_path, dst=renamed_model_path)
+
+          model = aiplatform.Model.upload_xgboost_model_file(
+              model_file_path=renamed_model_path,
+              xgboost_version=xgboost_version,
+
+              display_name=display_name,
+              description=description,
+
+              # instance_schema_uri=instance_schema_uri,
+              # parameters_schema_uri=parameters_schema_uri,
+              # prediction_schema_uri=prediction_schema_uri,
+              # explanation_metadata=explanation_metadata,
+              # explanation_parameters=explanation_parameters,
+
+              project=project,
+              location=location,
+              labels=labels,
+              # encryption_spec_key_name=encryption_spec_key_name,
+              staging_bucket=staging_bucket,
+          )
+          model_json = json.dumps(model.to_dict(), indent=2)
+          print(model_json)
+          return (model.resource_name, model_json)
+
+      def _serialize_json(obj) -> str:
+          if isinstance(obj, str):
+              return obj
+          import json
+          def default_serializer(obj):
+              if hasattr(obj, 'to_struct'):
+                  return obj.to_struct()
+              else:
+                  raise TypeError("Object of type '%s' is not JSON serializable and does not have .to_struct() method." % obj.__class__.__name__)
+          return json.dumps(obj, default=default_serializer, sort_keys=True)
+
+      def _serialize_str(str_value: str) -> str:
+          if not isinstance(str_value, str):
+              raise TypeError('Value "{}" has type "{}" instead of str.'.format(str(str_value), str(type(str_value))))
+          return str_value
+
+      import json
+      import argparse
+      _parser = argparse.ArgumentParser(prog='Upload XGBoost model to Google Cloud Vertex AI', description='')
+      _parser.add_argument("--model", dest="model_path", type=str, required=True, default=argparse.SUPPRESS)
+      _parser.add_argument("--xgboost-version", dest="xgboost_version", type=str, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--display-name", dest="display_name", type=str, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--description", dest="description", type=str, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--project", dest="project", type=str, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--location", dest="location", type=str, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--labels", dest="labels", type=json.loads, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("--staging-bucket", dest="staging_bucket", type=str, required=False, default=argparse.SUPPRESS)
+      _parser.add_argument("----output-paths", dest="_output_paths", type=str, nargs=2)
+      _parsed_args = vars(_parser.parse_args())
+      _output_files = _parsed_args.pop("_output_paths", [])
+
+      _outputs = upload_XGBoost_model_to_Google_Cloud_Vertex_AI(**_parsed_args)
+
+      _output_serializers = [
+          _serialize_str,
+          _serialize_json,
+
+      ]
+
+      import os
+      for idx, output_file in enumerate(_output_files):
+          try:
+              os.makedirs(os.path.dirname(output_file))
+          except OSError:
+              pass
+          with open(output_file, 'w') as f:
+              f.write(_output_serializers[idx](_outputs[idx]))
+    args:
+    - --model
+    - {inputPath: model}
+    - if:
+        cond: {isPresent: xgboost_version}
+        then:
+        - --xgboost-version
+        - {inputValue: xgboost_version}
+    - if:
+        cond: {isPresent: display_name}
+        then:
+        - --display-name
+        - {inputValue: display_name}
+    - if:
+        cond: {isPresent: description}
+        then:
+        - --description
+        - {inputValue: description}
+    - if:
+        cond: {isPresent: project}
+        then:
+        - --project
+        - {inputValue: project}
+    - if:
+        cond: {isPresent: location}
+        then:
+        - --location
+        - {inputValue: location}
+    - if:
+        cond: {isPresent: labels}
+        then:
+        - --labels
+        - {inputValue: labels}
+    - if:
+        cond: {isPresent: staging_bucket}
+        then:
+        - --staging-bucket
+        - {inputValue: staging_bucket}
+    - '----output-paths'
+    - {outputPath: model_name}
+    - {outputPath: model_dict}

community-content/pipeline_components/google-cloud/storage/download/component.yaml

@@ -0,0 +1,35 @@
+name: Download from GCS
+inputs:
+- {name: GCS path, type: String}
+outputs:
+- {name: Data}
+metadata:
+  annotations:
+    author: Alexey Volkov <alexey.volkov@ark-kun.com>
+    canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml'
+implementation:
+    container:
+        image: google/cloud-sdk
+        command:
+        - bash # Pattern comparison only works in Bash
+        - -ex
+        - -c
+        - |
+            if [ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then
+                gcloud auth activate-service-account --key-file="${GOOGLE_APPLICATION_CREDENTIALS}"
+            fi
+
+            uri="$0"
+            output_path="$1"
+
+            # Checking whether the URI points to a single blob, a directory or a URI pattern
+            # URI points to a blob when that URI does not end with slash and listing that URI only yields the same URI
+            if [[ "$uri" != */ ]] && (gsutil ls "$uri" | grep --fixed-strings --line-regexp "$uri"); then
+                mkdir -p "$(dirname "$output_path")"
+                gsutil -m cp -r "$uri" "$output_path"
+            else
+                mkdir -p "$output_path" # When source path is a directory, gsutil requires the destination to also be a directory
+                gsutil -m rsync -r "$uri" "$output_path" # gsutil cp has different path handling than Linux cp. It always puts the source directory (name) inside the destination directory. gsutil rsync does not have that problem.
+            fi
+        - inputValue: GCS path
+        - outputPath: Data

community-content/pipeline_components/image_ml_model_training/load_image_classification_model/component.yaml

@@ -0,0 +1,112 @@
+name: Load image classification model from tfhub
+description: |
+  Loads specified model from TFHub, creates layer to receive additional (3 channel) imagery data.
+  Args:
+    class_names (Sequence[str]):
+        Sequence of strings of categories for classification corresponding to input data.
+    loaded_model_path (str):
+        Output path for the loaded model.
+    image_size_path (str):
+        Output path for the model expected image size.
+    model_name (Optional[str]):
+        Name of the pre-trained image classification model to load from TFHub.
+        Eligible model_name:
+        - efficientnetv2-s
+        - efficientnetv2-m
+        - efficientnetv2-l
+        - efficientnetv2-s-21k
+        - efficientnetv2-m-21k
+        - efficientnetv2-l-21k
+        - efficientnetv2-xl-21k
+        - efficientnetv2-b0-21k
+        - efficientnetv2-b1-21k
+        - efficientnetv2-b2-21k
+        - efficientnetv2-b3-21k
+        - efficientnetv2-s-21k-ft1k
+        - efficientnetv2-m-21k-ft1k
+        - efficientnetv2-l-21k-ft1k
+        - efficientnetv2-xl-21k-ft1k
+        - efficientnetv2-b0-21k-ft1k
+        - efficientnetv2-b1-21k-ft1k
+        - efficientnetv2-b2-21k-ft1k
+        - efficientnetv2-b3-21k-ft1k
+        - efficientnetv2-b0
+        - efficientnetv2-b1
+        - efficientnetv2-b2
+        - efficientnetv2-b3
+        - efficientnet_b0
+        - efficientnet_b1
+        - efficientnet_b2
+        - efficientnet_b3
+        - efficientnet_b4
+        - efficientnet_b5
+        - efficientnet_b6
+        - efficientnet_b7
+        - bit_s-r50x1
+        - inception_v3
+        - inception_resnet_v2
+        - resnet_v1_50
+        - resnet_v1_101
+        - resnet_v1_152
+        - resnet_v2_50
+        - resnet_v2_101
+        - resnet_v2_152
+        - nasnet_large
+        - nasnet_mobile
+        - pnasnet_large
+        - mobilenet_v2_100_224
+        - mobilenet_v2_130_224
+        - mobilenet_v2_140_224
+        - mobilenet_v3_small_100_224
+        - mobilenet_v3_small_075_224
+        - mobilenet_v3_large_100_224
+        - mobilenet_v3_large_075_224
+    dropout_rate (Optional[float]):
+        Fraction of input units to drop in the last layer. Value should be between 0.0 and 1.0.
+    trainable (Optional[bool]):
+        If true fine tuning will be performed on entire Hub model. If false only additional
+        layers will be trained.
+    l2_regularization_penalty (Optional[float]):
+        l2 regularization penalty.
+inputs:
+- {name: class_names, type: 'typing.List[str]', description: List of class names corresponding
+    to the input image data}
+- {name: model_name, type: String, description: Name of the TFHub model to load, default: efficientnetv2-xl-21k,
+  optional: true}
+- {name: dropout_rate, type: Float, description: Dropout rate, default: '0.2', optional: true}
+- name: trainable
+  type: Boolean
+  description: True if fine tuning should be performed
+  default: "True"
+  optional: true
+- {name: l2_regularization_penalty, type: Float, description: Regularization penalty,
+  default: '0.0001', optional: true}
+outputs:
+- {name: loaded_model_path, type: TensorflowSavedModel, description: Output path for
+    the loaded model}
+- {name: image_size_path, type: HeightWidth}
+implementation:
+  container:
+    image: us-docker.pkg.dev/vertex-ai/ready-to-go-image-classification/image-components:v0.1
+    # command is a list of strings (command-line arguments).
+    # The YAML language has two syntaxes for lists and you can use either of them.
+    # Here we use the "flow syntax" - comma-separated strings inside square brackets.
+    command: [
+      python3,
+      # Path of the program inside the container
+      /pipelines/component/src/loading_component.py,
+      --loaded-model-path,
+      {outputPath: loaded_model_path},
+      --class-names,
+      {inputValue: class_names},
+      --model-name,
+      {inputValue: model_name},
+      --dropout-rate,
+      {inputValue: dropout_rate},
+      --trainable,
+      {inputValue: trainable},
+      --l2-regularization-penalty,
+      {inputValue: l2_regularization_penalty},
+      --image-size-path,
+      {outputPath: image_size_path},
+    ]

community-content/pipeline_components/image_ml_model_training/pipeline.py

@@ -0,0 +1,62 @@
+# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
+from kfp import components
+from kfp.v2 import dsl
+
+# %% Loading components
+upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url('https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/component.yaml')
+deploy_model_to_endpoint_op = components.load_component_from_url('https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml')
+transcode_imagedataset_tfrecord_from_csv_op = components.load_component_from_url('https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/community-content/pipeline_components/image_ml_model_training/transcode_tfrecord_image_dataset_from_csv/component.yaml')
+load_image_classification_model_from_tfhub_op = components.load_component_from_url('https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/b5b65198a6c2ffe8c0fa2aa70127e3325752df68/community-content/pipeline_components/image_ml_model_training/load_image_classification_model/component.yaml')
+preprocess_image_data_op = components.load_component_from_url('https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/community-content/pipeline_components/image_ml_model_training/preprocess_image_data/component.yaml')
+train_tensorflow_image_classification_model_op = components.load_component_from_url('https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/community-content/pipeline_components/image_ml_model_training/train_image_classification_model/component.yaml')
+
+
+# %% Pipeline definition
+def image_classification_pipeline():
+    class_names = ['daisy', 'dandelion', 'roses', 'sunflowers', 'tulips']
+    csv_image_data_path = 'gs://cloud-samples-data/ai-platform/flowers/flowers.csv'
+    deploy_model = False
+
+    image_data = dsl.importer(
+        artifact_uri=csv_image_data_path, artifact_class=dsl.Dataset).output
+
+    image_tfrecord_data = transcode_imagedataset_tfrecord_from_csv_op(
+        csv_image_data_path=image_data,
+        class_names=class_names
+    ).outputs['tfrecord_image_data_path']
+
+    loaded_model_outputs = load_image_classification_model_from_tfhub_op(
+        class_names=class_names,
+    ).outputs
+
+    preprocessed_data = preprocess_image_data_op(
+        image_tfrecord_data,
+        height_width_path=loaded_model_outputs['image_size_path'],
+    ).outputs
+
+    trained_model = (train_tensorflow_image_classification_model_op(
+        preprocessed_training_data_path = preprocessed_data['preprocessed_training_data_path'],
+        preprocessed_validation_data_path = preprocessed_data['preprocessed_validation_data_path'],
+        model_path=loaded_model_outputs['loaded_model_path']).
+                   set_cpu_limit('96').
+                   set_memory_limit('128G').
+                   add_node_selector_constraint('cloud.google.com/gke-accelerator', 'NVIDIA_TESLA_A100').
+                   set_gpu_limit('8').
+                   outputs['trained_model_path'])
+
+    vertex_model_name = upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op(
+        model=trained_model,
+    ).outputs['model_name']
+
+    # Deploying the model might incur additional costs over time
+    if deploy_model:
+        vertex_endpoint_name = deploy_model_to_endpoint_op(
+            model_name=vertex_model_name,
+        ).outputs['endpoint_name']
+
+pipeline_func = image_classification_pipeline
+
+# %% Pipeline submission
+if __name__ == '__main__':
+    from google.cloud import aiplatform
+    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

community-content/pipeline_components/image_ml_model_training/preprocess_image_data/component.yaml

@@ -0,0 +1,57 @@
+name: Preprocess image data
+description: |
+  Preprocess the image data and split between train and validation.
+  Args:
+    input_data_path (str):
+        Input path for the TFRecord image data. Data will be formatted as 'label' (encoded image
+        label), and 'image_raw' (the binary string of the image data).
+    height_width_path (str):
+        Path to square height and width to resize images to. File should contain single float value.
+        Value is dependent on training model.
+    preprocessed_training_data_path (str):
+        Output path for the TFRecord training data. Data will be formatted as 'label' (encoded image
+        label), and 'image_raw' (the binary string of the image data).
+    preprocessed_validation_data_path (str):
+        Output path for the TFRecord validation data. Data will be formatted as 'label' (encoded
+        image label), and 'image_raw' (the binary string of the image data).
+    validation_split (Optional[float]):
+        Fraction of data that will make up validation dataset. Value should be between 0.0 and 1.0.
+    seed (Optional[int]):
+        The global random seed to ensure the system gets a unique random sequence
+        that is deterministic (https://www.tensorflow.org/api_docs/python/tf/random/set_seed).
+inputs:
+- {name: input_data_path, type: ImageDatasetTFRecord, description: 'Input path for
+    the TFRecord image data,'}
+- {name: height_width_path, type: HeightWidth, description: 'Path to square height and width to
+    resize images to,'}
+- {name: validation_split, type: Float, description: 'Fraction of data that will make
+    up validation dataset,', default: '0.2', optional: true}
+- {name: seed, type: Integer, description: Random seed, default: '0', optional: true}
+outputs:
+- {name: preprocessed_training_data_path, type: ImageDatasetTFRecord, description: 'Output
+    path for the training data,'}
+- {name: preprocessed_validation_data_path, type: ImageDatasetTFRecord, description: 'Output
+    path for the validation data,'}
+implementation:
+  container:
+    image: us-docker.pkg.dev/vertex-ai/ready-to-go-image-classification/image-components:v0.1
+    # command is a list of strings (command-line arguments).
+    # The YAML language has two syntaxes for lists and you can use either of them.
+    # Here we use the "flow syntax" - comma-separated strings inside square brackets.
+    command: [
+      python3,
+      # Path of the program inside the container
+      /pipelines/component/src/preprocessing_component.py,
+      --input-data-path,
+      {inputPath: input_data_path},
+      --height-width-path,
+      {inputPath: height_width_path},
+      --validation-split,
+      {inputValue: validation_split},
+      --seed,
+      {inputValue: seed},
+      --preprocessed-training-data-path,
+      {outputPath: preprocessed_training_data_path},
+      --preprocessed-validation-data-path,
+      {outputPath: preprocessed_validation_data_path},
+    ]

community-content/pipeline_components/image_ml_model_training/train_image_classification_model/component.yaml

@@ -0,0 +1,90 @@
+name: Train tensorflow image classification model
+description: |
+  Creates a trained image classification TensorFlow model.
+  Args:
+    preprocessed_training_data_path (str):
+        Input path to the TFRecord training data. Data will be formatted as 'label' (encoded image
+        label), and 'image_raw' (the binary string of the image data).
+    preprocessed_validation_data_path (str):
+        Input path to the TFRecord validation data. Data will be formatted as 'label' (encoded
+        image label), and 'image_raw' (the binary string of the image data).
+    model_path (str):
+        Input path to the loaded pre-trained model.
+    trained_model_path (str):
+        Output path to save the trained model to.
+    optimizer_name (Optional[str]):
+        Name of the tf.keras optimizer. Available optimizers are listed at
+        https://keras.io/api/optimizers/
+    optimizer_parameters (Optional[Dict[str, str]]):
+        Optimizer parameters.
+    loss_function_name (Optional[str]):
+        Name of the loss function.
+    loss_function_parameters (Optional[Dict[str, str]]):
+        Loss function parameters.
+    number_of_epochs (Optional[int]):
+        Number of training iterations over data.
+    metric_names (Optional[Sequence[str]]):
+        List of tf.keras.metrics to be evaluated by the model during training and testing. Available
+        metrics are listed at https://keras.io/api/metrics/.
+    seed Optional(int):
+        The global random seed to ensure the system gets a unique random sequence
+        that is deterministic (https://www.tensorflow.org/api_docs/python/tf/random/set_seed).
+inputs:
+- {name: preprocessed_training_data_path, type: ImageDatasetTFRecord, description: 'Input
+    path for the training data,'}
+- {name: preprocessed_validation_data_path, type: ImageDatasetTFRecord, description: 'Input
+    path for the validation data,'}
+- {name: model_path, type: TensorflowSavedModel, description: 'Input path for the
+    model,'}
+- {name: optimizer_name, type: String, description: 'Name of the optimizer,', default: SGD,
+  optional: true}
+- {name: optimizer_parameters, type: 'typing.Dict[str, str]', description: 'Optimizer
+    parameters,', default: '{}', optional: true}
+- {name: loss_function_name, type: String, description: 'Name of the loss function,',
+  default: CategoricalCrossentropy, optional: true}
+- {name: loss_function_parameters, type: 'typing.Dict[str, str]', description: 'Loss
+    function parameters,', default: '{}', optional: true}
+- {name: number_of_epochs, type: Integer, description: 'Number of epochs,', default: '10',
+  optional: true}
+- {name: metric_names, type: 'typing.List[str]', description: 'List of metrics to
+    use,', default: '["accuracy"]', optional: true}
+- {name: seed, type: Integer, description: 'Random seed,', default: '0', optional: true}
+- {name: batch_size, type: Integer, description: Batch size, default: '16', optional: true}
+outputs:
+- {name: trained_model_path, type: TensorflowSavedModel, description: 'Output path
+    for the saved model,'}
+implementation:
+  container:
+    image: us-docker.pkg.dev/vertex-ai/ready-to-go-image-classification/image-components:v0.1
+    # command is a list of strings (command-line arguments).
+    # The YAML language has two syntaxes for lists and you can use either of them.
+    # Here we use the "flow syntax" - comma-separated strings inside square brackets.
+    command: [
+      python3,
+      # Path of the program inside the container
+      /pipelines/component/src/training_component.py,
+      --preprocessed-training-data-path,
+      {inputPath: preprocessed_training_data_path},
+      --preprocessed-validation-data-path,
+      {inputPath: preprocessed_validation_data_path},
+      --model-path,
+      {inputPath: model_path},
+      --trained-model-path,
+      {outputPath: trained_model_path},
+      --optimizer-name,
+      {inputValue: optimizer_name},
+      --loss-function-name,
+      {inputValue: loss_function_name},
+      --number-of-epochs,
+      {inputValue: number_of_epochs},
+      --seed,
+      {inputValue: seed},
+      --batch-size,
+      {inputValue: batch_size},
+      --metric-names,
+      {inputValue: metric_names},
+      --optimizer-parameters,
+      {inputValue: optimizer_parameters},
+      --loss-function-parameters,
+      {inputValue: loss_function_parameters},
+    ]

community-content/pipeline_components/image_ml_model_training/transcode_tfrecord_image_dataset_from_csv/component.yaml

@@ -0,0 +1,37 @@
+name: Transcode imagedataset tfrecord from csv
+description: |
+  Transcodes CSV Data into TFRecord file of TFExamples.
+  Args:
+    csv_image_data_path (str):
+        Path to the CSV image data. Data must include 'image_filepath' (Path to image file) and
+        'image_label' (output for a prediction) fields.
+    class_names (Sequence[str]):
+        Sequence of strings of categories for classification corresponding to input data.
+    tfrecord_image_data_path (str):
+        Output path for the TFRecord image data. Data will be formatted as 'label' (encoded image
+        label), and 'image_raw' (the binary string of the image data).
+inputs:
+- {name: csv_image_data_path, type: ImageDatasetCSV, description: Input path for the
+    CSV image data}
+- {name: class_names, type: 'typing.List[str]', description: List of class names corresponding
+    to the input image data}
+outputs:
+- {name: tfrecord_image_data_path, type: ImageDatasetTFRecord, description: Output
+    path for the TFRecord image data}
+implementation:
+  container:
+    image: us-docker.pkg.dev/vertex-ai/ready-to-go-image-classification/image-components:v0.1
+    # command is a list of strings (command-line arguments).
+    # The YAML language has two syntaxes for lists and you can use either of them.
+    # Here we use the "flow syntax" - comma-separated strings inside square brackets.
+    command: [
+      python3,
+      # Path of the program inside the container
+      /pipelines/component/src/transcoding_csv_component.py,
+      --csv-image-data-path,
+      {inputPath: csv_image_data_path},
+      --tfrecord-image-data-path,
+      {outputPath: tfrecord_image_data_path},
+      --class-names,
+      {inputValue: class_names},
+    ]

community-content/pipeline_components/image_ml_model_training/transcode_tfrecord_image_dataset_from_jsonl/component.yaml

@@ -0,0 +1,39 @@
+name: Transcode imagedataset tfrecord from jsonlines
+description: |
+  Transcodes JSONL Data into TFRecord file of TFExamples.
+  Args:
+    jsonl_image_data_path (str):
+        Input path for the JSONL image data
+        Path to the JSONL image data. Each line corresponds to a JSON input describing an image.
+        Schema follows AutoML image classification JSONL format
+        https://cloud.google.com/vertex-ai/docs/image-data/classification/prepare-data#json-lines.
+    class_names (Sequence[str]):
+        Sequence of strings of categories for classification corresponding to input data.
+    tfrecord_image_data_path (str):
+        Output path for the TFRecord image data. Data will be formatted as 'label' (encoded image
+        label), and 'image_raw' (the binary string of the image data).
+inputs:
+- {name: jsonl_image_data_path, type: ImageDatasetJsonLines, description: Input path
+    for the JSONL image data}
+- {name: class_names, type: 'typing.List[str]', description: List of class names corresponding
+    to the input image data}
+outputs:
+- {name: tfrecord_image_data_path, type: ImageDatasetTFRecord, description: Output
+    path for the TFRecord image data}
+implementation:
+  container:
+    image: us-docker.pkg.dev/vertex-ai/ready-to-go-image-classification/image-components:v0.1
+    # command is a list of strings (command-line arguments).
+    # The YAML language has two syntaxes for lists and you can use either of them.
+    # Here we use the "flow syntax" - comma-separated strings inside square brackets.
+    command: [
+      python3,
+      # Path of the program inside the container
+      /pipelines/component/src/transcoding_jsonl_component.py,
+      --jsonl-image-data-path,
+      {inputPath: jsonl_image_data_path},
+      --tfrecord-image-data-path,
+      {outputPath: tfrecord_image_data_path},
+      --class-names,
+      {inputValue: class_names},
+    ]