Merge branch 'main' into batch_tabular_model

.cloud-build/cleanup/cleanup.py

@@ -1,29 +1,10 @@
 from typing import List
 from ratemate import RateLimit
-import argparse
-
-parser = argparse.ArgumentParser()
-parser.add_argument("--dry_run",
-                    type=bool,
-                    default=False)
-args = parser.parse_args()
-
-
 from resource_cleanup_manager import (
     DatasetResourceCleanupManager,
     ModelResourceCleanupManager,
     EndpointResourceCleanupManager,
     ResourceCleanupManager,
-    MatchingEngineIndexEndpointResourceCleanupManager,
-    MatchingEngineIndexResourceCleanupManager,
-    FeatureStoreCleanupManager,
-    PipelineJobCleanupManager,
-    TrainingJobCleanupManager,
-    HyperparameterTuningCleanupManager,
-    BatchPredictionJobCleanupManager,
-    ExperimentCleanupManager,
-    BucketCleanupManager,
-    ArtifactRegistryCleanupManager
 )
 
 rate_limit = RateLimit(max_count=25, per=60, greedy=False)
@@ -40,6 +21,7 @@ def run_cleanup_managers(managers: List[ResourceCleanupManager], is_dry_run: boo
             try:
                 if not manager.is_deletable(resource):
                     continue
+
                 if is_dry_run:
                     resource_name = manager.resource_name(resource)
                     print(f"Will delete '{type_name}': {resource_name}")
@@ -52,24 +34,16 @@ def run_cleanup_managers(managers: List[ResourceCleanupManager], is_dry_run: boo
         print("")
 
 
-if args.dry_run:
+is_dry_run = False
+
+if is_dry_run:
     print("Starting cleanup in dry run mode...")
 
 # List of all cleanup managers
-managers: List[ResourceCleanupManager] = [
+managers = [
     DatasetResourceCleanupManager(),
     EndpointResourceCleanupManager(),
     ModelResourceCleanupManager(),  # ModelResourceCleanupManager must follow EndpointResourceCleanupManager due to deployed models blocking model deletion.
-    MatchingEngineIndexEndpointResourceCleanupManager(),
-    MatchingEngineIndexResourceCleanupManager(),
-    FeatureStoreCleanupManager(),
-    PipelineJobCleanupManager(),
-    TrainingJobCleanupManager(),
-    HyperparameterTuningCleanupManager(),
-    BatchPredictionJobCleanupManager(),
-    ExperimentCleanupManager(), # Experiment missing _resource_noun
-    BucketCleanupManager(),
-    ArtifactRegistryCleanupManager()
 ]
 
-run_cleanup_managers(managers=managers, is_dry_run=args.dry_run)
+run_cleanup_managers(managers=managers, is_dry_run=is_dry_run)

.cloud-build/cleanup/resource_cleanup_manager.py

@@ -1,18 +1,8 @@
-'''
-READ FIRST BEFORE MAKING CHANGES
--  Create a convention for resources created from vertex-ai-samples GH. We already have one IIRC
-- Only delete those objects as part of our clean-up script.
-- Don't run any tests on python-docs-samples-tests project, especially ones that affect resources created outside of our purview
-- Add --dry-run option to the clean-up script. This option will just output the list of resources the script will delete instead of actually deleting the resources.
-- Have a larger conversation in DEE before touching any resources that were not created as part of vertex-ai-samples
-'''
-import os
 import abc
 from typing import Any, Type
 
 from google.cloud import aiplatform
 from google.cloud.aiplatform import base
-from google.cloud import storage
 from proto.datetime_helpers import DatetimeWithNanoseconds
 
 # If a resource was updated within this number of seconds, do not delete.
@@ -79,7 +69,7 @@ class VertexAIResourceCleanupManager(ResourceCleanupManager):
     def delete(self, resource):
         resource.delete()
 
-    def get_seconds_since_modification(self, resource: Any) -> float:
+    def get_seconds_since_modification(self, resource: Any) -> bool:
         update_time = resource.update_time
         current_time = DatetimeWithNanoseconds.now(tz=update_time.tzinfo)
         return (current_time - update_time).total_seconds()
@@ -107,154 +97,15 @@ class EndpointResourceCleanupManager(VertexAIResourceCleanupManager):
     vertex_ai_resource = aiplatform.Endpoint
 
     def delete(self, resource):
+        # TODO: Remove this once https://github.com/googleapis/python-aiplatform/issues/1441 is fixed
+        resource._sync_gca_resource()
         for deployed_model_id in [
             models.id for models in resource._gca_resource.deployed_models
         ]:
             resource._undeploy(deployed_model_id=deployed_model_id)
+
         resource.delete(force=True)
 
 
 class ModelResourceCleanupManager(VertexAIResourceCleanupManager):
     vertex_ai_resource = aiplatform.Model
-
-
-class MatchingEngineIndexResourceCleanupManager(VertexAIResourceCleanupManager):
-    vertex_ai_resource = aiplatform.MatchingEngineIndex
-
-
-class MatchingEngineIndexEndpointResourceCleanupManager(VertexAIResourceCleanupManager):
-    vertex_ai_resource = aiplatform.MatchingEngineIndexEndpoint
-
-    def delete(self, resource):
-        resource.undeploy_all()
-        resource.delete(force=True)
-
-class FeatureStoreCleanupManager(VertexAIResourceCleanupManager):
-    vertex_ai_resource = aiplatform.Featurestore
-
-    def resource_name(self, resource: Any) -> str:
-        return resource.name
-
-class PipelineJobCleanupManager(VertexAIResourceCleanupManager):
-    vertex_ai_resource = aiplatform.PipelineJob
-
-class TrainingJobCleanupManager(VertexAIResourceCleanupManager):
-    vertex_ai_resource = aiplatform.training_jobs._CustomTrainingJob
-
-    job_types = [
-            aiplatform.AutoMLImageTrainingJob,
-            aiplatform.AutoMLTextTrainingJob,
-            aiplatform.AutoMLTabularTrainingJob,
-            aiplatform.AutoMLVideoTrainingJob,
-            aiplatform.AutoMLForecastingTrainingJob,
-            aiplatform.CustomJob,
-            aiplatform.CustomTrainingJob,
-            aiplatform.CustomContainerTrainingJob,
-            aiplatform.CustomPythonPackageTrainingJob
-    ]
-
-    def list(self) -> Any:
-        return [
-            job
-            for job_type in self.job_types
-            for job in job_type.list()
-        ]
-
-class HyperparameterTuningCleanupManager(VertexAIResourceCleanupManager):
-    vertex_ai_resource = aiplatform.HyperparameterTuningJob
-
-
-class BatchPredictionJobCleanupManager(VertexAIResourceCleanupManager):
-    vertex_ai_resource = aiplatform.BatchPredictionJob
-
-class ExperimentCleanupManager(VertexAIResourceCleanupManager):
-    vertex_ai_resource = aiplatform.Experiment
-
-    @property
-    def type_name(self) -> str:
-        return "Experiment"
-
-    def resource_name(self, resource: Any) -> str:
-        return resource.name
-
-    def get_seconds_since_modification(self, resource: Any) -> float:
-        update_time = resource._metadata_context.update_time
-        current_time = DatetimeWithNanoseconds.now()
-        return float(current_time.timestamp() - update_time.timestamp())
-
-class BucketCleanupManager(ResourceCleanupManager):
-    vertex_ai_resource = storage.bucket.Bucket
-
-    def list(self) -> Any:
-        storage_client = storage.Client()
-        return list(storage_client.list_buckets())
-
-    def delete(self, resource):
-        try:
-            resource.delete(force=True)
-        except Exception as e:
-            print(e)
-
-    @property
-    def type_name(self) -> str:
-        return "Bucket"
-
-    def get_seconds_since_modification(self, resource: Any) -> float:
-        # Bucket has no last_update property, only time created
-        created_time = resource.time_created
-        current_time = DatetimeWithNanoseconds.now()
-        return float(current_time.timestamp() - created_time.timestamp())
-
-    def resource_name(self, resource: Any) -> str:
-        return resource.name
-
-    def is_deletable(self, resource: Any) -> bool:
-        time_difference = self.get_seconds_since_modification(resource)
-
-        if not self.resource_name(resource).startswith('your-bucket-name'):
-            print(f"Skipping '{resource}' not a Vertex AI notebook bucket")
-            return False
-
-        # Check that it wasn't created too recently, to prevent race conditions
-        if time_difference <= RESOURCE_UPDATE_BUFFER_IN_SECONDS:
-            print(
-                f"Skipping '{resource}' due to update_time being '{time_difference}', which is less than '{RESOURCE_UPDATE_BUFFER_IN_SECONDS}'."
-            )
-            return False
-        return True
-
-class ArtifactRegistryCleanupManager(ResourceCleanupManager):
-    vertex_ai_resource = "Artifact Registry"
-
-    def list(self) -> Any:
-        import subprocess
-
-        result = subprocess.run(["gcloud artifacts repositories list --location=us-central1"], 
-                                shell=True, capture_output=True, text=True)
-
-        ret = []
-        lines = result.stdout.split('\n')[2:]
-        for line in lines:
-            repo = line.split(' ')[0]
-            if repo.startswith("my-docker-repo"):
-                ret.append(repo)
-
-        return ret
-
-    def delete(self, resource):
-        os.system(f"! gcloud artifacts repositories delete {resource} --location=us-central1")
-
-    @property
-    def type_name(self) -> str:
-        return "ArtifactRepository"
-
-    def resource_name(self, resource: Any) -> str:
-        return resource
-
-    # delete repository regardless of age
-    def get_seconds_since_modification(self, resource: Any) -> float:
-        return RESOURCE_UPDATE_BUFFER_IN_SECONDS + 1
-    
-    def is_deleteable(self, resource: Any) -> bool:
-        return True
-

.cloud-build/execute_changed_notebooks_cli.py

@@ -17,8 +17,6 @@
 
 import argparse
 import pathlib
-import os
-import csv
 
 import execute_changed_notebooks_helper
 
@@ -38,22 +36,9 @@ parser = argparse.ArgumentParser(description="Run changed notebooks.")
 parser.add_argument(
     "--test_paths_file",
     type=pathlib.Path,
-    help="The path to the file that has newline-delimited folders of notebooks that should be tested.",
+    help="The path to the file that has newline-limited folders of notebooks that should be tested.",
     required=True,
 )
-parser.add_argument(
-    "--test_percent",
-    type=int,
-    help="The percent of notebooks to be tested (between 1 and 100).",
-    required=False,
-    default=100,
-)
-parser.add_argument(
-    "--build_id",
-    type=str,
-    help="The build id (which may be a Cloud Build job specific or user explicit.",
-    required=True
-)
 parser.add_argument(
     "--base_branch",
     help="The base git branch to diff against to find changed files.",
@@ -122,90 +107,24 @@ parser.add_argument(
     default=True,
     help="Should run notebooks in parallel.",
 )
-parser.add_argument(
-    "--concurrent_notebooks",
-    type=int,
-    help="Maximum number of parallel notebook executions per minute",
-    default=10,
-    required=False,
-)
-parser.add_argument(
-    "--run_first_file",
-    type=pathlib.Path,
-    help="The path to the file that has newline-delimited of notebooks to run in the first batch",
-    default=None,
-    required=False,
-)
-parser.add_argument(
-    "--dry_run",
-    type=str2bool,
-    default=False,
-    help="Dry run for testing - no execution",
-)
 
 args = parser.parse_args()
 
-changed_notebooks = execute_changed_notebooks_helper.get_changed_notebooks(
+notebooks = execute_changed_notebooks_helper.get_changed_notebooks(
     test_paths_file=args.test_paths_file,
     base_branch=args.base_branch,
 )
 
-
-results_bucket = f"{args.artifacts_bucket}"
-# artifacts_bucket may get set by trigger to a full gs:// folder path
-if results_bucket.startswith("gs://"):
-    results_bucket = results_bucket[5:]
-results_bucket = results_bucket.split('/')[0]
-results_file = f"build_results/{args.build_id}.json"
-
-if args.test_percent == 100:
-    notebooks = changed_notebooks
-    accumulative_results = {}
-else:
-    accumulative_results = execute_changed_notebooks_helper.load_results(results_bucket, results_file)
-
-    notebooks = [changed_notebook for changed_notebook in changed_notebooks if execute_changed_notebooks_helper.select_notebook(changed_notebook, accumulative_results, args.test_percent)]
-    # cap the number of notebooks to the specified percentage
-    max_notebooks = int((len(changed_notebooks) * args.test_percent))
-    if (len(notebooks) > max_notebooks):
-        notebooks = notebooks[:max_notebooks]
-
-run_first = []
-if args.run_first_file:
-    if not os.path.isfile(args.run_first_file):
-        print("Error: file does not exist", args.run_first_file)
-    else:
-        with open(args.run_first_file, 'r') as csvfile:
-            reader = csv.reader(csvfile)
-            for row in reader:
-                notebook = row[0]
-                run_first.append(notebook)
-
-    for notebook in run_first:
-        if notebook in notebooks:
-            # remove from existing list
-            notebooks.remove(notebook)
-            # add back to the front of the list
-            notebooks.insert(0, notebook)
-            print(f"Run first: {notebook}")
-
-if args.dry_run:
-    print("Dry run ...\n")
-    for notebook in notebooks:
-        print(f"Would execute: {notebook}")
-else:
-    execute_changed_notebooks_helper.process_and_execute_notebooks(
-        notebooks=notebooks,
-        container_uri=args.container_uri,
-        staging_bucket=args.staging_bucket,
-        artifacts_bucket=args.artifacts_bucket,
-        results_file=results_file,
-        should_parallelize=args.should_parallelize,
-        timeout=args.timeout,
-        variable_project_id=args.variable_project_id,
-        variable_region=args.variable_region,
-        variable_service_account=args.variable_service_account,
-        variable_vpc_network=args.variable_vpc_network,
-        private_pool_id=args.private_pool_id,
-        concurrent_notebooks=args.concurrent_notebooks,
+execute_changed_notebooks_helper.process_and_execute_notebooks(
+    notebooks=notebooks,
+    container_uri=args.container_uri,
+    staging_bucket=args.staging_bucket,
+    artifacts_bucket=args.artifacts_bucket,
+    should_parallelize=args.should_parallelize,
+    timeout=args.timeout,
+    variable_project_id=args.variable_project_id,
+    variable_region=args.variable_region,
+    variable_service_account=args.variable_service_account,
+    variable_vpc_network=args.variable_vpc_network,
+    private_pool_id=args.private_pool_id,
 )

.cloud-build/execute_changed_notebooks_helper.py

@@ -17,36 +17,24 @@ import concurrent
 import dataclasses
 import datetime
 import functools
-import json
 import git
 import operator
 import os
-import io
-import json
 import pathlib
 import re
 import subprocess
-import random
-from google.cloud import storage
-import utils
-from typing import List, Optional, Dict, Any
-from utils import util
+from typing import List, Optional
 
 import execute_notebook_helper
 import execute_notebook_remote
 import nbformat
 from google.cloud.devtools.cloudbuild_v1.types import BuildOperationMetadata
+from ratemate import RateLimit
 from tabulate import tabulate
 from utils import NotebookProcessors, util
 
 # A buffer so that workers finish before the orchestrating job
 WORKER_TIMEOUT_BUFFER_IN_SECONDS: int = 60 * 60
-PYTHON_VERSION = "3.9"  # Set default python version
-
-# rolling time window for accumulating build results for selecting notebooks
-MAX_RESULTS_AGE_SECONDS: int = (60 * 60) * 24 * 60  # 60 days
-# maximum time since last run to force a run on the current build
-MAX_AGE_BEFORE_FORCE_RUN: int = (60 * 60) * 24 * 30
 
 
 def format_timedelta(delta: datetime.timedelta) -> str:
@@ -73,14 +61,11 @@ def format_timedelta(delta: datetime.timedelta) -> str:
 @dataclasses.dataclass
 class NotebookExecutionResult:
     name: str
-    path: str
     duration: datetime.timedelta
-    start_time: datetime.datetime
     is_pass: bool
     log_url: str
     output_uri: str
     build_id: str
-    logs_bucket: str
     error_message: Optional[str]
 
     @property
@@ -91,97 +76,6 @@ class NotebookExecutionResult:
             return None
 
 
-def load_results(results_bucket: str,
-                 results_file: str) -> Dict[str, Any]:
-    '''
-    Load accumulated notebook test results
-    '''
-
-    print("Loading existing accumulative results ...")
-    accumulative_results = {}
-    try:
-        client = storage.Client()
-        bucket = client.bucket(results_bucket)
-
-        build_results_dir = os.path.dirname(results_file)
-        blobs = client.list_blobs(results_bucket, prefix=build_results_dir)
-        for blob in blobs:
-            time_created = blob.time_created.replace(tzinfo=None)
-            if (datetime.datetime.now().replace(tzinfo=None) - time_created).total_seconds() > MAX_RESULTS_AGE_SECONDS:
-                continue
-
-            content = util.download_blob_into_memory(results_bucket, blob.name, download_as_text=True)
-
-            try:
-                build_results = json.loads(content)
-            except:
-                continue  # skip corrupted build results files
-            for notebook in build_results:
-                if notebook in accumulative_results:
-                    accumulative_results[notebook]['passed'] += build_results[notebook]['passed']
-                    accumulative_results[notebook]['failed'] += build_results[notebook]['failed']
-                    if accumulative_results[notebook]['last_time_ran'] > time_created:
-                        accumulative_results[notebook]['last_time_ran'] = time_created
-                else:
-                    accumulative_results[notebook] = build_results[notebook]
-                    accumulative_results[notebook]['failed_on_latest_run'] = build_results[notebook]['failed']
-                    accumulative_results[notebook]['last_time_ran'] = time_created
-
-        print(accumulative_results)
-    except Exception as e:
-        print(e)
-
-    # If there are no accumulative results, an empty dict is returned
-    return accumulative_results
-
-def select_notebook(changed_notebook: str,
-                    accumulative_results: Dict[str, Any],
-                    test_percent: int) -> bool:
-    '''
-    Algorithm to randomly select a notebook, but weight the propbability of selected based on past failures
-    '''
-
-    if changed_notebook in accumulative_results:
-        pass_count = accumulative_results[changed_notebook]['passed']
-        fail_count = accumulative_results[changed_notebook]['failed']
-        failed_on_latest_run = accumulative_results[changed_notebook]['failed_on_latest_run']
-        last_time_ran = accumulative_results[changed_notebook]['last_time_ran']
-    else:
-        pass_count = 1
-        fail_count = 0
-        failed_on_latest_run = 0
-        last_time_ran = datetime.datetime.now().replace(tzinfo=None)
-
-    # If notebook has not been ran in a long time, force running it
-    if (datetime.datetime.now().replace(tzinfo=None) - last_time_ran).total_seconds() > MAX_AGE_BEFORE_FORCE_RUN:
-        should_test_do_to_age = True
-    else:
-        should_test_do_to_age  = False
-
-
-    # if failed on the last time it was ran, select the notebook
-    if failed_on_latest_run:
-        inferred_failure_rate = 1
-    # otherwise, calculate the frequency of failure
-    else:
-        inferred_failure_rate = fail_count / (pass_count + fail_count)
-
-    # If failure rate is high, the chance of testing should be higher
-    should_test_due_to_failure = random.uniform(0, 1) <= inferred_failure_rate
-
-    #if accumulative_resultsi[changed_notebook]['latest_date_ran']
-
-    # Additionally, only test a percentage of these
-    should_test_due_to_random_subset = random.uniform(0, 1) <= (test_percent / 100)
-
-    if should_test_due_to_failure or should_test_due_to_random_subset or should_test_do_to_age:
-        print(f"Selected: {changed_notebook}, {should_test_due_to_failure}, {should_test_due_to_random_subset}")
-        return True
-    else:
-        print(f"Not Selected: {changed_notebook}, pass {pass_count}, fail {fail_count}")
-        return False
-
-
 def _process_notebook(
     notebook_path: str,
     variable_project_id: str,
@@ -203,7 +97,6 @@ def _process_notebook(
             "VPC_NETWORK": variable_vpc_network,
         },
     )
-    unique_strings_preprocessor = NotebookProcessors.UniqueStringsPreprocessor()
 
     # Use no-execute preprocessor
     (
@@ -212,41 +105,11 @@ def _process_notebook(
     ) = remove_no_execute_cells_preprocessor.preprocess(nb)
 
     (nb, resources) = update_variables_preprocessor.preprocess(nb, resources)
-    (nb, resources) = unique_strings_preprocessor.preprocess(nb, resources)
 
     with open(notebook_path, mode="w", encoding="utf-8") as new_file:
         nbformat.write(nb, new_file)
 
 
-def _get_notebook_python_version(notebook_path: str) -> str:
-    """
-    Get the python version for running the notebook if it is specified in
-    the notebook.
-    """
-    python_version = PYTHON_VERSION
-
-    # Load the notebook
-    file = open(notebook_path)
-    src = file.read()
-    nb_json = json.loads(src)
-
-    # Iterate over the cells in the ipynb
-    for cell in nb_json["cells"]:
-        if cell["cell_type"] == "markdown":
-            markdown = str.join("", cell["source"])
-
-            # Look for the python version specification pattern
-            re_match = re.search(
-                "python version = (\d\.\d)", markdown, flags=re.IGNORECASE
-            )
-            if re_match:
-                # get the version number
-                python_version = re_match.group(1)
-                break
-
-    return python_version
-
-
 def _create_tag(filepath: str) -> str:
     tag = os.path.basename(os.path.normpath(filepath))
     tag = re.sub("[^0-9a-zA-Z_.-]+", "-", tag)
@@ -257,6 +120,7 @@ def _create_tag(filepath: str) -> str:
     return tag
 
 
+rate_limit = RateLimit(max_count=50, per=60, greedy=True)
 
 
 def process_and_execute_notebook(
@@ -272,6 +136,7 @@ def process_and_execute_notebook(
     notebook: str,
     should_get_tail_logs: bool = False,
 ) -> NotebookExecutionResult:
+    rate_limit.wait()  # wait before creating the task
 
     print(f"Running notebook: {notebook}")
 
@@ -290,26 +155,18 @@ def process_and_execute_notebook(
 
     result = NotebookExecutionResult(
         name=tag,
-        path=notebook,
         duration=datetime.timedelta(seconds=0),
-        start_time=datetime.datetime.now(),
         is_pass=False,
         output_uri=notebook_output_uri,
         log_url="",
         build_id="",
-        logs_bucket="",
         error_message=None,
     )
 
     # TODO: Handle cases where multiple notebooks have the same name
+    time_start = datetime.datetime.now()
     operation = None
     try:
-        # Get the python version for running the notebook if specified
-        notebook_exec_python_version = _get_notebook_python_version(
-            notebook_path=notebook
-        )
-        print(f"Running notebook with python {notebook_exec_python_version}")
-
         # Pre-process notebook by substituting variable names
         _process_notebook(
             notebook_path=notebook,
@@ -336,21 +193,18 @@ def process_and_execute_notebook(
             private_pool_id=private_pool_id,
             private_pool_region=variable_region,
             timeout_in_seconds=timeout_in_seconds,
-            python_version=notebook_exec_python_version,
         )
 
         operation_metadata = BuildOperationMetadata(mapping=operation.metadata)
         result.build_id = operation_metadata.build.id
         result.log_url = operation_metadata.build.log_url
-        result.logs_bucket = operation_metadata.build.logs_bucket
 
         # Block and wait for the result
-        operation_result = operation.result(timeout=timeout_in_seconds)
+        operation_result = operation.result()
 
-        result.duration = datetime.datetime.now() - result.start_time
+        result.duration = datetime.datetime.now() - time_start
         result.is_pass = True
         print(f"{notebook} PASSED in {format_timedelta(result.duration)}.")
-
     except Exception as error:
         result.error_message = str(error)
 
@@ -369,7 +223,7 @@ def process_and_execute_notebook(
             except Exception as error:
                 result.error_message = str(error)
 
-        result.duration = datetime.datetime.now() - result.start_time
+        result.duration = datetime.datetime.now() - time_start
         result.is_pass = False
 
         print(
@@ -437,59 +291,12 @@ def get_changed_notebooks(
 
     return notebooks
 
-def _save_results(results: List[NotebookExecutionResult],
-                  artifacts_bucket: str,
-                  results_file: str):
-
-    artifacts_bucket = artifacts_bucket.replace("gs://", "").split('/')[0]
-
-    print("Updating build results ...")
-    build_results = {}
-    for result in results:
-        if result.is_pass:
-            pass_count = 1
-            fail_count = 0
-        else:
-            pass_count = 0
-            fail_count = 1
-        if result.error_message is None:
-            error_type = ''
-        elif 'INTERNAL' in result.error_message:
-            error_type = 'INTERNAL'
-        elif 'context deadline exceeded' in result.error_message:
-            error_type = 'TIMEOUT'
-        elif 'Quota' in result.error_message:
-            error_type = 'QUOTA'
-        elif 'ServiceUnavailable' in result.error_message:
-            error_type = 'SERVICEUNAVAILABLE'
-        elif 'ModuleNotFoundError' in result.error_message:
-            error_type = 'IMPORT'
-        else:
-            error_type = ''
-        build_results[result.path] = {
-                'duration': result.duration.total_seconds(),
-                'start_time': str(result.start_time),
-                'passed': pass_count,
-                'failed': fail_count,
-                'error_type': error_type
-        }
-        print(f"adding {result.path}")
-
-    print(f"Saving accumulative results to {results_file}, nentries {len(build_results)}")
-    content = json.dumps(build_results)
-
-    client = storage.Client()
-    bucket = client.get_bucket(artifacts_bucket)
-    bucket.blob(str(results_file)).upload_from_string(content, 'text/json')
-
-
 
 def process_and_execute_notebooks(
     notebooks: List[str],
     container_uri: str,
     staging_bucket: str,
     artifacts_bucket: str,
-    results_file: str,
     should_parallelize: bool,
     timeout: int,
     variable_project_id: str,
@@ -497,7 +304,6 @@ def process_and_execute_notebooks(
     variable_service_account: str,
     variable_vpc_network: Optional[str] = None,
     private_pool_id: Optional[str] = None,
-    concurrent_notebooks: Optional[int] = 10,
 ):
     """
     Run the notebooks that exist under the folders defined in the test_paths_file.
@@ -518,8 +324,6 @@ def process_and_execute_notebooks(
             Required. The GCS staging bucket to write source code to.
         artifacts_bucket (str):
             Required. The GCS staging bucket to write executed notebooks to.
-        results_file (str):
-            Required: The path to the artifacts bucket to save results 
         variable_project_id (str):
             Required. The value for PROJECT_ID to inject into notebooks.
         variable_region (str):
@@ -528,7 +332,6 @@ def process_and_execute_notebooks(
             Required. Should run notebooks in parallel using a thread pool as opposed to in sequence.
         timeout (str):
             Required. Timeout string according to https://cloud.google.com/build/docs/build-config-file-schema#timeout.
-        concurrent_notebooks (int): Max number of notebooks per minute to run in parallel.
     """
 
     # Calculate deadline
@@ -536,7 +339,7 @@ def process_and_execute_notebooks(
         seconds=max(timeout - WORKER_TIMEOUT_BUFFER_IN_SECONDS, 0)
     )
 
-    if len(notebooks) >= 1:
+    if len(notebooks) > 1:
         notebook_execution_results: List[NotebookExecutionResult] = []
 
         print(f"Found {len(notebooks)} modified notebooks: {notebooks}")
@@ -545,9 +348,7 @@ def process_and_execute_notebooks(
             print(
                 "Running notebooks in parallel, so no logs will be displayed. Please wait..."
             )
-
-
-            with concurrent.futures.ThreadPoolExecutor(max_workers=concurrent_notebooks) as executor:
+            with concurrent.futures.ThreadPoolExecutor(max_workers=100) as executor:
                 print(f"Max workers: {executor._max_workers}")
 
                 notebook_execution_results = list(
@@ -603,7 +404,6 @@ def process_and_execute_notebooks(
                         result.log_url,
                         result.output_uri,
                         result.output_uri_web,
-                        result.logs_bucket,
                     ]
                     for result in results_sorted
                 ],
@@ -614,39 +414,10 @@ def process_and_execute_notebooks(
                     "log_url",
                     "output_uri",
                     "output_uri_web",
-                    "logs_bucket",
                 ],
             )
         )
 
-        if len(notebooks) == 1:
-            print("=" * 100)
-            print("The notebook execution build log:\n")
-            print("=" * 100)
-
-            build_id = results_sorted[0].build_id
-            logs_bucket_name = (results_sorted[0].logs_bucket).replace("gs://", "")
-            log_file_name = f"log-{build_id}.txt"
-
-            log_contents = util.download_blob_into_memory(
-                bucket_name=logs_bucket_name,
-                blob_name=log_file_name,
-                download_as_text=True,
-            )
-
-            # Remove extra steps from the log
-            match = re.search("starting Step #4", log_contents, flags=re.IGNORECASE)
-
-            if match is not None:
-                match_index = match.span()[0]
-                print(log_contents[match_index:])
-            else:
-                print(log_contents)
-
-        _save_results(results_sorted, 
-                      artifacts_bucket, 
-                      results_file)
-
         print("\n=== END RESULTS===\n")
 
         total_notebook_duration = functools.reduce(
@@ -662,5 +433,25 @@ def process_and_execute_notebooks(
         # Raise error if any notebooks failed
         if not all([result.is_pass for result in results_sorted]):
             raise RuntimeError("Notebook failures detected. See logs for details")
+
+    elif len(notebooks) == 1:
+        notebook = notebooks[0]
+
+        # Pre-process notebook by substituting variable names
+        _process_notebook(
+            notebook_path=notebook,
+            variable_project_id=variable_project_id,
+            variable_region=variable_region,
+            variable_service_account=variable_service_account,
+            variable_vpc_network=variable_vpc_network,
+        )
+
+        execute_notebook_helper.execute_notebook(
+            notebook_source=notebook,
+            output_file_or_uri="/".join(
+                [artifacts_bucket, pathlib.Path(notebook).name]
+            ),
+            should_log_output=True,
+        )
     else:
         print("No notebooks modified in this pull request.")

.cloud-build/execute_notebook_remote.py

@@ -40,7 +40,6 @@ def execute_notebook_remote(
     private_pool_region: Optional[str],
     tag: Optional[str],
     timeout_in_seconds: Optional[int] = None,
-    python_version: Optional[str] = None
 ) -> operation.Operation:
     """Create and execute a single notebook on Google Cloud Build"""
     # Load build steps from YAML
@@ -51,12 +50,8 @@ def execute_notebook_remote(
         "_PYTHON_IMAGE": container_uri,
         "_NOTEBOOK_GCS_URI": notebook_uri,
         "_NOTEBOOK_OUTPUT_GCS_URI": notebook_output_uri,
-        "_PYTHON_VERSION" : f"python{python_version}"
     }
 
-    if python_version is not None:
-      substitutions["_PYTHON_VERSION"] = "python" + python_version
-
     build = cloudbuild_v1.Build()
 
     options: Optional[client_options.ClientOptions] = None

.cloud-build/notebook-execution-test-cloudbuild-single.yaml

@@ -10,21 +10,21 @@ steps:
     entrypoint: /bin/sh
     args:
     - -c
-    - ${_PYTHON_VERSION} .cloud-build/CheckPythonVersion.py -q
+    - python3 .cloud-build/CheckPythonVersion.py -q
   # Create a virtual environment
   - name: ${_PYTHON_IMAGE}
     entrypoint: /bin/sh
     args:
     - -c
-    - ${_PYTHON_VERSION} -m venv workspace/env
+    - python3 -m venv workspace/env
   # Install Python dependencies
   - name: ${_PYTHON_IMAGE}
     entrypoint: /bin/sh
     args:
     - -c
     - . workspace/env/bin/activate &&
-      python -m pip -q install -U pip &&
-      python -m pip -q install -U -r .cloud-build/requirements.txt
+      python3 -m pip -q install -U pip &&
+      python3 -m pip -q install -U -r .cloud-build/requirements.txt
   # Install Python dependencies and run testing script
   - name: ${_PYTHON_IMAGE}
     entrypoint: /bin/sh
@@ -32,7 +32,7 @@ steps:
     - -c
     - |
       . workspace/env/bin/activate &&
-      python .cloud-build/execute_notebook_cli.py --notebook_source "${_NOTEBOOK_GCS_URI}" --output_file_or_uri "${_NOTEBOOK_OUTPUT_GCS_URI}"
+      python3 .cloud-build/execute_notebook_cli.py --notebook_source "${_NOTEBOOK_GCS_URI}" --output_file_or_uri "${_NOTEBOOK_OUTPUT_GCS_URI}"
     env:
     - 'IS_TESTING=1'
 timeout: 86400s

.cloud-build/notebook-execution-test-cloudbuild.yaml

@@ -36,7 +36,7 @@ steps:
     - -c
     - |
       . workspace/env/bin/activate &&
-      python3 .cloud-build/execute_changed_notebooks_cli.py --test_paths_file "${_TEST_PATHS_FILE}" --base_branch "${_FORCED_BASE_BRANCH}" --container_uri ${_PYTHON_IMAGE} --staging_bucket ${_GCS_STAGING_BUCKET}  --artifacts_bucket ${_GCS_STAGING_BUCKET}/executed_notebooks/PR_${_PR_NUMBER}/BUILD_${BUILD_ID} --variable_project_id ${PROJECT_ID} --variable_region ${_GCP_REGION} --variable_service_account ${_GCP_SERVICE_ACCOUNT} --variable_vpc_network "${_GCP_VPC_NETWORK_NAME}" `if [ ! -z "${_PRIVATE_POOL_NAME}" ]; then echo "--private_pool_id ${_PRIVATE_POOL_NAME}"; fi` --build_id ${BUILD_ID} --test_percent=${_TEST_PERCENT} --concurrent_notebooks=${_CONCURRENT_NOTEBOOKS} --run_first_file=${_RUN_FIRST_FILE}
+      python3 .cloud-build/execute_changed_notebooks_cli.py --test_paths_file "${_TEST_PATHS_FILE}" --base_branch "${_FORCED_BASE_BRANCH}" --container_uri ${_PYTHON_IMAGE} --staging_bucket ${_GCS_STAGING_BUCKET}  --artifacts_bucket ${_GCS_STAGING_BUCKET}/executed_notebooks/PR_${_PR_NUMBER}/BUILD_${BUILD_ID} --variable_project_id ${PROJECT_ID} --variable_region ${_GCP_REGION} --variable_service_account ${_GCP_SERVICE_ACCOUNT} --variable_vpc_network "${_GPC_VPC_NETWORK_NAME}" `if [ ! -z "${_PRIVATE_POOL_NAME}" ]; then echo "--private_pool_id ${_PRIVATE_POOL_NAME}"; fi`
     env:
     - 'IS_TESTING=1'
 timeout: 86400s

.cloud-build/requirements.txt

@@ -3,15 +3,11 @@ numpy
 jupyter
 nbconvert
 papermill
+pandas
 matplotlib
 tabulate
 google-cloud-aiplatform
 google-cloud-storage
 google-cloud-build
-google-cloud-storage
 ratemate
-GitPython
-tqdm
-fsspec
-pandas
-
+GitPython

.cloud-build/run_first.csv

@@ -1,6 +0,0 @@
-notebooks/official/pipelines/google_cloud_pipeline_components_automl_text.ipynb
-notebooks/official/generative_ai/rlhf_tune_llm.ipynb
-notebooks/official/generative_ai/tune_peft.ipynb
-notebooks/official/prediction/llm_streaming_prediction.ipynb
-notebooks/official/migration/sdk-automl-text-classification-batch-prediction.ipynb
-notebooks/official/vizier/get_started_vertex_vizier.ipynb

.cloud-build/test_notebook_vm.txt

@@ -1,5 +1,5 @@
 notebooks/official/vizier/gapic-vizier-multi-objective-optimization.ipynb
 notebooks/official/pipelines/lightweight_functions_component_io_kfp.ipynb
 notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-locally-trained-models.ipynb
-notebooks/official/custom/custom-tabular-bq-managed-dataset.ipynb
-.cloud-build/tests/python_version_test.ipynb
+notebooks/official/pipelines/metrics_viz_run_compare_kfp.ipynb
+notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb

.cloud-build/test_single_notebook_vm.txt

@@ -1 +1 @@
-notebooks/official/custom/custom-tabular-bq-managed-dataset.ipynb
+notebooks/official/pipelines/metrics_viz_run_compare_kfp.ipynb

.cloud-build/tests/flaky.py

@@ -1,46 +0,0 @@
-# grep PASSED tests.txt | cut -c 10-100 >passed.txt
-
-import os
-
-repo_dir = '/home/jupyter/vertex-ai-samples/'
-repo_dir_len = len(repo_dir)
-official_dir = repo_dir + 'notebooks/official'
-
-entries = os.scandir(official_dir)
-folders = []
-for entry in entries:
-    if entry.is_dir():
-        folders.append(entry.path)
-
-# Passing
-with open('passed.txt', 'r') as pass_file:
-    notebook_names = pass_file.readlines()
-        
-notebooks = []
-for folder in folders:
-    entries = os.scandir(folder)
-    for entry in entries:
-        for notebook in notebook_names:
-            if entry.name == notebook.rstrip():
-                notebooks.append(entry.path[repo_dir_len:])
-                
-with open('passing_tests.txt', 'w') as f:
-    for notebook in notebooks:
-        f.write(notebook + '\n')
-        
-        
-# Failing
-with open('failed.txt', 'r') as fail_file:
-    notebook_names = fail_file.readlines()
-        
-notebooks = []
-for folder in folders:
-    entries = os.scandir(folder)
-    for entry in entries:
-        for notebook in notebook_names:
-            if entry.name == notebook.rstrip():
-                notebooks.append(entry.path[repo_dir_len:])
-                
-with open('failing_tests.txt', 'w') as f:
-    for notebook in notebooks:
-        f.write(notebook + '\n')

.cloud-build/tests/python_version_test.ipynb

@@ -1,61 +0,0 @@
-{
-  "cells": [
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "57a3d44ed8a8"
-      },
-      "source": [
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**_NOTE_**: This notebook has been tested in the following environment:\n",
-        "\n",
-        "* Python version = 3.7\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI API and Compute Engine API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component).\n",
-        "\n",
-        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "c6516f90311b"
-      },
-      "outputs": [],
-      "source": [
-        "# test if the right python version is being used\n",
-        "import sys\n",
-        "\n",
-        "actual_python_version = f\"{sys.version_info.major}.{sys.version_info.minor}\"\n",
-        "print(f\"Runtime python version: {actual_python_version}\")\n",
-        "\n",
-        "assert actual_python_version == \"3.7\", \"Wrong python version!\""
-      ]
-    }
-  ],
-  "metadata": {
-    "colab": {
-      "name": "python_version_test.ipynb",
-      "toc_visible": true
-    },
-    "kernelspec": {
-      "display_name": "Python 3",
-      "name": "python3"
-    }
-  },
-  "nbformat": 4,
-  "nbformat_minor": 0
-}

.cloud-build/tests/select_notebooks_test.py

@@ -1,33 +0,0 @@
-import sys
-
-from execute_changed_notebooks_helper import (load_results, select_notebook)
-
-
-def test_load_results():
-    bucket: str = "cloud-build-notebooks-presubmit"
-    bucket_file: str = "build_results"
-
-    accum = load_results(bucket, bucket_file)
-
-    print(accum)
-
-    assert len(accum) > 0
-
-def test_select_notebook():
-    bucket: str = "cloud-build-notebooks-presubmit"
-    bucket_file: str = "build_results"
-
-    accum = load_results(bucket, bucket_file)
-
-    n_select = 0
-    n_notselect = 0
-    for notebook in accum:
-        if select_notebook(notebook, accum, 50):
-            n_select += 1
-        else:
-            n_notselect += 1
-
-    print(f"SELECTED {n_select}, NOT SELECTED {n_notselect}")
-
-    assert n_select > 0
-    assert n_notselect > 0

.cloud-build/utils/NotebookProcessors.py

@@ -14,8 +14,6 @@
 # limitations under the License.
 
 from typing import Dict
-import random
-import string
 
 from nbconvert.preprocessors import Preprocessor
 
@@ -35,7 +33,7 @@ class RemoveNoExecuteCells(Preprocessor):
 
 
 class UpdateVariablesPreprocessor(Preprocessor):
-    def __init__(self, replacement_map: Dict[str, str]):
+    def __init__(self, replacement_map: Dict):
         self._replacement_map = replacement_map
 
     @staticmethod
@@ -65,36 +63,3 @@ class UpdateVariablesPreprocessor(Preprocessor):
             executable_cells.append(cell)
         notebook.cells = executable_cells
         return notebook, resources
-
-
-# Generate a uuid of a specifed length
-def generate_uuid(length: int = 8) -> str:
-    return "".join(random.choices(string.ascii_lowercase + string.digits, k=length))
-
-
-class UniqueStringsPreprocessor(Preprocessor):
-    # A preprocessor that replaces strings that end with "-unique" or "_unique" with a uuid.
-
-    @staticmethod
-    def update_unique_strings(content: str):
-        # Replace strings that end with "-unique" or "_unique" with a uuid.
-
-        unique_id = generate_uuid()
-        return (
-            content.replace('-unique"', f'-{unique_id}"')
-            .replace("-unique'", f'-{unique_id}"')
-            .replace('_unique"', f'_{unique_id}"')
-            .replace("_unique'", f'_{unique_id}"')
-        )
-
-    def preprocess(self, notebook, resources=None):
-        executable_cells = []
-        for cell in notebook.cells:
-            if cell.cell_type == "code":
-                cell.source = self.update_unique_strings(
-                    content=cell.source,
-                )
-
-            executable_cells.append(cell)
-        notebook.cells = executable_cells
-        return notebook, resources

.cloud-build/utils/UpdateNotebookVariables.py

@@ -40,3 +40,65 @@ def get_updated_value(content: str, variable_name: str, variable_value: str) ->
         content,
         flags=re.M,
     )
+
+
+def test_update_value():
+    new_content = get_updated_value(
+        content='asdf\nPROJECT_ID = "[your-project-id]" #@param {type:"string"} \nasdf',
+        variable_name="PROJECT_ID",
+        variable_value="sample-project",
+    )
+    assert (
+        new_content
+        == 'asdf\nPROJECT_ID = "sample-project" #@param {type:"string"} \nasdf'
+    )
+
+
+def test_update_value_single_quotes():
+    new_content = get_updated_value(
+        content="PROJECT_ID = '[your-project-id]'",
+        variable_name="PROJECT_ID",
+        variable_value="sample-project",
+    )
+    assert new_content == "PROJECT_ID = 'sample-project'"
+
+
+def test_update_value_avoidance():
+    new_content = get_updated_value(
+        content="PROJECT_ID = shell_output[0] ",
+        variable_name="PROJECT_ID",
+        variable_value="sample-project",
+    )
+    assert new_content == "PROJECT_ID = shell_output[0] "
+
+
+def test_region():
+    new_content = get_updated_value(
+        content='REGION = "[your-region]"  # @param {type:"string"}',
+        variable_name="REGION",
+        variable_value="us-central1",
+    )
+    assert new_content == 'REGION = "us-central1"  # @param {type:"string"}'
+
+
+def test_region_equal_equals_ignore():
+    # Tests that == is ignored
+    new_content = get_updated_value(
+        content='REGION == "[your-region]"  # @param {type:"string"}',
+        variable_name="REGION",
+        variable_value="us-central1",
+    )
+    assert new_content == 'REGION == "[your-region]"  # @param {type:"string"}'
+
+
+def test_service_account():
+    # Tests that == is ignored
+    new_content = get_updated_value(
+        content='SERVICE_ACCOUNT = "[your-service-account]"  # @param {type:"string"}',
+        variable_name="SERVICE_ACCOUNT",
+        variable_value="12345-compute@developer.gserviceaccount.com",
+    )
+    assert (
+        new_content
+        == 'SERVICE_ACCOUNT = "12345-compute@developer.gserviceaccount.com"  # @param {type:"string"}'
+    )

.cloud-build/utils/build_results_viewer.py

@@ -1,30 +0,0 @@
-'''
-Viewer for the weekly regression testing of the official notebooks
-
-Cloud Storage location: gs://cloud-build-notebooks-presubmit/build_results/
-'''
-import argparse
-import json
-from  util import download_file
-
-
-parser = argparse.ArgumentParser()
-parser.add_argument('--file', dest='file',
-                    default='build.json', type=str, help='build results file')
-args = parser.parse_args()
-
-if args.file.startswith("gs://"):
-    path = args.file[5:]
-    bucket = path.split('/')[0]
-    file   = path[len(bucket)+1:]
-    download_file(bucket, file, "build.json")
-    args.file = "build.json"
-
-with open(args.file, 'r') as f:
-    results = json.load(f)
-
-for item in results.items():
-    if item[1]['passed']:
-        print(f"{item[0]},PASSED")
-    else:
-        print(f"{item[0]},FAILED")

.cloud-build/utils/tests/test_preprocessors.py

@@ -1,14 +0,0 @@
-from utils import NotebookProcessors
-
-
-def test_update_value():
-    # Test that the content was updated
-    preprocessor = NotebookProcessors.UniqueStringsPreprocessor()
-
-    content = 'PROJECT_ID = "your-project-id-unique"'
-
-    new_content = preprocessor.update_unique_strings(content)
-
-    assert new_content != content
-    assert new_content.startswith('PROJECT_ID = "your-project-id-')
-    assert new_content.endswith('"')

.cloud-build/utils/tests/test_update_notebook_variables.py

@@ -1,63 +0,0 @@
-from utils import UpdateNotebookVariables
-
-
-def test_update_value():
-    new_content = UpdateNotebookVariables.get_updated_value(
-        content='asdf\nPROJECT_ID = "[your-project-id]" #@param {type:"string"} \nasdf',
-        variable_name="PROJECT_ID",
-        variable_value="sample-project",
-    )
-    assert (
-        new_content
-        == 'asdf\nPROJECT_ID = "sample-project" #@param {type:"string"} \nasdf'
-    )
-
-
-def test_update_value_single_quotes():
-    new_content = UpdateNotebookVariables.get_updated_value(
-        content="PROJECT_ID = '[your-project-id]'",
-        variable_name="PROJECT_ID",
-        variable_value="sample-project",
-    )
-    assert new_content == "PROJECT_ID = 'sample-project'"
-
-
-def test_update_value_avoidance():
-    new_content = UpdateNotebookVariables.get_updated_value(
-        content="PROJECT_ID = shell_output[0] ",
-        variable_name="PROJECT_ID",
-        variable_value="sample-project",
-    )
-    assert new_content == "PROJECT_ID = shell_output[0] "
-
-
-def test_region():
-    new_content = UpdateNotebookVariables.get_updated_value(
-        content='REGION = "[your-region]"  # @param {type:"string"}',
-        variable_name="REGION",
-        variable_value="us-central1",
-    )
-    assert new_content == 'REGION = "us-central1"  # @param {type:"string"}'
-
-
-def test_region_equal_equals_ignore():
-    # Tests that == is ignored
-    new_content = UpdateNotebookVariables.get_updated_value(
-        content='REGION == "[your-region]"  # @param {type:"string"}',
-        variable_name="REGION",
-        variable_value="us-central1",
-    )
-    assert new_content == 'REGION == "[your-region]"  # @param {type:"string"}'
-
-
-def test_service_account():
-    # Tests that == is ignored
-    new_content = UpdateNotebookVariables.get_updated_value(
-        content='SERVICE_ACCOUNT = "[your-service-account]"  # @param {type:"string"}',
-        variable_name="SERVICE_ACCOUNT",
-        variable_value="12345-compute@developer.gserviceaccount.com",
-    )
-    assert (
-        new_content
-        == 'SERVICE_ACCOUNT = "12345-compute@developer.gserviceaccount.com"  # @param {type:"string"}'
-    )

.cloud-build/utils/util.py

@@ -3,7 +3,7 @@ import subprocess
 import tarfile
 import uuid
 from datetime import datetime
-from typing import Optional, Union
+from typing import Optional
 
 from google.auth import credentials as auth_credentials
 from google.cloud import storage
@@ -58,29 +58,3 @@ def archive_code_and_upload(staging_bucket: str):
     print(f"Uploaded source code archive to {source_archived_file_gcs}")
 
     return source_archived_file_gcs
-
-
-def download_blob_into_memory(
-    bucket_name: str, blob_name: str, download_as_text: Optional[bool] = False
-) -> Union[bytes, str]:
-    """
-    Downloads a blob into memory as byte or as text if
-    download_as_text is set to True.
-    """
-
-    storage_client = storage.Client()
-
-    bucket = storage_client.bucket(bucket_name)
-
-    # Construct a client side representation of a blob.
-    blob = bucket.blob(blob_name)
-
-    # Download the blob content
-    if download_as_text:
-        contents = blob.download_as_text()
-    else:
-        contents = blob.download_as_bytes()
-
-    print(f"Downloaded storage object {blob_name} from bucket {bucket_name}.")
-
-    return contents

.cloud-build/web-index/do_not_index.txt

@@ -1,11 +0,0 @@
-sdk2_remote_tabnet_training.ipynb
-remote_hyperparameter_tuning.ipynb
-remote_prediction.ipynb
-remote_training_bigframes_pytorch.ipynb
-remote_training_bigframes_sklearn.ipynb
-remote_training_bigframes_tensorflow.ipynb
-remote_training_lightning.ipynb
-remote_training_pytorch.ipynb
-remote_training_sklearn.ipynb
-remote_training_tensorflow_with_autologging.ipynb
-

.cloud-build/web-index/notebook-index-cloudbuild.yaml

@@ -1,23 +0,0 @@
-steps:
-  # Fetch full repo for diff purposes
-  - name: gcr.io/cloud-builders/git
-    args: [fetch, --unshallow, --quiet]
-  # Create a virtual environment
-  - name: ${_PYTHON_IMAGE}
-    entrypoint: /bin/sh
-    args:
-    - -c
-    - python3 -m venv workspace/env
-  # Install Python dependencies and run testing script
-  - name: ${_PYTHON_IMAGE}
-    entrypoint: /bin/sh
-    args:
-    - -c
-    - |
-      . workspace/env/bin/activate &&
-      python3 notebooks/notebook_template_review.py --web --title --steps --desc --linkback --notebook-dir=notebooks/official --skip-file=${_DO_NOT_INDEX_FILE} >web.html
-artifacts:
-  objects:
-          location: gs://${_GCS_ARTIFACTS_BUCKET}/webdoc
-          paths: ['web.html']
-timeout: 86400s

.github/workflows/linter/Dockerfile

@@ -1,20 +0,0 @@
-# To use this image, run this command with the desired notebook args from the top-level vertex-ai-samples directory:
-# 1. To lint all changed notebooks:
-# docker run -v ${PWD}:/setup/app gcr.io/python-docs-samples-tests/notebook_linter:latest
-# 2. To lint specific notebooks:
-# docker run -v ${PWD}:/setup/app gcr.io/python-docs-samples-tests/notebook_linter:latest notebooks/1.ipynb notebooks/2.ipynb
-
-FROM python:3.11
-
-WORKDIR setup
-
-COPY ./requirements.txt .
-COPY ./run_linter.sh .
-
-# Install dependencies.
-RUN pip install --upgrade pip
-RUN pip install -r requirements.txt
-
-WORKDIR app
-
-ENTRYPOINT ["/setup/run_linter.sh"]

.github/workflows/linter/requirements.txt

@@ -2,9 +2,9 @@ git+https://github.com/tensorflow/docs
 ipython
 jupyter
 nbconvert
-black==23.10.1
-pyupgrade==3.13.0
-isort==5.12.0
-flake8==6.1.0
-nbqa==1.7.0
+black==22.6.0
+pyupgrade==2.34.0
+isort==5.10.1
+flake8==4.0.1
+nbqa==1.4.0
 

.github/workflows/linter/run_linter.sh

@@ -47,22 +47,12 @@ done
 
 echo "Test mode: $is_test"
 
-# Read in user-provided notebooks
-notebooks=()
-for arg in "$@"; do
-    if [[ $arg == *.ipynb ]]; then
-        notebooks+=("$arg")
-    fi
-done
-
 # Only check notebooks in test folders modified in this pull request.
 # Note: Use process substitution to persist the data in the array
-if [ ${#notebooks[@]} -eq 0 ]; then
-    echo "Checking for changed notebooked using git"
-    while read -r file || [ -n "$line" ]; do
-        notebooks+=("$file")
-    done < <(git diff --name-only main... | grep '\.ipynb$')
-fi
+notebooks=()
+while read -r file || [ -n "$line" ]; do
+    notebooks+=("$file")
+done < <(git diff --name-only main... | grep '\.ipynb$')
 
 problematic_notebooks=()
 if [ ${#notebooks[@]} -gt 0 ]; then
@@ -84,7 +74,7 @@ if [ ${#notebooks[@]} -gt 0 ]; then
                 # python3 -m nbqa black "$notebook" --check
                 # BLACK_RTN=$?
                 echo "Running pyupgrade..."
-                python3 -m nbqa pyupgrade --exit-zero-even-if-changed "$notebook"
+                python3 -m nbqa pyupgrade "$notebook"
                 PYUPGRADE_RTN=$?
                 echo "Running isort..."
                 python3 -m nbqa isort "$notebook" --check
@@ -97,7 +87,7 @@ if [ ${#notebooks[@]} -gt 0 ]; then
                 python3 -m nbqa black "$notebook"
                 BLACK_RTN=$?
                 echo "Running pyupgrade..."
-                python3 -m nbqa pyupgrade --exit-zero-even-if-changed "$notebook"
+                python3 -m nbqa pyupgrade "$notebook"
                 PYUPGRADE_RTN=$?
                 echo "Running isort..."
                 python3 -m nbqa isort "$notebook"

CONTRIBUTING.md

@@ -44,10 +44,12 @@ Finally, run this code block to check for errors. Each step will attempt to
 automatically fix any issues. If the fixes can't be performed automatically,
 then you will need to manually address them before submitting your PR.
 
-Note: For official, only submit one notebook per PR.
-
 ```shell
-docker run -v ${PWD}:/setup/app gcr.io/cloud-devrel-public-resources/notebook_linter:latest your_notebook
+nbqa black "$notebook"
+nbqa pyupgrade "$notebook"
+nbqa isort "$notebook"
+nbqa flake8 "$notebook" --extend-ignore=W391,E501,F821,E402,F404,W503,E203,E722,W293,W291
+python3 -m tensorflow_docs.tools.nbfmt --remove_outputs "$notebook"
 ```
 
 ## Code Reviews

community-content/CODEOWNERS

@@ -6,21 +6,3 @@
 /sklearn_text_classification_from_script_using_vertex_sdk       @maxhardt
 /pluto_on_workbench @wkharold
 /cpr-examples @samthrasher
-/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines @Ark-kun
-/pipeline_components @Ark-kun
-/pipeline_components/image_ml_model_training @lakeyk
-/prediction_featurestore_integration @googleapis/vertex-prediction-team
-/vertex_vision_model_garden/model_oss/util @weigary
-/vertex_vision_model_garden/model_oss/diffusers @weigary
-/vertex_vision_model_garden/model_oss/keras @dstnluong-google
-/vertex_vision_model_garden/model_oss/transformers @dstnluong-google
-/vertex_vision_model_garden/model_oss/pic2word @jismailyan-google
-/vertex_vision_model_garden/model_oss/open_clip @lydhr
-/vertex_vision_model_garden/model_oss/movinet @KCFindstr
-/vertex_vision_model_garden/model_oss/data_converter @KCFindstr
-/vertex_vision_model_garden/model_oss/peft @weigary
-/vertex_vision_model_garden/model_oss/lm-evaluation-harness @kathyyu-google
-/vertex_vision_model_garden/model_oss/tfvision @dstnluong-google
-/vertex_vision_model_garden/model_oss/fvlm @minwoo33park
-/vertex_vision_model_garden/model_oss/imagebind @kathyyu-google
-

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_logistic_regression_model_using_Scikit_learn_and_import_to_Vertex_AI/pipeline.component.yaml

@@ -1,83 +0,0 @@
-name: Train tabular classification logistic regression model using Scikit learn pipeline
-metadata:
-  annotations:
-    author: Alexey Volkov <alexey.volkov@ark-kun.com>
-    canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_classification_logistic_regression_model_using_Scikit_learn_and_import_to_Vertex_AI/pipeline.component.yaml
-    sdk: https://cloud-pipelines.net/pipeline-editor/
-implementation:
-  graph:
-    tasks:
-      Download from GCS:
-        componentRef:
-          digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
-        arguments:
-          GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
-        annotations:
-          editor.position: '{"x":40,"y":40,"width":180,"height":40}'
-      Select columns using Pandas on CSV data:
-        componentRef:
-          digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: Data
-              taskId: Download from GCS
-          column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
-        annotations:
-          editor.position: '{"x":40,"y":140,"width":180,"height":54}'
-      Fill all missing values using Pandas on CSV data:
-        componentRef:
-          digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: transformed_table
-              taskId: Select columns using Pandas on CSV data
-              type: CSV
-          replacement_value: '0'
-        annotations:
-          editor.position: '{"x":40,"y":250,"width":180,"height":54}'
-      Binarize column using Pandas on CSV data:
-        componentRef:
-          digest: d699afd4d7cae862708717cc160f4394ed0c04e536e9515923ef1e8865f01d44
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: transformed_table
-              taskId: Fill all missing values using Pandas on CSV data
-              type: CSV
-          column_name: tips
-          predicate: '> 0'
-          new_column_name: class
-        annotations:
-          editor.position: '{"x":40,"y":380,"width":180,"height":54}'
-      Train logistic regression model using scikit learn from CSV:
-        componentRef:
-          digest: a864625a822e4b1c8ef6fe4ae1454fd90f15438f70a6712bb4c30e0dda4d35b7
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/cb44b75c9c062fcc40c2b905b2024b4493dbc62b/components/ML_frameworks/Scikit_learn/Train_logistic_regression_model/from_CSV/component.yaml
-        arguments:
-          dataset:
-            taskOutput:
-              outputName: transformed_table
-              taskId: Binarize column using Pandas on CSV data
-              type: CSV
-          label_column_name: class
-        annotations:
-          editor.position: '{"x":40,"y":510,"width":180,"height":70}'
-      Upload Scikit learn pickle model to Google Cloud Vertex AI:
-        componentRef:
-          digest: 81c91c8d7d21ec97e0872f669d68bd89edea87279d703685db54aa94743bebcd
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/workaround_for_buggy_KFPv2_compiler/component.yaml
-        arguments:
-          model:
-            taskOutput:
-              outputName: model
-              taskId: Train logistic regression model using scikit learn from CSV
-              type: ScikitLearnPickleModel
-        annotations:
-          editor.position: '{"x":40,"y":660,"width":180,"height":70}'
-    outputValues: {}

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_logistic_regression_model_using_Scikit_learn_and_import_to_Vertex_AI/pipeline.py

@@ -1,73 +0,0 @@
-# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
-from kfp import components
-
-# %% Loading components
-download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/storage/download/component.yaml")
-select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml")
-fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
-binarize_column_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Binarize_column/in_CSV_format/component.yaml")
-train_logistic_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/1f5cf6e06409b704064b2086c0a705e4e6b4fcde/community-content/pipeline_components/ML_frameworks/Scikit_learn/Train_logistic_regression_model/from_CSV/component.yaml")
-upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/component.yaml")
-deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
-
-# %% Pipeline definition
-def train_tabular_classification_logistic_regression_model_using_Scikit_learn_pipeline():
-    dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
-    feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]  # Excluded "trip_total"
-    label_column = "tips"
-    # Deploying the model might incur additional costs over time
-    deploy_model = False
-
-    classification_label_column = "class"
-    all_columns = [label_column] + feature_columns
-
-    training_data = download_from_gcs_op(
-        gcs_path=dataset_gcs_uri
-    ).outputs["Data"]
-
-    training_data = select_columns_using_Pandas_on_CSV_data_op(
-        table=training_data,
-        column_names=all_columns,
-    ).outputs["transformed_table"]
-
-    # Cleaning the NaN values.
-    training_data = fill_all_missing_values_using_Pandas_on_CSV_data_op(
-        table=training_data,
-        replacement_value="0",
-        #replacement_type_name="float",
-    ).outputs["transformed_table"]
-
-    classification_training_data = binarize_column_using_Pandas_on_CSV_data_op(
-        table=training_data,
-        column_name=label_column,
-        predicate="> 0",
-        new_column_name=classification_label_column,
-    ).outputs["transformed_table"]
-
-    model = train_logistic_regression_model_using_scikit_learn_from_CSV_op(
-        dataset=classification_training_data,
-        label_column_name=classification_label_column,
-        # Optional:
-        #penalty="l2",
-        #solver="lbfgs",
-        #max_iterations=100,
-        #multi_class_mode="auto",
-        #random_seed=0,
-    ).outputs["model"]
-
-    vertex_model_name = upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op(
-        model=model,
-    ).outputs["model_name"]
-
-    # Deploying the model might incur additional costs over time
-    if deploy_model:
-        sklearn_vertex_endpoint_name = deploy_model_to_endpoint_op(
-            model_name=vertex_model_name,
-        ).outputs["endpoint_name"]
-
-pipeline_func = train_tabular_classification_logistic_regression_model_using_Scikit_learn_pipeline
-
-# %% Pipeline submission
-if __name__ == '__main__':
-    from google.cloud import aiplatform
-    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_model_using_PyTorch_and_import_to_Vertex_AI/pipeline.component.yaml

@@ -1,114 +0,0 @@
-name: Train tabular classification model using PyTorch pipeline
-metadata:
-  annotations:
-    author: Alexey Volkov <alexey.volkov@ark-kun.com>
-    canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_classification_model_using_PyTorch_and_import_to_Vertex_AI/pipeline.component.yaml
-    sdk: https://cloud-pipelines.net/pipeline-editor/
-implementation:
-  graph:
-    tasks:
-      Download from GCS:
-        componentRef:
-          digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
-        arguments:
-          GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
-        annotations:
-          editor.position: '{"x":240,"y":40,"width":180,"height":40}'
-      Select columns using Pandas on CSV data:
-        componentRef:
-          digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: Data
-              taskId: Download from GCS
-          column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
-        annotations:
-          editor.position: '{"x":240,"y":140,"width":180,"height":54}'
-      Fill all missing values using Pandas on CSV data:
-        componentRef:
-          digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: transformed_table
-              taskId: Select columns using Pandas on CSV data
-              type: CSV
-          replacement_value: '0'
-        annotations:
-          editor.position: '{"x":240,"y":250,"width":180,"height":54}'
-      Create fully connected pytorch network:
-        componentRef:
-          digest: d03d8248fd358a0275ec33568ee7dd7dce576cc112b09dfafe2651e4d97e04a9
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1a2ef3eeb77bc278f33cad0dd29008ea2431e191/components/PyTorch/Create_fully_connected_network/component.yaml
-        arguments:
-          input_size: '7'
-          hidden_layer_sizes: '[10]'
-          activation_name: elu
-          output_activation_name: sigmoid
-        annotations:
-          editor.position: '{"x":40,"y":360,"width":180,"height":54}'
-      Binarize column using Pandas on CSV data:
-        componentRef:
-          digest: d699afd4d7cae862708717cc160f4394ed0c04e536e9515923ef1e8865f01d44
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: transformed_table
-              taskId: Fill all missing values using Pandas on CSV data
-              type: CSV
-          column_name: tips
-          predicate: ' > 0'
-          new_column_name: class
-        annotations:
-          editor.position: '{"x":240,"y":360,"width":180,"height":54}'
-      Train pytorch model from csv:
-        componentRef:
-          digest: 40f3185eb61e9727f41a4e0c05dd3d3b44bd802aa0f378cfc31756560033949a
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml
-        arguments:
-          model:
-            taskOutput:
-              outputName: model
-              taskId: Create fully connected pytorch network
-              type: PyTorchScriptModule
-          training_data:
-            taskOutput:
-              outputName: transformed_table
-              taskId: Binarize column using Pandas on CSV data
-              type: CSV
-          label_column_name: class
-          loss_function_name: binary_cross_entropy
-        annotations:
-          editor.position: '{"x":240,"y":490,"width":180,"height":40}'
-      Create PyTorch Model Archive with base handler:
-        componentRef:
-          digest: 8298b5ee1b0f0879f893add4cf352c8dec7cf9e21bb9db134c91a2d046cdb0ec
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/46d51383e6554b7f3ab4fd8cf614d8c2b422fb22/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml
-        arguments:
-          Model:
-            taskOutput:
-              outputName: trained_model
-              taskId: Train pytorch model from csv
-              type: PyTorchScriptModule
-          Model name: model
-          Model version: '1.0'
-        annotations:
-          editor.position: '{"x":240,"y":590,"width":180,"height":54}'
-      Upload PyTorch model archive to Google Cloud Vertex AI:
-        componentRef:
-          digest: 4450212fae7b9001482aca7eb78b28413c205506eccf08a04e7754a8dfa99004
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/workaround_for_buggy_KFPv2_compiler/component.yaml
-        arguments:
-          model_archive:
-            taskOutput:
-              outputName: Model archive
-              taskId: Create PyTorch Model Archive with base handler
-              type: PyTorchModelArchive
-        annotations:
-          editor.position: '{"x":240,"y":720,"width":180,"height":70}'
-    outputValues: {}

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_model_using_PyTorch_and_import_to_Vertex_AI/pipeline.py

@@ -1,95 +0,0 @@
-# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
-from kfp import components
-
-# %% Loading components
-download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/storage/download/component.yaml")
-select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml")
-fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
-binarize_column_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Binarize_column/in_CSV_format/component.yaml")
-create_fully_connected_pytorch_network_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Create_fully_connected_network/component.yaml")
-train_pytorch_model_from_csv_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml")
-create_pytorch_model_archive_with_base_handler_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml")
-upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/component.yaml")
-deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
-
-# %% Pipeline definition
-def train_tabular_classification_model_using_PyTorch_pipeline():
-    dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
-    feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]  # Excluded "trip_total"
-    label_column = "tips"
-    # Deploying the model might incur additional costs over time
-    deploy_model = False
-
-    classification_label_column = "class"
-    all_columns = [label_column] + feature_columns
-
-    training_data = download_from_gcs_op(
-        gcs_path=dataset_gcs_uri
-    ).outputs["Data"]
-
-    training_data = select_columns_using_Pandas_on_CSV_data_op(
-        table=training_data,
-        column_names=all_columns,
-    ).outputs["transformed_table"]
-
-    # Cleaning the NaN values.
-    training_data = fill_all_missing_values_using_Pandas_on_CSV_data_op(
-        table=training_data,
-        replacement_value="0",
-        #replacement_type_name="float",
-    ).outputs["transformed_table"]
-
-    classification_training_data = binarize_column_using_Pandas_on_CSV_data_op(
-        table=training_data,
-        column_name=label_column,
-        predicate=" > 0",
-        new_column_name=classification_label_column,
-    ).outputs["transformed_table"]
-
-    network = create_fully_connected_pytorch_network_op(
-        input_size=len(feature_columns),
-        # Optional:
-        hidden_layer_sizes=[10],
-        activation_name="elu",
-        output_activation_name="sigmoid",
-        # output_size=1,
-    ).outputs["model"]
-
-    model = train_pytorch_model_from_csv_op(
-        model=network,
-        training_data=classification_training_data,
-        label_column_name=classification_label_column,
-        loss_function_name="binary_cross_entropy",
-        # Optional:
-        #number_of_epochs=1,
-        #learning_rate=0.1,
-        #optimizer_name="Adadelta",
-        #optimizer_parameters={},
-        #batch_size=32,
-        #batch_log_interval=100,
-        #random_seed=0,
-    ).outputs["trained_model"]
-
-    model_archive = create_pytorch_model_archive_with_base_handler_op(
-        model=model,
-        # Optional:
-        # model_name="model",
-        # model_version="1.0",
-    ).outputs["Model archive"]
-
-    vertex_model_name = upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op(
-        model_archive=model_archive,
-    ).outputs["model_name"]
-
-    # Deploying the model might incur additional costs over time
-    if deploy_model:
-        vertex_endpoint_name = deploy_model_to_endpoint_op(
-            model_name=vertex_model_name,
-        ).outputs["endpoint_name"]
-
-pipeline_func=train_tabular_classification_model_using_PyTorch_pipeline
-
-# %% Pipeline submission
-if __name__ == '__main__':
-    from google.cloud import aiplatform
-    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_model_using_TensorFlow_and_import_to_Vertex_AI/pipeline.component.yaml

@@ -1,132 +0,0 @@
-name: Train tabular classification model using TensorFlow pipeline
-metadata:
-  annotations:
-    author: Alexey Volkov <alexey.volkov@ark-kun.com>
-    canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_classification_model_using_TensorFlow_and_import_to_Vertex_AI/pipeline.component.yaml
-    sdk: https://cloud-pipelines.net/pipeline-editor/
-implementation:
-  graph:
-    tasks:
-      Download from GCS:
-        componentRef:
-          digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
-        arguments:
-          GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
-        annotations:
-          editor.position: '{"x":40,"y":40,"width":180,"height":40}'
-      Select columns using Pandas on CSV data:
-        componentRef:
-          digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: Data
-              taskId: Download from GCS
-          column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
-        annotations:
-          editor.position: '{"x":40,"y":140,"width":180,"height":54}'
-      Fill all missing values using Pandas on CSV data:
-        componentRef:
-          digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: transformed_table
-              taskId: Select columns using Pandas on CSV data
-              type: CSV
-          replacement_value: '0'
-        annotations:
-          editor.position: '{"x":40,"y":250,"width":180,"height":54}'
-      Binarize column using Pandas on CSV data:
-        componentRef:
-          digest: d699afd4d7cae862708717cc160f4394ed0c04e536e9515923ef1e8865f01d44
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: transformed_table
-              taskId: Fill all missing values using Pandas on CSV data
-              type: CSV
-          column_name: tips
-          predicate: ' > 0'
-          new_column_name: class
-        annotations:
-          editor.position: '{"x":40,"y":370,"width":180,"height":54}'
-      Split rows into subsets:
-        componentRef:
-          digest: a609c3c9196484290f24a1174955f95b27f07a7b458aa5cb8cde28866cb2cb46
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: transformed_table
-              taskId: Binarize column using Pandas on CSV data
-              type: CSV
-          fraction_1: '0.8'
-        annotations:
-          editor.position: '{"x":170,"y":500,"width":180,"height":40}'
-      Create fully connected tensorflow network:
-        componentRef:
-          digest: bfcafbc5ce711b1f69cabf1338212d10d50136a73db9f9f7c984de7b80b4bfb0
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/9ca0f9eecf5f896f65b8538bbd809747052617d1/components/tensorflow/Create_fully_connected_network/component.yaml
-        arguments:
-          input_size: '7'
-          hidden_layer_sizes: '[10]'
-          activation_name: elu
-          output_activation_name: sigmoid
-        annotations:
-          editor.position: '{"x":370,"y":500,"width":180,"height":54}'
-      Train model using Keras on CSV:
-        componentRef:
-          digest: 42ae60c889034dbad74815653e95b4f7d576b5f47f803173e8679c7b54984609
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c504a4010348c50eaaf6d4337586ccc008f4dcef/components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml
-        arguments:
-          training_data:
-            taskOutput:
-              outputName: split_1
-              taskId: Split rows into subsets
-              type: CSV
-          model:
-            taskOutput:
-              outputName: model
-              taskId: Create fully connected tensorflow network
-              type: TensorflowSavedModel
-          label_column_name: class
-          loss_function_name: binary_crossentropy
-          number_of_epochs: '10'
-        annotations:
-          editor.position: '{"x":40,"y":620,"width":180,"height":54}'
-      Upload Tensorflow model to Google Cloud Vertex AI:
-        componentRef:
-          digest: 2e45263ff640b1a688e359b6936e27a81b2407749a84f340af2aa5547e0cb92c
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/workaround_for_buggy_KFPv2_compiler/component.yaml
-        arguments:
-          model:
-            taskOutput:
-              outputName: trained_model
-              taskId: Train model using Keras on CSV
-              type: TensorflowSavedModel
-        annotations:
-          editor.position: '{"x":40,"y":750,"width":180,"height":54}'
-      Predict with TensorFlow model on CSV data:
-        componentRef:
-          digest: 921bb1563e93a78233b8acceab87055b9154ccf5595d056028cf0396ca224cd4
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/59c759ce6f543184e30db6817d2a703879bc0f39/components/tensorflow/Predict/on_CSV/component.yaml
-        arguments:
-          dataset:
-            taskOutput:
-              outputName: split_2
-              taskId: Split rows into subsets
-              type: CSV
-          model:
-            taskOutput:
-              outputName: trained_model
-              taskId: Train model using Keras on CSV
-              type: TensorflowSavedModel
-          label_column_name: class
-        annotations:
-          editor.position: '{"x":240,"y":750,"width":180,"height":54}'
-    outputValues: {}

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_model_using_TensorFlow_and_import_to_Vertex_AI/pipeline.py

@@ -1,106 +0,0 @@
-# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
-from kfp import components
-
-# %% Loading components
-download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/storage/download/component.yaml")
-select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml")
-fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
-binarize_column_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Binarize_column/in_CSV_format/component.yaml")
-split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
-create_fully_connected_tensorflow_network_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Create_fully_connected_network/component.yaml")
-train_model_using_Keras_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml")
-predict_with_TensorFlow_model_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Predict/on_CSV/component.yaml")
-upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/component.yaml")
-deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
-
-# %% Pipeline definition
-def train_tabular_classification_model_using_TensorFlow_pipeline():
-    dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
-    feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]  # Excluded "trip_total"
-    label_column = "tips"
-    training_set_fraction = 0.8
-    # Deploying the model might incur additional costs over time
-    deploy_model = False
-
-    classification_label_column = "class"
-    all_columns = [label_column] + feature_columns
-
-    dataset = download_from_gcs_op(
-        gcs_path=dataset_gcs_uri
-    ).outputs["Data"]
-
-    dataset = select_columns_using_Pandas_on_CSV_data_op(
-        table=dataset,
-        column_names=all_columns,
-    ).outputs["transformed_table"]
-
-    dataset = fill_all_missing_values_using_Pandas_on_CSV_data_op(
-        table=dataset,
-        replacement_value="0",
-        # # Optional:
-        # column_names=None,  # =[...]
-    ).outputs["transformed_table"]
-
-    classification_dataset = binarize_column_using_Pandas_on_CSV_data_op(
-        table=dataset,
-        column_name=label_column,
-        predicate=" > 0",
-        new_column_name=classification_label_column,
-    ).outputs["transformed_table"]
-
-    split_task = split_rows_into_subsets_op(
-        table=classification_dataset,
-        fraction_1=training_set_fraction,
-    )
-    classification_training_data = split_task.outputs["split_1"]
-    classification_testing_data = split_task.outputs["split_2"]
-
-    network = create_fully_connected_tensorflow_network_op(
-        input_size=len(feature_columns),
-        # Optional:
-        hidden_layer_sizes=[10],
-        activation_name="elu",
-        output_activation_name="sigmoid",
-        # output_size=1,
-    ).outputs["model"]
-
-    model = train_model_using_Keras_on_CSV_op(
-        training_data=classification_training_data,
-        model=network,
-        label_column_name=classification_label_column,
-        # Optional:
-        loss_function_name="binary_crossentropy",
-        number_of_epochs=10,
-        #learning_rate=0.1,
-        #optimizer_name="Adadelta",
-        #optimizer_parameters={},
-        #batch_size=32,
-        #metric_names=["mean_absolute_error"],
-        #random_seed=0,
-    ).outputs["trained_model"]
-
-    predictions = predict_with_TensorFlow_model_on_CSV_data_op(
-        dataset=classification_testing_data,
-        model=model,
-        # label_column_name needs to be set when doing prediction on a dataset that has labels
-        label_column_name=classification_label_column,
-        # Optional:
-        # batch_size=1000,
-    ).outputs["predictions"]
-
-    vertex_model_name = upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op(
-        model=model,
-    ).outputs["model_name"]
-
-    # Deploying the model might incur additional costs over time
-    if deploy_model:
-        vertex_endpoint_name = deploy_model_to_endpoint_op(
-            model_name=vertex_model_name,
-        ).outputs["endpoint_name"]
-
-pipeline_func = train_tabular_classification_model_using_TensorFlow_pipeline
-
-# %% Pipeline submission
-if __name__ == '__main__':
-    from google.cloud import aiplatform
-    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_model_using_XGBoost_and_import_to_Vertex_AI/pipeline.component.yaml

@@ -1,115 +0,0 @@
-name: Train tabular classification model using XGBoost pipeline
-metadata:
-  annotations:
-    author: Alexey Volkov <alexey.volkov@ark-kun.com>
-    canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_classification_model_using_XGBoost_and_import_to_Vertex_AI/pipeline.component.yaml
-    sdk: https://cloud-pipelines.net/pipeline-editor/
-implementation:
-  graph:
-    tasks:
-      Download from GCS:
-        componentRef:
-          digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
-        arguments:
-          GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
-        annotations:
-          editor.position: '{"x":40,"y":40,"width":180,"height":40}'
-      Select columns using Pandas on CSV data:
-        componentRef:
-          digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: Data
-              taskId: Download from GCS
-          column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
-        annotations:
-          editor.position: '{"x":40,"y":140,"width":180,"height":54}'
-      Fill all missing values using Pandas on CSV data:
-        componentRef:
-          digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: transformed_table
-              taskId: Select columns using Pandas on CSV data
-              type: CSV
-          replacement_value: '0'
-        annotations:
-          editor.position: '{"x":40,"y":250,"width":180,"height":54}'
-      Binarize column using Pandas on CSV data:
-        componentRef:
-          digest: d699afd4d7cae862708717cc160f4394ed0c04e536e9515923ef1e8865f01d44
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: transformed_table
-              taskId: Fill all missing values using Pandas on CSV data
-              type: CSV
-          column_name: tips
-          predicate: '> 0'
-          new_column_name: class
-        annotations:
-          editor.position: '{"x":40,"y":380,"width":180,"height":54}'
-      Split rows into subsets:
-        componentRef:
-          digest: a609c3c9196484290f24a1174955f95b27f07a7b458aa5cb8cde28866cb2cb46
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: transformed_table
-              taskId: Binarize column using Pandas on CSV data
-              type: CSV
-          fraction_1: '0.8'
-        annotations:
-          editor.position: '{"x":170,"y":510,"width":180,"height":40}'
-      Train XGBoost model on CSV:
-        componentRef:
-          digest: 538c5a01eb38deaf532d619f0bbeaff4efc550fe1f0f776fc06791097b68ceac
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/58d3a47f904f32a64af8403330ba7e2134cae46d/components/XGBoost/Train/component.yaml
-        arguments:
-          training_data:
-            taskOutput:
-              outputName: split_1
-              taskId: Split rows into subsets
-              type: CSV
-          label_column_name: class
-          objective: binary:logistic
-        annotations:
-          editor.position: '{"x":40,"y":630,"width":180,"height":40}'
-      Upload XGBoost model to Google Cloud Vertex AI:
-        componentRef:
-          digest: 5a5a273c403670743820986c03a4175b7cb4595a556524fefcce403656286977
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/workaround_for_buggy_KFPv2_compiler/component.yaml
-        arguments:
-          model:
-            taskOutput:
-              outputName: model
-              taskId: Train XGBoost model on CSV
-              type: XGBoostModel
-        annotations:
-          editor.position: '{"x":40,"y":750,"width":180,"height":54}'
-      Xgboost predict on CSV:
-        componentRef:
-          digest: 0876233a0c7306fefec188bd70f059b46d1fb5aa57be231799570e3bbbdd0d95
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/4694ec97baccf59284c2a1db4aa2250c22291eab/components/XGBoost/Predict/component.yaml
-        arguments:
-          data:
-            taskOutput:
-              outputName: split_2
-              taskId: Split rows into subsets
-              type: CSV
-          model:
-            taskOutput:
-              outputName: model
-              taskId: Train XGBoost model on CSV
-              type: XGBoostModel
-          label_column_name: class
-        annotations:
-          editor.position: '{"x":240,"y":750,"width":180,"height":40}'
-    outputValues: {}

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_model_using_XGBoost_and_import_to_Vertex_AI/pipeline.py

@@ -1,94 +0,0 @@
-# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
-from kfp import components
-
-# %% Loading components
-download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/storage/download/component.yaml")
-select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml")
-fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
-binarize_column_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Binarize_column/in_CSV_format/component.yaml")
-split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
-train_XGBoost_model_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Train/component.yaml")
-xgboost_predict_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Predict/component.yaml")
-upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/d5c9918850a6cc70004c4269dae066cfe2e664eb/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/component.yaml")
-deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
-
-# %% Pipeline definition
-def train_tabular_classification_model_using_XGBoost_pipeline():
-    dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
-    feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]  # Excluded "trip_total"
-    label_column = "tips"
-    training_set_fraction = 0.8
-    # Deploying the model might incur additional costs over time
-    deploy_model = False
-
-    classification_label_column = "class"
-    all_columns = [label_column] + feature_columns
-
-    dataset = download_from_gcs_op(
-        gcs_path=dataset_gcs_uri
-    ).outputs["Data"]
-
-    dataset = select_columns_using_Pandas_on_CSV_data_op(
-        table=dataset,
-        column_names=all_columns,
-    ).outputs["transformed_table"]
-
-    dataset = fill_all_missing_values_using_Pandas_on_CSV_data_op(
-        table=dataset,
-        replacement_value="0",
-        # # Optional:
-        # column_names=None,  # =[...]
-    ).outputs["transformed_table"]
-
-    classification_dataset = binarize_column_using_Pandas_on_CSV_data_op(
-        table=dataset,
-        column_name=label_column,
-        predicate="> 0",
-        new_column_name=classification_label_column,
-    ).outputs["transformed_table"]
-
-    split_task = split_rows_into_subsets_op(
-        table=classification_dataset,
-        fraction_1=training_set_fraction,
-    )
-    classification_training_data = split_task.outputs["split_1"]
-    classification_testing_data = split_task.outputs["split_2"]
-
-    model = train_XGBoost_model_on_CSV_op(
-        training_data=classification_training_data,
-        label_column_name=classification_label_column,
-        objective="binary:logistic",
-        # Optional:
-        #starting_model=None,
-        #num_iterations=10,
-        #booster_params={},
-        #booster="gbtree",
-        #learning_rate=0.3,
-        #min_split_loss=0,
-        #max_depth=6,
-    ).outputs["model"]
-
-    # Predicting on the testing data
-    predictions = xgboost_predict_on_CSV_op(
-        data=classification_testing_data,
-        model=model,
-        # label_column needs to be set when doing prediction on a dataset that has labels
-        label_column_name=classification_label_column,
-    ).outputs["predictions"]
-
-    vertex_model_name = upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op(
-        model=model,
-    ).outputs["model_name"]
-
-    # Deploying the model might incur additional costs over time
-    if deploy_model:
-        vertex_endpoint_name = deploy_model_to_endpoint_op(
-            model_name=vertex_model_name,
-        ).outputs["endpoint_name"]
-
-pipeline_func = train_tabular_classification_model_using_XGBoost_pipeline
-
-# %% Pipeline submission
-if __name__ == '__main__':
-    from google.cloud import aiplatform
-    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_model_using_all_frameworks_and_import_to_Vertex_AI/pipeline.component.yaml

@@ -1,257 +0,0 @@
-name: Train tabular classification model using all frameworks pipeline
-metadata:
-  annotations:
-    author: Alexey Volkov <alexey.volkov@ark-kun.com>
-    canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_classification_model_using_all_frameworks_and_import_to_Vertex_AI/pipeline.component.yaml
-    sdk: https://cloud-pipelines.net/pipeline-editor/
-implementation:
-  graph:
-    tasks:
-      Download from GCS:
-        componentRef:
-          digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
-        arguments:
-          GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
-        annotations:
-          editor.position: '{"x":550,"y":40,"width":180,"height":40}'
-      Select columns using Pandas on CSV data:
-        componentRef:
-          digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: Data
-              taskId: Download from GCS
-          column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
-        annotations:
-          editor.position: '{"x":550,"y":140,"width":180,"height":54}'
-      Fill all missing values using Pandas on CSV data:
-        componentRef:
-          digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: transformed_table
-              taskId: Select columns using Pandas on CSV data
-              type: CSV
-          replacement_value: '0'
-        annotations:
-          editor.position: '{"x":550,"y":250,"width":180,"height":54}'
-      Binarize column using Pandas on CSV data:
-        componentRef:
-          digest: d699afd4d7cae862708717cc160f4394ed0c04e536e9515923ef1e8865f01d44
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: transformed_table
-              taskId: Fill all missing values using Pandas on CSV data
-              type: CSV
-          column_name: tips
-          predicate: ' > 0'
-          new_column_name: class
-        annotations:
-          editor.position: '{"x":550,"y":380,"width":180,"height":54}'
-      Split rows into subsets:
-        componentRef:
-          digest: a609c3c9196484290f24a1174955f95b27f07a7b458aa5cb8cde28866cb2cb46
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: transformed_table
-              taskId: Binarize column using Pandas on CSV data
-              type: CSV
-          fraction_1: '0.8'
-        annotations:
-          editor.position: '{"x":550,"y":490,"width":180,"height":40}'
-      Create fully connected pytorch network:
-        componentRef:
-          digest: d03d8248fd358a0275ec33568ee7dd7dce576cc112b09dfafe2651e4d97e04a9
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1a2ef3eeb77bc278f33cad0dd29008ea2431e191/components/PyTorch/Create_fully_connected_network/component.yaml
-        arguments:
-          input_size: '7'
-          hidden_layer_sizes: '[10]'
-          activation_name: elu
-          output_activation_name: sigmoid
-        annotations:
-          editor.position: '{"x":380,"y":620,"width":180,"height":54}'
-      Create fully connected tensorflow network:
-        componentRef:
-          digest: bfcafbc5ce711b1f69cabf1338212d10d50136a73db9f9f7c984de7b80b4bfb0
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/9ca0f9eecf5f896f65b8538bbd809747052617d1/components/tensorflow/Create_fully_connected_network/component.yaml
-        arguments:
-          input_size: '7'
-          hidden_layer_sizes: '[10]'
-          activation_name: elu
-          output_activation_name: sigmoid
-        annotations:
-          editor.position: '{"x":40,"y":630,"width":180,"height":54}'
-      Train model using Keras on CSV:
-        componentRef:
-          digest: 42ae60c889034dbad74815653e95b4f7d576b5f47f803173e8679c7b54984609
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c504a4010348c50eaaf6d4337586ccc008f4dcef/components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml
-        arguments:
-          training_data:
-            taskOutput:
-              outputName: split_1
-              taskId: Split rows into subsets
-              type: CSV
-          model:
-            taskOutput:
-              outputName: model
-              taskId: Create fully connected tensorflow network
-              type: TensorflowSavedModel
-          label_column_name: class
-          loss_function_name: binary_crossentropy
-          number_of_epochs: '10'
-        annotations:
-          editor.position: '{"x":40,"y":750,"width":180,"height":54}'
-      Train pytorch model from csv:
-        componentRef:
-          digest: 40f3185eb61e9727f41a4e0c05dd3d3b44bd802aa0f378cfc31756560033949a
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml
-        arguments:
-          model:
-            taskOutput:
-              outputName: model
-              taskId: Create fully connected pytorch network
-              type: PyTorchScriptModule
-          training_data:
-            taskOutput:
-              outputName: split_1
-              taskId: Split rows into subsets
-              type: CSV
-          label_column_name: class
-          loss_function_name: binary_cross_entropy
-        annotations:
-          editor.position: '{"x":380,"y":750,"width":180,"height":40}'
-      Train XGBoost model on CSV:
-        componentRef:
-          digest: 538c5a01eb38deaf532d619f0bbeaff4efc550fe1f0f776fc06791097b68ceac
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/58d3a47f904f32a64af8403330ba7e2134cae46d/components/XGBoost/Train/component.yaml
-        arguments:
-          training_data:
-            taskOutput:
-              outputName: split_1
-              taskId: Split rows into subsets
-              type: CSV
-          label_column_name: class
-          objective: binary:logistic
-        annotations:
-          editor.position: '{"x":720,"y":750,"width":180,"height":40}'
-      Train logistic regression model using scikit learn from CSV:
-        componentRef:
-          digest: a864625a822e4b1c8ef6fe4ae1454fd90f15438f70a6712bb4c30e0dda4d35b7
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/cb44b75c9c062fcc40c2b905b2024b4493dbc62b/components/ML_frameworks/Scikit_learn/Train_logistic_regression_model/from_CSV/component.yaml
-        arguments:
-          dataset:
-            taskOutput:
-              outputName: split_1
-              taskId: Split rows into subsets
-              type: CSV
-          label_column_name: class
-        annotations:
-          editor.position: '{"x":1030,"y":750,"width":180,"height":70}'
-      Predict with TensorFlow model on CSV data:
-        componentRef:
-          digest: 921bb1563e93a78233b8acceab87055b9154ccf5595d056028cf0396ca224cd4
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/59c759ce6f543184e30db6817d2a703879bc0f39/components/tensorflow/Predict/on_CSV/component.yaml
-        arguments:
-          dataset:
-            taskOutput:
-              outputName: split_2
-              taskId: Split rows into subsets
-              type: CSV
-          model:
-            taskOutput:
-              outputName: trained_model
-              taskId: Train model using Keras on CSV
-              type: TensorflowSavedModel
-          label_column_name: class
-        annotations:
-          editor.position: '{"x":160,"y":880,"width":180,"height":54}'
-      Create PyTorch Model Archive with base handler:
-        componentRef:
-          digest: 8298b5ee1b0f0879f893add4cf352c8dec7cf9e21bb9db134c91a2d046cdb0ec
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/46d51383e6554b7f3ab4fd8cf614d8c2b422fb22/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml
-        arguments:
-          Model:
-            taskOutput:
-              outputName: trained_model
-              taskId: Train pytorch model from csv
-              type: PyTorchScriptModule
-          Model name: model
-          Model version: '1.0'
-        annotations:
-          editor.position: '{"x":380,"y":880,"width":180,"height":54}'
-      Xgboost predict on CSV:
-        componentRef:
-          digest: 0876233a0c7306fefec188bd70f059b46d1fb5aa57be231799570e3bbbdd0d95
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/4694ec97baccf59284c2a1db4aa2250c22291eab/components/XGBoost/Predict/component.yaml
-        arguments:
-          data:
-            taskOutput:
-              outputName: split_2
-              taskId: Split rows into subsets
-              type: CSV
-          model:
-            taskOutput:
-              outputName: model
-              taskId: Train XGBoost model on CSV
-              type: XGBoostModel
-          label_column_name: class
-        annotations:
-          editor.position: '{"x":810,"y":880,"width":180,"height":40}'
-      Upload Scikit learn pickle model to Google Cloud Vertex AI:
-        componentRef:
-          digest: 81c91c8d7d21ec97e0872f669d68bd89edea87279d703685db54aa94743bebcd
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/workaround_for_buggy_KFPv2_compiler/component.yaml
-        arguments:
-          model:
-            taskOutput:
-              outputName: model
-              taskId: Train logistic regression model using scikit learn from CSV
-              type: ScikitLearnPickleModel
-        annotations:
-          editor.position: '{"x":1030,"y":880,"width":180,"height":70}'
-      Upload Tensorflow model to Google Cloud Vertex AI:
-        componentRef:
-          digest: 2e45263ff640b1a688e359b6936e27a81b2407749a84f340af2aa5547e0cb92c
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/workaround_for_buggy_KFPv2_compiler/component.yaml
-        arguments:
-          model:
-            taskOutput:
-              outputName: trained_model
-              taskId: Train model using Keras on CSV
-              type: TensorflowSavedModel
-        annotations:
-          editor.position: '{"x":40,"y":1010,"width":180,"height":54}'
-      Upload PyTorch model archive to Google Cloud Vertex AI:
-        componentRef:
-          digest: 4450212fae7b9001482aca7eb78b28413c205506eccf08a04e7754a8dfa99004
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/workaround_for_buggy_KFPv2_compiler/component.yaml
-        arguments:
-          model_archive:
-            taskOutput:
-              outputName: Model archive
-              taskId: Create PyTorch Model Archive with base handler
-              type: PyTorchModelArchive
-        annotations:
-          editor.position: '{"x":380,"y":1010,"width":180,"height":70}'
-      Upload XGBoost model to Google Cloud Vertex AI:
-        componentRef:
-          digest: 5a5a273c403670743820986c03a4175b7cb4595a556524fefcce403656286977
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/workaround_for_buggy_KFPv2_compiler/component.yaml
-        arguments:
-          model:
-            taskOutput:
-              outputName: model
-              taskId: Train XGBoost model on CSV
-              type: XGBoostModel
-        annotations:
-          editor.position: '{"x":720,"y":1010,"width":180,"height":54}'
-    outputValues: {}

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_model_using_all_frameworks_and_import_to_Vertex_AI/pipeline.py

@@ -1,224 +0,0 @@
-# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
-from kfp import components
-
-# %% Loading components
-download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/storage/download/component.yaml")
-select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml")
-fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
-binarize_column_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Binarize_column/in_CSV_format/component.yaml")
-split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
-
-# TensorFlow
-create_fully_connected_tensorflow_network_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Create_fully_connected_network/component.yaml")
-train_model_using_Keras_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml")
-predict_with_TensorFlow_model_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Predict/on_CSV/component.yaml")
-upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/component.yaml")
-
-# PyTorch
-create_fully_connected_pytorch_network_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Create_fully_connected_network/component.yaml")
-train_pytorch_model_from_csv_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml")
-create_pytorch_model_archive_with_base_handler_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml")
-upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/component.yaml")
-
-# XGBoost
-train_XGBoost_model_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Train/component.yaml")
-xgboost_predict_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Predict/component.yaml")
-upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/d5c9918850a6cc70004c4269dae066cfe2e664eb/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/component.yaml")
-
-# Scikit-learn
-#train_linear_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/1f5cf6e06409b704064b2086c0a705e4e6b4fcde/community-content/pipeline_components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml")
-train_logistic_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/1f5cf6e06409b704064b2086c0a705e4e6b4fcde/community-content/pipeline_components/ML_frameworks/Scikit_learn/Train_logistic_regression_model/from_CSV/component.yaml")
-upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/component.yaml")
-
-# Vertex AI
-deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
-
-# %% Pipeline definition
-def train_tabular_classification_model_using_all_frameworks_pipeline():
-    dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
-    feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]  # Excluded "trip_total"
-    label_column = "tips"
-    training_set_fraction = 0.8
-    # Deploying the model might incur additional costs over time
-    deploy_model = False
-
-    classification_label_column = "class"
-    all_columns = [label_column] + feature_columns
-
-    dataset = download_from_gcs_op(
-        gcs_path=dataset_gcs_uri
-    ).outputs["Data"]
-
-    dataset = select_columns_using_Pandas_on_CSV_data_op(
-        table=dataset,
-        column_names=all_columns,
-    ).outputs["transformed_table"]
-
-    dataset = fill_all_missing_values_using_Pandas_on_CSV_data_op(
-        table=dataset,
-        replacement_value="0",
-        # # Optional:
-        # column_names=None,  # =[...]
-    ).outputs["transformed_table"]
-
-    classification_dataset = binarize_column_using_Pandas_on_CSV_data_op(
-        table=dataset,
-        column_name=label_column,
-        predicate=" > 0",
-        new_column_name=classification_label_column,
-    ).outputs["transformed_table"]
-
-    split_task = split_rows_into_subsets_op(
-        table=classification_dataset,
-        fraction_1=training_set_fraction,
-    )
-    classification_training_data = split_task.outputs["split_1"]
-    classification_testing_data = split_task.outputs["split_2"]
-
-    # TensorFlow
-    tensorflow_network = create_fully_connected_tensorflow_network_op(
-        input_size=len(feature_columns),
-        # Optional:
-        hidden_layer_sizes=[10],
-        activation_name="elu",
-        output_activation_name="sigmoid",
-        # output_size=1,
-    ).outputs["model"]
-
-    tensorflow_model = train_model_using_Keras_on_CSV_op(
-        training_data=classification_training_data,
-        model=tensorflow_network,
-        label_column_name=classification_label_column,
-        # Optional:
-        loss_function_name="binary_crossentropy",
-        number_of_epochs=10,
-        #learning_rate=0.1,
-        #optimizer_name="Adadelta",
-        #optimizer_parameters={},
-        #batch_size=32,
-        #metric_names=["mean_absolute_error"],
-        #random_seed=0,
-    ).outputs["trained_model"]
-
-    tensorflow_predictions = predict_with_TensorFlow_model_on_CSV_data_op(
-        dataset=classification_testing_data,
-        model=tensorflow_model,
-        # label_column_name needs to be set when doing prediction on a dataset that has labels
-        label_column_name=classification_label_column,
-        # Optional:
-        # batch_size=1000,
-    ).outputs["predictions"]
-
-    tensorflow_vertex_model_name = upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op(
-        model=tensorflow_model,
-    ).outputs["model_name"]
-
-    # Deploying the model might incur additional costs over time
-    if deploy_model:
-        tensorflow_vertex_endpoint_name = deploy_model_to_endpoint_op(
-            model_name=tensorflow_vertex_model_name,
-        ).outputs["endpoint_name"]
-
-    # PyTorch
-    pytorch_network = create_fully_connected_pytorch_network_op(
-        input_size=len(feature_columns),
-        # Optional:
-        hidden_layer_sizes=[10],
-        activation_name="elu",
-        output_activation_name="sigmoid",
-        # output_size=1,
-    ).outputs["model"]
-
-    pytorch_model = train_pytorch_model_from_csv_op(
-        model=pytorch_network,
-        training_data=classification_training_data,
-        label_column_name=classification_label_column,
-        loss_function_name="binary_cross_entropy",
-        # Optional:
-        #number_of_epochs=1,
-        #learning_rate=0.1,
-        #optimizer_name="Adadelta",
-        #optimizer_parameters={},
-        #batch_size=32,
-        #batch_log_interval=100,
-        #random_seed=0,
-    ).outputs["trained_model"]
-
-    pytorch_model_archive = create_pytorch_model_archive_with_base_handler_op(
-        model=pytorch_model,
-        # Optional:
-        # model_name="model",
-        # model_version="1.0",
-    ).outputs["Model archive"]
-
-    pytorch_vertex_model_name = upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op(
-        model_archive=pytorch_model_archive,
-    ).outputs["model_name"]
-
-    # Deploying the model might incur additional costs over time
-    if deploy_model:
-        pytorch_vertex_endpoint_name = deploy_model_to_endpoint_op(
-            model_name=pytorch_vertex_model_name,
-        ).outputs["endpoint_name"]
-
-    # XGBoost
-    xgboost_model = train_XGBoost_model_on_CSV_op(
-        training_data=classification_training_data,
-        label_column_name=classification_label_column,
-        objective="binary:logistic",
-        # Optional:
-        #starting_model=None,
-        #num_iterations=10,
-        #booster_params={},
-        #booster="gbtree",
-        #learning_rate=0.3,
-        #min_split_loss=0,
-        #max_depth=6,
-    ).outputs["model"]
-
-    # Predicting on the testing data
-    xgboost_predictions = xgboost_predict_on_CSV_op(
-        data=classification_testing_data,
-        model=xgboost_model,
-        # label_column needs to be set when doing prediction on a dataset that has labels
-        label_column_name=classification_label_column,
-    ).outputs["predictions"]
-
-    xgboost_vertex_model_name = upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op(
-        model=xgboost_model,
-    ).outputs["model_name"]
-
-    # Deploying the model might incur additional costs over time
-    if deploy_model:
-        xgboost_vertex_endpoint_name = deploy_model_to_endpoint_op(
-            model_name=xgboost_vertex_model_name,
-        ).outputs["endpoint_name"]
-
-    # Scikit-learn
-    sklearn_model = train_logistic_regression_model_using_scikit_learn_from_CSV_op(
-        dataset=classification_training_data,
-        label_column_name=classification_label_column,
-        # Optional:
-        #penalty="l2",
-        #solver="lbfgs",
-        #max_iterations=100,
-        #multi_class_mode="auto",
-        #random_seed=0,
-    ).outputs["model"]
-
-    sklearn_vertex_model_name = upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op(
-        model=sklearn_model,
-    ).outputs["model_name"]
-
-    # Deploying the model might incur additional costs over time
-    if deploy_model:
-        sklearn_vertex_endpoint_name = deploy_model_to_endpoint_op(
-            model_name=sklearn_vertex_model_name,
-        ).outputs["endpoint_name"]
-
-pipeline_func=train_tabular_classification_model_using_all_frameworks_pipeline
-
-# %% Pipeline submission
-if __name__ == '__main__':
-    from google.cloud import aiplatform
-    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_regression_linear_model_using_Scikit_learn_and_import_to_Vertex_AI/pipeline.component.yaml

@@ -1,68 +0,0 @@
-name: Train tabular regression linear model using Scikit learn pipeline
-metadata:
-  annotations:
-    author: Alexey Volkov <alexey.volkov@ark-kun.com>
-    canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_regression_linear_model_using_Scikit_learn_and_import_to_Vertex_AI/pipeline.component.yaml
-    sdk: https://cloud-pipelines.net/pipeline-editor/
-implementation:
-  graph:
-    tasks:
-      Download from GCS:
-        componentRef:
-          digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
-        arguments:
-          GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
-        annotations:
-          editor.position: '{"x":40,"y":40,"width":180,"height":40}'
-      Select columns using Pandas on CSV data:
-        componentRef:
-          digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: Data
-              taskId: Download from GCS
-          column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
-        annotations:
-          editor.position: '{"x":40,"y":140,"width":180,"height":54}'
-      Fill all missing values using Pandas on CSV data:
-        componentRef:
-          digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: transformed_table
-              taskId: Select columns using Pandas on CSV data
-              type: CSV
-          replacement_value: '0'
-        annotations:
-          editor.position: '{"x":40,"y":250,"width":180,"height":54}'
-      Train linear regression model using scikit learn from CSV:
-        componentRef:
-          digest: c7fe7912ab0d1fb45d201d452e9ce6be5544e7d8c6d229db7a4b931ff58560f3
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/f807e02b54d4886c65a05f40848fd51c72407f40/components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml
-        arguments:
-          dataset:
-            taskOutput:
-              outputName: transformed_table
-              taskId: Fill all missing values using Pandas on CSV data
-              type: CSV
-          label_column_name: tips
-        annotations:
-          editor.position: '{"x":40,"y":360,"width":180,"height":54}'
-      Upload Scikit learn pickle model to Google Cloud Vertex AI:
-        componentRef:
-          digest: 81c91c8d7d21ec97e0872f669d68bd89edea87279d703685db54aa94743bebcd
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/workaround_for_buggy_KFPv2_compiler/component.yaml
-        arguments:
-          model:
-            taskOutput:
-              outputName: model
-              taskId: Train linear regression model using scikit learn from CSV
-              type: ScikitLearnPickleModel
-        annotations:
-          editor.position: '{"x":40,"y":490,"width":180,"height":70}'
-    outputValues: {}

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_regression_linear_model_using_Scikit_learn_and_import_to_Vertex_AI/pipeline.py

@@ -1,57 +0,0 @@
-# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
-from kfp import components
-
-# %% Loading components
-download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/storage/download/component.yaml")
-select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml")
-fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
-train_linear_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/1f5cf6e06409b704064b2086c0a705e4e6b4fcde/community-content/pipeline_components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml")
-upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/component.yaml")
-deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
-
-# %% Pipeline definition
-def train_tabular_regression_linear_model_using_Scikit_learn_pipeline():
-    dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
-    feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]  # Excluded "trip_total"
-    label_column = "tips"
-    all_columns = [label_column] + feature_columns
-    # Deploying the model might incur additional costs over time
-    deploy_model = False
-
-    training_data = download_from_gcs_op(
-        gcs_path=dataset_gcs_uri
-    ).outputs["Data"]
-
-    training_data = select_columns_using_Pandas_on_CSV_data_op(
-        table=training_data,
-        column_names=all_columns,
-    ).outputs["transformed_table"]
-
-    # Cleaning the NaN values.
-    training_data = fill_all_missing_values_using_Pandas_on_CSV_data_op(
-        table=training_data,
-        replacement_value="0",
-        #replacement_type_name="float",
-    ).outputs["transformed_table"]
-
-    model = train_linear_regression_model_using_scikit_learn_from_CSV_op(
-        dataset=training_data,
-        label_column_name=label_column,
-    ).outputs["model"]
-
-    vertex_model_name = upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op(
-        model=model,
-    ).outputs["model_name"]
-
-    # Deploying the model might incur additional costs over time
-    if deploy_model:
-        sklearn_vertex_endpoint_name = deploy_model_to_endpoint_op(
-            model_name=vertex_model_name,
-        ).outputs["endpoint_name"]
-
-pipeline_func = train_tabular_regression_linear_model_using_Scikit_learn_pipeline
-
-# %% Pipeline submission
-if __name__ == '__main__':
-    from google.cloud import aiplatform
-    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_regression_model_using_PyTorch_and_import_to_Vertex_AI/pipeline.component.yaml

@@ -1,97 +0,0 @@
-name: Train tabular regression model using PyTorch pipeline
-metadata:
-  annotations:
-    author: Alexey Volkov <alexey.volkov@ark-kun.com>
-    canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_regression_model_using_PyTorch_and_import_to_Vertex_AI/pipeline.component.yaml
-    sdk: https://cloud-pipelines.net/pipeline-editor/
-implementation:
-  graph:
-    tasks:
-      Download from GCS:
-        componentRef:
-          digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
-        arguments:
-          GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
-        annotations:
-          editor.position: '{"x":240,"y":40,"width":180,"height":40}'
-      Select columns using Pandas on CSV data:
-        componentRef:
-          digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: Data
-              taskId: Download from GCS
-          column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
-        annotations:
-          editor.position: '{"x":240,"y":130,"width":180,"height":54}'
-      Create fully connected pytorch network:
-        componentRef:
-          digest: d03d8248fd358a0275ec33568ee7dd7dce576cc112b09dfafe2651e4d97e04a9
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1a2ef3eeb77bc278f33cad0dd29008ea2431e191/components/PyTorch/Create_fully_connected_network/component.yaml
-        arguments:
-          input_size: '7'
-          hidden_layer_sizes: '[10]'
-          activation_name: elu
-        annotations:
-          editor.position: '{"x":40,"y":240,"width":180,"height":54}'
-      Fill all missing values using Pandas on CSV data:
-        componentRef:
-          digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: transformed_table
-              taskId: Select columns using Pandas on CSV data
-              type: CSV
-          replacement_value: '0'
-        annotations:
-          editor.position: '{"x":240,"y":240,"width":180,"height":54}'
-      Train pytorch model from csv:
-        componentRef:
-          digest: 40f3185eb61e9727f41a4e0c05dd3d3b44bd802aa0f378cfc31756560033949a
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml
-        arguments:
-          model:
-            taskOutput:
-              outputName: model
-              taskId: Create fully connected pytorch network
-              type: PyTorchScriptModule
-          training_data:
-            taskOutput:
-              outputName: transformed_table
-              taskId: Fill all missing values using Pandas on CSV data
-              type: CSV
-          label_column_name: tips
-        annotations:
-          editor.position: '{"x":240,"y":380,"width":180,"height":40}'
-      Create PyTorch Model Archive with base handler:
-        componentRef:
-          digest: 8298b5ee1b0f0879f893add4cf352c8dec7cf9e21bb9db134c91a2d046cdb0ec
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/46d51383e6554b7f3ab4fd8cf614d8c2b422fb22/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml
-        arguments:
-          Model:
-            taskOutput:
-              outputName: trained_model
-              taskId: Train pytorch model from csv
-              type: PyTorchScriptModule
-          Model name: model
-          Model version: '1.0'
-        annotations:
-          editor.position: '{"x":240,"y":500,"width":180,"height":54}'
-      Upload PyTorch model archive to Google Cloud Vertex AI:
-        componentRef:
-          digest: 4450212fae7b9001482aca7eb78b28413c205506eccf08a04e7754a8dfa99004
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/workaround_for_buggy_KFPv2_compiler/component.yaml
-        arguments:
-          model_archive:
-            taskOutput:
-              outputName: Model archive
-              taskId: Create PyTorch Model Archive with base handler
-              type: PyTorchModelArchive
-        annotations:
-          editor.position: '{"x":240,"y":630,"width":180,"height":70}'
-    outputValues: {}

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_regression_model_using_PyTorch_and_import_to_Vertex_AI/pipeline.py

@@ -1,85 +0,0 @@
-# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
-from kfp import components
-
-# %% Loading components
-download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/storage/download/component.yaml")
-select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml")
-fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
-create_fully_connected_pytorch_network_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Create_fully_connected_network/component.yaml")
-train_pytorch_model_from_csv_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml")
-create_pytorch_model_archive_with_base_handler_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml")
-upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/component.yaml")
-deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
-
-# %% Pipeline definition
-def train_tabular_regression_model_using_PyTorch_pipeline():
-    dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
-    feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]  # Excluded "trip_total"
-    label_column = "tips"
-    all_columns = [label_column] + feature_columns
-    # Deploying the model might incur additional costs over time
-    deploy_model = False
-
-    training_data = download_from_gcs_op(
-        gcs_path=dataset_gcs_uri
-    ).outputs["Data"]
-
-    training_data = select_columns_using_Pandas_on_CSV_data_op(
-        table=training_data,
-        column_names=all_columns,
-    ).outputs["transformed_table"]
-
-    # Cleaning the NaN values.
-    training_data = fill_all_missing_values_using_Pandas_on_CSV_data_op(
-        table=training_data,
-        replacement_value="0",
-        #replacement_type_name="float",
-    ).outputs["transformed_table"]
-
-    network = create_fully_connected_pytorch_network_op(
-        input_size=len(feature_columns),
-        # Optional:
-        hidden_layer_sizes=[10],
-        activation_name="elu",
-        # output_activation_name=None,
-        # output_size=1,
-    ).outputs["model"]
-
-    model = train_pytorch_model_from_csv_op(
-        model=network,
-        training_data=training_data,
-        label_column_name=label_column,
-        # Optional:
-        #loss_function_name="mse_loss",
-        #number_of_epochs=1,
-        #learning_rate=0.1,
-        #optimizer_name="Adadelta",
-        #optimizer_parameters={},
-        #batch_size=32,
-        #batch_log_interval=100,
-        #random_seed=0,
-    ).outputs["trained_model"]
-
-    model_archive = create_pytorch_model_archive_with_base_handler_op(
-        model=model,
-        # Optional:
-        # model_name="model",
-        # model_version="1.0",
-    ).outputs["Model archive"]
-
-    vertex_model_name = upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op(
-        model_archive=model_archive,
-    ).outputs["model_name"]
-
-    # Deploying the model might incur additional costs over time
-    if deploy_model:
-        vertex_endpoint_name = deploy_model_to_endpoint_op(
-            model_name=vertex_model_name,
-        ).outputs["endpoint_name"]
-
-pipeline_func=train_tabular_regression_model_using_PyTorch_pipeline
-
-# %% Pipeline submission
-if __name__ == '__main__':
-    from google.cloud import aiplatform
-    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_regression_model_using_TensorFlow_and_import_to_Vertex_AI/pipeline.component.yaml

@@ -1,116 +0,0 @@
-name: Train tabular regression model using Tensorflow pipeline
-metadata:
-  annotations:
-    author: Alexey Volkov <alexey.volkov@ark-kun.com>
-    canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_regression_model_using_TensorFlow_and_import_to_Vertex_AI/pipeline.component.yaml
-    sdk: https://cloud-pipelines.net/pipeline-editor/
-implementation:
-  graph:
-    tasks:
-      Download from GCS:
-        componentRef:
-          digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
-        arguments:
-          GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
-        annotations:
-          editor.position: '{"x":40,"y":40,"width":180,"height":40}'
-      Select columns using Pandas on CSV data:
-        componentRef:
-          digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: Data
-              taskId: Download from GCS
-          column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
-        annotations:
-          editor.position: '{"x":40,"y":140,"width":180,"height":54}'
-      Fill all missing values using Pandas on CSV data:
-        componentRef:
-          digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: transformed_table
-              taskId: Select columns using Pandas on CSV data
-              type: CSV
-          replacement_value: '0'
-        annotations:
-          editor.position: '{"x":40,"y":250,"width":180,"height":54}'
-      Split rows into subsets:
-        componentRef:
-          digest: a609c3c9196484290f24a1174955f95b27f07a7b458aa5cb8cde28866cb2cb46
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: transformed_table
-              taskId: Fill all missing values using Pandas on CSV data
-              type: CSV
-          fraction_1: '0.8'
-        annotations:
-          editor.position: '{"x":170,"y":380,"width":180,"height":40}'
-      Create fully connected tensorflow network:
-        componentRef:
-          digest: bfcafbc5ce711b1f69cabf1338212d10d50136a73db9f9f7c984de7b80b4bfb0
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/9ca0f9eecf5f896f65b8538bbd809747052617d1/components/tensorflow/Create_fully_connected_network/component.yaml
-        arguments:
-          input_size: '7'
-          hidden_layer_sizes: '[10]'
-          activation_name: elu
-        annotations:
-          editor.position: '{"x":370,"y":380,"width":180,"height":54}'
-      Train model using Keras on CSV:
-        componentRef:
-          digest: 42ae60c889034dbad74815653e95b4f7d576b5f47f803173e8679c7b54984609
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c504a4010348c50eaaf6d4337586ccc008f4dcef/components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml
-        arguments:
-          training_data:
-            taskOutput:
-              outputName: split_1
-              taskId: Split rows into subsets
-              type: CSV
-          model:
-            taskOutput:
-              outputName: model
-              taskId: Create fully connected tensorflow network
-              type: TensorflowSavedModel
-          label_column_name: tips
-          number_of_epochs: '10'
-          metric_names: '["mean_absolute_error"]'
-        annotations:
-          editor.position: '{"x":40,"y":500,"width":180,"height":54}'
-      Upload Tensorflow model to Google Cloud Vertex AI:
-        componentRef:
-          digest: 2e45263ff640b1a688e359b6936e27a81b2407749a84f340af2aa5547e0cb92c
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/workaround_for_buggy_KFPv2_compiler/component.yaml
-        arguments:
-          model:
-            taskOutput:
-              outputName: trained_model
-              taskId: Train model using Keras on CSV
-              type: TensorflowSavedModel
-        annotations:
-          editor.position: '{"x":40,"y":630,"width":180,"height":54}'
-      Predict with TensorFlow model on CSV data:
-        componentRef:
-          digest: 921bb1563e93a78233b8acceab87055b9154ccf5595d056028cf0396ca224cd4
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/59c759ce6f543184e30db6817d2a703879bc0f39/components/tensorflow/Predict/on_CSV/component.yaml
-        arguments:
-          dataset:
-            taskOutput:
-              outputName: split_2
-              taskId: Split rows into subsets
-              type: CSV
-          model:
-            taskOutput:
-              outputName: trained_model
-              taskId: Train model using Keras on CSV
-              type: TensorflowSavedModel
-          label_column_name: tips
-        annotations:
-          editor.position: '{"x":240,"y":630,"width":180,"height":54}'
-    outputValues: {}

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_regression_model_using_TensorFlow_and_import_to_Vertex_AI/pipeline.py

@@ -1,97 +0,0 @@
-# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
-from kfp import components
-
-# %% Loading components
-download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/storage/download/component.yaml")
-select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml")
-fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
-split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
-create_fully_connected_tensorflow_network_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Create_fully_connected_network/component.yaml")
-train_model_using_Keras_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml")
-predict_with_TensorFlow_model_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Predict/on_CSV/component.yaml")
-upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/component.yaml")
-deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
-
-# %% Pipeline definition
-def train_tabular_regression_model_using_Tensorflow_pipeline():
-    dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
-    feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]  # Excluded "trip_total"
-    label_column = "tips"
-    training_set_fraction = 0.8
-    # Deploying the model might incur additional costs over time
-    deploy_model = False
-
-    all_columns = [label_column] + feature_columns
-
-    dataset = download_from_gcs_op(
-        gcs_path=dataset_gcs_uri
-    ).outputs["Data"]
-
-    dataset = select_columns_using_Pandas_on_CSV_data_op(
-        table=dataset,
-        column_names=all_columns,
-    ).outputs["transformed_table"]
-
-    dataset = fill_all_missing_values_using_Pandas_on_CSV_data_op(
-        table=dataset,
-        replacement_value="0",
-        # # Optional:
-        # column_names=None,  # =[...]
-    ).outputs["transformed_table"]
-
-    split_task = split_rows_into_subsets_op(
-        table=dataset,
-        fraction_1=training_set_fraction,
-    )
-    training_data = split_task.outputs["split_1"]
-    testing_data = split_task.outputs["split_2"]
-
-    network = create_fully_connected_tensorflow_network_op(
-        input_size=len(feature_columns),
-        # Optional:
-        hidden_layer_sizes=[10],
-        activation_name="elu",
-        # output_activation_name=None,
-        # output_size=1,
-    ).outputs["model"]
-
-    model = train_model_using_Keras_on_CSV_op(
-        training_data=training_data,
-        model=network,
-        label_column_name=label_column,
-        # Optional:
-        #loss_function_name="mean_squared_error",
-        number_of_epochs=10,
-        #learning_rate=0.1,
-        #optimizer_name="Adadelta",
-        #optimizer_parameters={},
-        #batch_size=32,
-        metric_names=["mean_absolute_error"],
-        #random_seed=0,
-    ).outputs["trained_model"]
-
-    predictions = predict_with_TensorFlow_model_on_CSV_data_op(
-        dataset=testing_data,
-        model=model,
-        # label_column_name needs to be set when doing prediction on a dataset that has labels
-        label_column_name=label_column,
-        # Optional:
-        # batch_size=1000,
-    ).outputs["predictions"]
-
-    vertex_model_name = upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op(
-        model=model,
-    ).outputs["model_name"]
-
-    # Deploying the model might incur additional costs over time
-    if deploy_model:
-        vertex_endpoint_name = deploy_model_to_endpoint_op(
-            model_name=vertex_model_name,
-        ).outputs["endpoint_name"]
-
-pipeline_func=train_tabular_regression_model_using_Tensorflow_pipeline
-
-# %% Pipeline submission
-if __name__ == '__main__':
-    from google.cloud import aiplatform
-    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_regression_model_using_XGBoost_and_import_to_Vertex_AI/pipeline.component.yaml

@@ -1,99 +0,0 @@
-name: Train tabular regression model using XGBoost pipeline
-metadata:
-  annotations:
-    author: Alexey Volkov <alexey.volkov@ark-kun.com>
-    canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_regression_model_using_XGBoost_and_import_to_Vertex_AI/pipeline.component.yaml
-    sdk: https://cloud-pipelines.net/pipeline-editor/
-implementation:
-  graph:
-    tasks:
-      Download from GCS:
-        componentRef:
-          digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
-        arguments:
-          GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
-        annotations:
-          editor.position: '{"x":40,"y":40,"width":180,"height":40}'
-      Select columns using Pandas on CSV data:
-        componentRef:
-          digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: Data
-              taskId: Download from GCS
-          column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
-        annotations:
-          editor.position: '{"x":40,"y":140,"width":180,"height":54}'
-      Fill all missing values using Pandas on CSV data:
-        componentRef:
-          digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: transformed_table
-              taskId: Select columns using Pandas on CSV data
-              type: CSV
-          replacement_value: '0'
-        annotations:
-          editor.position: '{"x":40,"y":250,"width":180,"height":54}'
-      Split rows into subsets:
-        componentRef:
-          digest: a609c3c9196484290f24a1174955f95b27f07a7b458aa5cb8cde28866cb2cb46
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: transformed_table
-              taskId: Fill all missing values using Pandas on CSV data
-              type: CSV
-          fraction_1: '0.8'
-        annotations:
-          editor.position: '{"x":170,"y":360,"width":180,"height":40}'
-      Train XGBoost model on CSV:
-        componentRef:
-          digest: 538c5a01eb38deaf532d619f0bbeaff4efc550fe1f0f776fc06791097b68ceac
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/58d3a47f904f32a64af8403330ba7e2134cae46d/components/XGBoost/Train/component.yaml
-        arguments:
-          training_data:
-            taskOutput:
-              outputName: split_1
-              taskId: Split rows into subsets
-              type: CSV
-          label_column_name: tips
-        annotations:
-          editor.position: '{"x":40,"y":480,"width":180,"height":40}'
-      Upload XGBoost model to Google Cloud Vertex AI:
-        componentRef:
-          digest: 5a5a273c403670743820986c03a4175b7cb4595a556524fefcce403656286977
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/workaround_for_buggy_KFPv2_compiler/component.yaml
-        arguments:
-          model:
-            taskOutput:
-              outputName: model
-              taskId: Train XGBoost model on CSV
-              type: XGBoostModel
-        annotations:
-          editor.position: '{"x":40,"y":600,"width":180,"height":54}'
-      Xgboost predict on CSV:
-        componentRef:
-          digest: 0876233a0c7306fefec188bd70f059b46d1fb5aa57be231799570e3bbbdd0d95
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/4694ec97baccf59284c2a1db4aa2250c22291eab/components/XGBoost/Predict/component.yaml
-        arguments:
-          data:
-            taskOutput:
-              outputName: split_2
-              taskId: Split rows into subsets
-              type: CSV
-          model:
-            taskOutput:
-              outputName: model
-              taskId: Train XGBoost model on CSV
-              type: XGBoostModel
-          label_column_name: tips
-        annotations:
-          editor.position: '{"x":240,"y":600,"width":180,"height":40}'
-    outputValues: {}

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_regression_model_using_XGBoost_and_import_to_Vertex_AI/pipeline.py

@@ -1,85 +0,0 @@
-# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
-from kfp import components
-
-# %% Loading components
-download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/storage/download/component.yaml")
-select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml")
-fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
-split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
-train_XGBoost_model_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Train/component.yaml")
-xgboost_predict_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Predict/component.yaml")
-upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/d5c9918850a6cc70004c4269dae066cfe2e664eb/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/component.yaml")
-deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
-
-# %% Pipeline definition
-def train_tabular_regression_model_using_XGBoost_pipeline():
-    dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
-    feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]  # Excluded "trip_total"
-    label_column = "tips"
-    training_set_fraction = 0.8
-    # Deploying the model might incur additional costs over time
-    deploy_model = False
-
-    all_columns = [label_column] + feature_columns
-
-    dataset = download_from_gcs_op(
-        gcs_path=dataset_gcs_uri
-    ).outputs["Data"]
-
-    dataset = select_columns_using_Pandas_on_CSV_data_op(
-        table=dataset,
-        column_names=all_columns,
-    ).outputs["transformed_table"]
-
-    dataset = fill_all_missing_values_using_Pandas_on_CSV_data_op(
-        table=dataset,
-        replacement_value="0",
-        # # Optional:
-        # column_names=None,  # =[...]
-    ).outputs["transformed_table"]
-
-    split_task = split_rows_into_subsets_op(
-        table=dataset,
-        fraction_1=training_set_fraction,
-    )
-    training_data = split_task.outputs["split_1"]
-    testing_data = split_task.outputs["split_2"]
-
-    model = train_XGBoost_model_on_CSV_op(
-        training_data=training_data,
-        label_column_name=label_column,
-        # Optional:
-        #starting_model=None,
-        #num_iterations=10,
-        #booster_params={},
-        #objective="reg:squarederror",
-        #booster="gbtree",
-        #learning_rate=0.3,
-        #min_split_loss=0,
-        #max_depth=6,
-    ).outputs["model"]
-
-    # Predicting on the testing data
-    predictions = xgboost_predict_on_CSV_op(
-        data=testing_data,
-        model=model,
-        # label_column needs to be set when doing prediction on a dataset that has labels
-        label_column_name=label_column,
-    ).outputs["predictions"]
-
-    vertex_model_name = upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op(
-        model=model,
-    ).outputs["model_name"]
-
-    # Deploying the model might incur additional costs over time
-    if deploy_model:
-        vertex_endpoint_name = deploy_model_to_endpoint_op(
-            model_name=vertex_model_name,
-        ).outputs["endpoint_name"]
-
-pipeline_func = train_tabular_regression_model_using_XGBoost_pipeline
-
-# %% Pipeline submission
-if __name__ == '__main__':
-    from google.cloud import aiplatform
-    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_regression_model_using_all_frameworks_and_import_to_Vertex_AI/pipeline.component.yaml

@@ -1,238 +0,0 @@
-name: Train tabular regression model using all frameworks pipeline
-metadata:
-  annotations:
-    author: Alexey Volkov <alexey.volkov@ark-kun.com>
-    canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_regression_model_using_all_frameworks_and_import_to_Vertex_AI/pipeline.component.yaml
-    sdk: https://cloud-pipelines.net/pipeline-editor/
-implementation:
-  graph:
-    tasks:
-      Download from GCS:
-        componentRef:
-          digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
-        arguments:
-          GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
-        annotations:
-          editor.position: '{"x":550,"y":40,"width":180,"height":40}'
-      Select columns using Pandas on CSV data:
-        componentRef:
-          digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: Data
-              taskId: Download from GCS
-          column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
-        annotations:
-          editor.position: '{"x":550,"y":140,"width":180,"height":54}'
-      Fill all missing values using Pandas on CSV data:
-        componentRef:
-          digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: transformed_table
-              taskId: Select columns using Pandas on CSV data
-              type: CSV
-          replacement_value: '0'
-        annotations:
-          editor.position: '{"x":550,"y":250,"width":180,"height":54}'
-      Split rows into subsets:
-        componentRef:
-          digest: a609c3c9196484290f24a1174955f95b27f07a7b458aa5cb8cde28866cb2cb46
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml
-        arguments:
-          table:
-            taskOutput:
-              outputName: transformed_table
-              taskId: Fill all missing values using Pandas on CSV data
-              type: CSV
-          fraction_1: '0.8'
-        annotations:
-          editor.position: '{"x":550,"y":360,"width":180,"height":40}'
-      Create fully connected pytorch network:
-        componentRef:
-          digest: d03d8248fd358a0275ec33568ee7dd7dce576cc112b09dfafe2651e4d97e04a9
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1a2ef3eeb77bc278f33cad0dd29008ea2431e191/components/PyTorch/Create_fully_connected_network/component.yaml
-        arguments:
-          input_size: '7'
-          hidden_layer_sizes: '[10]'
-          activation_name: elu
-        annotations:
-          editor.position: '{"x":380,"y":490,"width":180,"height":54}'
-      Create fully connected tensorflow network:
-        componentRef:
-          digest: bfcafbc5ce711b1f69cabf1338212d10d50136a73db9f9f7c984de7b80b4bfb0
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/9ca0f9eecf5f896f65b8538bbd809747052617d1/components/tensorflow/Create_fully_connected_network/component.yaml
-        arguments:
-          input_size: '7'
-          hidden_layer_sizes: '[10]'
-          activation_name: elu
-        annotations:
-          editor.position: '{"x":40,"y":500,"width":180,"height":54}'
-      Train model using Keras on CSV:
-        componentRef:
-          digest: 42ae60c889034dbad74815653e95b4f7d576b5f47f803173e8679c7b54984609
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c504a4010348c50eaaf6d4337586ccc008f4dcef/components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml
-        arguments:
-          training_data:
-            taskOutput:
-              outputName: split_1
-              taskId: Split rows into subsets
-              type: CSV
-          model:
-            taskOutput:
-              outputName: model
-              taskId: Create fully connected tensorflow network
-              type: TensorflowSavedModel
-          label_column_name: tips
-          number_of_epochs: '10'
-          metric_names: '["mean_absolute_error"]'
-        annotations:
-          editor.position: '{"x":40,"y":620,"width":180,"height":54}'
-      Train pytorch model from csv:
-        componentRef:
-          digest: 40f3185eb61e9727f41a4e0c05dd3d3b44bd802aa0f378cfc31756560033949a
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml
-        arguments:
-          model:
-            taskOutput:
-              outputName: model
-              taskId: Create fully connected pytorch network
-              type: PyTorchScriptModule
-          training_data:
-            taskOutput:
-              outputName: split_1
-              taskId: Split rows into subsets
-              type: CSV
-          label_column_name: tips
-        annotations:
-          editor.position: '{"x":380,"y":620,"width":180,"height":40}'
-      Train XGBoost model on CSV:
-        componentRef:
-          digest: 538c5a01eb38deaf532d619f0bbeaff4efc550fe1f0f776fc06791097b68ceac
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/58d3a47f904f32a64af8403330ba7e2134cae46d/components/XGBoost/Train/component.yaml
-        arguments:
-          training_data:
-            taskOutput:
-              outputName: split_1
-              taskId: Split rows into subsets
-              type: CSV
-          label_column_name: tips
-        annotations:
-          editor.position: '{"x":720,"y":620,"width":180,"height":40}'
-      Train linear regression model using scikit learn from CSV:
-        componentRef:
-          digest: c7fe7912ab0d1fb45d201d452e9ce6be5544e7d8c6d229db7a4b931ff58560f3
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/f807e02b54d4886c65a05f40848fd51c72407f40/components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml
-        arguments:
-          dataset:
-            taskOutput:
-              outputName: split_1
-              taskId: Split rows into subsets
-              type: CSV
-          label_column_name: tips
-        annotations:
-          editor.position: '{"x":1030,"y":620,"width":180,"height":54}'
-      Predict with TensorFlow model on CSV data:
-        componentRef:
-          digest: 921bb1563e93a78233b8acceab87055b9154ccf5595d056028cf0396ca224cd4
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/59c759ce6f543184e30db6817d2a703879bc0f39/components/tensorflow/Predict/on_CSV/component.yaml
-        arguments:
-          dataset:
-            taskOutput:
-              outputName: split_2
-              taskId: Split rows into subsets
-              type: CSV
-          model:
-            taskOutput:
-              outputName: trained_model
-              taskId: Train model using Keras on CSV
-              type: TensorflowSavedModel
-          label_column_name: tips
-        annotations:
-          editor.position: '{"x":160,"y":750,"width":180,"height":54}'
-      Create PyTorch Model Archive with base handler:
-        componentRef:
-          digest: 8298b5ee1b0f0879f893add4cf352c8dec7cf9e21bb9db134c91a2d046cdb0ec
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/46d51383e6554b7f3ab4fd8cf614d8c2b422fb22/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml
-        arguments:
-          Model:
-            taskOutput:
-              outputName: trained_model
-              taskId: Train pytorch model from csv
-              type: PyTorchScriptModule
-          Model name: model
-          Model version: '1.0'
-        annotations:
-          editor.position: '{"x":380,"y":750,"width":180,"height":54}'
-      Xgboost predict on CSV:
-        componentRef:
-          digest: 0876233a0c7306fefec188bd70f059b46d1fb5aa57be231799570e3bbbdd0d95
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/4694ec97baccf59284c2a1db4aa2250c22291eab/components/XGBoost/Predict/component.yaml
-        arguments:
-          data:
-            taskOutput:
-              outputName: split_2
-              taskId: Split rows into subsets
-              type: CSV
-          model:
-            taskOutput:
-              outputName: model
-              taskId: Train XGBoost model on CSV
-              type: XGBoostModel
-          label_column_name: tips
-        annotations:
-          editor.position: '{"x":810,"y":750,"width":180,"height":40}'
-      Upload Scikit learn pickle model to Google Cloud Vertex AI:
-        componentRef:
-          digest: 81c91c8d7d21ec97e0872f669d68bd89edea87279d703685db54aa94743bebcd
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/workaround_for_buggy_KFPv2_compiler/component.yaml
-        arguments:
-          model:
-            taskOutput:
-              outputName: model
-              taskId: Train linear regression model using scikit learn from CSV
-              type: ScikitLearnPickleModel
-        annotations:
-          editor.position: '{"x":1030,"y":750,"width":180,"height":70}'
-      Upload Tensorflow model to Google Cloud Vertex AI:
-        componentRef:
-          digest: 2e45263ff640b1a688e359b6936e27a81b2407749a84f340af2aa5547e0cb92c
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/workaround_for_buggy_KFPv2_compiler/component.yaml
-        arguments:
-          model:
-            taskOutput:
-              outputName: trained_model
-              taskId: Train model using Keras on CSV
-              type: TensorflowSavedModel
-        annotations:
-          editor.position: '{"x":40,"y":880,"width":180,"height":54}'
-      Upload PyTorch model archive to Google Cloud Vertex AI:
-        componentRef:
-          digest: 4450212fae7b9001482aca7eb78b28413c205506eccf08a04e7754a8dfa99004
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/workaround_for_buggy_KFPv2_compiler/component.yaml
-        arguments:
-          model_archive:
-            taskOutput:
-              outputName: Model archive
-              taskId: Create PyTorch Model Archive with base handler
-              type: PyTorchModelArchive
-        annotations:
-          editor.position: '{"x":380,"y":880,"width":180,"height":70}'
-      Upload XGBoost model to Google Cloud Vertex AI:
-        componentRef:
-          digest: 5a5a273c403670743820986c03a4175b7cb4595a556524fefcce403656286977
-          url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/workaround_for_buggy_KFPv2_compiler/component.yaml
-        arguments:
-          model:
-            taskOutput:
-              outputName: model
-              taskId: Train XGBoost model on CSV
-              type: XGBoostModel
-        annotations:
-          editor.position: '{"x":720,"y":880,"width":180,"height":54}'
-    outputValues: {}

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_regression_model_using_all_frameworks_and_import_to_Vertex_AI/pipeline.py

@@ -1,208 +0,0 @@
-# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
-from kfp import components
-
-# %% Loading components
-download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/storage/download/component.yaml")
-select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml")
-fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
-split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
-
-# TensorFlow
-create_fully_connected_tensorflow_network_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Create_fully_connected_network/component.yaml")
-train_model_using_Keras_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml")
-predict_with_TensorFlow_model_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Predict/on_CSV/component.yaml")
-upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/component.yaml")
-
-# PyTorch
-create_fully_connected_pytorch_network_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Create_fully_connected_network/component.yaml")
-train_pytorch_model_from_csv_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml")
-create_pytorch_model_archive_with_base_handler_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml")
-upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/component.yaml")
-
-# XGBoost
-train_XGBoost_model_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Train/component.yaml")
-xgboost_predict_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Predict/component.yaml")
-upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/d5c9918850a6cc70004c4269dae066cfe2e664eb/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/component.yaml")
-
-# Scikit-learn
-train_linear_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/1f5cf6e06409b704064b2086c0a705e4e6b4fcde/community-content/pipeline_components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml")
-upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/component.yaml")
-
-# Vertex AI
-deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
-
-# %% Pipeline definition
-def train_tabular_regression_model_using_all_frameworks_pipeline():
-    dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
-    feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]  # Excluded "trip_total"
-    label_column = "tips"
-    training_set_fraction = 0.8
-    # Deploying the model might incur additional costs over time
-    deploy_model = False
-
-    all_columns = [label_column] + feature_columns
-
-    dataset = download_from_gcs_op(
-        gcs_path=dataset_gcs_uri
-    ).outputs["Data"]
-
-    dataset = select_columns_using_Pandas_on_CSV_data_op(
-        table=dataset,
-        column_names=all_columns,
-    ).outputs["transformed_table"]
-
-    dataset = fill_all_missing_values_using_Pandas_on_CSV_data_op(
-        table=dataset,
-        replacement_value="0",
-        # # Optional:
-        # column_names=None,  # =[...]
-    ).outputs["transformed_table"]
-
-    split_task = split_rows_into_subsets_op(
-        table=dataset,
-        fraction_1=training_set_fraction,
-    )
-    training_data = split_task.outputs["split_1"]
-    testing_data = split_task.outputs["split_2"]
-
-    # TensorFlow
-    tensorflow_network = create_fully_connected_tensorflow_network_op(
-        input_size=len(feature_columns),
-        # Optional:
-        hidden_layer_sizes=[10],
-        activation_name="elu",
-        # output_activation_name=None,
-        # output_size=1,
-    ).outputs["model"]
-
-    tensorflow_model = train_model_using_Keras_on_CSV_op(
-        training_data=training_data,
-        model=tensorflow_network,
-        label_column_name=label_column,
-        # Optional:
-        #loss_function_name="mean_squared_error",
-        number_of_epochs=10,
-        #learning_rate=0.1,
-        #optimizer_name="Adadelta",
-        #optimizer_parameters={},
-        #batch_size=32,
-        metric_names=["mean_absolute_error"],
-        #random_seed=0,
-    ).outputs["trained_model"]
-
-    tensorflow_predictions = predict_with_TensorFlow_model_on_CSV_data_op(
-        dataset=testing_data,
-        model=tensorflow_model,
-        # label_column_name needs to be set when doing prediction on a dataset that has labels
-        label_column_name=label_column,
-        # Optional:
-        # batch_size=1000,
-    ).outputs["predictions"]
-
-    tensorflow_vertex_model_name = upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op(
-        model=tensorflow_model,
-    ).outputs["model_name"]
-
-    # Deploying the model might incur additional costs over time
-    if deploy_model:
-        tensorflow_vertex_endpoint_name = deploy_model_to_endpoint_op(
-            model_name=tensorflow_vertex_model_name,
-        ).outputs["endpoint_name"]
-
-    # PyTorch
-    pytorch_network = create_fully_connected_pytorch_network_op(
-        input_size=len(feature_columns),
-        # Optional:
-        hidden_layer_sizes=[10],
-        activation_name="elu",
-        # output_activation_name=None,
-        # output_size=1,
-    ).outputs["model"]
-
-    pytorch_model = train_pytorch_model_from_csv_op(
-        model=pytorch_network,
-        training_data=training_data,
-        label_column_name=label_column,
-        # Optional:
-        #loss_function_name="mse_loss",
-        #number_of_epochs=1,
-        #learning_rate=0.1,
-        #optimizer_name="Adadelta",
-        #optimizer_parameters={},
-        #batch_size=32,
-        #batch_log_interval=100,
-        #random_seed=0,
-    ).outputs["trained_model"]
-
-    pytorch_model_archive = create_pytorch_model_archive_with_base_handler_op(
-        model=pytorch_model,
-        # Optional:
-        # model_name="model",
-        # model_version="1.0",
-    ).outputs["Model archive"]
-
-    pytorch_vertex_model_name = upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op(
-        model_archive=pytorch_model_archive,
-    ).outputs["model_name"]
-
-    # Deploying the model might incur additional costs over time
-    if deploy_model:
-        pytorch_vertex_endpoint_name = deploy_model_to_endpoint_op(
-            model_name=pytorch_vertex_model_name,
-        ).outputs["endpoint_name"]
-
-    # XGBoost
-    xgboost_model = train_XGBoost_model_on_CSV_op(
-        training_data=training_data,
-        label_column_name=label_column,
-        # Optional:
-        #starting_model=None,
-        #num_iterations=10,
-        #booster_params={},
-        #objective="reg:squarederror",
-        #booster="gbtree",
-        #learning_rate=0.3,
-        #min_split_loss=0,
-        #max_depth=6,
-    ).outputs["model"]
-
-    # Predicting on the testing data
-    xgboost_predictions = xgboost_predict_on_CSV_op(
-        data=testing_data,
-        model=xgboost_model,
-        # label_column needs to be set when doing prediction on a dataset that has labels
-        label_column_name=label_column,
-    ).outputs["predictions"]
-
-    xgboost_vertex_model_name = upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op(
-        model=xgboost_model,
-    ).outputs["model_name"]
-
-    # Deploying the model might incur additional costs over time
-    if deploy_model:
-        xgboost_vertex_endpoint_name = deploy_model_to_endpoint_op(
-            model_name=xgboost_vertex_model_name,
-        ).outputs["endpoint_name"]
-
-    # Scikit-learn
-    sklearn_model = train_linear_regression_model_using_scikit_learn_from_CSV_op(
-        dataset=training_data,
-        label_column_name=label_column,
-    ).outputs["model"]
-
-    sklearn_vertex_model_name = upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op(
-        model=sklearn_model,
-    ).outputs["model_name"]
-
-    # Deploying the model might incur additional costs over time
-    if deploy_model:
-        sklearn_vertex_endpoint_name = deploy_model_to_endpoint_op(
-            model_name=sklearn_vertex_model_name,
-        ).outputs["endpoint_name"]
-
-pipeline_func=train_tabular_regression_model_using_all_frameworks_pipeline
-
-# %% Pipeline submission
-if __name__ == '__main__':
-    from google.cloud import aiplatform
-    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

community-content/cpr-examples/timm_serving/.gitignore

@@ -2,5 +2,4 @@ cpr_model_server.py
 entrypoint.py
 state_dict.pth
 config.json
-**/__pycache__
-!testdata/**
+**/__pycache__

community-content/cpr-examples/timm_serving/README.md

@@ -2,7 +2,7 @@
 
 ## About CPR
 
-CPR ([custom prediction routines](https://github.com/googleapis/python-aiplatform/blob/main/google/cloud/aiplatform/prediction/README.md)) is a framework designed by Google Cloud developers to make it easier to combine machine learning models with custom preprocessing and postprocessing logic in a real-time serving application. 
+CPR ([custom prediction routines](https://github.com/googleapis/python-aiplatform/blob/custom-prediction-routine/google/cloud/aiplatform/prediction/README.md)) is a framework designed by Google Cloud developers to make it easier to combine machine learning models with custom preprocessing and postprocessing logic in a real-time serving application. 
 
 ## Using this example
 
@@ -34,23 +34,6 @@ Finally, install the Python modules required to build and run the model server:
 pip install -r requirements.txt
 ```
 
-### Auth
-
-This example uses Google Cloud Storage for hosting model artifacts and Artifact Registry to store the container image. 
-You'll need to authorize yourself before you can interact with these.
-
-First, log in to GCP with application default credentials:
-```sh
-gcloud auth application-default login
-```
-
-Next, if you haven't done so already, set up the [gcloud credential helper](https://cloud.google.com/artifact-registry/docs/docker/authentication)
-for the Artifact Registry region where you intend to host the image.  
-```
-gcloud auth configure-docker <region>-docker.pkg.dev
-```
-
-
 ### Predictor
 
 The `TimmPredictor` class in `timm_serving/predictor.py` implements most of the important logic for the server.

community-content/cpr-examples/timm_serving/config.py

@@ -60,9 +60,9 @@ class CPRConfig(object):
     image: str = "timm_predictor:latest"
     artifact_local_dir: str = ""
     region: str = "us-central1"
-    project_id: str = "<your project ID here>"
+    project_id: str = "samthrasher-experimental"
     repository: str = "cpr-images"
-    artifact_gcs_dir: str = "gs://<your bucket ID here>/timm-vit224/"
+    artifact_gcs_dir: str = "gs://samthrasher-cpr-example/timm-vit224/"
     model_name: str = ""
     endpoint_name: str = ""
     machine_type: str = "n1-standard-2"

community-content/cpr-examples/timm_serving/requirements.txt

@@ -5,4 +5,4 @@ timm==0.5.4
 smart_open==6.0.0
 
 google-cloud-storage>=1.26.0,<2.0.0dev
-google-cloud-aiplatform[prediction]>=1.16.0
+google-cloud-aiplatform[prediction] @ git+https://github.com/googleapis/python-aiplatform.git@custom-prediction-routine

community-content/cpr-examples/timm_serving/test.py

@@ -70,10 +70,7 @@ class PredictorUnitTests(absltest.TestCase):
     def setUp(self):
         super().setUp()
         self.config = CPRConfig()
-        try:
-            self.config.load()
-        except FileNotFoundError:
-            logging.info("No saved config file found, using default values.")
+        self.config.load()
         self.predictor = predictor.TimmPredictor()
 
     def test_load_from_saved_state_dict_ok(self):
@@ -173,10 +170,7 @@ class ServerEndToEndTests(absltest.TestCase):
     def setUp(self):
         super().setUp()
         self.config = CPRConfig()
-        try:
-            self.config.load()
-        except FileNotFoundError:
-            logging.info("No saved config file found, using default values.")
+        self.config.load()
         self.local_model = cpr.LocalModel(
             serving_container_spec=aiplatform.gapic.ModelContainerSpec(
                 image_uri=self.config.image

community-content/cpr-examples/timm_serving/testdata/bad_model_1/state_dict.pth

@@ -1 +0,0 @@
-blah

community-content/pipeline_components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml

@@ -1,64 +0,0 @@
-name: Train linear regression model using scikit learn from CSV
-metadata:
-  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml'}
-inputs:
-- {name: dataset, type: CSV}
-- {name: label_column_name, type: String}
-outputs:
-- {name: model, type: ScikitLearnPickleModel}
-implementation:
-  container:
-    image: python:3.9
-    command:
-    - sh
-    - -c
-    - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
-      'scikit-learn==1.0.2' 'pandas==1.4.3' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3
-      -m pip install --quiet --no-warn-script-location 'scikit-learn==1.0.2' 'pandas==1.4.3'
-      --user) && "$0" "$@"
-    - sh
-    - -ec
-    - |
-      program_path=$(mktemp)
-      printf "%s" "$0" > "$program_path"
-      python3 -u "$program_path" "$@"
-    - |
-      def _make_parent_dirs_and_return_path(file_path: str):
-          import os
-          os.makedirs(os.path.dirname(file_path), exist_ok=True)
-          return file_path
-
-      def train_linear_regression_model_using_scikit_learn_from_CSV(
-          dataset_path,
-          model_path,
-          label_column_name,
-      ):
-          import pandas
-          import pickle
-          from sklearn import linear_model
-
-          df = pandas.read_csv(dataset_path)
-          model = linear_model.LinearRegression()
-          model.fit(
-              X=df.drop(columns=label_column_name),
-              y=df[label_column_name],
-          )
-
-          with open(model_path, "wb") as f:
-              pickle.dump(model, f)
-
-      import argparse
-      _parser = argparse.ArgumentParser(prog='Train linear regression model using scikit learn from CSV', description='')
-      _parser.add_argument("--dataset", dest="dataset_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--label-column-name", dest="label_column_name", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--model", dest="model_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
-      _parsed_args = vars(_parser.parse_args())
-
-      _outputs = train_linear_regression_model_using_scikit_learn_from_CSV(**_parsed_args)
-    args:
-    - --dataset
-    - {inputPath: dataset}
-    - --label-column-name
-    - {inputValue: label_column_name}
-    - --model
-    - {outputPath: model}

community-content/pipeline_components/ML_frameworks/Scikit_learn/Train_logistic_regression_model/from_CSV/component.yaml

@@ -1,163 +0,0 @@
-name: Train logistic regression model using scikit learn from CSV
-description: Train logistic regression model using Scikit-learn
-metadata:
-  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/ML_frameworks/Scikit_learn/Train_logistic_regression_model/from_CSV/component.yaml'}
-inputs:
-- {name: dataset, type: CSV}
-- {name: label_column_name, type: String}
-- {name: penalty, type: String, default: l2, optional: true}
-- {name: solver, type: String, default: lbfgs, optional: true}
-- {name: max_iterations, type: Integer, default: '100', optional: true}
-- {name: multi_class_mode, type: String, default: auto, optional: true}
-- {name: random_seed, type: Integer, default: '0', optional: true}
-outputs:
-- {name: model, type: ScikitLearnPickleModel}
-- {name: model_parameters, type: JsonObject}
-implementation:
-  container:
-    image: python:3.9
-    command:
-    - sh
-    - -c
-    - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
-      'scikit-learn==1.0.2' 'pandas==1.4.3' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3
-      -m pip install --quiet --no-warn-script-location 'scikit-learn==1.0.2' 'pandas==1.4.3'
-      --user) && "$0" "$@"
-    - sh
-    - -ec
-    - |
-      program_path=$(mktemp)
-      printf "%s" "$0" > "$program_path"
-      python3 -u "$program_path" "$@"
-    - |
-      def _make_parent_dirs_and_return_path(file_path: str):
-          import os
-          os.makedirs(os.path.dirname(file_path), exist_ok=True)
-          return file_path
-
-      def train_logistic_regression_model_using_scikit_learn_from_CSV(
-          dataset_path,
-          model_path,
-          label_column_name,
-          penalty = "l2", # l1, l2, elasticnet, none
-          solver = "lbfgs", # newton-cg, lbfgs, liblinear, sag, saga
-          max_iterations = 100,
-          multi_class_mode = "auto", # auto, ovr, multinomial
-          random_seed = 0,
-      ):
-          """Train logistic regression model using Scikit-learn
-
-          See https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html
-          """
-          import json
-          import pandas
-          import pickle
-          from sklearn import linear_model
-
-          df = pandas.read_csv(dataset_path)
-          model = linear_model.LogisticRegression(
-              penalty=penalty,
-              #dual=False,
-              #tol=1e-4,
-              #C=1.0,
-              #fit_intercept=True,
-              #intercept_scaling=1,
-              #class_weight=None,
-              random_state=random_seed,
-              solver=solver,
-              max_iter=max_iterations,
-              multi_class=multi_class_mode,
-              #l1_ratio=None,
-              verbose=1,
-          )
-
-          model_parameters = model.get_params()
-          model_parameters_json = json.dumps(model_parameters, indent=2)
-          print("Model parameters:")
-          print(model_parameters_json)
-          print()
-
-          model.fit(
-              X=df.drop(columns=label_column_name),
-              y=df[label_column_name],
-          )
-
-          with open(model_path, "wb") as f:
-              pickle.dump(model, f)
-
-          return (model_parameters_json,)
-
-      def _serialize_json(obj) -> str:
-          if isinstance(obj, str):
-              return obj
-          import json
-          def default_serializer(obj):
-              if hasattr(obj, 'to_struct'):
-                  return obj.to_struct()
-              else:
-                  raise TypeError("Object of type '%s' is not JSON serializable and does not have .to_struct() method." % obj.__class__.__name__)
-          return json.dumps(obj, default=default_serializer, sort_keys=True)
-
-      import argparse
-      _parser = argparse.ArgumentParser(prog='Train logistic regression model using scikit learn from CSV', description='Train logistic regression model using Scikit-learn')
-      _parser.add_argument("--dataset", dest="dataset_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--label-column-name", dest="label_column_name", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--penalty", dest="penalty", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--solver", dest="solver", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--max-iterations", dest="max_iterations", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--multi-class-mode", dest="multi_class_mode", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--random-seed", dest="random_seed", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--model", dest="model_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("----output-paths", dest="_output_paths", type=str, nargs=1)
-      _parsed_args = vars(_parser.parse_args())
-      _output_files = _parsed_args.pop("_output_paths", [])
-
-      _outputs = train_logistic_regression_model_using_scikit_learn_from_CSV(**_parsed_args)
-
-      _output_serializers = [
-          _serialize_json,
-
-      ]
-
-      import os
-      for idx, output_file in enumerate(_output_files):
-          try:
-              os.makedirs(os.path.dirname(output_file))
-          except OSError:
-              pass
-          with open(output_file, 'w') as f:
-              f.write(_output_serializers[idx](_outputs[idx]))
-    args:
-    - --dataset
-    - {inputPath: dataset}
-    - --label-column-name
-    - {inputValue: label_column_name}
-    - if:
-        cond: {isPresent: penalty}
-        then:
-        - --penalty
-        - {inputValue: penalty}
-    - if:
-        cond: {isPresent: solver}
-        then:
-        - --solver
-        - {inputValue: solver}
-    - if:
-        cond: {isPresent: max_iterations}
-        then:
-        - --max-iterations
-        - {inputValue: max_iterations}
-    - if:
-        cond: {isPresent: multi_class_mode}
-        then:
-        - --multi-class-mode
-        - {inputValue: multi_class_mode}
-    - if:
-        cond: {isPresent: random_seed}
-        then:
-        - --random-seed
-        - {inputValue: random_seed}
-    - --model
-    - {outputPath: model}
-    - '----output-paths'
-    - {outputPath: model_parameters}

community-content/pipeline_components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml

@@ -1,41 +0,0 @@
-name: Create PyTorch Model Archive with base handler
-inputs:
-- {name: Model, type: PyTorchScriptModule}
-- {name: Model name, type: String, default: model}
-- {name: Model version, type: String, default: "1.0"}
-outputs:
-- {name: Model archive, type: PyTorchModelArchive}
-metadata:
-  annotations:
-    author: Alexey Volkov <alexey.volkov@ark-kun.com>
-    canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml'
-implementation:
-  container:
-    image: pytorch/torchserve:0.6.0-cpu
-    command:
-    - bash
-    - -exc
-    - |
-      model_path=$0
-      model_name=$1
-      model_version=$2
-      output_model_archive_path=$3
-
-      mkdir -p "$(dirname "$output_model_archive_path")"
-
-      # TODO: Use the built-in base_handler once my fix is merged: https://github.com/pytorch/serve/pull/1682
-      echo '
-      from ts.torch_handler import base_handler
-      class BaseHandler(base_handler.BaseHandler):
-          pass
-      ' > base_handler.py  # torch-model-archiver needs the handler to have .py extension
-      torch-model-archiver --model-name "$model_name" --version "$model_version" --serialized-file "$model_path" --handler base_handler.py
-
-      # torch-model-archiver does not allow specifying the output path, but always writes to "${model_name}.<format>"
-      expected_model_archive_path="${model_name}.mar"
-      mv "$expected_model_archive_path" "$output_model_archive_path"
-
-    - {inputPath: Model}
-    - {inputValue: Model name}
-    - {inputValue: Model version}
-    - {outputPath: Model archive}

community-content/pipeline_components/PyTorch/Create_fully_connected_network/component.yaml

@@ -1,117 +0,0 @@
-name: Create fully connected pytorch network
-description: Creates fully-connected network in PyTorch ScriptModule format
-metadata:
-  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/PyTorch/Create_fully_connected_network/component.yaml'}
-inputs:
-- {name: input_size, type: Integer}
-- {name: hidden_layer_sizes, type: JsonArray, default: '[]', optional: true}
-- {name: output_size, type: Integer, default: '1', optional: true}
-- {name: activation_name, type: String, default: relu, optional: true}
-- {name: output_activation_name, type: String, optional: true}
-- {name: random_seed, type: Integer, default: '0', optional: true}
-outputs:
-- {name: model, type: PyTorchScriptModule}
-implementation:
-  container:
-    image: pytorch/pytorch:1.7.1-cuda11.0-cudnn8-runtime
-    command:
-    - sh
-    - -ec
-    - |
-      program_path=$(mktemp)
-      printf "%s" "$0" > "$program_path"
-      python3 -u "$program_path" "$@"
-    - |
-      def _make_parent_dirs_and_return_path(file_path: str):
-          import os
-          os.makedirs(os.path.dirname(file_path), exist_ok=True)
-          return file_path
-
-      def create_fully_connected_pytorch_network(
-          input_size,
-          model_path,
-          hidden_layer_sizes = [],
-          output_size = 1,
-          activation_name = 'relu',
-          output_activation_name = None,
-          random_seed = 0,
-      ):
-          '''Creates fully-connected network in PyTorch ScriptModule format'''
-          import torch
-          torch.manual_seed(random_seed)
-
-          activation = getattr(torch, activation_name, None) or getattr(torch.nn.functional, activation_name, None)
-          if not activation:
-              raise ValueError(f'Activation "{activation_name}" was not found.')
-
-          class ActivationLayer(torch.nn.Module):
-              def forward(self, input):
-                  return activation(input)
-
-          layers = []
-          prev_layer_size = input_size
-          for layer_size in hidden_layer_sizes:
-              layer = torch.nn.Linear(prev_layer_size, layer_size)
-              prev_layer_size = layer_size
-              layers.append(layer)
-              layers.append(ActivationLayer())
-
-          # Adding the output layer
-          layers.append(torch.nn.Linear(prev_layer_size, output_size))
-
-          # Adding the optional activation after the output layer
-          if output_activation_name:
-              output_activation = getattr(torch, output_activation_name, None) or getattr(torch.nn.functional, output_activation_name, None)
-              class OutputActivationLayer(torch.nn.Module):
-                  def forward(self, input):
-                      return output_activation(input)
-              layers.append(OutputActivationLayer())
-
-          network = torch.nn.Sequential(*layers)
-          script_module = torch.jit.script(network)
-          print(script_module)
-          script_module.save(model_path)
-
-      import json
-      import argparse
-      _parser = argparse.ArgumentParser(prog='Create fully connected pytorch network', description='Creates fully-connected network in PyTorch ScriptModule format')
-      _parser.add_argument("--input-size", dest="input_size", type=int, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--hidden-layer-sizes", dest="hidden_layer_sizes", type=json.loads, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--output-size", dest="output_size", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--activation-name", dest="activation_name", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--output-activation-name", dest="output_activation_name", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--random-seed", dest="random_seed", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--model", dest="model_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
-      _parsed_args = vars(_parser.parse_args())
-
-      _outputs = create_fully_connected_pytorch_network(**_parsed_args)
-    args:
-    - --input-size
-    - {inputValue: input_size}
-    - if:
-        cond: {isPresent: hidden_layer_sizes}
-        then:
-        - --hidden-layer-sizes
-        - {inputValue: hidden_layer_sizes}
-    - if:
-        cond: {isPresent: output_size}
-        then:
-        - --output-size
-        - {inputValue: output_size}
-    - if:
-        cond: {isPresent: activation_name}
-        then:
-        - --activation-name
-        - {inputValue: activation_name}
-    - if:
-        cond: {isPresent: output_activation_name}
-        then:
-        - --output-activation-name
-        - {inputValue: output_activation_name}
-    - if:
-        cond: {isPresent: random_seed}
-        then:
-        - --random-seed
-        - {inputValue: random_seed}
-    - --model
-    - {outputPath: model}

community-content/pipeline_components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml

@@ -1,209 +0,0 @@
-name: Train pytorch model from csv
-description: Trains PyTorch model
-metadata:
-  annotations:
-    author: Alexey Volkov <alexey.volkov@ark-kun.com>
-    canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml'
-inputs:
-- {name: model, type: PyTorchScriptModule}
-- {name: training_data, type: CSV}
-- {name: label_column_name, type: String}
-- {name: loss_function_name, type: String, default: mse_loss, optional: true}
-- {name: number_of_epochs, type: Integer, default: '1', optional: true}
-- {name: learning_rate, type: Float, default: '0.1', optional: true}
-- {name: optimizer_name, type: String, default: Adadelta, optional: true}
-- {name: optimizer_parameters, type: JsonObject, optional: true}
-- {name: batch_size, type: Integer, default: '32', optional: true}
-- {name: batch_log_interval, type: Integer, default: '100', optional: true}
-- {name: random_seed, type: Integer, default: '0', optional: true}
-outputs:
-- {name: trained_model, type: PyTorchScriptModule}
-implementation:
-  container:
-    image: pytorch/pytorch:1.7.1-cuda11.0-cudnn8-runtime
-    command:
-    - sh
-    - -c
-    - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
-      'pandas==1.4.3' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet
-      --no-warn-script-location 'pandas==1.4.3' --user) && "$0" "$@"
-    - sh
-    - -ec
-    - |
-      program_path=$(mktemp)
-      printf "%s" "$0" > "$program_path"
-      python3 -u "$program_path" "$@"
-    - |
-      def _make_parent_dirs_and_return_path(file_path: str):
-          import os
-          os.makedirs(os.path.dirname(file_path), exist_ok=True)
-          return file_path
-
-      def train_pytorch_model_from_csv(
-          model_path,
-          training_data_path,
-          trained_model_path,
-          label_column_name,
-          loss_function_name = 'mse_loss',
-          number_of_epochs = 1,
-          learning_rate = 0.1,
-          optimizer_name = 'Adadelta',
-          optimizer_parameters = None,
-          batch_size = 32,
-          batch_log_interval = 100,
-          random_seed = 0,
-      ):
-          '''Trains PyTorch model'''
-          import pandas
-          import torch
-
-          torch.manual_seed(random_seed)
-
-          use_cuda = torch.cuda.is_available()
-          device = torch.device("cuda" if use_cuda else "cpu")
-
-          model = torch.jit.load(model_path)
-          model.to(device)
-          model.train()
-
-          optimizer_class = getattr(torch.optim, optimizer_name, None)
-          if not optimizer_class:
-              raise ValueError(f'Optimizer "{optimizer_name}" was not found.')
-
-          optimizer_parameters = optimizer_parameters or {}
-          optimizer_parameters['lr'] = learning_rate
-          optimizer = optimizer_class(model.parameters(), **optimizer_parameters)
-
-          loss_function = getattr(torch, loss_function_name, None) or getattr(torch.nn, loss_function_name, None) or getattr(torch.nn.functional, loss_function_name, None)
-          if not loss_function:
-              raise ValueError(f'Loss function "{loss_function_name}" was not found.')
-
-          class CsvDataset(torch.utils.data.Dataset):
-
-              def __init__(self, file_path, label_column_name, drop_nan_columns_or_rows = 'columns'):
-                  dataframe = pandas.read_csv(file_path).convert_dtypes()
-                  # Preventing error: default_collate: batch must contain tensors, numpy arrays, numbers, dicts or lists; found object
-                  if drop_nan_columns_or_rows == 'columns':
-                      non_nan_data = dataframe.dropna(axis='columns')
-                      removed_columns = set(dataframe.columns) - set(non_nan_data.columns)
-                      if removed_columns:
-                          print('Skipping columns with NaNs: ' + str(removed_columns))
-                      dataframe = non_nan_data
-                  if drop_nan_columns_or_rows == 'rows':
-                      non_nan_data = dataframe.dropna(axis='index')
-                      number_of_removed_rows = len(dataframe) - len(non_nan_data)
-                      if number_of_removed_rows:
-                          print(f'Skipped {number_of_removed_rows} rows with NaNs.')
-                      dataframe = non_nan_data
-                  numerical_data = dataframe.select_dtypes(include='number')
-                  non_numerical_data = dataframe.select_dtypes(exclude='number')
-                  if not non_numerical_data.empty:
-                      print('Skipping non-number columns:')
-                      print(non_numerical_data.dtypes)
-                  self._dataframe = dataframe
-                  self.labels = numerical_data[[label_column_name]]
-                  self.features = numerical_data.drop(columns=[label_column_name])
-
-              def __len__(self):
-                  return len(self._dataframe)
-
-              def __getitem__(self, index):
-                  return [self.features.loc[index].to_numpy(dtype='float32'), self.labels.loc[index].to_numpy(dtype='float32')]
-
-          dataset = CsvDataset(
-              file_path=training_data_path,
-              label_column_name=label_column_name,
-          )
-          train_loader = torch.utils.data.DataLoader(
-              dataset=dataset,
-              batch_size=batch_size,
-              shuffle=True,
-          )
-
-          last_full_batch_loss = None
-          for epoch in range(1, number_of_epochs + 1):
-              for batch_idx, (data, target) in enumerate(train_loader):
-                  data, target = data.to(device), target.to(device)
-                  optimizer.zero_grad()
-                  output = model(data)
-                  loss = loss_function(output, target)
-                  loss.backward()
-                  optimizer.step()
-                  if len(data) == batch_size:
-                      last_full_batch_loss = loss.item()
-                  if batch_idx % batch_log_interval == 0:
-                      print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
-                          epoch, batch_idx * len(data), len(train_loader.dataset),
-                          100. * batch_idx / len(train_loader), loss.item()))
-              print(f'Training epoch {epoch} completed. Last full batch loss: {last_full_batch_loss:.6f}')
-
-          # print(optimizer.state_dict())
-          model.save(trained_model_path)
-
-      import json
-      import argparse
-      _parser = argparse.ArgumentParser(prog='Train pytorch model from csv', description='Trains PyTorch model')
-      _parser.add_argument("--model", dest="model_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--training-data", dest="training_data_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--label-column-name", dest="label_column_name", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--loss-function-name", dest="loss_function_name", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--number-of-epochs", dest="number_of_epochs", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--learning-rate", dest="learning_rate", type=float, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--optimizer-name", dest="optimizer_name", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--optimizer-parameters", dest="optimizer_parameters", type=json.loads, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--batch-size", dest="batch_size", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--batch-log-interval", dest="batch_log_interval", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--random-seed", dest="random_seed", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--trained-model", dest="trained_model_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
-      _parsed_args = vars(_parser.parse_args())
-
-      _outputs = train_pytorch_model_from_csv(**_parsed_args)
-    args:
-    - --model
-    - {inputPath: model}
-    - --training-data
-    - {inputPath: training_data}
-    - --label-column-name
-    - {inputValue: label_column_name}
-    - if:
-        cond: {isPresent: loss_function_name}
-        then:
-        - --loss-function-name
-        - {inputValue: loss_function_name}
-    - if:
-        cond: {isPresent: number_of_epochs}
-        then:
-        - --number-of-epochs
-        - {inputValue: number_of_epochs}
-    - if:
-        cond: {isPresent: learning_rate}
-        then:
-        - --learning-rate
-        - {inputValue: learning_rate}
-    - if:
-        cond: {isPresent: optimizer_name}
-        then:
-        - --optimizer-name
-        - {inputValue: optimizer_name}
-    - if:
-        cond: {isPresent: optimizer_parameters}
-        then:
-        - --optimizer-parameters
-        - {inputValue: optimizer_parameters}
-    - if:
-        cond: {isPresent: batch_size}
-        then:
-        - --batch-size
-        - {inputValue: batch_size}
-    - if:
-        cond: {isPresent: batch_log_interval}
-        then:
-        - --batch-log-interval
-        - {inputValue: batch_log_interval}
-    - if:
-        cond: {isPresent: random_seed}
-        then:
-        - --random-seed
-        - {inputValue: random_seed}
-    - --trained-model
-    - {outputPath: trained_model}

community-content/pipeline_components/XGBoost/Predict/component.yaml

@@ -1,110 +0,0 @@
-name: Xgboost predict on CSV
-description: Makes predictions using a trained XGBoost model.
-metadata:
-  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/XGBoost/Predict/component.yaml'}
-inputs:
-- {name: data, type: CSV, description: Feature data in Apache Parquet format.}
-- {name: model, type: XGBoostModel, description: Trained model in binary XGBoost format.}
-- {name: label_column_name, type: String, description: Optional. Name of the column
-    containing the label data that is excluded during the prediction., optional: true}
-outputs:
-- {name: predictions, description: Model predictions.}
-implementation:
-  container:
-    image: python:3.10
-    command:
-    - sh
-    - -c
-    - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
-      'xgboost==1.6.1' 'pandas==1.4.3' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3
-      -m pip install --quiet --no-warn-script-location 'xgboost==1.6.1' 'pandas==1.4.3'
-      --user) && "$0" "$@"
-    - sh
-    - -ec
-    - |
-      program_path=$(mktemp)
-      printf "%s" "$0" > "$program_path"
-      python3 -u "$program_path" "$@"
-    - |
-      def _make_parent_dirs_and_return_path(file_path: str):
-          import os
-          os.makedirs(os.path.dirname(file_path), exist_ok=True)
-          return file_path
-
-      def xgboost_predict_on_CSV(
-          data_path,
-          model_path,
-          predictions_path,
-          label_column_name = None,
-      ):
-          """Makes predictions using a trained XGBoost model.
-
-          Args:
-              data_path: Feature data in Apache Parquet format.
-              model_path: Trained model in binary XGBoost format.
-              predictions_path: Model predictions.
-              label_column_name: Optional. Name of the column containing the label data that is excluded during the prediction.
-
-          Annotations:
-              author: Alexey Volkov <alexey.volkov@ark-kun.com>
-          """
-          from pathlib import Path
-
-          import numpy
-          import pandas
-          import xgboost
-
-          df = pandas.read_csv(
-              data_path,
-          ).convert_dtypes()
-          print("Evaluation data information:")
-          df.info(verbose=True)
-          # Converting column types that XGBoost does not support
-          for column_name, dtype in df.dtypes.items():
-              if dtype in ["string", "object"]:
-                  print(f"Treating the {dtype.name} column '{column_name}' as categorical.")
-                  df[column_name] = df[column_name].astype("category")
-                  print(f"Inferred {len(df[column_name].cat.categories)} categories for the '{column_name}' column.")
-              # Working around the XGBoost issue with nullable floats: https://github.com/dmlc/xgboost/issues/8213
-              if pandas.api.types.is_float_dtype(dtype):
-                  # Converting from "Float64" to "float64"
-                  df[column_name] = df[column_name].astype(dtype.name.lower())
-          print("Final evaluation data information:")
-          df.info(verbose=True)
-
-          if label_column_name is not None:
-              df = df.drop(columns=[label_column_name])
-
-          testing_data = xgboost.DMatrix(
-              data=df,
-              enable_categorical=True,
-          )
-
-          model = xgboost.Booster(model_file=model_path)
-
-          predictions = model.predict(testing_data)
-
-          Path(predictions_path).parent.mkdir(parents=True, exist_ok=True)
-          numpy.savetxt(predictions_path, predictions)
-
-      import argparse
-      _parser = argparse.ArgumentParser(prog='Xgboost predict on CSV', description='Makes predictions using a trained XGBoost model.')
-      _parser.add_argument("--data", dest="data_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--model", dest="model_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--label-column-name", dest="label_column_name", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--predictions", dest="predictions_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
-      _parsed_args = vars(_parser.parse_args())
-
-      _outputs = xgboost_predict_on_CSV(**_parsed_args)
-    args:
-    - --data
-    - {inputPath: data}
-    - --model
-    - {inputPath: model}
-    - if:
-        cond: {isPresent: label_column_name}
-        then:
-        - --label-column-name
-        - {inputValue: label_column_name}
-    - --predictions
-    - {outputPath: predictions}

community-content/pipeline_components/XGBoost/Train/component.yaml

@@ -1,241 +0,0 @@
-name: Train XGBoost model on CSV
-description: Trains an XGBoost model.
-metadata:
-  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/XGBoost/Train/component.yaml'}
-inputs:
-- {name: training_data, type: CSV, description: Training data in CSV format.}
-- {name: label_column_name, type: String, description: Name of the column containing
-    the label data.}
-- {name: starting_model, type: XGBoostModel, description: Existing trained model to
-    start from (in the binary XGBoost format)., optional: true}
-- {name: num_iterations, type: Integer, description: Number of boosting iterations.,
-  default: '10', optional: true}
-- name: objective
-  type: String
-  description: |-
-    The learning task and the corresponding learning objective.
-    See https://xgboost.readthedocs.io/en/latest/parameter.html#learning-task-parameters
-    The most common values are:
-    "reg:squarederror" - Regression with squared loss (default).
-    "reg:logistic" - Logistic regression.
-    "binary:logistic" - Logistic regression for binary classification, output probability.
-    "binary:logitraw" - Logistic regression for binary classification, output score before logistic transformation
-    "rank:pairwise" - Use LambdaMART to perform pairwise ranking where the pairwise loss is minimized
-    "rank:ndcg" - Use LambdaMART to perform list-wise ranking where Normalized Discounted Cumulative Gain (NDCG) is maximized
-  default: reg:squarederror
-  optional: true
-- {name: booster, type: String, description: 'The booster to use. Can be `gbtree`,
-    `gblinear` or `dart`; `gbtree` and `dart` use tree based models while `gblinear`
-    uses linear functions.', default: gbtree, optional: true}
-- {name: learning_rate, type: Float, description: 'Step size shrinkage used in update
-    to prevents overfitting. Range: [0,1].', default: '0.3', optional: true}
-- name: min_split_loss
-  type: Float
-  description: |-
-    Minimum loss reduction required to make a further partition on a leaf node of the tree.
-    The larger `min_split_loss` is, the more conservative the algorithm will be. Range: [0,Inf].
-  default: '0'
-  optional: true
-- name: max_depth
-  type: Integer
-  description: |-
-    Maximum depth of a tree. Increasing this value will make the model more complex and more likely to overfit.
-    0 indicates no limit on depth. Range: [0,Inf].
-  default: '6'
-  optional: true
-- {name: booster_params, type: JsonObject, description: 'Parameters for the booster.
-    See https://xgboost.readthedocs.io/en/latest/parameter.html', optional: true}
-outputs:
-- {name: model, type: XGBoostModel, description: Trained model in the binary XGBoost
-    format.}
-- {name: model_config, type: XGBoostModelConfig, description: The internal parameter
-    configuration of Booster as a JSON string.}
-implementation:
-  container:
-    image: python:3.10
-    command:
-    - sh
-    - -c
-    - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
-      'xgboost==1.6.1' 'pandas==1.4.3' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3
-      -m pip install --quiet --no-warn-script-location 'xgboost==1.6.1' 'pandas==1.4.3'
-      --user) && "$0" "$@"
-    - sh
-    - -ec
-    - |
-      program_path=$(mktemp)
-      printf "%s" "$0" > "$program_path"
-      python3 -u "$program_path" "$@"
-    - |
-      def _make_parent_dirs_and_return_path(file_path: str):
-          import os
-          os.makedirs(os.path.dirname(file_path), exist_ok=True)
-          return file_path
-
-      def train_XGBoost_model_on_CSV(
-          training_data_path,
-          model_path,
-          model_config_path,
-          label_column_name,
-          starting_model_path = None,
-          num_iterations = 10,
-          # Booster parameters
-          objective = "reg:squarederror",
-          booster = "gbtree",
-          learning_rate = 0.3,
-          min_split_loss = 0,
-          max_depth = 6,
-          booster_params = None,
-      ):
-          """Trains an XGBoost model.
-
-          Args:
-              training_data_path: Training data in CSV format.
-              model_path: Trained model in the binary XGBoost format.
-              model_config_path: The internal parameter configuration of Booster as a JSON string.
-              starting_model_path: Existing trained model to start from (in the binary XGBoost format).
-              label_column_name: Name of the column containing the label data.
-              num_iterations: Number of boosting iterations.
-              booster_params: Parameters for the booster. See https://xgboost.readthedocs.io/en/latest/parameter.html
-              objective: The learning task and the corresponding learning objective.
-                  See https://xgboost.readthedocs.io/en/latest/parameter.html#learning-task-parameters
-                  The most common values are:
-                  "reg:squarederror" - Regression with squared loss (default).
-                  "reg:logistic" - Logistic regression.
-                  "binary:logistic" - Logistic regression for binary classification, output probability.
-                  "binary:logitraw" - Logistic regression for binary classification, output score before logistic transformation
-                  "rank:pairwise" - Use LambdaMART to perform pairwise ranking where the pairwise loss is minimized
-                  "rank:ndcg" - Use LambdaMART to perform list-wise ranking where Normalized Discounted Cumulative Gain (NDCG) is maximized
-              booster: The booster to use. Can be `gbtree`, `gblinear` or `dart`; `gbtree` and `dart` use tree based models while `gblinear` uses linear functions.
-              learning_rate: Step size shrinkage used in update to prevents overfitting. Range: [0,1].
-              min_split_loss: Minimum loss reduction required to make a further partition on a leaf node of the tree.
-                  The larger `min_split_loss` is, the more conservative the algorithm will be. Range: [0,Inf].
-              max_depth: Maximum depth of a tree. Increasing this value will make the model more complex and more likely to overfit.
-                  0 indicates no limit on depth. Range: [0,Inf].
-
-          Annotations:
-              author: Alexey Volkov <alexey.volkov@ark-kun.com>
-          """
-          import pandas
-          import xgboost
-
-          df = pandas.read_csv(
-              training_data_path,
-          ).convert_dtypes()
-          print("Training data information:")
-          df.info(verbose=True)
-          # Converting column types that XGBoost does not support
-          for column_name, dtype in df.dtypes.items():
-              if dtype in ["string", "object"]:
-                  print(f"Treating the {dtype.name} column '{column_name}' as categorical.")
-                  df[column_name] = df[column_name].astype("category")
-                  print(f"Inferred {len(df[column_name].cat.categories)} categories for the '{column_name}' column.")
-              # Working around the XGBoost issue with nullable floats: https://github.com/dmlc/xgboost/issues/8213
-              if pandas.api.types.is_float_dtype(dtype):
-                  # Converting from "Float64" to "float64"
-                  df[column_name] = df[column_name].astype(dtype.name.lower())
-          print()
-          print("Final training data information:")
-          df.info(verbose=True)
-
-          training_data = xgboost.DMatrix(
-              data=df.drop(columns=[label_column_name]),
-              label=df[[label_column_name]],
-              enable_categorical=True,
-          )
-
-          booster_params = booster_params or {}
-          booster_params.setdefault("objective", objective)
-          booster_params.setdefault("booster", booster)
-          booster_params.setdefault("learning_rate", learning_rate)
-          booster_params.setdefault("min_split_loss", min_split_loss)
-          booster_params.setdefault("max_depth", max_depth)
-
-          starting_model = None
-          if starting_model_path:
-              starting_model = xgboost.Booster(model_file=starting_model_path)
-
-          print()
-          print("Training the model:")
-          model = xgboost.train(
-              params=booster_params,
-              dtrain=training_data,
-              num_boost_round=num_iterations,
-              xgb_model=starting_model,
-              evals=[(training_data, "training_data")],
-          )
-
-          # Saving the model in binary format
-          model.save_model(model_path)
-
-          model_config_str = model.save_config()
-          with open(model_config_path, "w") as model_config_file:
-              model_config_file.write(model_config_str)
-
-      import json
-      import argparse
-      _parser = argparse.ArgumentParser(prog='Train XGBoost model on CSV', description='Trains an XGBoost model.')
-      _parser.add_argument("--training-data", dest="training_data_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--label-column-name", dest="label_column_name", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--starting-model", dest="starting_model_path", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--num-iterations", dest="num_iterations", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--objective", dest="objective", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--booster", dest="booster", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--learning-rate", dest="learning_rate", type=float, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--min-split-loss", dest="min_split_loss", type=float, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--max-depth", dest="max_depth", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--booster-params", dest="booster_params", type=json.loads, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--model", dest="model_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--model-config", dest="model_config_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
-      _parsed_args = vars(_parser.parse_args())
-
-      _outputs = train_XGBoost_model_on_CSV(**_parsed_args)
-    args:
-    - --training-data
-    - {inputPath: training_data}
-    - --label-column-name
-    - {inputValue: label_column_name}
-    - if:
-        cond: {isPresent: starting_model}
-        then:
-        - --starting-model
-        - {inputPath: starting_model}
-    - if:
-        cond: {isPresent: num_iterations}
-        then:
-        - --num-iterations
-        - {inputValue: num_iterations}
-    - if:
-        cond: {isPresent: objective}
-        then:
-        - --objective
-        - {inputValue: objective}
-    - if:
-        cond: {isPresent: booster}
-        then:
-        - --booster
-        - {inputValue: booster}
-    - if:
-        cond: {isPresent: learning_rate}
-        then:
-        - --learning-rate
-        - {inputValue: learning_rate}
-    - if:
-        cond: {isPresent: min_split_loss}
-        then:
-        - --min-split-loss
-        - {inputValue: min_split_loss}
-    - if:
-        cond: {isPresent: max_depth}
-        then:
-        - --max-depth
-        - {inputValue: max_depth}
-    - if:
-        cond: {isPresent: booster_params}
-        then:
-        - --booster-params
-        - {inputValue: booster_params}
-    - --model
-    - {outputPath: model}
-    - --model-config
-    - {outputPath: model_config}

community-content/pipeline_components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml

@@ -1,204 +0,0 @@
-name: Split rows into subsets
-description: Splits the data table according to the split fractions.
-metadata:
-  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml'}
-inputs:
-- {name: table, type: CSV}
-- {name: fraction_1, type: Float, description: 'The proportion of the lines to put
-    into the 1st split. Range: [0, 1]'}
-- name: fraction_2
-  type: Float
-  description: |-
-    The proportion of the lines to put into the 2nd split. Range: [0, 1]
-    If fraction_2 is not specified, then fraction_2 = 1 - fraction_1.
-    The remaining lines go to the 3rd split (if any).
-  optional: true
-- {name: random_seed, type: Integer, default: '0', optional: true}
-outputs:
-- {name: split_1, type: CSV}
-- {name: split_2, type: CSV}
-- {name: split_3, type: CSV}
-- {name: split_1_count, type: Integer}
-- {name: split_2_count, type: Integer}
-- {name: split_3_count, type: Integer}
-implementation:
-  container:
-    image: python:3.9
-    command:
-    - sh
-    - -ec
-    - |
-      program_path=$(mktemp)
-      printf "%s" "$0" > "$program_path"
-      python3 -u "$program_path" "$@"
-    - |
-      def _make_parent_dirs_and_return_path(file_path: str):
-          import os
-          os.makedirs(os.path.dirname(file_path), exist_ok=True)
-          return file_path
-
-      def split_rows_into_subsets(
-          table_path,
-          split_1_path,
-          split_2_path,
-          split_3_path,
-          fraction_1,
-          fraction_2 = None,
-          random_seed = 0,
-      ):
-          """Splits the data table according to the split fractions.
-
-          Args:
-              fraction_1: The proportion of the lines to put into the 1st split. Range: [0, 1]
-              fraction_2: The proportion of the lines to put into the 2nd split. Range: [0, 1]
-                  If fraction_2 is not specified, then fraction_2 = 1 - fraction_1.
-                  The remaining lines go to the 3rd split (if any).
-          """
-          import random
-
-          random.seed(random_seed)
-
-          SHUFFLE_BUFFER_SIZE = 10000
-
-          num_splits = 3
-
-          if fraction_1 < 0 or fraction_1 > 1:
-              raise ValueError("fraction_1 must be in between 0 and 1.")
-
-          if fraction_2 is None:
-              fraction_2 = 1 - fraction_1
-          if fraction_2 < 0 or fraction_2 > 1:
-              raise ValueError("fraction_2 must be in between 0 and 1.")
-
-          fraction_3 = 1 - fraction_1 - fraction_2
-
-          fractions = [
-              fraction_1,
-              fraction_2,
-              fraction_3,
-          ]
-
-          assert sum(fractions) == 1
-
-          written_line_counts = [0] * num_splits
-
-          output_files = [
-              open(split_1_path, "wb"),
-              open(split_2_path, "wb"),
-              open(split_3_path, "wb"),
-          ]
-
-          with open(table_path, "rb") as input_file:
-              # Writing the headers
-              header_line = input_file.readline()
-              for output_file in output_files:
-                  output_file.write(header_line)
-
-              while True:
-                  line_buffer = []
-                  for i in range(SHUFFLE_BUFFER_SIZE):
-                      line = input_file.readline()
-                      if not line:
-                          break
-                      line_buffer.append(line)
-
-                  # We need to exactly partition the lines between the output files
-                  # To overcome possible systematic bias, we could calculate the total numbers
-                  # of lines written to each file and take that into account.
-                  num_read_lines = len(line_buffer)
-                  number_of_lines_for_files = [0] * num_splits
-                  # List that will have the index of the destination file for each line
-                  file_index_for_line = []
-                  remaining_lines = num_read_lines
-                  remaining_fraction = 1
-                  for i in range(num_splits):
-                      number_of_lines_for_file = (
-                          round(remaining_lines * (fractions[i] / remaining_fraction))
-                          if remaining_fraction > 0
-                          else 0
-                      )
-                      number_of_lines_for_files[i] = number_of_lines_for_file
-                      remaining_lines -= number_of_lines_for_file
-                      remaining_fraction -= fractions[i]
-                      file_index_for_line.extend([i] * number_of_lines_for_file)
-
-                  assert remaining_lines == 0, f"{remaining_lines}"
-                  assert len(file_index_for_line) == num_read_lines
-
-                  random.shuffle(file_index_for_line)
-
-                  for i in range(num_read_lines):
-                      output_files[file_index_for_line[i]].write(line_buffer[i])
-                      written_line_counts[file_index_for_line[i]] += 1
-
-                  # Exit if the file ended before we were able to fully fill the buffer
-                  if len(line_buffer) != SHUFFLE_BUFFER_SIZE:
-                      break
-
-          for output_file in output_files:
-              output_file.close()
-
-          return written_line_counts
-
-      def _serialize_int(int_value: int) -> str:
-          if isinstance(int_value, str):
-              return int_value
-          if not isinstance(int_value, int):
-              raise TypeError('Value "{}" has type "{}" instead of int.'.format(str(int_value), str(type(int_value))))
-          return str(int_value)
-
-      import argparse
-      _parser = argparse.ArgumentParser(prog='Split rows into subsets', description='Splits the data table according to the split fractions.')
-      _parser.add_argument("--table", dest="table_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--fraction-1", dest="fraction_1", type=float, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--fraction-2", dest="fraction_2", type=float, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--random-seed", dest="random_seed", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--split-1", dest="split_1_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--split-2", dest="split_2_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--split-3", dest="split_3_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("----output-paths", dest="_output_paths", type=str, nargs=3)
-      _parsed_args = vars(_parser.parse_args())
-      _output_files = _parsed_args.pop("_output_paths", [])
-
-      _outputs = split_rows_into_subsets(**_parsed_args)
-
-      _output_serializers = [
-          _serialize_int,
-          _serialize_int,
-          _serialize_int,
-
-      ]
-
-      import os
-      for idx, output_file in enumerate(_output_files):
-          try:
-              os.makedirs(os.path.dirname(output_file))
-          except OSError:
-              pass
-          with open(output_file, 'w') as f:
-              f.write(_output_serializers[idx](_outputs[idx]))
-    args:
-    - --table
-    - {inputPath: table}
-    - --fraction-1
-    - {inputValue: fraction_1}
-    - if:
-        cond: {isPresent: fraction_2}
-        then:
-        - --fraction-2
-        - {inputValue: fraction_2}
-    - if:
-        cond: {isPresent: random_seed}
-        then:
-        - --random-seed
-        - {inputValue: random_seed}
-    - --split-1
-    - {outputPath: split_1}
-    - --split-2
-    - {outputPath: split_2}
-    - --split-3
-    - {outputPath: split_3}
-    - '----output-paths'
-    - {outputPath: split_1_count}
-    - {outputPath: split_2_count}
-    - {outputPath: split_3_count}

community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml

@@ -1,241 +0,0 @@
-name: Deploy model to endpoint for Google Cloud Vertex AI Model
-description: Deploys Google Cloud Vertex AI Model to a Google Cloud Vertex AI Endpoint.
-metadata:
-  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/KFPv2_hell/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/workaround_for_buggy_KFPv2_compiler/component.yaml'}
-inputs:
-- {name: model_name, type: String, description: Full resource name of a Google Cloud
-    Vertex AI Model}
-- name: endpoint_name
-  type: String
-  description: |-
-    Optional. Full name of Google Cloud Vertex Endpoint. A new
-    endpoint is created if the name is not passed.
-  optional: true
-- name: machine_type
-  type: String
-  description: |-
-    The type of the machine. See the [list of machine types
-    supported for prediction
-    ](https://cloud.google.com/vertex-ai/docs/predictions/configure-compute#machine-types).
-    Defaults to "n1-standard-2"
-  default: n1-standard-2
-  optional: true
-- name: min_replica_count
-  type: Integer
-  description: |-
-    Optional. The minimum number of machine replicas this deployed
-    model will be always deployed on. If traffic against it increases,
-    it may dynamically be deployed onto more replicas, and as traffic
-    decreases, some of these extra replicas may be freed.
-  default: '1'
-  optional: true
-- name: max_replica_count
-  type: Integer
-  description: |-
-    Optional. The maximum number of replicas this deployed model may
-    be deployed on when the traffic against it increases. If requested
-    value is too large, the deployment will error, but if deployment
-    succeeds then the ability to scale the model to that many replicas
-    is guaranteed (barring service outages). If traffic against the
-    deployed model increases beyond what its replicas at maximum may
-    handle, a portion of the traffic will be dropped. If this value
-    is not provided, the smaller value of min_replica_count or 1 will
-    be used.
-  default: '1'
-  optional: true
-- name: accelerator_type
-  type: String
-  description: |-
-    Optional. Hardware accelerator type. Must also set accelerator_count if used.
-    One of ACCELERATOR_TYPE_UNSPECIFIED, NVIDIA_TESLA_K80, NVIDIA_TESLA_P100,
-    NVIDIA_TESLA_V100, NVIDIA_TESLA_P4, NVIDIA_TESLA_T4
-  optional: true
-- {name: accelerator_count, type: Integer, description: Optional. The number of accelerators
-    to attach to a worker replica., optional: true}
-outputs:
-- {name: endpoint_name, type: String}
-- {name: endpoint_dict, type: JsonObject}
-implementation:
-  container:
-    image: python:3.9
-    command:
-    - sh
-    - -c
-    - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
-      'google-cloud-aiplatform==1.7.0' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3
-      -m pip install --quiet --no-warn-script-location 'google-cloud-aiplatform==1.7.0'
-      --user) && "$0" "$@"
-    - sh
-    - -ec
-    - |
-      program_path=$(mktemp)
-      printf "%s" "$0" > "$program_path"
-      python3 -u "$program_path" "$@"
-    - |
-      def deploy_model_to_endpoint_for_Google_Cloud_Vertex_AI_Model(
-          model_name,
-          endpoint_name = None,
-          machine_type = "n1-standard-2",
-          min_replica_count = 1,
-          max_replica_count = 1,
-          accelerator_type = None,
-          accelerator_count = None,
-          #
-          # Uncomment when anyone requests these:
-          # deployed_model_display_name: str = None,
-          # traffic_percentage: int = 0,
-          # traffic_split: dict = None,
-          # service_account: str = None,
-          # explanation_metadata: "google.cloud.aiplatform_v1.types.explanation_metadata.ExplanationMetadata" = None,
-          # explanation_parameters: "google.cloud.aiplatform_v1.types.explanation.ExplanationParameters" = None,
-          #
-          # encryption_spec_key_name: str = None,
-      ):
-          """Deploys Google Cloud Vertex AI Model to a Google Cloud Vertex AI Endpoint.
-
-          Args:
-              model_name: Full resource name of a Google Cloud Vertex AI Model
-              endpoint_name: Optional. Full name of Google Cloud Vertex Endpoint. A new
-                  endpoint is created if the name is not passed.
-              machine_type: The type of the machine. See the [list of machine types
-                  supported for prediction
-                  ](https://cloud.google.com/vertex-ai/docs/predictions/configure-compute#machine-types).
-                  Defaults to "n1-standard-2"
-              min_replica_count (int):
-                  Optional. The minimum number of machine replicas this deployed
-                  model will be always deployed on. If traffic against it increases,
-                  it may dynamically be deployed onto more replicas, and as traffic
-                  decreases, some of these extra replicas may be freed.
-              max_replica_count (int):
-                  Optional. The maximum number of replicas this deployed model may
-                  be deployed on when the traffic against it increases. If requested
-                  value is too large, the deployment will error, but if deployment
-                  succeeds then the ability to scale the model to that many replicas
-                  is guaranteed (barring service outages). If traffic against the
-                  deployed model increases beyond what its replicas at maximum may
-                  handle, a portion of the traffic will be dropped. If this value
-                  is not provided, the smaller value of min_replica_count or 1 will
-                  be used.
-              accelerator_type (str):
-                  Optional. Hardware accelerator type. Must also set accelerator_count if used.
-                  One of ACCELERATOR_TYPE_UNSPECIFIED, NVIDIA_TESLA_K80, NVIDIA_TESLA_P100,
-                  NVIDIA_TESLA_V100, NVIDIA_TESLA_P4, NVIDIA_TESLA_T4
-              accelerator_count (int):
-                  Optional. The number of accelerators to attach to a worker replica.
-          """
-          import json
-          from google.cloud import aiplatform
-
-          model = aiplatform.Model(model_name=model_name)
-
-          if endpoint_name:
-              endpoint = aiplatform.Endpoint(endpoint_name=endpoint_name)
-          else:
-              endpoint_display_name = model.display_name[:118] + "_endpoint"
-              endpoint = aiplatform.Endpoint.create(
-                  display_name=endpoint_display_name,
-                  project=model.project,
-                  location=model.location,
-                  # encryption_spec_key_name=encryption_spec_key_name,
-                  labels={"component-source": "github-com-ark-kun-pipeline-components"},
-              )
-
-          endpoint = model.deploy(
-              endpoint=endpoint,
-              # deployed_model_display_name=deployed_model_display_name,
-              machine_type=machine_type,
-              min_replica_count=min_replica_count,
-              max_replica_count=max_replica_count,
-              accelerator_type=accelerator_type,
-              accelerator_count=accelerator_count,
-              # service_account=service_account,
-              # explanation_metadata=explanation_metadata,
-              # explanation_parameters=explanation_parameters,
-              # encryption_spec_key_name=encryption_spec_key_name,
-          )
-
-          endpoint_json = json.dumps(endpoint.to_dict(), indent=2)
-          print(endpoint_json)
-          return (endpoint.resource_name, endpoint_json)
-
-      def _serialize_json(obj) -> str:
-          if isinstance(obj, str):
-              return obj
-          import json
-          def default_serializer(obj):
-              if hasattr(obj, 'to_struct'):
-                  return obj.to_struct()
-              else:
-                  raise TypeError("Object of type '%s' is not JSON serializable and does not have .to_struct() method." % obj.__class__.__name__)
-          return json.dumps(obj, default=default_serializer, sort_keys=True)
-
-      def _serialize_str(str_value: str) -> str:
-          if not isinstance(str_value, str):
-              raise TypeError('Value "{}" has type "{}" instead of str.'.format(str(str_value), str(type(str_value))))
-          return str_value
-
-      import argparse
-      _parser = argparse.ArgumentParser(prog='Deploy model to endpoint for Google Cloud Vertex AI Model', description='Deploys Google Cloud Vertex AI Model to a Google Cloud Vertex AI Endpoint.')
-      _parser.add_argument("--model-name", dest="model_name", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--endpoint-name", dest="endpoint_name", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--machine-type", dest="machine_type", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--min-replica-count", dest="min_replica_count", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--max-replica-count", dest="max_replica_count", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--accelerator-type", dest="accelerator_type", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--accelerator-count", dest="accelerator_count", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("----output-paths", dest="_output_paths", type=str, nargs=2)
-      _parsed_args = vars(_parser.parse_args())
-      _output_files = _parsed_args.pop("_output_paths", [])
-
-      _outputs = deploy_model_to_endpoint_for_Google_Cloud_Vertex_AI_Model(**_parsed_args)
-
-      _output_serializers = [
-          _serialize_str,
-          _serialize_json,
-
-      ]
-
-      import os
-      for idx, output_file in enumerate(_output_files):
-          try:
-              os.makedirs(os.path.dirname(output_file))
-          except OSError:
-              pass
-          with open(output_file, 'w') as f:
-              f.write(_output_serializers[idx](_outputs[idx]))
-    args:
-    - --model-name
-    - {inputValue: model_name}
-    - if:
-        cond: {isPresent: endpoint_name}
-        then:
-        - --endpoint-name
-        - {inputValue: endpoint_name}
-    - if:
-        cond: {isPresent: machine_type}
-        then:
-        - --machine-type
-        - {inputValue: machine_type}
-    - if:
-        cond: {isPresent: min_replica_count}
-        then:
-        - --min-replica-count
-        - {inputValue: min_replica_count}
-    - if:
-        cond: {isPresent: max_replica_count}
-        then:
-        - --max-replica-count
-        - {inputValue: max_replica_count}
-    - if:
-        cond: {isPresent: accelerator_type}
-        then:
-        - --accelerator-type
-        - {inputValue: accelerator_type}
-    - if:
-        cond: {isPresent: accelerator_count}
-        then:
-        - --accelerator-count
-        - {inputValue: accelerator_count}
-    - '----output-paths'
-    - {outputPath: endpoint_name}
-    - {outputPath: endpoint_dict}

community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/component.yaml

@@ -1,297 +0,0 @@
-name: Upload PyTorch model archive to Google Cloud Vertex AI
-metadata:
-  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/KFPv2_hell/components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/workaround_for_buggy_KFPv2_compiler/component.yaml'}
-inputs:
-- {name: model_archive, type: PyTorchModelArchive}
-- {name: torchserve_version, type: String, default: 0.6.0, optional: true}
-- name: use_gpu
-  type: Boolean
-  default: "False"
-  optional: true
-- {name: display_name, type: String, optional: true}
-- {name: description, type: String, optional: true}
-- {name: project, type: String, optional: true}
-- {name: location, type: String, optional: true}
-- {name: labels, type: JsonObject, optional: true}
-- {name: staging_bucket, type: String, optional: true}
-outputs:
-- {name: model_name, type: String}
-- {name: model_dict, type: JsonObject}
-implementation:
-  container:
-    image: python:3.9
-    command:
-    - sh
-    - -c
-    - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
-      'google-cloud-aiplatform==1.13.1' 'google-cloud-build==3.8.3' || PIP_DISABLE_PIP_VERSION_CHECK=1
-      python3 -m pip install --quiet --no-warn-script-location 'google-cloud-aiplatform==1.13.1'
-      'google-cloud-build==3.8.3' --user) && "$0" "$@"
-    - sh
-    - -ec
-    - |
-      program_path=$(mktemp)
-      printf "%s" "$0" > "$program_path"
-      python3 -u "$program_path" "$@"
-    - |
-      def upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI(
-          model_archive_path,
-          torchserve_version = "0.6.0",
-          use_gpu = False,
-
-          display_name = None,
-          description = None,
-
-          # Uncomment when anyone requests these:
-          # instance_schema_uri: str = None,
-          # parameters_schema_uri: str = None,
-          # prediction_schema_uri: str = None,
-          # explanation_metadata: "google.cloud.aiplatform_v1.types.explanation_metadata.ExplanationMetadata" = None,
-          # explanation_parameters: "google.cloud.aiplatform_v1.types.explanation.ExplanationParameters" = None,
-
-          project = None,
-          location = None,
-          labels = None,
-          # encryption_spec_key_name: str = None,
-          staging_bucket = None,
-      ):
-          import json
-          import os
-          from google.cloud import aiplatform
-
-          if not location:
-              location = os.environ.get("CLOUD_ML_REGION")
-
-          if not labels:
-              labels = {}
-          labels["component-source"] = "github-com-ark-kun-pipeline-components"
-
-          container_image_tag = torchserve_version + "-" + ("gpu" if use_gpu else "cpu")
-          container_image_uri = f"pytorch/torchserve:{container_image_tag}"
-
-          # Vertex Endpoints refuse to support non-Google container registries.
-          # We have to work around this to reduce user frustration
-          # TODO: Remove this code when Vertex Endpoints service starts supporting other container registries.
-          def copy_container_image(
-              src_container_image_uri,
-              dst_container_image_uri,
-              project_id,
-          ):
-              from google.cloud.devtools import cloudbuild
-              from google import protobuf
-              build_client = cloudbuild.CloudBuildClient()
-              build_config = cloudbuild.Build(
-                  images=[dst_container_image_uri],
-                  steps=[
-                      cloudbuild.BuildStep(
-                          name="gcr.io/cloud-builders/docker",
-                          entrypoint="bash",
-                          args=[
-                              "-exc",
-                              'docker pull --quiet "$0" && docker tag "$0" "$1"',
-                              src_container_image_uri,
-                              dst_container_image_uri,
-                          ],
-                      ),
-                  ],
-                  timeout=protobuf.duration_pb2.Duration(
-                      seconds=1800,
-                  ),
-              )
-              build_operation = build_client.create_build(
-                  project_id=project_id,
-                  build=build_config,
-              )
-              try:
-                  result = build_operation.result()
-              except:
-                  print(f"Logs are available at [{build_operation.metadata.build.log_url}].")
-                  raise
-              return result
-
-          project_id = aiplatform.initializer.global_config.project
-          mirrored_container_uri = f"gcr.io/{project_id}/container_mirror/{container_image_uri}"
-          # FIX: Only mirror when image does not exist
-          # docker does is unable to get the registry data from inside container (it cannot connecto to docker socket):
-          # docker.errors.DockerException: Error while fetching server API version: ('Connection aborted.', FileNotFoundError(2, 'No such file or directory'))
-          # import docker
-          # try:
-          #     docker_client = docker.from_env()
-          #     docker_client.images.get_registry_data(mirrored_container_uri)
-          # except docker.errors.NotFound:
-          if True:
-              print(f"Mirroring {container_image_uri} to {mirrored_container_uri}")
-              copy_container_image(
-                  src_container_image_uri=container_image_uri,
-                  dst_container_image_uri=mirrored_container_uri,
-                  project_id=project_id,
-              )
-          container_image_uri = mirrored_container_uri
-          # End of container image mirroring code
-
-          model_archive_file_name = os.path.basename(model_archive_path)
-          model_archive_dir = os.path.dirname(model_archive_path)
-
-          model = aiplatform.Model.upload(
-              # FIX: Use public image or mirror the official image
-              #serving_container_image_uri="gcr.io/avolkov-31337/mirror/pytorch/torchserve",
-              serving_container_image_uri=container_image_uri,
-              artifact_uri=model_archive_dir,
-              serving_container_command=[
-                  "bash",
-                  "-exc",
-                  '''
-      model_archive_uri="$0"
-      #model_archive_local_path=$(mktemp --suffix ".mar")
-      # For some reason the model must already be inside the model-store directory.
-      model_archive_local_path=./model-store/model.mar
-
-      # Downloading the model archive from GCS
-      # TODO: Fix gsutil bugs (requires project ID, has auth issues) and use gsutil instead.
-      # gsutil cp "$model_archive_uri" "$model_archive_local_path"
-      pip install google-cloud-storage
-      python -c '
-      import sys
-      from google.cloud import storage
-
-      model_archive_uri = sys.argv[1]
-      model_archive_local_path = sys.argv[2]
-
-      storage_client = storage.Client()
-      blob = storage.Blob.from_string(uri=model_archive_uri, client=storage_client)
-      blob.download_to_filename(filename=model_archive_local_path)
-      ' "$model_archive_uri" "$model_archive_local_path"
-
-      #Note: config.properties is owned by root. Our user is not root.
-      echo "
-      service_envelope=json
-      # Needed for external access
-      inference_address=http://0.0.0.0:8080
-      management_address=http://0.0.0.0:8081
-      " > config2.properties
-      torchserve --start --foreground --no-config-snapshots --models main-model="$model_archive_local_path" --model-store ./model-store/ --ts-config config2.properties
-                  ''',
-                  "$(AIP_STORAGE_URI)/" + model_archive_file_name,
-              ],
-              serving_container_predict_route="/predictions/main-model",
-              #serving_container_predict_route="/v1/models/main-model:predict",
-              serving_container_health_route="/ping",
-              serving_container_ports=[8080],
-
-              display_name=display_name,
-              description=description,
-
-              # instance_schema_uri=instance_schema_uri,
-              # parameters_schema_uri=parameters_schema_uri,
-              # prediction_schema_uri=prediction_schema_uri,
-              # explanation_metadata=explanation_metadata,
-              # explanation_parameters=explanation_parameters,
-
-              project=project,
-              location=location,
-              labels=labels,
-              # encryption_spec_key_name=encryption_spec_key_name,
-              staging_bucket=staging_bucket,
-          )
-          model_json = json.dumps(model.to_dict(), indent=2)
-          print(model_json)
-          return (model.resource_name, model_json)
-
-      def _deserialize_bool(s) -> bool:
-          from distutils.util import strtobool
-          return strtobool(s) == 1
-
-      def _serialize_json(obj) -> str:
-          if isinstance(obj, str):
-              return obj
-          import json
-          def default_serializer(obj):
-              if hasattr(obj, 'to_struct'):
-                  return obj.to_struct()
-              else:
-                  raise TypeError("Object of type '%s' is not JSON serializable and does not have .to_struct() method." % obj.__class__.__name__)
-          return json.dumps(obj, default=default_serializer, sort_keys=True)
-
-      def _serialize_str(str_value: str) -> str:
-          if not isinstance(str_value, str):
-              raise TypeError('Value "{}" has type "{}" instead of str.'.format(str(str_value), str(type(str_value))))
-          return str_value
-
-      import json
-      import argparse
-      _parser = argparse.ArgumentParser(prog='Upload PyTorch model archive to Google Cloud Vertex AI', description='')
-      _parser.add_argument("--model-archive", dest="model_archive_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--torchserve-version", dest="torchserve_version", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--use-gpu", dest="use_gpu", type=_deserialize_bool, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--display-name", dest="display_name", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--description", dest="description", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--project", dest="project", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--location", dest="location", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--labels", dest="labels", type=json.loads, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--staging-bucket", dest="staging_bucket", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("----output-paths", dest="_output_paths", type=str, nargs=2)
-      _parsed_args = vars(_parser.parse_args())
-      _output_files = _parsed_args.pop("_output_paths", [])
-
-      _outputs = upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI(**_parsed_args)
-
-      _output_serializers = [
-          _serialize_str,
-          _serialize_json,
-
-      ]
-
-      import os
-      for idx, output_file in enumerate(_output_files):
-          try:
-              os.makedirs(os.path.dirname(output_file))
-          except OSError:
-              pass
-          with open(output_file, 'w') as f:
-              f.write(_output_serializers[idx](_outputs[idx]))
-    args:
-    - --model-archive
-    - {inputPath: model_archive}
-    - if:
-        cond: {isPresent: torchserve_version}
-        then:
-        - --torchserve-version
-        - {inputValue: torchserve_version}
-    - if:
-        cond: {isPresent: use_gpu}
-        then:
-        - --use-gpu
-        - {inputValue: use_gpu}
-    - if:
-        cond: {isPresent: display_name}
-        then:
-        - --display-name
-        - {inputValue: display_name}
-    - if:
-        cond: {isPresent: description}
-        then:
-        - --description
-        - {inputValue: description}
-    - if:
-        cond: {isPresent: project}
-        then:
-        - --project
-        - {inputValue: project}
-    - if:
-        cond: {isPresent: location}
-        then:
-        - --location
-        - {inputValue: location}
-    - if:
-        cond: {isPresent: labels}
-        then:
-        - --labels
-        - {inputValue: labels}
-    - if:
-        cond: {isPresent: staging_bucket}
-        then:
-        - --staging-bucket
-        - {inputValue: staging_bucket}
-    - '----output-paths'
-    - {outputPath: model_name}
-    - {outputPath: model_dict}

community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/component.yaml

@@ -1,181 +0,0 @@
-name: Upload Scikit learn pickle model to Google Cloud Vertex AI
-metadata:
-  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/KFPv2_hell/components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/workaround_for_buggy_KFPv2_compiler/component.yaml'}
-inputs:
-- {name: model, type: ScikitLearnPickleModel}
-- {name: sklearn_version, type: String, optional: true}
-- {name: display_name, type: String, optional: true}
-- {name: description, type: String, optional: true}
-- {name: project, type: String, optional: true}
-- {name: location, type: String, optional: true}
-- {name: labels, type: JsonObject, optional: true}
-- {name: staging_bucket, type: String, optional: true}
-outputs:
-- {name: model_name, type: String}
-- {name: model_dict, type: JsonObject}
-implementation:
-  container:
-    image: python:3.9
-    command:
-    - sh
-    - -c
-    - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
-      'google-cloud-aiplatform==1.16.0' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3
-      -m pip install --quiet --no-warn-script-location 'google-cloud-aiplatform==1.16.0'
-      --user) && "$0" "$@"
-    - sh
-    - -ec
-    - |
-      program_path=$(mktemp)
-      printf "%s" "$0" > "$program_path"
-      python3 -u "$program_path" "$@"
-    - |
-      def upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI(
-          model_path,
-          sklearn_version = None,
-
-          display_name = None,
-          description = None,
-
-          # Uncomment when anyone requests these:
-          # instance_schema_uri: str = None,
-          # parameters_schema_uri: str = None,
-          # prediction_schema_uri: str = None,
-          # explanation_metadata: "google.cloud.aiplatform_v1.types.explanation_metadata.ExplanationMetadata" = None,
-          # explanation_parameters: "google.cloud.aiplatform_v1.types.explanation.ExplanationParameters" = None,
-
-          project = None,
-          location = None,
-          labels = None,
-          # encryption_spec_key_name: str = None,
-          staging_bucket = None,
-      ):
-          import json
-          import os
-          import shutil
-          import tempfile
-          from google.cloud import aiplatform
-
-          if not location:
-              location = os.environ.get("CLOUD_ML_REGION")
-
-          if not labels:
-              labels = {}
-          labels["component-source"] = "github-com-ark-kun-pipeline-components"
-
-          # The serving container decides the model type based on the model file extension.
-          # So we need to rename the mode file (e.g. /tmp/inputs/model/data) to *.pkl
-          _, renamed_model_path = tempfile.mkstemp(suffix=".pkl")
-          shutil.copyfile(src=model_path, dst=renamed_model_path)
-
-          model = aiplatform.Model.upload_scikit_learn_model_file(
-              model_file_path=renamed_model_path,
-              sklearn_version=sklearn_version,
-
-              display_name=display_name,
-              description=description,
-
-              # instance_schema_uri=instance_schema_uri,
-              # parameters_schema_uri=parameters_schema_uri,
-              # prediction_schema_uri=prediction_schema_uri,
-              # explanation_metadata=explanation_metadata,
-              # explanation_parameters=explanation_parameters,
-
-              project=project,
-              location=location,
-              labels=labels,
-              # encryption_spec_key_name=encryption_spec_key_name,
-              staging_bucket=staging_bucket,
-          )
-          model_json = json.dumps(model.to_dict(), indent=2)
-          print(model_json)
-          return (model.resource_name, model_json)
-
-      def _serialize_json(obj) -> str:
-          if isinstance(obj, str):
-              return obj
-          import json
-          def default_serializer(obj):
-              if hasattr(obj, 'to_struct'):
-                  return obj.to_struct()
-              else:
-                  raise TypeError("Object of type '%s' is not JSON serializable and does not have .to_struct() method." % obj.__class__.__name__)
-          return json.dumps(obj, default=default_serializer, sort_keys=True)
-
-      def _serialize_str(str_value: str) -> str:
-          if not isinstance(str_value, str):
-              raise TypeError('Value "{}" has type "{}" instead of str.'.format(str(str_value), str(type(str_value))))
-          return str_value
-
-      import json
-      import argparse
-      _parser = argparse.ArgumentParser(prog='Upload Scikit learn pickle model to Google Cloud Vertex AI', description='')
-      _parser.add_argument("--model", dest="model_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--sklearn-version", dest="sklearn_version", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--display-name", dest="display_name", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--description", dest="description", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--project", dest="project", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--location", dest="location", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--labels", dest="labels", type=json.loads, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--staging-bucket", dest="staging_bucket", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("----output-paths", dest="_output_paths", type=str, nargs=2)
-      _parsed_args = vars(_parser.parse_args())
-      _output_files = _parsed_args.pop("_output_paths", [])
-
-      _outputs = upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI(**_parsed_args)
-
-      _output_serializers = [
-          _serialize_str,
-          _serialize_json,
-
-      ]
-
-      import os
-      for idx, output_file in enumerate(_output_files):
-          try:
-              os.makedirs(os.path.dirname(output_file))
-          except OSError:
-              pass
-          with open(output_file, 'w') as f:
-              f.write(_output_serializers[idx](_outputs[idx]))
-    args:
-    - --model
-    - {inputPath: model}
-    - if:
-        cond: {isPresent: sklearn_version}
-        then:
-        - --sklearn-version
-        - {inputValue: sklearn_version}
-    - if:
-        cond: {isPresent: display_name}
-        then:
-        - --display-name
-        - {inputValue: display_name}
-    - if:
-        cond: {isPresent: description}
-        then:
-        - --description
-        - {inputValue: description}
-    - if:
-        cond: {isPresent: project}
-        then:
-        - --project
-        - {inputValue: project}
-    - if:
-        cond: {isPresent: location}
-        then:
-        - --location
-        - {inputValue: location}
-    - if:
-        cond: {isPresent: labels}
-        then:
-        - --labels
-        - {inputValue: labels}
-    - if:
-        cond: {isPresent: staging_bucket}
-        then:
-        - --staging-bucket
-        - {inputValue: staging_bucket}
-    - '----output-paths'
-    - {outputPath: model_name}
-    - {outputPath: model_dict}

community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/component.yaml

@@ -1,190 +0,0 @@
-name: Upload Tensorflow model to Google Cloud Vertex AI
-metadata:
-  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/KFPv2_hell/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/workaround_for_buggy_KFPv2_compiler/component.yaml'}
-inputs:
-- {name: model, type: TensorflowSavedModel}
-- {name: tensorflow_version, type: String, optional: true}
-- name: use_gpu
-  type: Boolean
-  default: "False"
-  optional: true
-- {name: display_name, type: String, optional: true}
-- {name: description, type: String, optional: true}
-- {name: project, type: String, optional: true}
-- {name: location, type: String, optional: true}
-- {name: labels, type: JsonObject, optional: true}
-- {name: staging_bucket, type: String, optional: true}
-outputs:
-- {name: model_name, type: String}
-- {name: model_dict, type: JsonObject}
-implementation:
-  container:
-    image: python:3.9
-    command:
-    - sh
-    - -c
-    - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
-      'google-cloud-aiplatform==1.16.0' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3
-      -m pip install --quiet --no-warn-script-location 'google-cloud-aiplatform==1.16.0'
-      --user) && "$0" "$@"
-    - sh
-    - -ec
-    - |
-      program_path=$(mktemp)
-      printf "%s" "$0" > "$program_path"
-      python3 -u "$program_path" "$@"
-    - |
-      def upload_Tensorflow_model_to_Google_Cloud_Vertex_AI(
-          model_path,
-          tensorflow_version = None,
-          use_gpu = False,
-
-          display_name = None,
-          description = None,
-
-          # Uncomment when anyone requests these:
-          # instance_schema_uri: str = None,
-          # parameters_schema_uri: str = None,
-          # prediction_schema_uri: str = None,
-          # explanation_metadata: "google.cloud.aiplatform_v1.types.explanation_metadata.ExplanationMetadata" = None,
-          # explanation_parameters: "google.cloud.aiplatform_v1.types.explanation.ExplanationParameters" = None,
-
-          project = None,
-          location = None,
-          labels = None,
-          # encryption_spec_key_name: str = None,
-          staging_bucket = None,
-      ):
-          import json
-          import os
-          from google.cloud import aiplatform
-
-          if not location:
-              location = os.environ.get("CLOUD_ML_REGION")
-
-          if not labels:
-              labels = {}
-          labels["component-source"] = "github-com-ark-kun-pipeline-components"
-
-          model = aiplatform.Model.upload_tensorflow_saved_model(
-              saved_model_dir=model_path,
-              tensorflow_version=tensorflow_version,
-              use_gpu=use_gpu,
-
-              display_name=display_name,
-              description=description,
-
-              # instance_schema_uri=instance_schema_uri,
-              # parameters_schema_uri=parameters_schema_uri,
-              # prediction_schema_uri=prediction_schema_uri,
-              # explanation_metadata=explanation_metadata,
-              # explanation_parameters=explanation_parameters,
-
-              project=project,
-              location=location,
-              labels=labels,
-              # encryption_spec_key_name=encryption_spec_key_name,
-              staging_bucket=staging_bucket,
-          )
-          model_json = json.dumps(model.to_dict(), indent=2)
-          print(model_json)
-          return (model.resource_name, model_json)
-
-      def _deserialize_bool(s) -> bool:
-          from distutils.util import strtobool
-          return strtobool(s) == 1
-
-      def _serialize_json(obj) -> str:
-          if isinstance(obj, str):
-              return obj
-          import json
-          def default_serializer(obj):
-              if hasattr(obj, 'to_struct'):
-                  return obj.to_struct()
-              else:
-                  raise TypeError("Object of type '%s' is not JSON serializable and does not have .to_struct() method." % obj.__class__.__name__)
-          return json.dumps(obj, default=default_serializer, sort_keys=True)
-
-      def _serialize_str(str_value: str) -> str:
-          if not isinstance(str_value, str):
-              raise TypeError('Value "{}" has type "{}" instead of str.'.format(str(str_value), str(type(str_value))))
-          return str_value
-
-      import json
-      import argparse
-      _parser = argparse.ArgumentParser(prog='Upload Tensorflow model to Google Cloud Vertex AI', description='')
-      _parser.add_argument("--model", dest="model_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--tensorflow-version", dest="tensorflow_version", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--use-gpu", dest="use_gpu", type=_deserialize_bool, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--display-name", dest="display_name", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--description", dest="description", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--project", dest="project", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--location", dest="location", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--labels", dest="labels", type=json.loads, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--staging-bucket", dest="staging_bucket", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("----output-paths", dest="_output_paths", type=str, nargs=2)
-      _parsed_args = vars(_parser.parse_args())
-      _output_files = _parsed_args.pop("_output_paths", [])
-
-      _outputs = upload_Tensorflow_model_to_Google_Cloud_Vertex_AI(**_parsed_args)
-
-      _output_serializers = [
-          _serialize_str,
-          _serialize_json,
-
-      ]
-
-      import os
-      for idx, output_file in enumerate(_output_files):
-          try:
-              os.makedirs(os.path.dirname(output_file))
-          except OSError:
-              pass
-          with open(output_file, 'w') as f:
-              f.write(_output_serializers[idx](_outputs[idx]))
-    args:
-    - --model
-    - {inputPath: model}
-    - if:
-        cond: {isPresent: tensorflow_version}
-        then:
-        - --tensorflow-version
-        - {inputValue: tensorflow_version}
-    - if:
-        cond: {isPresent: use_gpu}
-        then:
-        - --use-gpu
-        - {inputValue: use_gpu}
-    - if:
-        cond: {isPresent: display_name}
-        then:
-        - --display-name
-        - {inputValue: display_name}
-    - if:
-        cond: {isPresent: description}
-        then:
-        - --description
-        - {inputValue: description}
-    - if:
-        cond: {isPresent: project}
-        then:
-        - --project
-        - {inputValue: project}
-    - if:
-        cond: {isPresent: location}
-        then:
-        - --location
-        - {inputValue: location}
-    - if:
-        cond: {isPresent: labels}
-        then:
-        - --labels
-        - {inputValue: labels}
-    - if:
-        cond: {isPresent: staging_bucket}
-        then:
-        - --staging-bucket
-        - {inputValue: staging_bucket}
-    - '----output-paths'
-    - {outputPath: model_name}
-    - {outputPath: model_dict}

community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/component.yaml

@@ -1,181 +0,0 @@
-name: Upload XGBoost model to Google Cloud Vertex AI
-metadata:
-  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/workaround_for_buggy_KFPv2_compiler/component.yaml'}
-inputs:
-- {name: model, type: XGBoostModel}
-- {name: xgboost_version, type: String, optional: true}
-- {name: display_name, type: String, optional: true}
-- {name: description, type: String, optional: true}
-- {name: project, type: String, optional: true}
-- {name: location, type: String, optional: true}
-- {name: labels, type: JsonObject, optional: true}
-- {name: staging_bucket, type: String, optional: true}
-outputs:
-- {name: model_name, type: String}
-- {name: model_dict, type: JsonObject}
-implementation:
-  container:
-    image: python:3.9
-    command:
-    - sh
-    - -c
-    - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
-      'google-cloud-aiplatform==1.16.0' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3
-      -m pip install --quiet --no-warn-script-location 'google-cloud-aiplatform==1.16.0'
-      --user) && "$0" "$@"
-    - sh
-    - -ec
-    - |
-      program_path=$(mktemp)
-      printf "%s" "$0" > "$program_path"
-      python3 -u "$program_path" "$@"
-    - |
-      def upload_XGBoost_model_to_Google_Cloud_Vertex_AI(
-          model_path,
-          xgboost_version = None,
-
-          display_name = None,
-          description = None,
-
-          # Uncomment when anyone requests these:
-          # instance_schema_uri: str = None,
-          # parameters_schema_uri: str = None,
-          # prediction_schema_uri: str = None,
-          # explanation_metadata: "google.cloud.aiplatform_v1.types.explanation_metadata.ExplanationMetadata" = None,
-          # explanation_parameters: "google.cloud.aiplatform_v1.types.explanation.ExplanationParameters" = None,
-
-          project = None,
-          location = None,
-          labels = None,
-          # encryption_spec_key_name: str = None,
-          staging_bucket = None,
-      ):
-          import json
-          import os
-          import shutil
-          import tempfile
-          from google.cloud import aiplatform
-
-          if not location:
-              location = os.environ.get("CLOUD_ML_REGION")
-
-          if not labels:
-              labels = {}
-          labels["component-source"] = "github-com-ark-kun-pipeline-components"
-
-          # The serving container decides the model type based on the model file extension.
-          # So we need to rename the mode file (e.g. /tmp/inputs/model/data) to *.bst
-          _, renamed_model_path = tempfile.mkstemp(suffix=".bst")
-          shutil.copyfile(src=model_path, dst=renamed_model_path)
-
-          model = aiplatform.Model.upload_xgboost_model_file(
-              model_file_path=renamed_model_path,
-              xgboost_version=xgboost_version,
-
-              display_name=display_name,
-              description=description,
-
-              # instance_schema_uri=instance_schema_uri,
-              # parameters_schema_uri=parameters_schema_uri,
-              # prediction_schema_uri=prediction_schema_uri,
-              # explanation_metadata=explanation_metadata,
-              # explanation_parameters=explanation_parameters,
-
-              project=project,
-              location=location,
-              labels=labels,
-              # encryption_spec_key_name=encryption_spec_key_name,
-              staging_bucket=staging_bucket,
-          )
-          model_json = json.dumps(model.to_dict(), indent=2)
-          print(model_json)
-          return (model.resource_name, model_json)
-
-      def _serialize_json(obj) -> str:
-          if isinstance(obj, str):
-              return obj
-          import json
-          def default_serializer(obj):
-              if hasattr(obj, 'to_struct'):
-                  return obj.to_struct()
-              else:
-                  raise TypeError("Object of type '%s' is not JSON serializable and does not have .to_struct() method." % obj.__class__.__name__)
-          return json.dumps(obj, default=default_serializer, sort_keys=True)
-
-      def _serialize_str(str_value: str) -> str:
-          if not isinstance(str_value, str):
-              raise TypeError('Value "{}" has type "{}" instead of str.'.format(str(str_value), str(type(str_value))))
-          return str_value
-
-      import json
-      import argparse
-      _parser = argparse.ArgumentParser(prog='Upload XGBoost model to Google Cloud Vertex AI', description='')
-      _parser.add_argument("--model", dest="model_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--xgboost-version", dest="xgboost_version", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--display-name", dest="display_name", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--description", dest="description", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--project", dest="project", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--location", dest="location", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--labels", dest="labels", type=json.loads, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--staging-bucket", dest="staging_bucket", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("----output-paths", dest="_output_paths", type=str, nargs=2)
-      _parsed_args = vars(_parser.parse_args())
-      _output_files = _parsed_args.pop("_output_paths", [])
-
-      _outputs = upload_XGBoost_model_to_Google_Cloud_Vertex_AI(**_parsed_args)
-
-      _output_serializers = [
-          _serialize_str,
-          _serialize_json,
-
-      ]
-
-      import os
-      for idx, output_file in enumerate(_output_files):
-          try:
-              os.makedirs(os.path.dirname(output_file))
-          except OSError:
-              pass
-          with open(output_file, 'w') as f:
-              f.write(_output_serializers[idx](_outputs[idx]))
-    args:
-    - --model
-    - {inputPath: model}
-    - if:
-        cond: {isPresent: xgboost_version}
-        then:
-        - --xgboost-version
-        - {inputValue: xgboost_version}
-    - if:
-        cond: {isPresent: display_name}
-        then:
-        - --display-name
-        - {inputValue: display_name}
-    - if:
-        cond: {isPresent: description}
-        then:
-        - --description
-        - {inputValue: description}
-    - if:
-        cond: {isPresent: project}
-        then:
-        - --project
-        - {inputValue: project}
-    - if:
-        cond: {isPresent: location}
-        then:
-        - --location
-        - {inputValue: location}
-    - if:
-        cond: {isPresent: labels}
-        then:
-        - --labels
-        - {inputValue: labels}
-    - if:
-        cond: {isPresent: staging_bucket}
-        then:
-        - --staging-bucket
-        - {inputValue: staging_bucket}
-    - '----output-paths'
-    - {outputPath: model_name}
-    - {outputPath: model_dict}

community-content/pipeline_components/google-cloud/storage/download/component.yaml

@@ -1,35 +0,0 @@
-name: Download from GCS
-inputs:
-- {name: GCS path, type: String}
-outputs:
-- {name: Data}
-metadata:
-  annotations:
-    author: Alexey Volkov <alexey.volkov@ark-kun.com>
-    canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml'
-implementation:
-    container:
-        image: google/cloud-sdk
-        command:
-        - bash # Pattern comparison only works in Bash
-        - -ex
-        - -c
-        - |
-            if [ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then
-                gcloud auth activate-service-account --key-file="${GOOGLE_APPLICATION_CREDENTIALS}"
-            fi
-
-            uri="$0"
-            output_path="$1"
-
-            # Checking whether the URI points to a single blob, a directory or a URI pattern
-            # URI points to a blob when that URI does not end with slash and listing that URI only yields the same URI
-            if [[ "$uri" != */ ]] && (gsutil ls "$uri" | grep --fixed-strings --line-regexp "$uri"); then
-                mkdir -p "$(dirname "$output_path")"
-                gsutil -m cp -r "$uri" "$output_path"
-            else
-                mkdir -p "$output_path" # When source path is a directory, gsutil requires the destination to also be a directory
-                gsutil -m rsync -r "$uri" "$output_path" # gsutil cp has different path handling than Linux cp. It always puts the source directory (name) inside the destination directory. gsutil rsync does not have that problem.
-            fi
-        - inputValue: GCS path
-        - outputPath: Data

community-content/pipeline_components/image_ml_model_training/load_image_classification_model/component.yaml

@@ -1,112 +0,0 @@
-name: Load image classification model from tfhub
-description: |
-  Loads specified model from TFHub, creates layer to receive additional (3 channel) imagery data.
-  Args:
-    class_names (Sequence[str]):
-        Sequence of strings of categories for classification corresponding to input data.
-    loaded_model_path (str):
-        Output path for the loaded model.
-    image_size_path (str):
-        Output path for the model expected image size.
-    model_name (Optional[str]):
-        Name of the pre-trained image classification model to load from TFHub.
-        Eligible model_name:
-        - efficientnetv2-s
-        - efficientnetv2-m
-        - efficientnetv2-l
-        - efficientnetv2-s-21k
-        - efficientnetv2-m-21k
-        - efficientnetv2-l-21k
-        - efficientnetv2-xl-21k
-        - efficientnetv2-b0-21k
-        - efficientnetv2-b1-21k
-        - efficientnetv2-b2-21k
-        - efficientnetv2-b3-21k
-        - efficientnetv2-s-21k-ft1k
-        - efficientnetv2-m-21k-ft1k
-        - efficientnetv2-l-21k-ft1k
-        - efficientnetv2-xl-21k-ft1k
-        - efficientnetv2-b0-21k-ft1k
-        - efficientnetv2-b1-21k-ft1k
-        - efficientnetv2-b2-21k-ft1k
-        - efficientnetv2-b3-21k-ft1k
-        - efficientnetv2-b0
-        - efficientnetv2-b1
-        - efficientnetv2-b2
-        - efficientnetv2-b3
-        - efficientnet_b0
-        - efficientnet_b1
-        - efficientnet_b2
-        - efficientnet_b3
-        - efficientnet_b4
-        - efficientnet_b5
-        - efficientnet_b6
-        - efficientnet_b7
-        - bit_s-r50x1
-        - inception_v3
-        - inception_resnet_v2
-        - resnet_v1_50
-        - resnet_v1_101
-        - resnet_v1_152
-        - resnet_v2_50
-        - resnet_v2_101
-        - resnet_v2_152
-        - nasnet_large
-        - nasnet_mobile
-        - pnasnet_large
-        - mobilenet_v2_100_224
-        - mobilenet_v2_130_224
-        - mobilenet_v2_140_224
-        - mobilenet_v3_small_100_224
-        - mobilenet_v3_small_075_224
-        - mobilenet_v3_large_100_224
-        - mobilenet_v3_large_075_224
-    dropout_rate (Optional[float]):
-        Fraction of input units to drop in the last layer. Value should be between 0.0 and 1.0.
-    trainable (Optional[bool]):
-        If true fine tuning will be performed on entire Hub model. If false only additional
-        layers will be trained.
-    l2_regularization_penalty (Optional[float]):
-        l2 regularization penalty.
-inputs:
-- {name: class_names, type: 'typing.List[str]', description: List of class names corresponding
-    to the input image data}
-- {name: model_name, type: String, description: Name of the TFHub model to load, default: efficientnetv2-xl-21k,
-  optional: true}
-- {name: dropout_rate, type: Float, description: Dropout rate, default: '0.2', optional: true}
-- name: trainable
-  type: Boolean
-  description: True if fine tuning should be performed
-  default: "True"
-  optional: true
-- {name: l2_regularization_penalty, type: Float, description: Regularization penalty,
-  default: '0.0001', optional: true}
-outputs:
-- {name: loaded_model_path, type: TensorflowSavedModel, description: Output path for
-    the loaded model}
-- {name: image_size_path, type: HeightWidth}
-implementation:
-  container:
-    image: us-docker.pkg.dev/vertex-ai/ready-to-go-image-classification/image-components:v0.2
-    # command is a list of strings (command-line arguments).
-    # The YAML language has two syntaxes for lists and you can use either of them.
-    # Here we use the "flow syntax" - comma-separated strings inside square brackets.
-    command: [
-      python3,
-      # Path of the program inside the container
-      /pipelines/component/src/loading_component.py,
-      --loaded-model-path,
-      {outputPath: loaded_model_path},
-      --class-names,
-      {inputValue: class_names},
-      --model-name,
-      {inputValue: model_name},
-      --dropout-rate,
-      {inputValue: dropout_rate},
-      --trainable,
-      {inputValue: trainable},
-      --l2-regularization-penalty,
-      {inputValue: l2_regularization_penalty},
-      --image-size-path,
-      {outputPath: image_size_path},
-    ]

community-content/pipeline_components/image_ml_model_training/pipeline.py

@@ -1,62 +0,0 @@
-# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
-from kfp import components
-from kfp.v2 import dsl
-
-# %% Loading components
-upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url('https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/component.yaml')
-deploy_model_to_endpoint_op = components.load_component_from_url('https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml')
-transcode_imagedataset_tfrecord_from_csv_op = components.load_component_from_url('https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/community-content/pipeline_components/image_ml_model_training/transcode_tfrecord_image_dataset_from_csv/component.yaml')
-load_image_classification_model_from_tfhub_op = components.load_component_from_url('https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/b5b65198a6c2ffe8c0fa2aa70127e3325752df68/community-content/pipeline_components/image_ml_model_training/load_image_classification_model/component.yaml')
-preprocess_image_data_op = components.load_component_from_url('https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/community-content/pipeline_components/image_ml_model_training/preprocess_image_data/component.yaml')
-train_tensorflow_image_classification_model_op = components.load_component_from_url('https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/community-content/pipeline_components/image_ml_model_training/train_image_classification_model/component.yaml')
-
-
-# %% Pipeline definition
-def image_classification_pipeline():
-    class_names = ['daisy', 'dandelion', 'roses', 'sunflowers', 'tulips']
-    csv_image_data_path = 'gs://cloud-samples-data/ai-platform/flowers/flowers.csv'
-    deploy_model = False
-
-    image_data = dsl.importer(
-        artifact_uri=csv_image_data_path, artifact_class=dsl.Dataset).output
-
-    image_tfrecord_data = transcode_imagedataset_tfrecord_from_csv_op(
-        csv_image_data_path=image_data,
-        class_names=class_names
-    ).outputs['tfrecord_image_data_path']
-
-    loaded_model_outputs = load_image_classification_model_from_tfhub_op(
-        class_names=class_names,
-    ).outputs
-
-    preprocessed_data = preprocess_image_data_op(
-        image_tfrecord_data,
-        height_width_path=loaded_model_outputs['image_size_path'],
-    ).outputs
-
-    trained_model = (train_tensorflow_image_classification_model_op(
-        preprocessed_training_data_path = preprocessed_data['preprocessed_training_data_path'],
-        preprocessed_validation_data_path = preprocessed_data['preprocessed_validation_data_path'],
-        model_path=loaded_model_outputs['loaded_model_path']).
-                   set_cpu_limit('96').
-                   set_memory_limit('128G').
-                   add_node_selector_constraint('cloud.google.com/gke-accelerator', 'NVIDIA_TESLA_A100').
-                   set_gpu_limit('8').
-                   outputs['trained_model_path'])
-
-    vertex_model_name = upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op(
-        model=trained_model,
-    ).outputs['model_name']
-
-    # Deploying the model might incur additional costs over time
-    if deploy_model:
-        vertex_endpoint_name = deploy_model_to_endpoint_op(
-            model_name=vertex_model_name,
-        ).outputs['endpoint_name']
-
-pipeline_func = image_classification_pipeline
-
-# %% Pipeline submission
-if __name__ == '__main__':
-    from google.cloud import aiplatform
-    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

community-content/pipeline_components/image_ml_model_training/preprocess_image_data/component.yaml

@@ -1,57 +0,0 @@
-name: Preprocess image data
-description: |
-  Preprocess the image data and split between train and validation.
-  Args:
-    input_data_path (str):
-        Input path for the TFRecord image data. Data will be formatted as 'label' (encoded image
-        label), and 'image_raw' (the binary string of the image data).
-    height_width_path (str):
-        Path to square height and width to resize images to. File should contain single float value.
-        Value is dependent on training model.
-    preprocessed_training_data_path (str):
-        Output path for the TFRecord training data. Data will be formatted as 'label' (encoded image
-        label), and 'image_raw' (the binary string of the image data).
-    preprocessed_validation_data_path (str):
-        Output path for the TFRecord validation data. Data will be formatted as 'label' (encoded
-        image label), and 'image_raw' (the binary string of the image data).
-    validation_split (Optional[float]):
-        Fraction of data that will make up validation dataset. Value should be between 0.0 and 1.0.
-    seed (Optional[int]):
-        The global random seed to ensure the system gets a unique random sequence
-        that is deterministic (https://www.tensorflow.org/api_docs/python/tf/random/set_seed).
-inputs:
-- {name: input_data_path, type: ImageDatasetTFRecord, description: 'Input path for
-    the TFRecord image data,'}
-- {name: height_width_path, type: HeightWidth, description: 'Path to square height and width to
-    resize images to,'}
-- {name: validation_split, type: Float, description: 'Fraction of data that will make
-    up validation dataset,', default: '0.2', optional: true}
-- {name: seed, type: Integer, description: Random seed, default: '0', optional: true}
-outputs:
-- {name: preprocessed_training_data_path, type: ImageDatasetTFRecord, description: 'Output
-    path for the training data,'}
-- {name: preprocessed_validation_data_path, type: ImageDatasetTFRecord, description: 'Output
-    path for the validation data,'}
-implementation:
-  container:
-    image: us-docker.pkg.dev/vertex-ai/ready-to-go-image-classification/image-components:v0.2
-    # command is a list of strings (command-line arguments).
-    # The YAML language has two syntaxes for lists and you can use either of them.
-    # Here we use the "flow syntax" - comma-separated strings inside square brackets.
-    command: [
-      python3,
-      # Path of the program inside the container
-      /pipelines/component/src/preprocessing_component.py,
-      --input-data-path,
-      {inputPath: input_data_path},
-      --height-width-path,
-      {inputPath: height_width_path},
-      --validation-split,
-      {inputValue: validation_split},
-      --seed,
-      {inputValue: seed},
-      --preprocessed-training-data-path,
-      {outputPath: preprocessed_training_data_path},
-      --preprocessed-validation-data-path,
-      {outputPath: preprocessed_validation_data_path},
-    ]

community-content/pipeline_components/image_ml_model_training/train_image_classification_model/component.yaml

@@ -1,90 +0,0 @@
-name: Train tensorflow image classification model
-description: |
-  Creates a trained image classification TensorFlow model.
-  Args:
-    preprocessed_training_data_path (str):
-        Input path to the TFRecord training data. Data will be formatted as 'label' (encoded image
-        label), and 'image_raw' (the binary string of the image data).
-    preprocessed_validation_data_path (str):
-        Input path to the TFRecord validation data. Data will be formatted as 'label' (encoded
-        image label), and 'image_raw' (the binary string of the image data).
-    model_path (str):
-        Input path to the loaded pre-trained model.
-    trained_model_path (str):
-        Output path to save the trained model to.
-    optimizer_name (Optional[str]):
-        Name of the tf.keras optimizer. Available optimizers are listed at
-        https://keras.io/api/optimizers/
-    optimizer_parameters (Optional[Dict[str, str]]):
-        Optimizer parameters.
-    loss_function_name (Optional[str]):
-        Name of the loss function.
-    loss_function_parameters (Optional[Dict[str, str]]):
-        Loss function parameters.
-    number_of_epochs (Optional[int]):
-        Number of training iterations over data.
-    metric_names (Optional[Sequence[str]]):
-        List of tf.keras.metrics to be evaluated by the model during training and testing. Available
-        metrics are listed at https://keras.io/api/metrics/.
-    seed Optional(int):
-        The global random seed to ensure the system gets a unique random sequence
-        that is deterministic (https://www.tensorflow.org/api_docs/python/tf/random/set_seed).
-inputs:
-- {name: preprocessed_training_data_path, type: ImageDatasetTFRecord, description: 'Input
-    path for the training data,'}
-- {name: preprocessed_validation_data_path, type: ImageDatasetTFRecord, description: 'Input
-    path for the validation data,'}
-- {name: model_path, type: TensorflowSavedModel, description: 'Input path for the
-    model,'}
-- {name: optimizer_name, type: String, description: 'Name of the optimizer,', default: SGD,
-  optional: true}
-- {name: optimizer_parameters, type: 'typing.Dict[str, str]', description: 'Optimizer
-    parameters,', default: '{}', optional: true}
-- {name: loss_function_name, type: String, description: 'Name of the loss function,',
-  default: CategoricalCrossentropy, optional: true}
-- {name: loss_function_parameters, type: 'typing.Dict[str, str]', description: 'Loss
-    function parameters,', default: '{}', optional: true}
-- {name: number_of_epochs, type: Integer, description: 'Number of epochs,', default: '10',
-  optional: true}
-- {name: metric_names, type: 'typing.List[str]', description: 'List of metrics to
-    use,', default: '["accuracy"]', optional: true}
-- {name: seed, type: Integer, description: 'Random seed,', default: '0', optional: true}
-- {name: batch_size, type: Integer, description: Batch size, default: '16', optional: true}
-outputs:
-- {name: trained_model_path, type: TensorflowSavedModel, description: 'Output path
-    for the saved model,'}
-implementation:
-  container:
-    image: us-docker.pkg.dev/vertex-ai/ready-to-go-image-classification/image-components:v0.2
-    # command is a list of strings (command-line arguments).
-    # The YAML language has two syntaxes for lists and you can use either of them.
-    # Here we use the "flow syntax" - comma-separated strings inside square brackets.
-    command: [
-      python3,
-      # Path of the program inside the container
-      /pipelines/component/src/training_component.py,
-      --preprocessed-training-data-path,
-      {inputPath: preprocessed_training_data_path},
-      --preprocessed-validation-data-path,
-      {inputPath: preprocessed_validation_data_path},
-      --model-path,
-      {inputPath: model_path},
-      --trained-model-path,
-      {outputPath: trained_model_path},
-      --optimizer-name,
-      {inputValue: optimizer_name},
-      --loss-function-name,
-      {inputValue: loss_function_name},
-      --number-of-epochs,
-      {inputValue: number_of_epochs},
-      --seed,
-      {inputValue: seed},
-      --batch-size,
-      {inputValue: batch_size},
-      --metric-names,
-      {inputValue: metric_names},
-      --optimizer-parameters,
-      {inputValue: optimizer_parameters},
-      --loss-function-parameters,
-      {inputValue: loss_function_parameters},
-    ]

community-content/pipeline_components/image_ml_model_training/transcode_tfrecord_image_dataset_from_csv/component.yaml

@@ -1,37 +0,0 @@
-name: Transcode imagedataset tfrecord from csv
-description: |
-  Transcodes CSV Data into TFRecord file of TFExamples.
-  Args:
-    csv_image_data_path (str):
-        Path to the CSV image data. Data must include 'image_filepath' (Path to image file) and
-        'image_label' (output for a prediction) fields.
-    class_names (Sequence[str]):
-        Sequence of strings of categories for classification corresponding to input data.
-    tfrecord_image_data_path (str):
-        Output path for the TFRecord image data. Data will be formatted as 'label' (encoded image
-        label), and 'image_raw' (the binary string of the image data).
-inputs:
-- {name: csv_image_data_path, type: ImageDatasetCSV, description: Input path for the
-    CSV image data}
-- {name: class_names, type: 'typing.List[str]', description: List of class names corresponding
-    to the input image data}
-outputs:
-- {name: tfrecord_image_data_path, type: ImageDatasetTFRecord, description: Output
-    path for the TFRecord image data}
-implementation:
-  container:
-    image: us-docker.pkg.dev/vertex-ai/ready-to-go-image-classification/image-components:v0.2
-    # command is a list of strings (command-line arguments).
-    # The YAML language has two syntaxes for lists and you can use either of them.
-    # Here we use the "flow syntax" - comma-separated strings inside square brackets.
-    command: [
-      python3,
-      # Path of the program inside the container
-      /pipelines/component/src/transcoding_csv_component.py,
-      --csv-image-data-path,
-      {inputPath: csv_image_data_path},
-      --tfrecord-image-data-path,
-      {outputPath: tfrecord_image_data_path},
-      --class-names,
-      {inputValue: class_names},
-    ]

community-content/pipeline_components/image_ml_model_training/transcode_tfrecord_image_dataset_from_jsonl/component.yaml

@@ -1,39 +0,0 @@
-name: Transcode imagedataset tfrecord from jsonlines
-description: |
-  Transcodes JSONL Data into TFRecord file of TFExamples.
-  Args:
-    jsonl_image_data_path (str):
-        Input path for the JSONL image data
-        Path to the JSONL image data. Each line corresponds to a JSON input describing an image.
-        Schema follows AutoML image classification JSONL format
-        https://cloud.google.com/vertex-ai/docs/image-data/classification/prepare-data#json-lines.
-    class_names (Sequence[str]):
-        Sequence of strings of categories for classification corresponding to input data.
-    tfrecord_image_data_path (str):
-        Output path for the TFRecord image data. Data will be formatted as 'label' (encoded image
-        label), and 'image_raw' (the binary string of the image data).
-inputs:
-- {name: jsonl_image_data_path, type: ImageDatasetJsonLines, description: Input path
-    for the JSONL image data}
-- {name: class_names, type: 'typing.List[str]', description: List of class names corresponding
-    to the input image data}
-outputs:
-- {name: tfrecord_image_data_path, type: ImageDatasetTFRecord, description: Output
-    path for the TFRecord image data}
-implementation:
-  container:
-    image: us-docker.pkg.dev/vertex-ai/ready-to-go-image-classification/image-components:v0.2
-    # command is a list of strings (command-line arguments).
-    # The YAML language has two syntaxes for lists and you can use either of them.
-    # Here we use the "flow syntax" - comma-separated strings inside square brackets.
-    command: [
-      python3,
-      # Path of the program inside the container
-      /pipelines/component/src/transcoding_jsonl_component.py,
-      --jsonl-image-data-path,
-      {inputPath: jsonl_image_data_path},
-      --tfrecord-image-data-path,
-      {outputPath: tfrecord_image_data_path},
-      --class-names,
-      {inputValue: class_names},
-    ]

community-content/pipeline_components/pandas/Binarize_column/in_CSV_format/component.yaml

@@ -1,113 +0,0 @@
-name: Binarize column using Pandas on CSV data
-metadata:
-  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/pandas/Binarize_column/in_CSV_format/component.yaml'}
-inputs:
-- {name: table, type: CSV}
-- {name: column_name, type: String}
-- {name: predicate, type: String, default: '> 0', optional: true}
-- {name: new_column_name, type: String, optional: true}
-- name: keep_original_column
-  type: Boolean
-  default: "False"
-  optional: true
-outputs:
-- {name: transformed_table, type: CSV}
-implementation:
-  container:
-    image: python:3.9
-    command:
-    - sh
-    - -c
-    - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
-      'pandas==1.4.3' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet
-      --no-warn-script-location 'pandas==1.4.3' --user) && "$0" "$@"
-    - sh
-    - -ec
-    - |
-      program_path=$(mktemp)
-      printf "%s" "$0" > "$program_path"
-      python3 -u "$program_path" "$@"
-    - |
-      def _make_parent_dirs_and_return_path(file_path: str):
-          import os
-          os.makedirs(os.path.dirname(file_path), exist_ok=True)
-          return file_path
-
-      def binarize_column_using_Pandas_on_CSV_data(
-          table_path,
-          transformed_table_path,
-          column_name,
-          predicate = "> 0",
-          new_column_name = None,
-          keep_original_column = False,
-      ):
-          import pandas
-
-          df = pandas.read_csv(table_path).convert_dtypes()
-          original_series = df[column_name]
-
-          # Dynamically executing the predicate code
-          # Variable namespace for code execution
-          namespace = dict(x=original_series)
-          # I though that there should be no space before `predicate` so that "dot" predicate methods like ".between(min, max)" work.
-          # However Python allows spaces before dot: `df .isna()`.
-          # So having a space is not a problem
-          transform_code = f"""new_series_boolean = x {predicate}"""
-          # Note: exec() takes no keyword arguments
-          # exec(__source=transform_code, __globals=namespace)
-          exec(transform_code, namespace)
-          new_series_boolean = namespace["new_series_boolean"]
-
-          # There are multiple ways to convert boolean column to integer.
-          # .apply(int) might be faster. https://stackoverflow.com/a/49804868/1497385
-          # TODO: Do a proper benchmark.
-          new_series = new_series_boolean.apply(int)
-          # new_series = new_series_boolean.astype(int)
-          # new_series = new_series_boolean.replace({False: 0, True: 1})
-
-          if new_column_name:
-              df.insert(loc=0, column=new_column_name, value=new_series)
-              if not keep_original_column:
-                  df = df.drop(columns=[column_name])
-          else:
-              df[column_name] = new_series
-
-          df.to_csv(transformed_table_path, index=False)
-
-      def _deserialize_bool(s) -> bool:
-          from distutils.util import strtobool
-          return strtobool(s) == 1
-
-      import argparse
-      _parser = argparse.ArgumentParser(prog='Binarize column using Pandas on CSV data', description='')
-      _parser.add_argument("--table", dest="table_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--column-name", dest="column_name", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--predicate", dest="predicate", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--new-column-name", dest="new_column_name", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--keep-original-column", dest="keep_original_column", type=_deserialize_bool, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--transformed-table", dest="transformed_table_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
-      _parsed_args = vars(_parser.parse_args())
-
-      _outputs = binarize_column_using_Pandas_on_CSV_data(**_parsed_args)
-    args:
-    - --table
-    - {inputPath: table}
-    - --column-name
-    - {inputValue: column_name}
-    - if:
-        cond: {isPresent: predicate}
-        then:
-        - --predicate
-        - {inputValue: predicate}
-    - if:
-        cond: {isPresent: new_column_name}
-        then:
-        - --new-column-name
-        - {inputValue: new_column_name}
-    - if:
-        cond: {isPresent: keep_original_column}
-        then:
-        - --keep-original-column
-        - {inputValue: keep_original_column}
-    - --transformed-table
-    - {outputPath: transformed_table}

community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml

@@ -1,75 +0,0 @@
-name: Fill all missing values using Pandas on CSV data
-metadata:
-  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml'}
-inputs:
-- {name: table, type: CSV}
-- {name: replacement_value, type: String, default: '0', optional: true}
-- {name: column_names, type: JsonArray, optional: true}
-outputs:
-- {name: transformed_table, type: CSV}
-implementation:
-  container:
-    image: python:3.9
-    command:
-    - sh
-    - -c
-    - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
-      'pandas==1.4.1' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet
-      --no-warn-script-location 'pandas==1.4.1' --user) && "$0" "$@"
-    - sh
-    - -ec
-    - |
-      program_path=$(mktemp)
-      printf "%s" "$0" > "$program_path"
-      python3 -u "$program_path" "$@"
-    - |
-      def _make_parent_dirs_and_return_path(file_path: str):
-          import os
-          os.makedirs(os.path.dirname(file_path), exist_ok=True)
-          return file_path
-
-      def fill_all_missing_values_using_Pandas_on_CSV_data(
-          table_path,
-          transformed_table_path,
-          replacement_value = "0",
-          column_names = None,
-      ):
-          import pandas
-
-          df = pandas.read_csv(
-              table_path,
-              dtype="string",
-          )
-
-          for column_name in column_names or df.columns:
-              df[column_name] = df[column_name].fillna(value=replacement_value)
-
-          df.to_csv(
-              transformed_table_path, index=False,
-          )
-
-      import json
-      import argparse
-      _parser = argparse.ArgumentParser(prog='Fill all missing values using Pandas on CSV data', description='')
-      _parser.add_argument("--table", dest="table_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--replacement-value", dest="replacement_value", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--column-names", dest="column_names", type=json.loads, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--transformed-table", dest="transformed_table_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
-      _parsed_args = vars(_parser.parse_args())
-
-      _outputs = fill_all_missing_values_using_Pandas_on_CSV_data(**_parsed_args)
-    args:
-    - --table
-    - {inputPath: table}
-    - if:
-        cond: {isPresent: replacement_value}
-        then:
-        - --replacement-value
-        - {inputValue: replacement_value}
-    - if:
-        cond: {isPresent: column_names}
-        then:
-        - --column-names
-        - {inputValue: column_names}
-    - --transformed-table
-    - {outputPath: transformed_table}

community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml

@@ -1,59 +0,0 @@
-name: Select columns using Pandas on CSV data
-metadata:
-  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/pandas/Select_columns/in_CSV_format/component.yaml'}
-inputs:
-- {name: table, type: CSV}
-- {name: column_names, type: JsonArray}
-outputs:
-- {name: transformed_table, type: CSV}
-implementation:
-  container:
-    image: python:3.9
-    command:
-    - sh
-    - -c
-    - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
-      'pandas==1.4.2' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet
-      --no-warn-script-location 'pandas==1.4.2' --user) && "$0" "$@"
-    - sh
-    - -ec
-    - |
-      program_path=$(mktemp)
-      printf "%s" "$0" > "$program_path"
-      python3 -u "$program_path" "$@"
-    - |
-      def _make_parent_dirs_and_return_path(file_path: str):
-          import os
-          os.makedirs(os.path.dirname(file_path), exist_ok=True)
-          return file_path
-
-      def select_columns_using_Pandas_on_CSV_data(
-          table_path,
-          transformed_table_path,
-          column_names,
-      ):
-          import pandas
-
-          df = pandas.read_csv(
-              table_path,
-              dtype="string",
-          )
-          df = df[column_names]
-          df.to_csv(transformed_table_path, index=False)
-
-      import json
-      import argparse
-      _parser = argparse.ArgumentParser(prog='Select columns using Pandas on CSV data', description='')
-      _parser.add_argument("--table", dest="table_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--column-names", dest="column_names", type=json.loads, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--transformed-table", dest="transformed_table_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
-      _parsed_args = vars(_parser.parse_args())
-
-      _outputs = select_columns_using_Pandas_on_CSV_data(**_parsed_args)
-    args:
-    - --table
-    - {inputPath: table}
-    - --column-names
-    - {inputValue: column_names}
-    - --transformed-table
-    - {outputPath: transformed_table}

community-content/pipeline_components/tensorflow/Create_fully_connected_network/component.yaml

@@ -1,102 +0,0 @@
-name: Create fully connected tensorflow network
-description: Creates fully-connected network in Tensorflow SavedModel format
-metadata:
-  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/tensorflow/Create_fully_connected_network/component.yaml'}
-inputs:
-- {name: input_size, type: Integer}
-- {name: hidden_layer_sizes, type: JsonArray, default: '[]', optional: true}
-- {name: output_size, type: Integer, default: '1', optional: true}
-- {name: activation_name, type: String, default: relu, optional: true}
-- {name: output_activation_name, type: String, optional: true}
-- {name: random_seed, type: Integer, default: '0', optional: true}
-outputs:
-- {name: model, type: TensorflowSavedModel}
-implementation:
-  container:
-    image: tensorflow/tensorflow:2.7.0
-    command:
-    - sh
-    - -ec
-    - |
-      program_path=$(mktemp)
-      printf "%s" "$0" > "$program_path"
-      python3 -u "$program_path" "$@"
-    - |
-      def _make_parent_dirs_and_return_path(file_path: str):
-          import os
-          os.makedirs(os.path.dirname(file_path), exist_ok=True)
-          return file_path
-
-      def create_fully_connected_tensorflow_network(
-          input_size,
-          model_path,
-          hidden_layer_sizes = [],
-          output_size = 1,
-          activation_name = "relu",
-          output_activation_name = None,
-          random_seed = 0,
-      ):
-          """Creates fully-connected network in Tensorflow SavedModel format"""
-          import tensorflow as tf
-          tf.random.set_seed(seed=random_seed)
-
-          model = tf.keras.models.Sequential()
-          model.add(tf.keras.Input(shape=(input_size,)))
-          for layer_size in hidden_layer_sizes:
-              model.add(tf.keras.layers.Dense(units=layer_size, activation=activation_name))
-          # The last layer is left without activation
-          model.add(tf.keras.layers.Dense(units=output_size, activation=output_activation_name))
-
-          print(model.summary())
-
-          # Using tf.keras.models.save_model instead of tf.saved_model.save to prevent downstream error:
-          #tf.saved_model.save(model, model_path)
-          # ValueError: Unable to create a Keras model from this SavedModel.
-          # This SavedModel was created with `tf.saved_model.save`, and lacks the Keras metadata.
-          # Please save your Keras model by calling `model.save`or `tf.keras.models.save_model`.
-          # See https://github.com/keras-team/keras/issues/16451
-          tf.keras.models.save_model(model, model_path)
-
-      import json
-      import argparse
-      _parser = argparse.ArgumentParser(prog='Create fully connected tensorflow network', description='Creates fully-connected network in Tensorflow SavedModel format')
-      _parser.add_argument("--input-size", dest="input_size", type=int, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--hidden-layer-sizes", dest="hidden_layer_sizes", type=json.loads, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--output-size", dest="output_size", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--activation-name", dest="activation_name", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--output-activation-name", dest="output_activation_name", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--random-seed", dest="random_seed", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--model", dest="model_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
-      _parsed_args = vars(_parser.parse_args())
-
-      _outputs = create_fully_connected_tensorflow_network(**_parsed_args)
-    args:
-    - --input-size
-    - {inputValue: input_size}
-    - if:
-        cond: {isPresent: hidden_layer_sizes}
-        then:
-        - --hidden-layer-sizes
-        - {inputValue: hidden_layer_sizes}
-    - if:
-        cond: {isPresent: output_size}
-        then:
-        - --output-size
-        - {inputValue: output_size}
-    - if:
-        cond: {isPresent: activation_name}
-        then:
-        - --activation-name
-        - {inputValue: activation_name}
-    - if:
-        cond: {isPresent: output_activation_name}
-        then:
-        - --output-activation-name
-        - {inputValue: output_activation_name}
-    - if:
-        cond: {isPresent: random_seed}
-        then:
-        - --random-seed
-        - {inputValue: random_seed}
-    - --model
-    - {outputPath: model}

community-content/pipeline_components/tensorflow/Predict/on_CSV/component.yaml

@@ -1,100 +0,0 @@
-name: Predict with TensorFlow model on CSV data
-metadata:
-  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/tensorflow/Predict/on_CSV/component.yaml'}
-inputs:
-- {name: dataset, type: CSV}
-- {name: model, type: TensorflowSavedModel}
-- {name: label_column_name, type: String, optional: true}
-- {name: batch_size, type: Integer, default: '1000', optional: true}
-outputs:
-- {name: predictions}
-implementation:
-  container:
-    image: tensorflow/tensorflow:2.9.1
-    command:
-    - sh
-    - -ec
-    - |
-      program_path=$(mktemp)
-      printf "%s" "$0" > "$program_path"
-      python3 -u "$program_path" "$@"
-    - |
-      def _make_parent_dirs_and_return_path(file_path: str):
-          import os
-          os.makedirs(os.path.dirname(file_path), exist_ok=True)
-          return file_path
-
-      def predict_with_TensorFlow_model_on_CSV_data(
-          dataset_path,
-          model_path,
-          predictions_path,
-          label_column_name = None,
-          batch_size = 1000,
-      ):
-          import numpy
-          import tensorflow as tf
-
-          model = tf.saved_model.load(export_dir=model_path)
-
-          dataset = tf.data.experimental.make_csv_dataset(
-              file_pattern=dataset_path,
-              batch_size=batch_size,
-              label_name=label_column_name,
-              header=True,
-              num_epochs=1,
-              shuffle=False,
-              ignore_errors=False,
-          )
-
-          def stack_feature_batches(features_batch):
-              # Need to stack individual feature columns to create a single feature tensor
-              # Need to cast all column tensor types to float to prevent errors.
-              list_of_feature_batches = list(
-                  tf.cast(x=feature_batch, dtype=tf.float32)
-                  for feature_batch in features_batch.values()
-              )
-              return tf.stack(list_of_feature_batches, axis=-1)
-
-          def transform_features_and_drop_labels(features_batch, labels_batch):
-              return stack_feature_batches(features_batch)
-
-          dataset_map_fn = (
-              transform_features_and_drop_labels
-              if label_column_name
-              else stack_feature_batches
-          )
-
-          dataset = dataset.map(dataset_map_fn)
-
-          with open(predictions_path, "w") as predictions_file:
-              for features_batch in dataset:
-                  predictions_tensor = model(features_batch)
-                  numpy.savetxt(predictions_file, predictions_tensor.numpy())
-
-      import argparse
-      _parser = argparse.ArgumentParser(prog='Predict with TensorFlow model on CSV data', description='')
-      _parser.add_argument("--dataset", dest="dataset_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--model", dest="model_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--label-column-name", dest="label_column_name", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--batch-size", dest="batch_size", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--predictions", dest="predictions_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
-      _parsed_args = vars(_parser.parse_args())
-
-      _outputs = predict_with_TensorFlow_model_on_CSV_data(**_parsed_args)
-    args:
-    - --dataset
-    - {inputPath: dataset}
-    - --model
-    - {inputPath: model}
-    - if:
-        cond: {isPresent: label_column_name}
-        then:
-        - --label-column-name
-        - {inputValue: label_column_name}
-    - if:
-        cond: {isPresent: batch_size}
-        then:
-        - --batch-size
-        - {inputValue: batch_size}
-    - --predictions
-    - {outputPath: predictions}

community-content/pipeline_components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml

@@ -1,170 +0,0 @@
-name: Train model using Keras on CSV
-metadata:
-  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml'}
-inputs:
-- {name: training_data, type: CSV}
-- {name: model, type: TensorflowSavedModel}
-- {name: label_column_name, type: String}
-- {name: loss_function_name, type: String, default: mean_squared_error, optional: true}
-- {name: number_of_epochs, type: Integer, default: '1', optional: true}
-- {name: learning_rate, type: Float, default: '0.1', optional: true}
-- {name: optimizer_name, type: String, default: Adadelta, optional: true}
-- {name: optimizer_parameters, type: JsonObject, optional: true}
-- {name: batch_size, type: Integer, default: '32', optional: true}
-- {name: metric_names, type: JsonArray, optional: true}
-- {name: random_seed, type: Integer, default: '0', optional: true}
-outputs:
-- {name: trained_model, type: TensorflowSavedModel}
-implementation:
-  container:
-    image: tensorflow/tensorflow:2.8.0
-    command:
-    - sh
-    - -ec
-    - |
-      program_path=$(mktemp)
-      printf "%s" "$0" > "$program_path"
-      python3 -u "$program_path" "$@"
-    - |
-      def _make_parent_dirs_and_return_path(file_path: str):
-          import os
-          os.makedirs(os.path.dirname(file_path), exist_ok=True)
-          return file_path
-
-      def train_model_using_Keras_on_CSV(
-          training_data_path,
-          model_path,
-          trained_model_path,
-          label_column_name,
-          loss_function_name = "mean_squared_error",
-          number_of_epochs = 1,
-          learning_rate = 0.1,
-          optimizer_name = "Adadelta",
-          optimizer_parameters = None,
-          batch_size = 32,
-          metric_names = None,
-          random_seed = 0,
-      ):
-          import tensorflow as tf
-          tf.random.set_seed(seed=random_seed)
-
-          # Loading model using Keras. Model loaded using TensorFlow does not have .fit.
-          #model = tf.saved_model.load(export_dir=model_path)
-          keras_model = tf.keras.models.load_model(filepath=model_path)
-
-          optimizer_parameters = optimizer_parameters or {}
-          optimizer_parameters["learning_rate"] = learning_rate
-          optimizer_config = {
-              "class_name": optimizer_name,
-              "config": optimizer_parameters,
-          }
-          optimizer = tf.keras.optimizers.get(optimizer_config)
-          loss = tf.keras.losses.get(loss_function_name)
-
-          training_dataset = tf.data.experimental.make_csv_dataset(
-              file_pattern=training_data_path,
-              batch_size=batch_size,
-              label_name=label_column_name,
-              header=True,
-              # Need to specify num_epochs=1 otherwise the training becomes infinite
-              num_epochs=1,
-              shuffle=True,
-              shuffle_seed=random_seed,
-              ignore_errors=True,
-          )
-          def stack_feature_batches(features_batch, labels_batch):
-              # Need to stack individual feature columns to create a single feature tensor
-              # Need to cast all column tensor types to float to prevent error:
-              # TypeError: Tensors in list passed to 'values' of 'Pack' Op have types [int32, float32, float32, int32, int32] that don't all match.
-              list_of_feature_batches = list(tf.cast(x=feature_batch, dtype=tf.float32) for feature_batch in features_batch.values())
-              return tf.stack(list_of_feature_batches, axis=-1), labels_batch
-
-          training_dataset = training_dataset.map(stack_feature_batches)
-
-          # Need to compile the model to prevent error:
-          # ValueError: No gradients provided for any variable: [..., ...].
-          keras_model.compile(
-              optimizer=optimizer,
-              loss=loss,
-              metrics=metric_names,
-          )
-          keras_model.fit(
-              training_dataset,
-              epochs=number_of_epochs,
-          )
-
-          # Using tf.keras.models.save_model instead of tf.saved_model.save to prevent downstream error:
-          #tf.saved_model.save(keras_model, trained_model_path)
-          # ValueError: Unable to create a Keras model from this SavedModel.
-          # This SavedModel was created with `tf.saved_model.save`, and lacks the Keras metadata.
-          # Please save your Keras model by calling `model.save`or `tf.keras.models.save_model`.
-          # See https://github.com/keras-team/keras/issues/16451
-          tf.keras.models.save_model(keras_model, trained_model_path)
-
-      import json
-      import argparse
-      _parser = argparse.ArgumentParser(prog='Train model using Keras on CSV', description='')
-      _parser.add_argument("--training-data", dest="training_data_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--model", dest="model_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--label-column-name", dest="label_column_name", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--loss-function-name", dest="loss_function_name", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--number-of-epochs", dest="number_of_epochs", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--learning-rate", dest="learning_rate", type=float, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--optimizer-name", dest="optimizer_name", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--optimizer-parameters", dest="optimizer_parameters", type=json.loads, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--batch-size", dest="batch_size", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--metric-names", dest="metric_names", type=json.loads, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--random-seed", dest="random_seed", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--trained-model", dest="trained_model_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
-      _parsed_args = vars(_parser.parse_args())
-
-      _outputs = train_model_using_Keras_on_CSV(**_parsed_args)
-    args:
-    - --training-data
-    - {inputPath: training_data}
-    - --model
-    - {inputPath: model}
-    - --label-column-name
-    - {inputValue: label_column_name}
-    - if:
-        cond: {isPresent: loss_function_name}
-        then:
-        - --loss-function-name
-        - {inputValue: loss_function_name}
-    - if:
-        cond: {isPresent: number_of_epochs}
-        then:
-        - --number-of-epochs
-        - {inputValue: number_of_epochs}
-    - if:
-        cond: {isPresent: learning_rate}
-        then:
-        - --learning-rate
-        - {inputValue: learning_rate}
-    - if:
-        cond: {isPresent: optimizer_name}
-        then:
-        - --optimizer-name
-        - {inputValue: optimizer_name}
-    - if:
-        cond: {isPresent: optimizer_parameters}
-        then:
-        - --optimizer-parameters
-        - {inputValue: optimizer_parameters}
-    - if:
-        cond: {isPresent: batch_size}
-        then:
-        - --batch-size
-        - {inputValue: batch_size}
-    - if:
-        cond: {isPresent: metric_names}
-        then:
-        - --metric-names
-        - {inputValue: metric_names}
-    - if:
-        cond: {isPresent: random_seed}
-        then:
-        - --random-seed
-        - {inputValue: random_seed}
-    - --trained-model
-    - {outputPath: trained_model}

community-content/pytorch_efficient_training/README.md

@@ -1,33 +0,0 @@
-# PyTorch Efficient Training Examples
-
-This folder provides PyTorch efficient training examples using ResNet-50 and ImageNet data.
-
-## Requirements
-
-```shell
-pip install --upgrade pip
-pip install -r requirements.txt
-```
-
-## Description
-
-* resnet.py - Train ResNet-50 on single GPU.
-* resnet_dp.py - Train ResNet-50 on single node multiple GPUs with `DataParallel` strategy.
-* resnet_ddp.py - Train ResNet-50 on single node multiple GPUs with `DistributedDataParallel` strategy.
-* resnet_ddp_wds.py - Train ResNet-50 on single node multiple GPUs with `DistributedDataParallel` strategy and `Webdataset`.
-* resnet_fsdp.py - Train ResNet-50 on single node multiple GPUs with `FullyShardedDataParallel` strategy.
-* resnet_fsdp_wds.py - Train ResNet-50 on single node multiple GPUs with `FullyShardedDataParallel` strategy and `Webdataset`.
-* shard_imagenet.py - Shard ImagNet individual files into `tar` files.
-
-## Benchmark
-
-When run the benchmark on Nvidia T4 GPUs using ImageNet validation dataset, you can get the result like:
-Strategy               | Seconds/Epoch - Local Data | Seconds/Epoch - Cloud Data
----------------------- | -------------------------- | --------------------------
-On 1 GPU               | 489                        | 804 (2x slower)
-On 4 GPUs (DP)         | 157                        | 738 (5x slower)
-On 4 GPUs (DDP)        | 134                        | 432 (3x slower)
-On 4 GPUs (DDP + WDS)  | 131                        | 133 (same performance)
-On 4 GPUs (FSDP)       | 139                        | 353 (3x slower)
-On 4 GPUs (FSDP + WDS) | 138                        | 135 (same performance)
-

community-content/pytorch_efficient_training/requirements.txt

@@ -1 +0,0 @@
-webdataset == 0.2.26

community-content/pytorch_efficient_training/resnet.py

@@ -1,197 +0,0 @@
-# Copyright 2022 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the \"License\");
-# you may not use this file except in compliance with the License.\n",
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an \"AS IS\" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Train resnet on single GPU."""
-
-import argparse
-import time
-
-from PIL import Image
-import torch
-from torch import nn
-import torchmetrics
-import torchvision
-from torchvision.models import resnet50
-
-
-class ImageFolder(torchvision.datasets.ImageFolder):
-  """Class for loading imagenet."""
-
-  def __init__(self, image_list_file, transform=None, target_transform=None):
-    self.samples = self._make_dataset(image_list_file)
-    self.loader = self._loader
-
-    self.imgs = self.samples
-    self.targets = [s[1] for s in self.samples]
-
-    self.transform = transform
-    self.target_transform = target_transform
-
-  def _make_dataset(self, image_list_file):
-    items = []
-    with open(image_list_file, 'r') as f:
-      for line in f:
-        item = line.strip().split(' ')
-        items.append((item[0], int(item[1])))
-    return items
-
-  def _loader(self, image_path):
-    with open(image_path, 'rb') as f:
-      img = Image.open(f)
-      img = img.convert('RGB')
-      return img
-
-
-def train(model, device, dataloader, optimizer):
-  model.train()
-  for image, target in dataloader:
-    image, target = image.to(device), target.to(device)
-    pred = model(image)
-    # pred.shape (N, C), target.shape (N)
-    loss = nn.functional.cross_entropy(pred, target)
-    optimizer.zero_grad()
-    loss.backward()
-    optimizer.step()
-  return loss
-
-
-def evaluate(model, device, dataloader, metric):
-  model.eval()
-  with torch.no_grad():
-    for image, target in dataloader:
-      image, target = image.to(device), target.to(device)
-      pred = model(image)
-      metric.update(pred, target)
-  accuracy = metric.compute()
-  metric.reset()
-  return accuracy
-
-
-def run_training(args):
-  """Run training and evaluation."""
-  # Create model.
-  model = resnet50(weights=None)
-  model = model.to(args.device)
-
-  # Create train dataloader.
-  train_dataset = ImageFolder(
-      image_list_file=args.train_data_path,
-      transform=torchvision.transforms.Compose([
-          torchvision.transforms.RandomResizedCrop(224),
-          torchvision.transforms.RandomHorizontalFlip(),
-          torchvision.transforms.ToTensor(),
-          torchvision.transforms.Normalize(
-              mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
-      ]))
-  train_dataloader = torch.utils.data.DataLoader(
-      dataset=train_dataset,
-      batch_size=args.train_batch_size,
-      shuffle=True,
-      num_workers=args.dataloader_num_workers,
-      pin_memory=True)
-  print(f'Train dataloader | samples: {len(train_dataloader.dataset)}, '
-        f'num workers: {train_dataloader.num_workers}, '
-        f'batch size: {args.train_batch_size}, '
-        f'batches/epoch: {len(train_dataloader)}')
-
-  # Create eval dataloader.
-  eval_dataset = ImageFolder(
-      image_list_file=args.eval_data_path,
-      transform=torchvision.transforms.Compose([
-          torchvision.transforms.Resize(256),
-          torchvision.transforms.CenterCrop(224),
-          torchvision.transforms.ToTensor(),
-          torchvision.transforms.Normalize(
-              mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
-      ]))
-  eval_dataloader = torch.utils.data.DataLoader(
-      dataset=eval_dataset,
-      batch_size=args.eval_batch_size,
-      shuffle=False,
-      num_workers=args.dataloader_num_workers,
-      pin_memory=True,
-      drop_last=True)
-  print(f'Eval dataloader | samples: {len(eval_dataloader.dataset)}, '
-        f'num workers: {eval_dataloader.num_workers}, '
-        f'batch size: {args.eval_batch_size}, '
-        f'batches/epoch: {len(eval_dataloader)}')
-
-  # Optimizer.
-  optimizer = torch.optim.SGD(model.parameters(), 0.1)
-
-  # Main loop.
-  metric = torchmetrics.classification.Accuracy(top_k=1).to(args.device)
-  for epoch in range(1, args.epochs + 1):
-    print(f'Running epoch {epoch}')
-
-    start = time.time()
-    train(model, args.device, train_dataloader, optimizer)
-    end = time.time()
-    print(f'Training finished in {(end - start):>0.3f} seconds')
-
-    start = time.time()
-    evaluate(model, args.device, eval_dataloader, metric)
-    end = time.time()
-    print(f'Evaluation finished in {(end - start):>0.3f} seconds')
-  print('Done')
-
-
-def create_args():
-  """Create main args."""
-  parser = argparse.ArgumentParser(
-      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-  parser.add_argument(
-      '--epochs',
-      default=1,
-      type=int,
-      help='number of total epochs to run')
-  parser.add_argument(
-      '--dataloader_num_workers',
-      default=2,
-      type=int,
-      help='number of workders for dataloader')
-  parser.add_argument(
-      '--train_data_path',
-      default='',
-      type=str,
-      help='path to training data')
-  parser.add_argument(
-      '--train_batch_size',
-      default=32,
-      type=int,
-      help='batch size for training')
-  parser.add_argument(
-      '--eval_data_path',
-      default='',
-      type=str,
-      help='path to evaluation data')
-  parser.add_argument(
-      '--eval_batch_size',
-      default=32,
-      type=int,
-      help='batch size for evaluation')
-  args = parser.parse_args()
-  return args
-
-
-def main():
-  args = create_args()
-  args.device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
-
-  print('Launch job on 1 GPU')
-  run_training(args)
-
-
-if __name__ == '__main__':
-  main()

community-content/pytorch_efficient_training/resnet_ddp.py

@@ -1,234 +0,0 @@
-# Copyright 2022 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the \"License\");
-# you may not use this file except in compliance with the License.\n",
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an \"AS IS\" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Train resnet on multiple GPUs with DDP."""
-
-import argparse
-import os
-import time
-
-from PIL import Image
-import torch
-from torch import nn
-import torch.distributed as dist
-import torch.multiprocessing as mp
-import torchmetrics
-import torchvision
-from torchvision.models import resnet50
-
-
-class ImageFolder(torchvision.datasets.ImageFolder):
-  """Class for loading imagenet."""
-
-  def __init__(self, image_list_file, transform=None, target_transform=None):
-    self.samples = self._make_dataset(image_list_file)
-    self.loader = self._loader
-
-    self.imgs = self.samples
-    self.targets = [s[1] for s in self.samples]
-
-    self.transform = transform
-    self.target_transform = target_transform
-
-  def _make_dataset(self, image_list_file):
-    items = []
-    with open(image_list_file, 'r') as f:
-      for line in f:
-        item = line.strip().split(' ')
-        items.append((item[0], int(item[1])))
-    return items
-
-  def _loader(self, image_path):
-    with open(image_path, 'rb') as f:
-      img = Image.open(f)
-      img = img.convert('RGB')
-      return img
-
-
-def train(model, device, dataloader, optimizer):
-  model.train()
-  for image, target in dataloader:
-    image = image.to(device, non_blocking=True)
-    target = target.to(device, non_blocking=True)
-    pred = model(image)
-    # pred.shape (N, C), target.shape (N)
-    loss = nn.functional.cross_entropy(pred, target)
-    optimizer.zero_grad()
-    loss.backward()
-    optimizer.step()
-  return loss
-
-
-def evaluate(model, device, dataloader, metric):
-  model.eval()
-  with torch.no_grad():
-    for image, target in dataloader:
-      image = image.to(device, non_blocking=True)
-      target = target.to(device, non_blocking=True)
-      pred = model(image)
-      metric.update(pred, target)
-  accuracy = metric.compute()
-  metric.reset()
-  return accuracy
-
-
-def worker(gpu, args):
-  """Run training and evaluation."""
-  # Init process group.
-  print(f'Initiating process {gpu}')
-  dist.init_process_group(
-      backend='nccl',
-      init_method='env://',
-      world_size=args.gpus,
-      rank=gpu)
-
-  # Create model.
-  model = resnet50(weights=None)
-  torch.cuda.set_device(gpu)
-  model.to(args.device)
-  model = torch.nn.SyncBatchNorm.convert_sync_batchnorm(model)
-  model = nn.parallel.DistributedDataParallel(model, device_ids=[gpu])
-
-  # Create train dataloader.
-  train_dataset = ImageFolder(
-      image_list_file=args.train_data_path,
-      transform=torchvision.transforms.Compose([
-          torchvision.transforms.RandomResizedCrop(224),
-          torchvision.transforms.RandomHorizontalFlip(),
-          torchvision.transforms.ToTensor(),
-          torchvision.transforms.Normalize(
-              mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
-      ]))
-  train_sampler = torch.utils.data.distributed.DistributedSampler(
-      train_dataset, num_replicas=args.gpus, rank=gpu)
-  train_dataloader = torch.utils.data.DataLoader(
-      dataset=train_dataset,
-      batch_size=args.train_batch_size,
-      shuffle=False,
-      num_workers=args.dataloader_num_workers,
-      pin_memory=True,
-      sampler=train_sampler)
-  if gpu == 0:
-    print(f'Train dataloader | samples: {len(train_dataloader.dataset)}, '
-          f'num workers: {train_dataloader.num_workers}, '
-          f'global batch size: {args.train_batch_size * args.gpus}, '
-          f'batches/epoch: {len(train_dataloader)}')
-
-  # Create eval dataloader.
-  eval_dataset = ImageFolder(
-      image_list_file=args.eval_data_path,
-      transform=torchvision.transforms.Compose([
-          torchvision.transforms.Resize(256),
-          torchvision.transforms.CenterCrop(224),
-          torchvision.transforms.ToTensor(),
-          torchvision.transforms.Normalize(
-              mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
-      ]))
-  eval_sampler = torch.utils.data.distributed.DistributedSampler(
-      eval_dataset, num_replicas=args.gpus, rank=gpu)
-  eval_dataloader = torch.utils.data.DataLoader(
-      dataset=eval_dataset,
-      batch_size=args.eval_batch_size,
-      shuffle=False,
-      num_workers=args.dataloader_num_workers,
-      pin_memory=True,
-      drop_last=True,
-      sampler=eval_sampler)
-  if gpu == 0:
-    print(f'Eval dataloader | samples: {len(eval_dataloader.dataset)}, '
-          f'num workers: {eval_dataloader.num_workers}, '
-          f'batch size: {args.eval_batch_size}, '
-          f'batches/epoch: {len(eval_dataloader)}')
-
-  # Optimizer.
-  optimizer = torch.optim.SGD(model.parameters(), 0.1)
-
-  # Main loop.
-  metric = torchmetrics.classification.Accuracy(top_k=1).to(args.device)
-  for epoch in range(1, args.epochs + 1):
-    if gpu == 0:
-      print(f'Running epoch {epoch}')
-    train_sampler.set_epoch(epoch)
-
-    start = time.time()
-    train(model, args.device, train_dataloader, optimizer)
-    end = time.time()
-    if gpu == 0:
-      print(f'Training finished in {(end - start):>0.3f} seconds')
-
-    start = time.time()
-    evaluate(model, args.device, eval_dataloader, metric)
-    end = time.time()
-    if gpu == 0:
-      print(f'Evaluation finished in {(end - start):>0.3f} seconds')
-
-  if gpu == 0:
-    print('Done')
-
-
-def create_args():
-  """Create main args."""
-  parser = argparse.ArgumentParser(
-      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-  parser.add_argument(
-      '--gpus',
-      default=4,
-      type=int,
-      help='number of gpus to use')
-  parser.add_argument(
-      '--epochs',
-      default=1,
-      type=int,
-      help='number of total epochs to run')
-  parser.add_argument(
-      '--dataloader_num_workers',
-      default=2,
-      type=int,
-      help='number of workders for dataloader')
-  parser.add_argument(
-      '--train_data_path',
-      default='',
-      type=str,
-      help='path to training data')
-  parser.add_argument(
-      '--train_batch_size',
-      default=32,
-      type=int,
-      help='batch size for training per gpu')
-  parser.add_argument(
-      '--eval_data_path',
-      default='',
-      type=str,
-      help='path to evaluation data')
-  parser.add_argument(
-      '--eval_batch_size',
-      default=32,
-      type=int,
-      help='batch size for evaluation per gpu')
-  args = parser.parse_args()
-  return args
-
-
-def main():
-  args = create_args()
-  os.environ['MASTER_ADDR'] = 'localhost'
-  os.environ['MASTER_PORT'] = '8888'
-
-  args.device = 'cuda' if torch.cuda.is_available() else 'cpu'
-  print(f'Launch job on {args.gpus} GPUs with DDP')
-  mp.spawn(worker, nprocs=args.gpus, args=(args,))
-
-
-if __name__ == '__main__':
-  main()

community-content/pytorch_efficient_training/resnet_ddp_wds.py

@@ -1,249 +0,0 @@
-# Copyright 2022 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the \"License\");
-# you may not use this file except in compliance with the License.\n",
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an \"AS IS\" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Train resnet on multiple GPUs with DDP."""
-
-import argparse
-import functools
-import itertools
-import math
-import os
-import time
-
-import torch
-from torch import nn
-import torch.distributed as dist
-import torch.multiprocessing as mp
-import torchmetrics
-from torchvision.models import resnet50
-from torchvision.transforms import transforms
-import webdataset as wds
-
-
-def wds_split(src, rank, world_size):
-  """Shards split function for webdataset."""
-  # The context of caller of this function is within multiple processes
-  # (by DDP world_size) and multiple workers (by dataloader_num_workers).
-  # So we totally have (world_size * num_workers) workers for processing data.
-  # NOTE: Raw data should be sharded to enough shards to make sure one process
-  # can handle at least one shard, otherwise the process may hang.
-  worker_id = 0
-  num_workers = 1
-  worker_info = torch.utils.data.get_worker_info()
-  if worker_info:
-    worker_id = worker_info.id
-    num_workers = worker_info.num_workers
-  for s in itertools.islice(src, rank * num_workers + worker_id, None,
-                            world_size * num_workers):
-    yield s
-
-
-def identity(x):
-  return x
-
-
-def create_wds_dataloader(rank, args, mode):
-  """Create webdataset dataset and dataloader."""
-  if mode == 'train':
-    transform = transforms.Compose([
-        transforms.RandomResizedCrop(224),
-        transforms.RandomHorizontalFlip(),
-        transforms.ToTensor(),
-        transforms.Normalize(
-            mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
-    ])
-    data_path = args.train_data_path
-    data_size = args.train_data_size
-    batch_size_local = args.train_batch_size
-    batch_size_global = args.train_batch_size * args.gpus
-    # Since webdataset disallows partial batch, we pad the last batch for train.
-    batches = int(math.ceil(data_size / batch_size_global))
-  else:
-    transform = transforms.Compose([
-        transforms.Resize(256),
-        transforms.CenterCrop(224),
-        transforms.ToTensor(),
-        transforms.Normalize(
-            mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
-    ])
-    data_path = args.eval_data_path
-    data_size = args.eval_data_size
-    batch_size_local = args.eval_batch_size
-    batch_size_global = args.eval_batch_size * args.gpus
-    # Since webdataset disallows partial batch, we drop the last batch for eval.
-    batches = int(data_size / batch_size_global)
-
-  dataset = wds.DataPipeline(
-      wds.SimpleShardList(data_path),
-      functools.partial(wds_split, rank=rank, world_size=args.gpus),
-      wds.tarfile_to_samples(),
-      wds.decode('pil'),
-      wds.to_tuple('jpg;png;jpeg cls'),
-      wds.map_tuple(transform, identity),
-      wds.batched(batch_size_local, partial=False),
-  )
-  num_workers = args.dataloader_num_workers
-  dataloader = wds.WebLoader(
-      dataset=dataset,
-      batch_size=None,
-      shuffle=False,
-      num_workers=num_workers,
-      persistent_workers=True if num_workers > 0 else False,
-      pin_memory=True).repeat(nbatches=batches)
-  print(f'{mode} dataloader | samples: {data_size}, '
-        f'num_workers: {num_workers}, '
-        f'local batch size: {batch_size_local}, '
-        f'global batch size: {batch_size_global}, '
-        f'batches: {batches}')
-  return dataloader
-
-
-def train(model, device, dataloader, optimizer):
-  model.train()
-  for image, target in dataloader:
-    image = image.to(device, non_blocking=True)
-    target = target.to(device, non_blocking=True)
-    pred = model(image)
-    # pred.shape (N, C), target.shape (N)
-    loss = nn.functional.cross_entropy(pred, target)
-    optimizer.zero_grad()
-    loss.backward()
-    optimizer.step()
-  return loss
-
-
-def evaluate(model, device, dataloader, metric):
-  model.eval()
-  with torch.no_grad():
-    for image, target in dataloader:
-      image = image.to(device, non_blocking=True)
-      target = target.to(device, non_blocking=True)
-      pred = model(image)
-      metric.update(pred, target)
-  accuracy = metric.compute()
-  metric.reset()
-  return accuracy
-
-
-def worker(gpu, args):
-  """Run training and evaluation."""
-  # Init process group.
-  print(f'Initiating process {gpu}')
-  dist.init_process_group(
-      backend='nccl',
-      init_method='env://',
-      world_size=args.gpus,
-      rank=gpu)
-
-  # Create model.
-  model = resnet50(weights=None)
-  torch.cuda.set_device(gpu)
-  model.to(args.device)
-  model = torch.nn.SyncBatchNorm.convert_sync_batchnorm(model)
-  model = nn.parallel.DistributedDataParallel(model, device_ids=[gpu])
-
-  # Create dataloader.
-  train_dataloader = create_wds_dataloader(gpu, args, 'train')
-  eval_dataloader = create_wds_dataloader(gpu, args, 'eval')
-
-  # Optimizer.
-  optimizer = torch.optim.SGD(model.parameters(), 0.1)
-
-  # Main loop.
-  metric = torchmetrics.classification.Accuracy(top_k=1).to(args.device)
-  for epoch in range(1, args.epochs + 1):
-    if gpu == 0:
-      print(f'Running epoch {epoch}')
-
-    start = time.time()
-    train(model, args.device, train_dataloader, optimizer)
-    end = time.time()
-    if gpu == 0:
-      print(f'Training finished in {(end - start):>0.3f} seconds')
-
-    start = time.time()
-    evaluate(model, args.device, eval_dataloader, metric)
-    end = time.time()
-    if gpu == 0:
-      print(f'Evaluation finished in {(end - start):>0.3f} seconds')
-
-  if gpu == 0:
-    print('Done')
-
-
-def create_args():
-  """Create main args."""
-  parser = argparse.ArgumentParser(
-      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-  parser.add_argument(
-      '--gpus',
-      default=4,
-      type=int,
-      help='number of gpus to use')
-  parser.add_argument(
-      '--epochs',
-      default=1,
-      type=int,
-      help='number of total epochs to run')
-  parser.add_argument(
-      '--dataloader_num_workers',
-      default=2,
-      type=int,
-      help='number of workders for dataloader')
-  parser.add_argument(
-      '--train_data_path',
-      default='',
-      type=str,
-      help='path to training data')
-  parser.add_argument(
-      '--train_batch_size',
-      default=32,
-      type=int,
-      help='batch size for training per gpu')
-  parser.add_argument(
-      '--train_data_size',
-      default=50000,
-      type=int,
-      help='data size for training')
-  parser.add_argument(
-      '--eval_data_path',
-      default='',
-      type=str,
-      help='path to evaluation data')
-  parser.add_argument(
-      '--eval_batch_size',
-      default=32,
-      type=int,
-      help='batch size for evaluation per gpu')
-  parser.add_argument(
-      '--eval_data_size',
-      default=50000,
-      type=int,
-      help='data size for evaluation')
-  args = parser.parse_args()
-  return args
-
-
-def main():
-  args = create_args()
-  os.environ['MASTER_ADDR'] = 'localhost'
-  os.environ['MASTER_PORT'] = '8888'
-
-  args.device = 'cuda' if torch.cuda.is_available() else 'cpu'
-  print(f'Launch job on {args.gpus} GPUs with DDP')
-  mp.spawn(worker, nprocs=args.gpus, args=(args,))
-
-
-if __name__ == '__main__':
-  main()

community-content/pytorch_efficient_training/resnet_dp.py

@@ -1,207 +0,0 @@
-# Copyright 2022 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the \"License\");
-# you may not use this file except in compliance with the License.\n",
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an \"AS IS\" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Train resnet on multiple GPUs with DP."""
-
-import argparse
-import time
-
-from PIL import Image
-import torch
-from torch import nn
-import torchmetrics
-import torchvision
-from torchvision.models import resnet50
-
-
-class ImageFolder(torchvision.datasets.ImageFolder):
-  """Class for loading imagenet."""
-
-  def __init__(self, image_list_file, transform=None, target_transform=None):
-    self.samples = self._make_dataset(image_list_file)
-    self.loader = self._loader
-
-    self.imgs = self.samples
-    self.targets = [s[1] for s in self.samples]
-
-    self.transform = transform
-    self.target_transform = target_transform
-
-  def _make_dataset(self, image_list_file):
-    items = []
-    with open(image_list_file, 'r') as f:
-      for line in f:
-        item = line.strip().split(' ')
-        items.append((item[0], int(item[1])))
-    return items
-
-  def _loader(self, image_path):
-    with open(image_path, 'rb') as f:
-      img = Image.open(f)
-      img = img.convert('RGB')
-      return img
-
-
-def train(model, device, dataloader, optimizer):
-  model.train()
-  for image, target in dataloader:
-    image, target = image.to(device), target.to(device)
-    pred = model(image)
-    # pred.shape (N, C), target.shape (N)
-    loss = nn.functional.cross_entropy(pred, target)
-    optimizer.zero_grad()
-    loss.backward()
-    optimizer.step()
-  return loss
-
-
-def evaluate(model, device, dataloader, metric):
-  model.eval()
-  with torch.no_grad():
-    for image, target in dataloader:
-      image, target = image.to(device), target.to(device)
-      pred = model(image)
-      metric.update(pred, target)
-  accuracy = metric.compute()
-  metric.reset()
-  return accuracy
-
-
-def run_training(args):
-  """Run training and evaluation."""
-  # Create model.
-  model = resnet50(weights=None)
-  model = nn.DataParallel(model)
-  model = model.to(args.device)
-
-  # Create train dataloader.
-  train_dataset = ImageFolder(
-      image_list_file=args.train_data_path,
-      transform=torchvision.transforms.Compose([
-          torchvision.transforms.RandomResizedCrop(224),
-          torchvision.transforms.RandomHorizontalFlip(),
-          torchvision.transforms.ToTensor(),
-          torchvision.transforms.Normalize(
-              mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
-      ]))
-  train_dataloader = torch.utils.data.DataLoader(
-      dataset=train_dataset,
-      batch_size=args.train_batch_size,
-      shuffle=True,
-      num_workers=args.dataloader_num_workers,
-      pin_memory=True)
-  print(f'Train dataloader | samples: {len(train_dataloader.dataset)}, '
-        f'num workers: {train_dataloader.num_workers}, '
-        f'global batch size: {args.train_batch_size}, '
-        f'batches/epoch: {len(train_dataloader)}')
-
-  # Create eval dataloader.
-  eval_dataset = ImageFolder(
-      image_list_file=args.eval_data_path,
-      transform=torchvision.transforms.Compose([
-          torchvision.transforms.Resize(256),
-          torchvision.transforms.CenterCrop(224),
-          torchvision.transforms.ToTensor(),
-          torchvision.transforms.Normalize(
-              mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
-      ]))
-  eval_dataloader = torch.utils.data.DataLoader(
-      dataset=eval_dataset,
-      batch_size=args.eval_batch_size,
-      shuffle=False,
-      num_workers=args.dataloader_num_workers,
-      pin_memory=True,
-      drop_last=True)
-  print(f'Eval dataloader | samples: {len(eval_dataloader.dataset)}, '
-        f'num workers: {eval_dataloader.num_workers}, '
-        f'global batch size: {args.eval_batch_size}, '
-        f'batches/epoch: {len(eval_dataloader)}')
-
-  # Optimizer.
-  optimizer = torch.optim.SGD(model.parameters(), 0.1)
-
-  # Main loop.
-  metric = torchmetrics.classification.Accuracy(top_k=1).to(args.device)
-  for epoch in range(1, args.epochs + 1):
-    print(f'Running epoch {epoch}')
-
-    start = time.time()
-    train(model, args.device, train_dataloader, optimizer)
-    end = time.time()
-    print(f'Training finished in {(end - start):>0.3f} seconds')
-
-    start = time.time()
-    evaluate(model, args.device, eval_dataloader, metric)
-    end = time.time()
-    print(f'Evaluation finished in {(end - start):>0.3f} seconds')
-  print('Done')
-
-
-def create_args():
-  """Create main args."""
-  parser = argparse.ArgumentParser(
-      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-  parser.add_argument(
-      '--gpus',
-      default=4,
-      type=int,
-      help='number of gpus to use')
-  parser.add_argument(
-      '--epochs',
-      default=1,
-      type=int,
-      help='number of total epochs to run')
-  parser.add_argument(
-      '--dataloader_num_workers',
-      default=2,
-      type=int,
-      help='number of workders for dataloader')
-  parser.add_argument(
-      '--train_data_path',
-      default='',
-      type=str,
-      help='path to training data')
-  parser.add_argument(
-      '--train_batch_size',
-      default=32,
-      type=int,
-      help='batch size for training per gpu')
-  parser.add_argument(
-      '--eval_data_path',
-      default='',
-      type=str,
-      help='path to evaluation data')
-  parser.add_argument(
-      '--eval_batch_size',
-      default=32,
-      type=int,
-      help='batch size for evaluation per gpu')
-  args = parser.parse_args()
-  return args
-
-
-def main():
-  args = create_args()
-
-  args.device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
-  args.train_batch_size *= args.gpus
-  args.eval_batch_size *= args.gpus
-  args.dataloader_num_workers *= args.gpus
-
-  print(f'Launch job on {args.gpus} GPU with nn.DataParallel')
-  run_training(args)
-
-
-if __name__ == '__main__':
-  main()

community-content/pytorch_efficient_training/resnet_fsdp.py

@@ -1,242 +0,0 @@
-# Copyright 2022 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the \"License\");
-# you may not use this file except in compliance with the License.\n",
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an \"AS IS\" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Train resnet on multiple GPUs with FSDP."""
-
-import argparse
-import functools
-import os
-import time
-
-from PIL import Image
-import torch
-from torch import nn
-import torch.distributed as dist
-from torch.distributed.fsdp import FullyShardedDataParallel as FSDP
-from torch.distributed.fsdp.wrap import size_based_auto_wrap_policy
-import torch.multiprocessing as mp
-import torchmetrics
-import torchvision
-from torchvision.models import resnet50
-
-
-class ImageFolder(torchvision.datasets.ImageFolder):
-  """Class for loading imagenet."""
-
-  def __init__(self, image_list_file, transform=None, target_transform=None):
-    self.samples = self._make_dataset(image_list_file)
-    self.loader = self._loader
-
-    self.imgs = self.samples
-    self.targets = [s[1] for s in self.samples]
-
-    self.transform = transform
-    self.target_transform = target_transform
-
-  def _make_dataset(self, image_list_file):
-    items = []
-    with open(image_list_file, 'r') as f:
-      for line in f:
-        item = line.strip().split(' ')
-        items.append((item[0], int(item[1])))
-    return items
-
-  def _loader(self, image_path):
-    with open(image_path, 'rb') as f:
-      img = Image.open(f)
-      img = img.convert('RGB')
-      return img
-
-
-def train(model, device, dataloader, optimizer):
-  model.train()
-  for image, target in dataloader:
-    image = image.to(device, non_blocking=True)
-    target = target.to(device, non_blocking=True)
-    pred = model(image)
-    # pred.shape (N, C), target.shape (N)
-    loss = nn.functional.cross_entropy(pred, target)
-    optimizer.zero_grad()
-    loss.backward()
-    optimizer.step()
-  return loss
-
-
-def evaluate(model, device, dataloader, metric):
-  model.eval()
-  with torch.no_grad():
-    for image, target in dataloader:
-      image = image.to(device, non_blocking=True)
-      target = target.to(device, non_blocking=True)
-      pred = model(image)
-      metric.update(pred, target)
-  accuracy = metric.compute()
-  metric.reset()
-  return accuracy
-
-
-def worker(gpu, args):
-  """Run training and evaluation."""
-  # Init process group.
-  print(f'Initiating process {gpu}')
-  dist.init_process_group(
-      backend='nccl',
-      init_method='env://',
-      world_size=args.gpus,
-      rank=gpu)
-
-  # Create train dataloader.
-  train_dataset = ImageFolder(
-      image_list_file=args.train_data_path,
-      transform=torchvision.transforms.Compose([
-          torchvision.transforms.RandomResizedCrop(224),
-          torchvision.transforms.RandomHorizontalFlip(),
-          torchvision.transforms.ToTensor(),
-          torchvision.transforms.Normalize(
-              mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
-      ]))
-  train_sampler = torch.utils.data.distributed.DistributedSampler(
-      train_dataset, num_replicas=args.gpus, rank=gpu)
-  train_dataloader = torch.utils.data.DataLoader(
-      dataset=train_dataset,
-      batch_size=args.train_batch_size,
-      shuffle=False,
-      num_workers=args.dataloader_num_workers,
-      pin_memory=True,
-      sampler=train_sampler)
-  if gpu == 0:
-    print(f'Train dataloader | samples: {len(train_dataloader.dataset)}, '
-          f'num workers: {train_dataloader.num_workers}, '
-          f'global batch size: {args.train_batch_size * args.gpus}, '
-          f'batches/epoch: {len(train_dataloader)}')
-
-  # Create eval dataloader.
-  eval_dataset = ImageFolder(
-      image_list_file=args.eval_data_path,
-      transform=torchvision.transforms.Compose([
-          torchvision.transforms.Resize(256),
-          torchvision.transforms.CenterCrop(224),
-          torchvision.transforms.ToTensor(),
-          torchvision.transforms.Normalize(
-              mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
-      ]))
-  eval_sampler = torch.utils.data.distributed.DistributedSampler(
-      eval_dataset, num_replicas=args.gpus, rank=gpu)
-  eval_dataloader = torch.utils.data.DataLoader(
-      dataset=eval_dataset,
-      batch_size=args.eval_batch_size,
-      shuffle=False,
-      num_workers=args.dataloader_num_workers,
-      pin_memory=True,
-      drop_last=True,
-      sampler=eval_sampler)
-  if gpu == 0:
-    print(f'Eval dataloader | samples: {len(eval_dataloader.dataset)}, '
-          f'num workers: {eval_dataloader.num_workers}, '
-          f'batch size: {args.eval_batch_size}, '
-          f'batches/epoch: {len(eval_dataloader)}')
-
-  # Wrap policy.
-  my_auto_wrap_policy = functools.partial(
-      size_based_auto_wrap_policy, min_num_params=100)
-  torch.cuda.set_device(gpu)
-
-  # Create model.
-  model = resnet50(weights=None)
-  model.to(args.device)
-  model = FSDP(model, auto_wrap_policy=my_auto_wrap_policy)
-
-  # Optimizer.
-  optimizer = torch.optim.SGD(model.parameters(), 0.1)
-
-  # Main loop.
-  metric = torchmetrics.classification.Accuracy(top_k=1).to(args.device)
-  for epoch in range(1, args.epochs + 1):
-    if gpu == 0:
-      print(f'Running epoch {epoch}')
-    train_sampler.set_epoch(epoch)
-
-    start = time.time()
-    train(model, args.device, train_dataloader, optimizer)
-    end = time.time()
-    if gpu == 0:
-      print(f'Training finished in {(end - start):>0.3f} seconds')
-
-    start = time.time()
-    evaluate(model, args.device, eval_dataloader, metric)
-    end = time.time()
-    if gpu == 0:
-      print(f'Evaluation finished in {(end - start):>0.3f} seconds')
-
-  if gpu == 0:
-    print('Done')
-  dist.destroy_process_group()
-
-
-def create_args():
-  """Create main args."""
-  parser = argparse.ArgumentParser(
-      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-  parser.add_argument(
-      '--gpus',
-      default=4,
-      type=int,
-      help='number of gpus to use')
-  parser.add_argument(
-      '--epochs',
-      default=2,
-      type=int,
-      help='number of total epochs to run')
-  parser.add_argument(
-      '--dataloader_num_workers',
-      default=2,
-      type=int,
-      help='number of workders for dataloader')
-  parser.add_argument(
-      '--train_data_path',
-      default='',
-      type=str,
-      help='path to training data')
-  parser.add_argument(
-      '--train_batch_size',
-      default=32,
-      type=int,
-      help='batch size for training per gpu')
-  parser.add_argument(
-      '--eval_data_path',
-      default='',
-      type=str,
-      help='path to evaluation data')
-  parser.add_argument(
-      '--eval_batch_size',
-      default=32,
-      type=int,
-      help='batch size for evaluation per gpu')
-  args = parser.parse_args()
-  return args
-
-
-def main():
-  args = create_args()
-
-  os.environ['MASTER_ADDR'] = 'localhost'
-  os.environ['MASTER_PORT'] = '8888'
-
-  args.device = 'cuda' if torch.cuda.is_available() else 'cpu'
-  print(f'Launch job on {args.gpus} GPUs with FSDP')
-  mp.spawn(worker, nprocs=args.gpus, args=(args,))
-
-
-if __name__ == '__main__':
-  main()

community-content/pytorch_efficient_training/resnet_fsdp_wds.py

@@ -1,240 +0,0 @@
-"""Train resnet on multiple GPUs with DDP."""
-
-import argparse
-import functools
-import itertools
-import math
-import os
-import time
-
-import torch
-from torch import nn
-import torch.distributed as dist
-from torch.distributed.fsdp import FullyShardedDataParallel as FSDP
-from torch.distributed.fsdp.wrap import size_based_auto_wrap_policy
-import torch.multiprocessing as mp
-import torchmetrics
-from torchvision.models import resnet50
-from torchvision.transforms import transforms
-import webdataset as wds
-
-
-def wds_split(src, rank, world_size):
-  """Shards split function for webdataset."""
-  # The context of caller of this function is within multiple processes
-  # (by DDP world_size) and multiple workers (by dataloader_num_workers).
-  # So we totally have (world_size * num_workers) workers for processing data.
-  # NOTE: Raw data should be sharded to enough shards to make sure one process
-  # can handle at least one shard, otherwise the process may hang.
-  worker_id = 0
-  num_workers = 1
-  worker_info = torch.utils.data.get_worker_info()
-  if worker_info:
-    worker_id = worker_info.id
-    num_workers = worker_info.num_workers
-  for s in itertools.islice(src, rank * num_workers + worker_id, None,
-                            world_size * num_workers):
-    yield s
-
-
-def identity(x):
-  return x
-
-
-def create_wds_dataloader(rank, args, mode):
-  """Create webdataset dataset and dataloader."""
-  if mode == 'train':
-    transform = transforms.Compose([
-        transforms.RandomResizedCrop(224),
-        transforms.RandomHorizontalFlip(),
-        transforms.ToTensor(),
-        transforms.Normalize(
-            mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
-    ])
-    data_path = args.train_data_path
-    data_size = args.train_data_size
-    batch_size_local = args.train_batch_size
-    batch_size_global = args.train_batch_size * args.gpus
-    # Since webdataset disallows partial batch, we pad the last batch for train.
-    batches = int(math.ceil(data_size / batch_size_global))
-  else:
-    transform = transforms.Compose([
-        transforms.Resize(256),
-        transforms.CenterCrop(224),
-        transforms.ToTensor(),
-        transforms.Normalize(
-            mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
-    ])
-    data_path = args.eval_data_path
-    data_size = args.eval_data_size
-    batch_size_local = args.eval_batch_size
-    batch_size_global = args.eval_batch_size * args.gpus
-    # Since webdataset disallows partial batch, we drop the last batch for eval.
-    batches = int(data_size / batch_size_global)
-
-  dataset = wds.DataPipeline(
-      wds.SimpleShardList(data_path),
-      functools.partial(wds_split, rank=rank, world_size=args.gpus),
-      wds.tarfile_to_samples(),
-      wds.decode('pil'),
-      wds.to_tuple('jpg;png;jpeg cls'),
-      wds.map_tuple(transform, identity),
-      wds.batched(batch_size_local, partial=False),
-  )
-  num_workers = args.dataloader_num_workers
-  dataloader = wds.WebLoader(
-      dataset=dataset,
-      batch_size=None,
-      shuffle=False,
-      num_workers=num_workers,
-      persistent_workers=True if num_workers > 0 else False,
-      pin_memory=True).repeat(nbatches=batches)
-  print(f'{mode} dataloader | samples: {data_size}, '
-        f'num_workers: {num_workers}, '
-        f'local batch size: {batch_size_local}, '
-        f'global batch size: {batch_size_global}, '
-        f'batches: {batches}')
-  return dataloader
-
-
-def train(model, device, dataloader, optimizer):
-  model.train()
-  for image, target in dataloader:
-    image = image.to(device, non_blocking=True)
-    target = target.to(device, non_blocking=True)
-    pred = model(image)
-    # pred.shape (N, C), target.shape (N)
-    loss = nn.functional.cross_entropy(pred, target)
-    optimizer.zero_grad()
-    loss.backward()
-    optimizer.step()
-  return loss
-
-
-def evaluate(model, device, dataloader, metric):
-  model.eval()
-  with torch.no_grad():
-    for image, target in dataloader:
-      image = image.to(device, non_blocking=True)
-      target = target.to(device, non_blocking=True)
-      pred = model(image)
-      metric.update(pred, target)
-  accuracy = metric.compute()
-  metric.reset()
-  return accuracy
-
-
-def worker(gpu, args):
-  """Run training and evaluation."""
-  # Init process group.
-  print(f'Initiating process {gpu}')
-  dist.init_process_group(
-      backend='nccl',
-      init_method='env://',
-      world_size=args.gpus,
-      rank=gpu)
-
-  # Create dataloader.
-  train_dataloader = create_wds_dataloader(gpu, args, 'train')
-  eval_dataloader = create_wds_dataloader(gpu, args, 'eval')
-
-  # Wrap policy.
-  my_auto_wrap_policy = functools.partial(
-      size_based_auto_wrap_policy, min_num_params=100)
-  torch.cuda.set_device(gpu)
-
-  # Create model.
-  model = resnet50(weights=None)
-  model.to(args.device)
-  model = FSDP(model, auto_wrap_policy=my_auto_wrap_policy)
-
-  # Optimizer.
-  optimizer = torch.optim.SGD(model.parameters(), 0.1)
-
-  # Main loop.
-  metric = torchmetrics.classification.Accuracy(top_k=1).to(args.device)
-  for epoch in range(1, args.epochs + 1):
-    if gpu == 0:
-      print(f'Running epoch {epoch}')
-
-    start = time.time()
-    train(model, args.device, train_dataloader, optimizer)
-    end = time.time()
-    if gpu == 0:
-      print(f'Training finished in {(end - start):>0.3f} seconds')
-
-    start = time.time()
-    evaluate(model, args.device, eval_dataloader, metric)
-    end = time.time()
-    if gpu == 0:
-      print(f'Evaluation finished in {(end - start):>0.3f} seconds')
-
-  if gpu == 0:
-    print('Done')
-
-
-def create_args():
-  """Create main args."""
-  parser = argparse.ArgumentParser(
-      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-  parser.add_argument(
-      '--gpus',
-      default=4,
-      type=int,
-      help='number of gpus to use')
-  parser.add_argument(
-      '--epochs',
-      default=2,
-      type=int,
-      help='number of total epochs to run')
-  parser.add_argument(
-      '--dataloader_num_workers',
-      default=2,
-      type=int,
-      help='number of workders for dataloader')
-  parser.add_argument(
-      '--train_data_path',
-      default='',
-      type=str,
-      help='path to training data')
-  parser.add_argument(
-      '--train_batch_size',
-      default=32,
-      type=int,
-      help='batch size for training per gpu')
-  parser.add_argument(
-      '--train_data_size',
-      default=50000,
-      type=int,
-      help='data size for training')
-  parser.add_argument(
-      '--eval_data_path',
-      default='',
-      type=str,
-      help='path to evaluation data')
-  parser.add_argument(
-      '--eval_batch_size',
-      default=32,
-      type=int,
-      help='batch size for evaluation per gpu')
-  parser.add_argument(
-      '--eval_data_size',
-      default=50000,
-      type=int,
-      help='data size for evaluation')
-  args = parser.parse_args()
-  return args
-
-
-def main():
-  args = create_args()
-  os.environ['MASTER_ADDR'] = 'localhost'
-  os.environ['MASTER_PORT'] = '8888'
-
-  args.device = 'cuda' if torch.cuda.is_available() else 'cpu'
-  print(f'Launch job on {args.gpus} GPUs with FSDP')
-  mp.spawn(worker, nprocs=args.gpus, args=(args,))
-
-
-if __name__ == '__main__':
-  main()

community-content/pytorch_efficient_training/shard_imagenet.py

@@ -1,98 +0,0 @@
-# Copyright 2022 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the \"License\");
-# you may not use this file except in compliance with the License.\n",
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an \"AS IS\" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-r"""Main function to shard ImageNet dataset.
-
-Example usage:
-python3 -u shard_imagenet.py \
---image_list_file=/home/jupyter/data/imagenet/train_list.txt \
---output_pattern=/home/jupyter/data/imagenet/validation-%06d.tar
-"""
-
-import argparse
-import os
-import random
-import webdataset as wds  # version: 0.2.26
-
-
-# NOTE: only supports writing to local path,
-# need gcsfuse mounting if want to write to gcs bucket.
-def write_shards(args):
-  """Shard individual data files."""
-  output_dir = os.path.dirname(args.output_pattern)
-  if not os.path.isdir(output_dir):
-    os.makedirs(output_dir)
-
-  items = []
-  # Image list file is a text file, each line is a pair (image_path, label).
-  with open(args.image_list_file, 'r') as f:
-    for line in f:
-      item = line.strip().split(' ')
-      items.append((item[0], int(item[1])))
-  # Shuffle items to avoid any large sequences of a single class
-  # in the dataset.
-  random.shuffle(items)
-
-  def _read_image(image_path):
-    with open(image_path, 'rb') as f:
-      return f.read()
-
-  with wds.ShardWriter(pattern=args.output_pattern,
-                       maxcount=args.max_images_per_shard,
-                       maxsize=args.max_bytes_per_shard) as sink:
-    for i, (image_path, target) in enumerate(items):
-      key = str(i)
-      image = _read_image(image_path)
-      sample = {'__key__': key, 'jpg': image, 'cls': target}
-      sink.write(sample)
-    if len(items) != sink.total:
-      raise ValueError('Items read {} != items written {}'.format(
-          len(items), sink.total))
-
-
-def create_args():
-  """Creates arg parser."""
-  parser = argparse.ArgumentParser(
-      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-  parser.add_argument(
-      '--image_list_file',
-      default='',
-      type=str,
-      help='path to image list file')
-  parser.add_argument(
-      '--output_pattern',
-      default='',
-      type=str,
-      help='the pattern for output shards, like /path/to/train-%06d.tar')
-  parser.add_argument(
-      '--max_images_per_shard',
-      default=10 * 1024,
-      type=int,
-      help='max number of images per shard')
-  parser.add_argument(
-      '--max_bytes_per_shard',
-      default=300 * 1024 * 1024,
-      type=int,
-      help='max bytes per shard')
-  args = parser.parse_args()
-  return args
-
-
-def main():
-  args = create_args()
-  write_shards(args)
-
-
-if __name__ == '__main__':
-  main()

community-content/pytorch_image_classification_distributed_data_parallel_training_with_vertex_sdk/trainer/requirements.txt

@@ -1,3 +1,3 @@
-torch==1.13.1
+torch==1.8.1
 torchvision==0.9.1
 tensorboard==2.5.0

community-content/pytorch_image_classification_single_gpu_with_vertex_sdk_and_torchserve/trainer/requirements.txt

@@ -1,3 +1,3 @@
-torch==1.13.1
+torch==1.8.1
 torchvision==0.9.1
 tensorboard==2.5.0

community-content/pytorch_pre_built_images_deployment/pytorch-text-classification-vertex-ai-deploy.ipynb

@@ -31,7 +31,17 @@
       "source": [
         "# Deploying a PyTorch Text Classification Model on [Vertex AI](https://cloud.google.com/vertex-ai)\n",
         "\n",
-        "**Kindly reach out to Vertex AI before you run any scale tests or you have any questions.**\n"
+        "**This is an Experimental release**, covered by the Pre-GA Offerings Terms of your Google Cloud Platform [Terms of Service](https://cloud.google.com/terms).\n",
+        "\n",
+        "Experiments are focused on validating a prototype and are not guaranteed to be released. They are not intended for production use or covered by any SLA, support obligation, or deprecation policy and might be subject to backward-incompatible changes.\n",
+        "\n",
+        "**Kindly drop us a note before you run any scale tests.**\n",
+        "\n",
+        "**Do not hesitate to contact vertexai-prediction-preview-feedback@google.com if you have any questions or run into any issues.**\n",
+        "\n",
+        "The usage of the product is free during the Experimental release period: you will still incur charges for other GCP products usage, such as storage.\n",
+        "\n",
+        "The projects need to be added to the allowlist in order to deploy PyTorch models using Vertex AI Prediction pre-built PyTorch images. If you are interested in the feature, please send an email to vertexai-prediction-preview-feedback@google.com to provide your project numbers OR project ids."
       ]
     },
     {

community-content/tf_agents_bandits_movie_recommendation_with_kfp_and_vertex_sdk/mlops_pipeline_tf_agents_bandits_movie_recommendation/src/logger/requirements.txt

@@ -1,4 +1,4 @@
 google-cloud-bigquery==2.20.0
 tensorflow==2.7.2
-pillow==10.0.1
+pillow==9.0.1
 tf-agents==0.8.0

community-content/tf_agents_bandits_movie_recommendation_with_kfp_and_vertex_sdk/mlops_pipeline_tf_agents_bandits_movie_recommendation/src/prediction_container/requirements.txt

@@ -1,4 +1,4 @@
 google-cloud-pubsub==2.5.0
-pillow==10.0.1
+pillow==9.0.1
 tf-agents==0.8.0
 tensorflow==2.7.2

community-content/tf_agents_bandits_movie_recommendation_with_kfp_and_vertex_sdk/mlops_pipeline_tf_agents_bandits_movie_recommendation/src/simulator/requirements.txt

@@ -1,5 +1,5 @@
 dataclasses==0.6
 google-cloud-aiplatform==1.8.1
 tensorflow==2.7.2
-pillow==10.0.1
+pillow==9.0.1
 tf-agents==0.8.0

community-content/vertex_vision_model_garden/.pylintrc

@@ -1,4 +0,0 @@
-[MASTER]
-
-generated-members=get_concrete_function,cv2.*
-ignored-modules=tensorflow,google.cloud

community-content/vertex_vision_model_garden/model_oss/data_converter/common_lib.py

@@ -1,623 +0,0 @@
-"""Library with functions to use for data conversion."""
-
-import json
-import os
-import random
-from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple, Union
-import uuid
-
-from absl import logging
-import apache_beam as beam
-import cv2
-import numpy as np
-import pandas as pd
-import PIL
-from PIL import Image
-import tensorflow as tf
-import yaml
-
-from util import constants
-from util import fileutils
-from apache_beam.options import pipeline_options
-
-REFORMATTED_CSV_SUFFIX = '-reformatted.csv'
-
-LABEL_MAP_NAME = 'label_map.yaml'
-
-_SPLIT_RATIO_ERROR_THRESHOLD = 1e-5
-# Internal constant. Only for distinguishing rows without ML use.
-ML_USE_UNASSIGNED = 'unassigned'
-ALL_ML_USES = (
-    constants.ML_USE_TRAINING,
-    constants.ML_USE_VALIDATION,
-    constants.ML_USE_TEST,
-    ML_USE_UNASSIGNED,
-)
-COLUMN_NAME_ML_USE = 'ml_use'
-COLUMN_NAME_GCS_FILE_PATH = 'gcs_file_path'
-COLUMN_NAME_LABEL = 'label'
-COLUMN_NAME_START_SEC = 'start_sec'
-COLUMN_NAME_END_SEC = 'end_sec'
-# Output filenames
-TRAIN_TFRECORD_NAME = 'train.tfrecord'
-VALIDATION_TFRECORD_NAME = 'val.tfrecord'
-TEST_TFRECORD_NAME = 'test.tfrecord'
-# Jsonl keys
-JSON_GCS_URI_KEY = 'imageGcsUri'
-JSON_RESOURCE_LABEL_KEY = 'dataItemResourceLabels'
-JSON_ML_USE_KEY = 'aiplatform.googleapis.com/ml_use'
-# I/O parameters
-READ_CHUNK_SIZE = 1024 * 1024 * 1024  # 1GB
-
-
-class WriteToTFRecord(beam.DoFn):
-  """DoFn to write TF examples to sharded TF record files."""
-
-  def __init__(
-      self,
-      output_prefix: str,
-      num_shards: int,
-      convert_fn: Callable[[Dict[str, Any]], tf.train.Example],
-  ):
-    self.output_prefix = output_prefix
-    self.num_shards = num_shards
-    self.writer: list[tf.io.TFRecordWriter] = []
-    self.sharded_files: list[str] = []
-    self.convert_fn = convert_fn
-    self.success_counter = beam.metrics.Metrics.counter(
-        self.__class__.__name__, 'Success'
-    )
-    self.failure_counter = beam.metrics.Metrics.counter(
-        self.__class__.__name__, 'Failure'
-    )
-
-  def start_bundle(self):
-    logging.info('Start writing TF Record to %s.', self.output_prefix)
-    unique_str = uuid.uuid4().hex
-    for i in range(self.num_shards):
-      uri = f'{self.output_prefix}-{i}-{unique_str}'
-      self.sharded_files.append(uri)
-      self.writer.append(tf.io.TFRecordWriter(uri))
-
-  def process(self, data: Dict[str, Any]) -> Iterable[Tuple[int, str]]:
-    try:
-      example = self.convert_fn(data)
-      data = example.SerializeToString()
-      idx = hash(data) % self.num_shards
-      self.writer[idx].write(data)
-      self.success_counter.inc()
-      yield (idx, self.sharded_files[idx])
-    # pylint: disable-next=broad-exception-caught
-    except Exception as err:
-      logging.error('Failed to process %s', data)
-      logging.exception(err)
-      self.failure_counter.inc()
-
-  def finish_bundle(self):
-    logging.info('Finish writing TF Record to %s.', self.output_prefix)
-    for writer in self.writer:
-      writer.close()
-    self.writer = []
-
-
-def convert_to_feature(
-    value: Union[List[Union[int, float, bytes]], int, float, bytes],
-    value_type: Optional[str] = None,
-) -> tf.train.Feature:
-  """Converts the given python object to a tf.train.Feature.
-
-  This is copied from tensorflow_models/official/vision/data/tfrecord_lib.py.
-
-  Args:
-    value: int, float, bytes or a list of them.
-    value_type: optional, if specified, forces the feature to be of the given
-      type. Otherwise, type is inferred automatically. Can be one of ['bytes',
-      'int64', 'float', 'bytes_list', 'int64_list', 'float_list']
-
-  Returns:
-    feature: A tf.train.Feature object.
-  """
-
-  if value_type is None:
-    element = value[0] if isinstance(value, list) else value
-
-    if isinstance(element, bytes):
-      value_type = 'bytes'
-
-    elif isinstance(element, (int, np.integer)):
-      value_type = 'int64'
-
-    elif isinstance(element, (float, np.floating)):
-      value_type = 'float'
-
-    else:
-      raise ValueError(
-          'Cannot convert type {} to feature'.format(type(element))
-      )
-
-    if isinstance(value, list):
-      value_type = value_type + '_list'
-
-  if value_type == 'int64':
-    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))
-
-  elif value_type == 'int64_list':
-    value = np.asarray(value).astype(np.int64).reshape(-1)
-    return tf.train.Feature(int64_list=tf.train.Int64List(value=value))
-
-  elif value_type == 'float':
-    return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))
-
-  elif value_type == 'float_list':
-    value = np.asarray(value).astype(np.float32).reshape(-1)
-    return tf.train.Feature(float_list=tf.train.FloatList(value=value))
-
-  elif value_type == 'bytes':
-    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
-
-  elif value_type == 'bytes_list':
-    return tf.train.Feature(bytes_list=tf.train.BytesList(value=value))
-
-  else:
-    raise ValueError('Unknown value_type parameter - {}'.format(value_type))
-
-
-def convert_to_string_feature(
-    value: str, encoding: str = 'utf-8'
-) -> tf.train.Feature:
-  """Returns a bytes_list from an encoded string."""
-  return convert_to_feature(value.encode(encoding))
-
-
-def convert_to_list_string_feature(
-    lst: list[str], encoding: str = 'utf-8'
-) -> tf.train.Feature:
-  """Returns a bytes_list from a list of encoded strings."""
-  return convert_to_feature([value.encode(encoding) for value in lst])
-
-
-def create_ml_use_array_with_split(
-    total_size: int,
-    split_ratio: Sequence[float],
-) -> list[str]:
-  """Create randomized list of 'training', 'validation', 'test'.
-
-  The list of will be of length total_size with ratios according to train_size,
-  validation_size, and test_size.
-
-  Args:
-    total_size: Length of sequence to return
-    split_ratio: Proportions to split into 'training', 'validation', and 'test'
-
-  Returns:
-    List containing 'training', 'validation', and 'test'
-  """
-  train_size, validation_size, _ = split_ratio
-  num_train = round(train_size * total_size)
-  num_validation = round(validation_size * total_size)
-  num_test = total_size - num_train - num_validation
-  ml_use_row = (
-      [constants.ML_USE_TRAINING] * num_train
-      + [constants.ML_USE_VALIDATION] * num_validation
-      + [constants.ML_USE_TEST] * num_test
-  )
-  random.shuffle(ml_use_row)
-  return ml_use_row
-
-
-def format_ml_use_column(df: pd.DataFrame):
-  df[COLUMN_NAME_ML_USE].replace(
-      # We need to support non-standard ML uses other than documented ones,
-      # since they are used by some existing datasets.
-      [r'(?i)^train(ing)?$', r'(?i)^test$', r'(?i)^validat(ion|e)$'],
-      [
-          constants.ML_USE_TRAINING,
-          constants.ML_USE_TEST,
-          constants.ML_USE_VALIDATION,
-      ],
-      inplace=True,
-      regex=True,
-  )
-
-
-def insert_missing_ml_use(df: pd.DataFrame) -> None:
-  """For every row that does not have ml_use as the first column, insert a column containing 'unassigned' to the front.
-
-  Args:
-    df: The DataFrame to process. The first column should be 'ml_use'.
-  """
-  df[COLUMN_NAME_ML_USE].fillna(ML_USE_UNASSIGNED, inplace=True)
-  rows_to_fill = ~df[COLUMN_NAME_ML_USE].isin(ALL_ML_USES)
-  df.loc[rows_to_fill] = df[rows_to_fill].shift(
-      axis=1, fill_value=ML_USE_UNASSIGNED
-  )
-
-
-def replace_unassigned_ml_use(
-    ml_uses: List[str],
-    split_ratio: Sequence[float],
-):
-  """Replace `unassigned` in ml_uses with `training`, `validation`, and `test` with ratios according to split_ratio.
-
-  Args:
-    ml_uses: List of ml_use string values.
-    split_ratio: Proportions to split into `training`, `validation`, and `test`.
-  """
-  unassigned_indices = [
-      i for i, ml_use in enumerate(ml_uses) if ml_use == ML_USE_UNASSIGNED
-  ]
-  ml_use_arr = create_ml_use_array_with_split(
-      len(unassigned_indices), split_ratio
-  )
-  for unassigned_index, ml_use in zip(unassigned_indices, ml_use_arr):
-    ml_uses[unassigned_index] = ml_use
-
-
-def merge_seq_into_dicts(
-    key: str, values: Sequence[Any], dicts: Sequence[Dict[Any, Any]]
-):
-  """Merges a list of values into a list of dicts, inserted with the given key.
-
-  Args:
-    key: Key to insert or overwrite in the dictionary.
-    values: A list of values to insert.
-    dicts: A list of dictionaries. Each value will be inserted into the
-      corresponding dictionary. The original value will be overwritten if the
-      key already existed.
-
-  Raises:
-    ValueError: The values and dicts have different lengths.
-  """
-  if len(values) != len(dicts):
-    raise ValueError(
-        f'Length of values and dicts must match, got {len(values)} and'
-        f' {len(dicts)}'
-    )
-  for val, d in zip(values, dicts):
-    d[key] = val
-
-
-def drop_invalid_rows(df: pd.DataFrame) -> int:
-  """Drops DataFrame rows missing the gcs_file_path column or the label column.
-
-  Args:
-    df: The DataFrame to process in place.
-
-  Returns:
-    The number of rows dropped.
-  """
-  original_rows = df.shape[0]
-  df.dropna(subset=[COLUMN_NAME_GCS_FILE_PATH, COLUMN_NAME_LABEL], inplace=True)
-  dropped_num = original_rows - df.shape[0]
-  if dropped_num > 0:
-    df.reset_index(drop=True, inplace=True)
-  return dropped_num
-
-
-def check_split_ratio(split_ratio: Sequence[float]):
-  """Checks if the give split ratio is valid.
-
-  Args:
-    split_ratio: Proportions to split into 'training', 'validation', and 'test'
-
-  Raises:
-    ValueError: Must have valid entries, correct length, and sum to 1.
-  """
-  if len(split_ratio) != 3:
-    raise ValueError('split_ratio must contain exactly 3 values.')
-  if abs(sum(split_ratio) - 1) > _SPLIT_RATIO_ERROR_THRESHOLD:
-    raise ValueError('split_ratio must sum to 1.')
-  if not all([0 <= val <= 1 for val in split_ratio]):
-    raise ValueError('Entries of split_ratio must be in the range [0, 1].')
-
-
-def check_num_shard(num_shard: Sequence[int]):
-  """Checks if the number of shards is valid.
-
-  Args:
-    num_shard: The number of shards for each tfrecord.
-
-  Raises:
-    ValueError: Must have valid entries and correct length.
-  """
-  if len(num_shard) != 3:
-    raise ValueError('num_shard must contain exactly 3 values.')
-  if not all([val >= 1 for val in num_shard]):
-    raise ValueError('Shards must be at least 1.')
-
-
-def create_label_map_yaml(meta_data_path: str, output_dir: str) -> None:
-  """Generate label_map.yaml from meta_data.yaml.
-
-  Args:
-    meta_data_path: Path to a meta_data.yaml file.
-    output_dir: Directory to output label_map.yaml.
-  """
-  tf.io.gfile.copy(
-      meta_data_path, os.path.join(output_dir, LABEL_MAP_NAME), overwrite=True
-  )
-
-
-def reformat_bbox(
-    bbox: Sequence[int], img_width: int, img_height: int
-) -> Tuple[float, float, float, float]:
-  """Converts XYWH unnormalized bounding box with to a normalized XYXY bounding box.
-
-  Args:
-    bbox: Relative bounding box with unnormalized coordinates as [x, y, width,
-      height].
-    img_width: Image's pixel width.
-    img_height: Image's pixel height.
-
-  Returns:
-    Absolute bounding box with normalized coordinates as
-      [xmin, ymin, xmax, ymax].
-  """
-  x, y, width, height = bbox
-  xmin = x / img_width
-  ymin = y / img_height
-  xmax = (x + width) / img_width
-  ymax = (y + height) / img_height
-  return xmin, ymin, xmax, ymax
-
-
-def encode_image(
-    filepath: str,
-    output_shape: Optional[Sequence[int]] = None,
-    image_format: str = 'png',
-) -> Tuple[bytes, Sequence[int]]:
-  """Encodes an image at the given path.
-
-  Args:
-    filepath: Path to the image.
-    output_shape: The output shape of the image, (height, width).
-    image_format: The format of the output image.
-
-  Returns:
-    The encoded image data in bytes and the shape of the image, (height, width).
-
-  Raises:
-    IOError: The image file is corrupt.
-  """
-  filepath = fileutils.force_gcs_fuse_path(filepath)
-  with open(filepath, 'rb') as f:
-    # If an output_shape is specified, resize the image and set data to the new
-    # bytes.
-    try:
-      img = Image.open(f)
-    except PIL.UnidentifiedImageError as e:
-      raise IOError(f'Failed to open {filepath}') from e
-
-    try:
-      if output_shape is not None:
-        rgb_img = img.resize((output_shape[1], output_shape[0])).convert('RGB')
-      else:
-        rgb_img = img.convert('RGB')
-      rgb_img = np.array(rgb_img)
-
-      _, data = cv2.imencode(f'.{image_format}', rgb_img)
-      data = data.tobytes()
-      return data, rgb_img.shape
-    except cv2.error as e:
-      raise IOError(f'Failed to encode {filepath}') from e
-    finally:
-      img.close()
-
-
-def encode_video(
-    filepath: str,
-    start_sec: float,
-    end_sec: float,
-    output_fps: int = 5,
-    output_shape: Optional[Sequence[int]] = None,
-    image_format: str = 'jpg',
-) -> Sequence[bytes]:
-  """Encodes a video clip at the given path with start and end timestamps.
-
-  Args:
-    filepath: Path to the video.
-    start_sec: Start timestamp of the video clip in seconds.
-    end_sec: End timestamp of the video clip in seconds.
-    output_fps: The output frame rate per second.
-    output_shape: The output shape of each frame, (height, width).
-    image_format: The format of the encoded frames.
-
-  Returns:
-    A list of the encoded frames data in bytes.
-
-  Raises:
-    IOError if the video file is corrupt.
-  """
-  filepath = fileutils.force_gcs_fuse_path(filepath)
-  video = None
-
-  try:
-    video = cv2.VideoCapture(filepath)
-    frames = []
-    frame_interval = 1 / output_fps
-    total_frames = video.get(cv2.CAP_PROP_FRAME_COUNT)
-    original_fps = video.get(cv2.CAP_PROP_FPS)
-    if not original_fps:
-      # 0 or None indicates the video is invalid
-      raise IOError(f'Failed to load {filepath}')
-    video_length = total_frames / original_fps
-    start_sec = max(start_sec, 0)
-    end_sec = min(end_sec, video_length)
-    for t in np.arange(start_sec, end_sec, frame_interval):
-      frame_idx = min(total_frames - 1, round(t * original_fps))
-      video.set(cv2.CAP_PROP_POS_FRAMES, frame_idx)
-      ret, frame = video.read()
-      if not ret:
-        raise IOError(f'Failed to load {filepath} at frame {frame_idx}')
-      if output_shape is not None:
-        frame = cv2.resize(frame, (output_shape[1], output_shape[0]))
-      _, data = cv2.imencode(f'.{image_format}', frame)
-      frames.append(data.tobytes())
-  except cv2.error as e:
-    raise IOError(f'Failed to load {filepath}') from e
-  finally:
-    if video:
-      video.release()
-  return frames
-
-
-def create_label_map(
-    labels: Sequence[str],
-) -> Tuple[Sequence[int], Dict[int, str]]:
-  """Creates a label map from a sequence of label strings.
-
-  Args:
-    labels: The sequence of labels to create label map from. Must not contain
-      invalid values, which means data without labels should be filtered first.
-
-  Returns:
-    The integer labels and the mapping from integers to the original strings.
-  """
-  inverse_label_map: Dict[str, int] = dict()
-  num_labels = 0
-  for label in labels:
-    if label not in inverse_label_map:
-      num_labels += 1
-      inverse_label_map[label] = num_labels
-  int_labels = [inverse_label_map[label] for label in labels]
-  label_map = {value: key for key, value in inverse_label_map.items()}
-  return int_labels, label_map
-
-
-def write_label_map(output_file: str, label_map: Dict[int, str]) -> None:
-  """Writes a label map to the output file, which can be a GCS uri."""
-  with tf.io.gfile.GFile(output_file, 'w') as f:
-    yaml.dump({'label_map': label_map}, f)
-
-
-def detectron_json_to_image_rows(input_json: str) -> list[Dict[str, Any]]:
-  """Converts a Detectron JSON file to a list of image rows.
-
-  Args:
-    input_json: A path to a Detectron JSON or JSONL file.
-
-  Returns:
-    A list of dictionaries, where each dictionary contains Detectron format
-    entry.
-
-  Raises:
-    ValueError: If the input JSON is invalid.
-  """
-
-  image_rows = []
-  with tf.io.gfile.GFile(input_json, 'r') as f:
-    for line in f:
-      json_data = json.loads(line)
-      if isinstance(json_data, dict):
-        image_rows.append(json_data)
-      elif isinstance(json_data, list):
-        image_rows.extend(json_data)
-      else:
-        raise ValueError(
-            'The input JSON is invalid. Dict or list is expected, but got '
-            f'{type(json_data)}.'
-        )
-  return image_rows
-
-
-def coco_json_to_image_rows(
-    input_json: str,
-) -> List[Dict[str, Any]]:
-  """Converts a COCO JSON file to a list of image rows.
-
-  Args:
-    input_json: A path to a COCO JSON or JSONL file.
-
-  Returns:
-    A list of dictionaries, where each dictionary contains COCO format entry.
-
-  Raises:
-    ValueError: If the input JSON is invalid.
-  """
-
-  with tf.io.gfile.GFile(input_json, 'r') as f:
-    coco_json = json.load(f)
-  if 'annotations' not in coco_json:
-    raise ValueError('"annotations" is not in the dataset.')
-  if 'images' not in coco_json:
-    raise ValueError('"images" is not in the dataset.')
-
-  images = coco_json['images']
-  return images
-
-
-def partition_by_ml_use(element: Dict[str, Any], num_partitions: int) -> int:
-  """Beam partition function to split data by ml_use."""
-  del num_partitions
-  try:
-    partition = ALL_ML_USES.index(element[COLUMN_NAME_ML_USE])
-  except Exception as e:
-    raise ValueError(f'Invalid ML use: {element[COLUMN_NAME_ML_USE]}') from e
-  return partition
-
-
-def run_beam_pipeline(pipeline: Any) -> None:
-  """Runs a beam pipeline. Works in both internal and docker environment."""
-  options = pipeline_options.PipelineOptions([
-      '--runner=FlinkRunner',
-      '--faster_copy',
-      '--max_parallelism', '8',
-  ])
-  p = beam.Pipeline(options=options)
-  pipeline(p)
-  result = p.run()
-  result.wait_until_finish()
-  for counter in result.metrics().query()['counters']:
-    logging.info('%s counter: %s.', counter.key.metric.name, counter)
-  logging.info('Completing beam pipeline.')
-
-
-def beam_convert_tfexamples(
-    root: beam.Pipeline,
-    data_list: Sequence[Dict[str, Any]],
-    convert_fn: Callable[[Dict[str, Any]], tf.train.Example],
-    output_dir: str,
-    num_shards: Sequence[int],
-) -> None:
-  """Constructs beam pipelines to convert train, val, test TF Examples."""
-  names = [TRAIN_TFRECORD_NAME, VALIDATION_TFRECORD_NAME, TEST_TFRECORD_NAME]
-  split_data = (
-      root
-      | 'Create PCollection' >> beam.Create(data_list)
-      | 'Data split' >> beam.Partition(partition_by_ml_use, 3)
-  )
-  for i in range(3):
-    ml_use: str = ALL_ML_USES[i]
-    num_shard = num_shards[i]
-    output_prefix = os.path.join(output_dir, names[i])
-    _ = (
-        split_data[i]
-        | f'Convert {ml_use} TF Examples'
-        >> beam.ParDo(WriteToTFRecord(output_prefix, num_shard, convert_fn))
-        | f'Group {ml_use} TF Record files' >> beam.GroupBy(lambda x: x[0])
-        | f'Merge {ml_use} TF Record files'
-        >> beam.Map(merge_tfrecords_func(output_prefix, num_shard))
-    )
-
-
-def merge_tfrecords_func(output_prefix: str, num_shard: int) -> ...:
-  """Returns a function to merge sharded worker output into expected shards."""
-  output_prefix = fileutils.force_gcs_fuse_path(output_prefix)
-
-  def merge_tfrecords(worker_output: Tuple[int, Sequence[Tuple[int, str]]]):
-    idx = worker_output[0]
-    files: Sequence[str] = np.unique([x[1] for x in worker_output[1]])
-    output_file = f'{output_prefix}-{idx:05d}-of-{num_shard:05d}'
-    with open(output_file, 'wb') as f:
-      for file in files:
-        logging.info('Merging %s.', file)
-        file = fileutils.force_gcs_fuse_path(file)
-        with open(file, 'rb') as fin:
-          while True:
-            data = fin.read(READ_CHUNK_SIZE)
-            if not data:
-              break
-            f.write(data)
-        os.remove(file)
-
-  return merge_tfrecords

community-content/vertex_vision_model_garden/model_oss/data_converter/convert_coco_label_map_iod.py

@@ -1,111 +0,0 @@
-r"""Converts COCO labels as yamls for model garden playground (IOD).
-"""
-
-import os
-import urllib.request
-
-from absl import app
-from absl import flags
-import tensorflow as tf
-import yaml
-
-from object_detection.utils import label_map_util
-
-_CONVERT_LABEL_TYPE_COCO_80 = 'coco_80'
-_CONVERT_LABEL_TYPE_COCO_91 = 'coco_91'
-
-_CONVERT_LABEL_TYPE = flags.DEFINE_enum(
-    'convert_label_type',
-    None,
-    [
-        _CONVERT_LABEL_TYPE_COCO_80,
-        _CONVERT_LABEL_TYPE_COCO_91,
-    ],
-    'Different types of label type conversion.',
-    required=True,
-)
-
-_TEMPORARY_PATH = flags.DEFINE_string(
-    'temporary_path',
-    None,
-    'The tempory path.',
-    required=True,
-)
-
-_OUTPUT_YAML_FILEPATH = flags.DEFINE_string(
-    'output_yaml_filepath',
-    None,
-    'The output yaml filepath.',
-    required=True,
-)
-
-
-def convert_coco_label_map_91(
-    output_yaml_filepath: str,
-) -> None:
-  """Converts coco label map 91."""
-  input_proto_filepath = 'https://raw.githubusercontent.com/tensorflow/models/master/research/object_detection/data/mscoco_label_map.pbtxt'
-  local_input_proto_filepath = os.path.join(
-      _TEMPORARY_PATH.value, 'mscoco_label_map.pbtxt'
-  )
-  with open(local_input_proto_filepath, 'w') as writer:
-    contents = (
-        urllib.request.urlopen(input_proto_filepath).read().decode('utf-8')
-    )
-    writer.write(contents)
-
-  label_map = label_map_util.load_labelmap(local_input_proto_filepath)
-  label_map_dict = label_map_util.get_label_map_dict(
-      label_map, use_display_name=True
-  )
-  swapped_label_map_dict = {v: k for k, v in label_map_dict.items()}
-  print(swapped_label_map_dict)
-
-  # Saves new label maps as yamls.
-  with tf.io.gfile.GFile(output_yaml_filepath, 'w') as writer:
-    writer.write(yaml.dump(swapped_label_map_dict))
-
-
-def convert_coco_label_map_80(
-    output_yaml_filepath: str,
-) -> None:
-  """Converts coco label map 80."""
-  # Loads label maps from texts.
-  input_text_filepath = 'https://gist.githubusercontent.com/AruniRC/7b3dadd004da04c80198557db5da4bda/raw/2f10965ace1e36c4a9dca76ead19b744f5eb7e88/ms_coco_classnames.txt'
-  local_input_text_filepath = os.path.join(
-      _TEMPORARY_PATH.value, 'ms_coco_classnames.txt'
-  )
-  with open(local_input_text_filepath, 'w') as writer:
-    contents = (
-        urllib.request.urlopen(input_text_filepath).read().decode('utf-8')
-    )
-    writer.write(contents)
-  with open(local_input_text_filepath, 'r') as file:
-    content = file.read()
-    label_map = yaml.safe_load(content)
-
-  # Removes background in label maps.
-  new_label_map = {}
-  for k, v in label_map.items():
-    if k == 0:
-      continue
-    new_label_map[k - 1] = v
-  print(new_label_map)
-  # Saves new label maps as yamls.
-  with tf.io.gfile.GFile(output_yaml_filepath, 'w') as writer:
-    writer.write(yaml.dump(new_label_map))
-
-
-def main(_) -> None:
-  if _CONVERT_LABEL_TYPE.value == _CONVERT_LABEL_TYPE_COCO_80:
-    convert_coco_label_map_80(_OUTPUT_YAML_FILEPATH.value)
-  elif _CONVERT_LABEL_TYPE.value == _CONVERT_LABEL_TYPE_COCO_91:
-    convert_coco_label_map_91(
-        _OUTPUT_YAML_FILEPATH.value,
-    )
-  else:
-    print('Not supported convert label type: ', _CONVERT_LABEL_TYPE.value)
-
-
-if __name__ == '__main__':
-  app.run(main)

community-content/vertex_vision_model_garden/model_oss/data_converter/convert_imagenet_label_map.py

@@ -1,86 +0,0 @@
-r"""Converts ImageNet label texts as yamls for model garden playground.
-
-# ImageNet1K will have label maps with background.
-"""
-
-import urllib.request
-from absl import app
-from absl import flags
-import tensorflow as tf
-import yaml
-
-
-_INPUT_TEXT_FILEPATH = flags.DEFINE_string(
-    'input_text_filepath',
-    None,
-    'The input text filepath.',
-    required=True,
-)
-
-_ADD_BACKGROUND_LABEL = flags.DEFINE_boolean(
-    'add_background_label',
-    None,
-    'Whether or not add background labels.',
-    required=True,
-)
-
-_ADD_IDS = flags.DEFINE_boolean(
-    'add_ids',
-    None,
-    'Whether or not add ids.',
-    required=True,
-)
-
-_OUTPUT_YAML_FILEPATH = flags.DEFINE_string(
-    'output_yaml_filepath',
-    None,
-    'The output yaml filepath.',
-    required=True,
-)
-
-
-def convert_imagenet_label_map_from_text_to_yaml(
-    input_text_filepath: str,
-    add_background_label: bool,
-    add_ids: bool,
-    output_yaml_filepath: str,
-) -> None:
-  """Converts imagenet label map from text to yamls."""
-  label_map = {}
-
-  # Shifts all keys by 1, and add 0 as 'background'.
-  if add_background_label:
-    label_map = yaml.safe_load(
-        urllib.request.urlopen(input_text_filepath).read()
-    )
-    new_label_map = {}
-    for key, value in label_map.items():
-      new_label_map[key + 1] = value
-    new_label_map[0] = 'background'
-    label_map = new_label_map
-
-  # Adds maps from id to each line.
-  if add_ids:
-    lines = urllib.request.urlopen(input_text_filepath).readlines()
-    current_id = 0
-    for line in lines:
-      label_map[current_id] = line.decode('ascii').strip()
-      print(label_map[current_id])
-      current_id += 1
-
-  # Saves new label maps as yamls.
-  with tf.io.gfile.GFile(output_yaml_filepath, 'w') as writer:
-    writer.write(yaml.dump(label_map))
-
-
-def main(_) -> None:
-  convert_imagenet_label_map_from_text_to_yaml(
-      _INPUT_TEXT_FILEPATH.value,
-      _ADD_BACKGROUND_LABEL.value,
-      _ADD_IDS.value,
-      _OUTPUT_YAML_FILEPATH.value,
-  )
-
-
-if __name__ == '__main__':
-  app.run(main)