Merge branch 'main' into auto_logging

.cloud-build/cleanup/cleanup.py

@@ -1,29 +1,10 @@
 from typing import List
 from ratemate import RateLimit
-import argparse
-
-parser = argparse.ArgumentParser()
-parser.add_argument("--dry_run",
-                    type=bool,
-                    default=False)
-args = parser.parse_args()
-
-
 from resource_cleanup_manager import (
     DatasetResourceCleanupManager,
     ModelResourceCleanupManager,
     EndpointResourceCleanupManager,
     ResourceCleanupManager,
-    MatchingEngineIndexEndpointResourceCleanupManager,
-    MatchingEngineIndexResourceCleanupManager,
-    FeatureStoreCleanupManager,
-    PipelineJobCleanupManager,
-    TrainingJobCleanupManager,
-    HyperparameterTuningCleanupManager,
-    BatchPredictionJobCleanupManager,
-    ExperimentCleanupManager,
-    BucketCleanupManager,
-    ArtifactRegistryCleanupManager
 )
 
 rate_limit = RateLimit(max_count=25, per=60, greedy=False)
@@ -40,6 +21,7 @@ def run_cleanup_managers(managers: List[ResourceCleanupManager], is_dry_run: boo
             try:
                 if not manager.is_deletable(resource):
                     continue
+
                 if is_dry_run:
                     resource_name = manager.resource_name(resource)
                     print(f"Will delete '{type_name}': {resource_name}")
@@ -52,24 +34,16 @@ def run_cleanup_managers(managers: List[ResourceCleanupManager], is_dry_run: boo
         print("")
 
 
-if args.dry_run:
+is_dry_run = False
+
+if is_dry_run:
     print("Starting cleanup in dry run mode...")
 
 # List of all cleanup managers
-managers: List[ResourceCleanupManager] = [
+managers = [
     DatasetResourceCleanupManager(),
     EndpointResourceCleanupManager(),
     ModelResourceCleanupManager(),  # ModelResourceCleanupManager must follow EndpointResourceCleanupManager due to deployed models blocking model deletion.
-    MatchingEngineIndexEndpointResourceCleanupManager(),
-    MatchingEngineIndexResourceCleanupManager(),
-    FeatureStoreCleanupManager(),
-    PipelineJobCleanupManager(),
-    TrainingJobCleanupManager(),
-    HyperparameterTuningCleanupManager(),
-    BatchPredictionJobCleanupManager(),
-    ExperimentCleanupManager(), # Experiment missing _resource_noun
-    BucketCleanupManager(),
-    ArtifactRegistryCleanupManager()
 ]
 
-run_cleanup_managers(managers=managers, is_dry_run=args.dry_run)
+run_cleanup_managers(managers=managers, is_dry_run=is_dry_run)

.cloud-build/cleanup/resource_cleanup_manager.py

@@ -1,18 +1,8 @@
-'''
-READ FIRST BEFORE MAKING CHANGES
--  Create a convention for resources created from vertex-ai-samples GH. We already have one IIRC
-- Only delete those objects as part of our clean-up script.
-- Don't run any tests on python-docs-samples-tests project, especially ones that affect resources created outside of our purview
-- Add --dry-run option to the clean-up script. This option will just output the list of resources the script will delete instead of actually deleting the resources.
-- Have a larger conversation in DEE before touching any resources that were not created as part of vertex-ai-samples
-'''
-import os
 import abc
 from typing import Any, Type
 
 from google.cloud import aiplatform
 from google.cloud.aiplatform import base
-from google.cloud import storage
 from proto.datetime_helpers import DatetimeWithNanoseconds
 
 # If a resource was updated within this number of seconds, do not delete.
@@ -79,7 +69,7 @@ class VertexAIResourceCleanupManager(ResourceCleanupManager):
     def delete(self, resource):
         resource.delete()
 
-    def get_seconds_since_modification(self, resource: Any) -> float:
+    def get_seconds_since_modification(self, resource: Any) -> bool:
         update_time = resource.update_time
         current_time = DatetimeWithNanoseconds.now(tz=update_time.tzinfo)
         return (current_time - update_time).total_seconds()
@@ -107,154 +97,15 @@ class EndpointResourceCleanupManager(VertexAIResourceCleanupManager):
     vertex_ai_resource = aiplatform.Endpoint
 
     def delete(self, resource):
+        # TODO: Remove this once https://github.com/googleapis/python-aiplatform/issues/1441 is fixed
+        resource._sync_gca_resource()
         for deployed_model_id in [
             models.id for models in resource._gca_resource.deployed_models
         ]:
             resource._undeploy(deployed_model_id=deployed_model_id)
+
         resource.delete(force=True)
 
 
 class ModelResourceCleanupManager(VertexAIResourceCleanupManager):
     vertex_ai_resource = aiplatform.Model
-
-
-class MatchingEngineIndexResourceCleanupManager(VertexAIResourceCleanupManager):
-    vertex_ai_resource = aiplatform.MatchingEngineIndex
-
-
-class MatchingEngineIndexEndpointResourceCleanupManager(VertexAIResourceCleanupManager):
-    vertex_ai_resource = aiplatform.MatchingEngineIndexEndpoint
-
-    def delete(self, resource):
-        resource.undeploy_all()
-        resource.delete(force=True)
-
-class FeatureStoreCleanupManager(VertexAIResourceCleanupManager):
-    vertex_ai_resource = aiplatform.Featurestore
-
-    def resource_name(self, resource: Any) -> str:
-        return resource.name
-
-class PipelineJobCleanupManager(VertexAIResourceCleanupManager):
-    vertex_ai_resource = aiplatform.PipelineJob
-
-class TrainingJobCleanupManager(VertexAIResourceCleanupManager):
-    vertex_ai_resource = aiplatform.training_jobs._CustomTrainingJob
-
-    job_types = [
-            aiplatform.AutoMLImageTrainingJob,
-            aiplatform.AutoMLTextTrainingJob,
-            aiplatform.AutoMLTabularTrainingJob,
-            aiplatform.AutoMLVideoTrainingJob,
-            aiplatform.AutoMLForecastingTrainingJob,
-            aiplatform.CustomJob,
-            aiplatform.CustomTrainingJob,
-            aiplatform.CustomContainerTrainingJob,
-            aiplatform.CustomPythonPackageTrainingJob
-    ]
-
-    def list(self) -> Any:
-        return [
-            job
-            for job_type in self.job_types
-            for job in job_type.list()
-        ]
-
-class HyperparameterTuningCleanupManager(VertexAIResourceCleanupManager):
-    vertex_ai_resource = aiplatform.HyperparameterTuningJob
-
-
-class BatchPredictionJobCleanupManager(VertexAIResourceCleanupManager):
-    vertex_ai_resource = aiplatform.BatchPredictionJob
-
-class ExperimentCleanupManager(VertexAIResourceCleanupManager):
-    vertex_ai_resource = aiplatform.Experiment
-
-    @property
-    def type_name(self) -> str:
-        return "Experiment"
-
-    def resource_name(self, resource: Any) -> str:
-        return resource.name
-
-    def get_seconds_since_modification(self, resource: Any) -> float:
-        update_time = resource._metadata_context.update_time
-        current_time = DatetimeWithNanoseconds.now()
-        return float(current_time.timestamp() - update_time.timestamp())
-
-class BucketCleanupManager(ResourceCleanupManager):
-    vertex_ai_resource = storage.bucket.Bucket
-
-    def list(self) -> Any:
-        storage_client = storage.Client()
-        return list(storage_client.list_buckets())
-
-    def delete(self, resource):
-        try:
-            resource.delete(force=True)
-        except Exception as e:
-            print(e)
-
-    @property
-    def type_name(self) -> str:
-        return "Bucket"
-
-    def get_seconds_since_modification(self, resource: Any) -> float:
-        # Bucket has no last_update property, only time created
-        created_time = resource.time_created
-        current_time = DatetimeWithNanoseconds.now()
-        return float(current_time.timestamp() - created_time.timestamp())
-
-    def resource_name(self, resource: Any) -> str:
-        return resource.name
-
-    def is_deletable(self, resource: Any) -> bool:
-        time_difference = self.get_seconds_since_modification(resource)
-
-        if not self.resource_name(resource).startswith('your-bucket-name'):
-            print(f"Skipping '{resource}' not a Vertex AI notebook bucket")
-            return False
-
-        # Check that it wasn't created too recently, to prevent race conditions
-        if time_difference <= RESOURCE_UPDATE_BUFFER_IN_SECONDS:
-            print(
-                f"Skipping '{resource}' due to update_time being '{time_difference}', which is less than '{RESOURCE_UPDATE_BUFFER_IN_SECONDS}'."
-            )
-            return False
-        return True
-
-class ArtifactRegistryCleanupManager(ResourceCleanupManager):
-    vertex_ai_resource = "Artifact Registry"
-
-    def list(self) -> Any:
-        import subprocess
-
-        result = subprocess.run(["gcloud artifacts repositories list --location=us-central1"], 
-                                shell=True, capture_output=True, text=True)
-
-        ret = []
-        lines = result.stdout.split('\n')[2:]
-        for line in lines:
-            repo = line.split(' ')[0]
-            if repo.startswith("my-docker-repo"):
-                ret.append(repo)
-
-        return ret
-
-    def delete(self, resource):
-        os.system(f"! gcloud artifacts repositories delete {resource} --location=us-central1")
-
-    @property
-    def type_name(self) -> str:
-        return "ArtifactRepository"
-
-    def resource_name(self, resource: Any) -> str:
-        return resource
-
-    # delete repository regardless of age
-    def get_seconds_since_modification(self, resource: Any) -> float:
-        return RESOURCE_UPDATE_BUFFER_IN_SECONDS + 1
-    
-    def is_deleteable(self, resource: Any) -> bool:
-        return True
-

.cloud-build/execute_changed_notebooks_cli.py

@@ -17,7 +17,6 @@
 
 import argparse
 import pathlib
-import os
 
 import execute_changed_notebooks_helper
 
@@ -40,19 +39,6 @@ parser.add_argument(
     help="The path to the file that has newline-limited folders of notebooks that should be tested.",
     required=True,
 )
-parser.add_argument(
-    "--test_percent",
-    type=int,
-    help="The percent of notebooks to be tested (between 1 and 100).",
-    required=False,
-    default=100,
-)
-parser.add_argument(
-    "--build_id",
-    type=str,
-    help="The build id (which may be a Cloud Build job specific or user explicit.",
-    required=True
-)
 parser.add_argument(
     "--base_branch",
     help="The base git branch to diff against to find changed files.",
@@ -121,60 +107,24 @@ parser.add_argument(
     default=True,
     help="Should run notebooks in parallel.",
 )
-parser.add_argument(
-    "--concurrent_notebooks",
-    type=int,
-    help="Maximum number of parallel notebook executions per minute",
-    default=10,
-    required=False,
-)
-parser.add_argument(
-    "--dry_run",
-    type=str2bool,
-    default=False,
-    help="Dry run for testing - no execution",
-)
 
 args = parser.parse_args()
 
-changed_notebooks = execute_changed_notebooks_helper.get_changed_notebooks(
+notebooks = execute_changed_notebooks_helper.get_changed_notebooks(
     test_paths_file=args.test_paths_file,
     base_branch=args.base_branch,
 )
 
-
-results_bucket = f"{args.artifacts_bucket}"
-# artifacts_bucket may get set by trigger to a full gs:// folder path
-if results_bucket.startswith("gs://"):
-    results_bucket = results_bucket[5:]
-results_bucket = results_bucket.split('/')[0]
-results_file = f"build_results/{args.build_id}.json"
-
-if args.test_percent == 100:
-    notebooks = changed_notebooks
-    accumulative_results = {}
-else:
-    accumulative_results = execute_changed_notebooks_helper.load_results(results_bucket, results_file)
-
-    notebooks = [changed_notebook for changed_notebook in changed_notebooks if execute_changed_notebooks_helper.select_notebook(changed_notebook, accumulative_results, args.test_percent)]
-
-if args.dry_run:
-    print("Dry run ...\n")
-    for notebook in notebooks:
-        print(f"Would execute: {notebook}")
-else:
-    execute_changed_notebooks_helper.process_and_execute_notebooks(
-        notebooks=notebooks,
-        container_uri=args.container_uri,
-        staging_bucket=args.staging_bucket,
-        artifacts_bucket=args.artifacts_bucket,
-        results_file=results_file,
-        should_parallelize=args.should_parallelize,
-        timeout=args.timeout,
-        variable_project_id=args.variable_project_id,
-        variable_region=args.variable_region,
-        variable_service_account=args.variable_service_account,
-        variable_vpc_network=args.variable_vpc_network,
-        private_pool_id=args.private_pool_id,
-        concurrent_notebooks=args.concurrent_notebooks,
+execute_changed_notebooks_helper.process_and_execute_notebooks(
+    notebooks=notebooks,
+    container_uri=args.container_uri,
+    staging_bucket=args.staging_bucket,
+    artifacts_bucket=args.artifacts_bucket,
+    should_parallelize=args.should_parallelize,
+    timeout=args.timeout,
+    variable_project_id=args.variable_project_id,
+    variable_region=args.variable_region,
+    variable_service_account=args.variable_service_account,
+    variable_vpc_network=args.variable_vpc_network,
+    private_pool_id=args.private_pool_id,
 )

.cloud-build/execute_changed_notebooks_helper.py

@@ -21,21 +21,18 @@ import json
 import git
 import operator
 import os
-import io
-import json
 import pathlib
 import re
 import subprocess
-import random
-from google.cloud import storage
 import utils
-from typing import List, Optional, Dict, Any
+from typing import List, Optional
 from utils import util
 
 import execute_notebook_helper
 import execute_notebook_remote
 import nbformat
 from google.cloud.devtools.cloudbuild_v1.types import BuildOperationMetadata
+from ratemate import RateLimit
 from tabulate import tabulate
 from utils import NotebookProcessors, util
 
@@ -43,9 +40,6 @@ from utils import NotebookProcessors, util
 WORKER_TIMEOUT_BUFFER_IN_SECONDS: int = 60 * 60
 PYTHON_VERSION = "3.9"  # Set default python version
 
-# rolling time window for accumulating build results for selecting notebooks
-MAX_RESULTS_AGE_SECONDS: int = (60 * 60) * 24 * 60  # 60 days
-
 
 def format_timedelta(delta: datetime.timedelta) -> str:
     """Formats a timedelta duration to [N days] %H:%M:%S format"""
@@ -71,9 +65,7 @@ def format_timedelta(delta: datetime.timedelta) -> str:
 @dataclasses.dataclass
 class NotebookExecutionResult:
     name: str
-    path: str
     duration: datetime.timedelta
-    start_time: datetime.datetime
     is_pass: bool
     log_url: str
     output_uri: str
@@ -89,75 +81,6 @@ class NotebookExecutionResult:
             return None
 
 
-def load_results(results_bucket: str,
-                 results_file: str) -> Dict[str, Any]:
-    '''
-    Load accumulated notebook test results
-    '''
-
-    print("Loading existing accumulative results ...")
-    accumulative_results = {}
-    try:
-        client = storage.Client()
-        bucket = client.bucket(results_bucket)
-
-        build_results_dir = os.path.dirname(results_file)
-        blobs = client.list_blobs(results_bucket, prefix=build_results_dir)
-        for blob in blobs:
-            time_created = blob.time_created.replace(tzinfo=None)
-            if (datetime.datetime.now().replace(tzinfo=None) - time_created).total_seconds() > MAX_RESULTS_AGE_SECONDS:
-                continue
-
-            content = util.download_blob_into_memory(results_bucket, blob.name, download_as_text=True)
-
-            try:
-                build_results = json.loads(content)
-            except:
-                continue  # skip corrupted build results files
-            for notebook in build_results:
-                if notebook in accumulative_results:
-                    accumulative_results[notebook]['passed'] += build_results[notebook]['passed']
-                    accumulative_results[notebook]['failed'] += build_results[notebook]['failed']
-                else:
-                    accumulative_results[notebook] = build_results[notebook]
-
-        print(accumulative_results)
-    except Exception as e:
-        print(e)
-
-    # If there are no accumulative results, an empty dict is returned
-    return accumulative_results
-
-def select_notebook(changed_notebook: str,
-                    accumulative_results: Dict[str, Any],
-                    test_percent: int) -> bool:
-    '''
-    Algorithm to randomly select a notebook, but weight the propbability of selected based on past failures
-    '''
-
-    if changed_notebook in accumulative_results:
-        pass_count = accumulative_results[changed_notebook]['passed']
-        fail_count = accumulative_results[changed_notebook]['failed']
-    else:
-        pass_count = 1
-        fail_count = 0
-
-    inferred_failure_rate = fail_count / (pass_count + fail_count)
-
-    # If failure rate is high, the chance of testing should be higher
-    should_test_due_to_failure = random.uniform(0, 1) <= inferred_failure_rate
-
-    # Additionally, only test a percentage of these
-    should_test_due_to_random_subset = random.uniform(0, 1) <= (test_percent / 100)
-
-    if should_test_due_to_failure or should_test_due_to_random_subset:
-        print(f"Selected: {changed_notebook}, {should_test_due_to_failure}, {should_test_due_to_random_subset}")
-        return True
-    else:
-        print(f"Not Selected: {changed_notebook}, pass {pass_count}, fail {fail_count}")
-        return False
-
-
 def _process_notebook(
     notebook_path: str,
     variable_project_id: str,
@@ -188,7 +111,6 @@ def _process_notebook(
     ) = remove_no_execute_cells_preprocessor.preprocess(nb)
 
     (nb, resources) = update_variables_preprocessor.preprocess(nb, resources)
-    (nb, resources) = unique_strings_preprocessor.preprocess(nb, resources)
 
     with open(notebook_path, mode="w", encoding="utf-8") as new_file:
         nbformat.write(nb, new_file)
@@ -233,6 +155,7 @@ def _create_tag(filepath: str) -> str:
     return tag
 
 
+rate_limit = RateLimit(max_count=50, per=60, greedy=True)
 
 
 def process_and_execute_notebook(
@@ -248,6 +171,7 @@ def process_and_execute_notebook(
     notebook: str,
     should_get_tail_logs: bool = False,
 ) -> NotebookExecutionResult:
+    rate_limit.wait()  # wait before creating the task
 
     print(f"Running notebook: {notebook}")
 
@@ -266,9 +190,7 @@ def process_and_execute_notebook(
 
     result = NotebookExecutionResult(
         name=tag,
-        path=notebook,
         duration=datetime.timedelta(seconds=0),
-        start_time=datetime.datetime.now(),
         is_pass=False,
         output_uri=notebook_output_uri,
         log_url="",
@@ -278,6 +200,7 @@ def process_and_execute_notebook(
     )
 
     # TODO: Handle cases where multiple notebooks have the same name
+    time_start = datetime.datetime.now()
     operation = None
     try:
         # Get the python version for running the notebook if specified
@@ -321,12 +244,11 @@ def process_and_execute_notebook(
         result.logs_bucket = operation_metadata.build.logs_bucket
 
         # Block and wait for the result
-        operation_result = operation.result(timeout=timeout_in_seconds)
+        operation_result = operation.result()
 
-        result.duration = datetime.datetime.now() - result.start_time
+        result.duration = datetime.datetime.now() - time_start
         result.is_pass = True
         print(f"{notebook} PASSED in {format_timedelta(result.duration)}.")
-
     except Exception as error:
         result.error_message = str(error)
 
@@ -345,7 +267,7 @@ def process_and_execute_notebook(
             except Exception as error:
                 result.error_message = str(error)
 
-        result.duration = datetime.datetime.now() - result.start_time
+        result.duration = datetime.datetime.now() - time_start
         result.is_pass = False
 
         print(
@@ -413,44 +335,12 @@ def get_changed_notebooks(
 
     return notebooks
 
-def _save_results(results: List[NotebookExecutionResult],
-                  artifacts_bucket: str,
-                  results_file: str):
-
-    artifacts_bucket = artifacts_bucket.replace("gs://", "").split('/')[0]
-
-    print("Updating build results ...")
-    build_results = {}
-    for result in results:
-        if result.is_pass:
-            pass_count = 1
-            fail_count = 0
-        else:
-            pass_count = 0
-            fail_count = 1
-        build_results[result.path] = {
-                'duration': result.duration.total_seconds(),
-                'start_time': str(result.start_time),
-                'passed': pass_count,
-                'failed': fail_count
-        }
-        print(f"adding {result.path}")
-
-    print("Saving accumulative results ...")
-    content = json.dumps(build_results)
-
-    client = storage.Client()
-    bucket = client.get_bucket(artifacts_bucket)
-    bucket.blob(str(results_file)).upload_from_string(content, 'text/json')
-
-
 
 def process_and_execute_notebooks(
     notebooks: List[str],
     container_uri: str,
     staging_bucket: str,
     artifacts_bucket: str,
-    results_file: str,
     should_parallelize: bool,
     timeout: int,
     variable_project_id: str,
@@ -458,7 +348,6 @@ def process_and_execute_notebooks(
     variable_service_account: str,
     variable_vpc_network: Optional[str] = None,
     private_pool_id: Optional[str] = None,
-    concurrent_notebooks: Optional[int] = 10,
 ):
     """
     Run the notebooks that exist under the folders defined in the test_paths_file.
@@ -479,8 +368,6 @@ def process_and_execute_notebooks(
             Required. The GCS staging bucket to write source code to.
         artifacts_bucket (str):
             Required. The GCS staging bucket to write executed notebooks to.
-        results_file (str):
-            Required: The path to the artifacts bucket to save results 
         variable_project_id (str):
             Required. The value for PROJECT_ID to inject into notebooks.
         variable_region (str):
@@ -489,7 +376,6 @@ def process_and_execute_notebooks(
             Required. Should run notebooks in parallel using a thread pool as opposed to in sequence.
         timeout (str):
             Required. Timeout string according to https://cloud.google.com/build/docs/build-config-file-schema#timeout.
-        concurrent_notebooks (int): Max number of notebooks per minute to run in parallel.
     """
 
     # Calculate deadline
@@ -506,9 +392,7 @@ def process_and_execute_notebooks(
             print(
                 "Running notebooks in parallel, so no logs will be displayed. Please wait..."
             )
-
-
-            with concurrent.futures.ThreadPoolExecutor(max_workers=concurrent_notebooks) as executor:
+            with concurrent.futures.ThreadPoolExecutor(max_workers=100) as executor:
                 print(f"Max workers: {executor._max_workers}")
 
                 notebook_execution_results = list(
@@ -586,7 +470,7 @@ def process_and_execute_notebooks(
             print("=" * 100)
 
             build_id = results_sorted[0].build_id
-            logs_bucket_name = (results_sorted[0].logs_bucket).replace("gs://", "")
+            logs_bucket_name = (results_sorted[0].logs_bucket).removeprefix("gs://")
             log_file_name = f"log-{build_id}.txt"
 
             log_contents = util.download_blob_into_memory(
@@ -604,10 +488,6 @@ def process_and_execute_notebooks(
             else:
                 print(log_contents)
 
-        _save_results(results_sorted, 
-                      artifacts_bucket, 
-                      results_file)
-
         print("\n=== END RESULTS===\n")
 
         total_notebook_duration = functools.reduce(

.cloud-build/notebook-execution-test-cloudbuild.yaml

@@ -36,7 +36,7 @@ steps:
     - -c
     - |
       . workspace/env/bin/activate &&
-      python3 .cloud-build/execute_changed_notebooks_cli.py --test_paths_file "${_TEST_PATHS_FILE}" --base_branch "${_FORCED_BASE_BRANCH}" --container_uri ${_PYTHON_IMAGE} --staging_bucket ${_GCS_STAGING_BUCKET}  --artifacts_bucket ${_GCS_STAGING_BUCKET}/executed_notebooks/PR_${_PR_NUMBER}/BUILD_${BUILD_ID} --variable_project_id ${PROJECT_ID} --variable_region ${_GCP_REGION} --variable_service_account ${_GCP_SERVICE_ACCOUNT} --variable_vpc_network "${_GPC_VPC_NETWORK_NAME}" `if [ ! -z "${_PRIVATE_POOL_NAME}" ]; then echo "--private_pool_id ${_PRIVATE_POOL_NAME}"; fi` --build_id ${BUILD_ID} --test_percent=${_TEST_PERCENT} --concurrent_notebooks=${_CONCURRENT_NOTEBOOKS}
+      python3 .cloud-build/execute_changed_notebooks_cli.py --test_paths_file "${_TEST_PATHS_FILE}" --base_branch "${_FORCED_BASE_BRANCH}" --container_uri ${_PYTHON_IMAGE} --staging_bucket ${_GCS_STAGING_BUCKET}  --artifacts_bucket ${_GCS_STAGING_BUCKET}/executed_notebooks/PR_${_PR_NUMBER}/BUILD_${BUILD_ID} --variable_project_id ${PROJECT_ID} --variable_region ${_GCP_REGION} --variable_service_account ${_GCP_SERVICE_ACCOUNT} --variable_vpc_network "${_GPC_VPC_NETWORK_NAME}" `if [ ! -z "${_PRIVATE_POOL_NAME}" ]; then echo "--private_pool_id ${_PRIVATE_POOL_NAME}"; fi`
     env:
     - 'IS_TESTING=1'
 timeout: 86400s

.cloud-build/requirements.txt

@@ -3,15 +3,11 @@ numpy
 jupyter
 nbconvert
 papermill
+pandas
 matplotlib
 tabulate
 google-cloud-aiplatform
 google-cloud-storage
 google-cloud-build
-google-cloud-storage
 ratemate
-GitPython
-tqdm
-fsspec
-pandas
-
+GitPython

.cloud-build/test_failing.txt

@@ -1,40 +0,0 @@
-notebooks/official/training/pytorch_gcs_data_training.ipynb
-notebooks/official/custom/custom_training_tensorboard_profiler.ipynb
-notebooks/official/custom/custom-tabular-bq-managed-dataset.ipynb
-notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-locally-trained-models.ipynb
-notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb
-notebooks/official/tabnet/tabnet_vertex_tutorial.ipynb
-notebooks/official/tabnet/get_started_with_tabnet.ipynb
-notebooks/official/pipelines/google_cloud_pipeline_components_automl_text.ipynb
-notebooks/official/pipelines/multicontender_vs_champion_deployment_method.ipynb
-notebooks/official/pipelines/google_cloud_pipeline_components_automl_images.ipynb
-notebooks/official/pipelines/rapid_prototyping_bqml_automl.ipynb
-notebooks/official/pipelines/challenger_vs_blessed_deployment_method.ipynb
-notebooks/official/matching_engine/sdk_matching_engine_create_stack_overflow_embeddings.ipynb
-notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb
-notebooks/official/matching_engine/sdk_matching_engine_create_text_to_image_embeddings.ipynb
-notebooks/official/explainable_ai/sdk_custom_image_classification_online_explain.ipynb
-notebooks/official/explainable_ai/xai_image_classification_feature_attributions.ipynb
-notebooks/official/explainable_ai/sdk_custom_image_classification_batch_explain.ipynb
-notebooks/official/tabular_workflows/tabnet_on_vertex_pipelines.ipynb
-notebooks/official/model_registry/get_started_with_model_registry.ipynb
-notebooks/official/model_registry/bqml_vertexai_model_registry.ipynb
-notebooks/official/sdk/SDK_Custom_Training_Python_Package_Managed_Text_Dataset_Tensorflow_Serving_Container.ipynb
-notebooks/official/model_monitoring/batch_prediction_model_monitoring.ipynb
-notebooks/official/model_monitoring/get_started_with_model_monitoring_setup.ipynb
-notebooks/official/model_monitoring/get_started_with_model_monitoring_custom.ipynb
-notebooks/official/model_monitoring/get_started_with_model_monitoring_custom_tf_serving.ipynb
-notebooks/official/model_monitoring/model_monitoring.ipynb
-notebooks/official/tensorboard/tensorboard_profiler_custom_training_with_prebuilt_container.ipynb
-notebooks/official/tensorboard/tensorboard_hyperparameter_tuning_with_hparams.ipynb
-notebooks/official/tensorboard/tensorboard_profiler_custom_training.ipynb
-notebooks/official/model_evaluation/custom_tabular_regression_model_evaluation.ipynb
-notebooks/official/model_evaluation/custom_tabular_classification_model_evaluation.ipynb
-notebooks/official/model_evaluation/automl_video_classification_model_evaluation.ipynb
-notebooks/official/experiments/comparing_local_trained_models.ipynb
-notebooks/official/automl/automl_image_classification_online_online_prediction.ipynb
-notebooks/official/automl/automl-text-classification.ipynb
-notebooks/official/automl/sdk_automl_video_object_tracking_batch.ipynb
-notebooks/official/feature_store/sdk-feature-store-pandas.ipynb
-notebooks/official/prediction/custom_batch_prediction_feature_filter.ipynb
-notebooks/official/prediction/pytorch_image_classification_with_prebuilt_serving_containers.ipynb

.cloud-build/test_passing.txt

@@ -1,80 +0,0 @@
-notebooks/official/training/hyperparameter_tuning_tensorflow.ipynb
-notebooks/official/training/get_started_with_vertex_distributed_training.ipynb
-notebooks/official/training/hyperparameter_tuning_xgboost.ipynb
-notebooks/official/training/multi_node_ddp_gloo_vertex_training_with_custom_container.ipynb
-notebooks/official/training/distributed_hyperparameter_tuning.ipynb
-notebooks/official/training/pytorch-text-sentiment-classification-custom-train-deploy.ipynb
-notebooks/official/training/xgboost_data_parallel_training_on_cpu_using_dask.ipynb
-notebooks/official/training/multi_node_ddp_nccl_vertex_training_with_custom_container.ipynb
-notebooks/official/bigquery_ml/get_started_with_bqml_training.ipynb
-notebooks/official/bigquery_ml/bqml-online-prediction.ipynb
-notebooks/official/custom/custom_training_container_and_model_registry.ipynb
-notebooks/official/custom/sdk-custom-image-classification-online.ipynb
-notebooks/official/custom/sdk-custom-image-classification-batch.ipynb
-notebooks/official/custom/SDK_FBProphet_Forecasting_Online.ipynb
-notebooks/official/custom/get_started_vertex_training_xgboost.ipynb
-notebooks/official/custom/get_started_with_vertex_endpoint_and_shared_vm.ipynb
-notebooks/official/custom/SDK_Custom_Container_Prediction.ipynb
-notebooks/official/reduction_server/pytorch_distributed_training_reduction_server.ipynb
-notebooks/official/tabnet/ai-explanations-tabnet-algorithm.ipynb
-notebooks/official/vizier/get_started_vertex_vizier.ipynb
-notebooks/official/vizier/gapic-vizier-multi-objective-optimization.ipynb
-notebooks/official/pipelines/get_started_with_hpt_pipeline_components.ipynb
-notebooks/official/pipelines/google_cloud_pipeline_components_automl_tabular.ipynb
-notebooks/official/pipelines/custom_tabular_train_batch_pred_bq_pipeline.ipynb
-notebooks/official/pipelines/metrics_viz_run_compare_kfp.ipynb
-notebooks/official/pipelines/google_cloud_pipeline_components_model_upload_predict_evaluate.ipynb
-notebooks/official/pipelines/google_cloud_pipeline_components_model_train_upload_deploy.ipynb
-notebooks/official/pipelines/get_started_with_machine_management.ipynb
-notebooks/official/pipelines/custom_model_training_and_batch_prediction.ipynb
-notebooks/official/pipelines/control_flow_kfp.ipynb
-notebooks/official/pipelines/lightweight_functions_component_io_kfp.ipynb
-notebooks/official/pipelines/google_cloud_pipeline_components_bqml_text.ipynb
-notebooks/official/pipelines/pipelines_intro_kfp.ipynb
-notebooks/official/pipelines/automl_tabular_classification_beans.ipynb
-notebooks/official/pipelines/google_cloud_pipeline_components_dataproc_tabular.ipynb
-notebooks/official/explainable_ai/sdk_automl_tabular_classification_online_explain.ipynb
-notebooks/official/explainable_ai/sdk_custom_tabular_regression_online_explain.ipynb
-notebooks/official/explainable_ai/sdk_automl_tabular_binary_classification_batch_explain.ipynb
-notebooks/official/explainable_ai/sdk_custom_tabular_regression_online_explain_get_metadata.ipynb
-notebooks/official/explainable_ai/sdk_custom_tabular_regression_batch_explain.ipynb
-notebooks/official/tabular_workflows/prophet_on_vertex_pipelines.ipynb
-notebooks/official/tabular_workflows/wide_and_deep_on_vertex_pipelines.ipynb
-notebooks/official/sdk/SDK_AutoML_Video_Classification.ipynb
-notebooks/official/model_monitoring/get_started_with_model_monitoring_automl.ipynb
-notebooks/official/model_monitoring/get_started_with_model_monitoring_automl_image_batch.ipynb
-notebooks/official/model_monitoring/get_started_with_model_monitoring_automl_image_online.ipynb
-notebooks/official/model_monitoring/get_started_with_model_monitoring_xgboost.ipynb
-notebooks/official/tensorboard/tensorboard_custom_training_with_custom_container.ipynb
-notebooks/official/tensorboard/tensorboard_custom_training_with_prebuilt_container.ipynb
-notebooks/official/tensorboard/tensorboard_vertex_ai_pipelines_integration.ipynb
-notebooks/official/model_evaluation/automl_text_classification_model_evaluation.ipynb
-notebooks/official/model_evaluation/get_started_with_custom_model_evaluation_import.ipynb
-notebooks/official/model_evaluation/automl_tabular_classification_model_evaluation.ipynb
-notebooks/official/model_evaluation/automl_tabular_regression_model_evaluation.ipynb
-notebooks/official/experiments/get_started_with_vertex_experiments.ipynb
-notebooks/official/experiments/comparing_pipeline_runs.ipynb
-notebooks/official/experiments/get_started_with_vertex_experiments_autologging.ipynb
-notebooks/official/experiments/build_model_experimentation_lineage_with_prebuild_code.ipynb
-notebooks/official/experiments/delete_outdated_tensorboard_experiments.ipynb
-notebooks/official/automl/sdk_automl_tabular_regression_batch_bq.ipynb
-notebooks/official/automl/sdk_automl_text_sentiment_analysis_online.ipynb
-notebooks/official/automl/sdk_automl_text_entity_extraction_online.ipynb
-notebooks/official/automl/sdk_automl_forecasting_hierarchical_batch.ipynb
-notebooks/official/automl/automl_text_entity_extraction_batch_prediction.ipynb
-notebooks/official/automl/automl_image_classification_batch_prediction.ipynb
-notebooks/official/automl/automl_text_sentiment_analysis_batch_prediction.ipynb
-notebooks/official/automl/sdk_automl_tabular_regression_online_bq.ipynb
-notebooks/official/automl/get_started_automl_training.ipynb
-notebooks/official/automl/automl-tabular-classification.ipynb
-notebooks/official/automl/automl_image_object_detection_export_edge.ipynb
-notebooks/official/automl/sdk_automl_image_object_detection_batch.ipynb
-notebooks/official/automl/automl_tabular_on_vertex_pipelines.ipynb
-notebooks/official/automl/sdk_automl_video_classification_batch.ipynb
-notebooks/official/automl/sdk_automl_video_action_recognition_batch.ipynb
-notebooks/official/automl/sdk_automl_tabular_forecasting_batch.ipynb
-notebooks/official/automl/automl_image_object_detection_online_prediction.ipynb
-notebooks/official/automl/automl_forecasting_bqml_arima_plus_comparison.ipynb
-notebooks/official/datasets/get_started_bq_datasets.ipynb
-notebooks/official/datasets/get_started_with_data_labeling.ipynb
-notebooks/official/feature_store/feature_store_streaming_ingestion_sdk.ipynb

.cloud-build/tests/flaky.py

@@ -1,46 +0,0 @@
-# grep PASSED tests.txt | cut -c 10-100 >passed.txt
-
-import os
-
-repo_dir = '/home/jupyter/vertex-ai-samples/'
-repo_dir_len = len(repo_dir)
-official_dir = repo_dir + 'notebooks/official'
-
-entries = os.scandir(official_dir)
-folders = []
-for entry in entries:
-    if entry.is_dir():
-        folders.append(entry.path)
-
-# Passing
-with open('passed.txt', 'r') as pass_file:
-    notebook_names = pass_file.readlines()
-        
-notebooks = []
-for folder in folders:
-    entries = os.scandir(folder)
-    for entry in entries:
-        for notebook in notebook_names:
-            if entry.name == notebook.rstrip():
-                notebooks.append(entry.path[repo_dir_len:])
-                
-with open('passing_tests.txt', 'w') as f:
-    for notebook in notebooks:
-        f.write(notebook + '\n')
-        
-        
-# Failing
-with open('failed.txt', 'r') as fail_file:
-    notebook_names = fail_file.readlines()
-        
-notebooks = []
-for folder in folders:
-    entries = os.scandir(folder)
-    for entry in entries:
-        for notebook in notebook_names:
-            if entry.name == notebook.rstrip():
-                notebooks.append(entry.path[repo_dir_len:])
-                
-with open('failing_tests.txt', 'w') as f:
-    for notebook in notebooks:
-        f.write(notebook + '\n')

.cloud-build/tests/select_notebooks_test.py

@@ -1,33 +0,0 @@
-import sys
-
-from execute_changed_notebooks_helper import (load_results, select_notebook)
-
-
-def test_load_results():
-    bucket: str = "cloud-build-notebooks-presubmit"
-    bucket_file: str = "build_results"
-
-    accum = load_results(bucket, bucket_file)
-
-    print(accum)
-
-    assert len(accum) > 0
-
-def test_select_notebook():
-    bucket: str = "cloud-build-notebooks-presubmit"
-    bucket_file: str = "build_results"
-
-    accum = load_results(bucket, bucket_file)
-
-    n_select = 0
-    n_notselect = 0
-    for notebook in accum:
-        if select_notebook(notebook, accum, 50):
-            n_select += 1
-        else:
-            n_notselect += 1
-
-    print(f"SELECTED {n_select}, NOT SELECTED {n_notselect}")
-
-    assert n_select > 0
-    assert n_notselect > 0

.cloud-build/utils/NotebookProcessors.py

@@ -73,19 +73,13 @@ def generate_uuid(length: int = 8) -> str:
 
 
 class UniqueStringsPreprocessor(Preprocessor):
-    # A preprocessor that replaces strings that end with "-unique" or "_unique" with a uuid.
+    # A preprocessor that replaces strings that end with "-unique" with a uuid.
 
     @staticmethod
     def update_unique_strings(content: str):
-        # Replace strings that end with "-unique" or "_unique" with a uuid.
+        # Replace strings that end with "-unique" with a uuid.
 
-        unique_id = generate_uuid()
-        return (
-            content.replace('-unique"', f'-{unique_id}"')
-            .replace("-unique'", f'-{unique_id}"')
-            .replace('_unique"', f'_{unique_id}"')
-            .replace("_unique'", f'_{unique_id}"')
-        )
+        return content.replace('-unique"', f'-{generate_uuid()}"')
 
     def preprocess(self, notebook, resources=None):
         executable_cells = []

.cloud-build/utils/build_results_viewer.py

@@ -1,22 +0,0 @@
-'''
-Viewer for the weekly regression testing of the official notebooks
-
-Cloud Storage location: gs://cloud-build-notebooks-presubmit/build_results/
-'''
-import argparse
-import json
-
-
-parser = argparse.ArgumentParser()
-parser.add_argument('--file', dest='file',
-                    default='build.json', type=str, help='build results file')
-import json
-
-with open('build.json', 'r') as f:
-    results = json.load(f)
-
-for item in results.items():
-    if item[1]['passed']:
-        print(f"{item[0]},PASSED")
-    else:
-        print(f"{item[0]},FAILED")

.cloud-build/utils/util.py

@@ -61,7 +61,9 @@ def archive_code_and_upload(staging_bucket: str):
 
 
 def download_blob_into_memory(
-    bucket_name: str, blob_name: str, download_as_text: Optional[bool] = False
+  bucket_name: str,
+  blob_name: str,
+  download_as_text: Optional[bool]=False
 ) -> Union[bytes, str]:
     """
     Downloads a blob into memory as byte or as text if
@@ -77,10 +79,13 @@ def download_blob_into_memory(
 
     # Download the blob content
     if download_as_text:
-        contents = blob.download_as_text()
+      contents = blob.download_as_text()
     else:
-        contents = blob.download_as_bytes()
+      contents = blob.download_as_bytes()
 
-    print(f"Downloaded storage object {blob_name} from bucket {bucket_name}.")
+    print(
+        f"Downloaded storage object {blob_name} from bucket {bucket_name}."
+    )
 
     return contents
+

.cloud-build/web-index/notebook-index-cloudbuild.yaml

@@ -1,23 +0,0 @@
-steps:
-  # Fetch full repo for diff purposes
-  - name: gcr.io/cloud-builders/git
-    args: [fetch, --unshallow, --quiet]
-  # Create a virtual environment
-  - name: ${_PYTHON_IMAGE}
-    entrypoint: /bin/sh
-    args:
-    - -c
-    - python3 -m venv workspace/env
-  # Install Python dependencies and run testing script
-  - name: ${_PYTHON_IMAGE}
-    entrypoint: /bin/sh
-    args:
-    - -c
-    - |
-      . workspace/env/bin/activate &&
-      python3 notebooks/notebook_template_review.py --web --title --steps --desc --linkback --notebook-dir=notebooks/official >web.html
-artifacts:
-  objects:
-          location: gs://${_GCS_ARTIFACTS_BUCKET}/webdoc
-          paths: ['web.html']
-timeout: 86400s

.github/workflows/linter/requirements.txt

@@ -2,9 +2,9 @@ git+https://github.com/tensorflow/docs
 ipython
 jupyter
 nbconvert
-black==23.3.0
-pyupgrade==3.7.0
-isort==5.12.0
-flake8==6.0.0
-nbqa==1.7.0
+black==22.6.0
+pyupgrade==2.34.0
+isort==5.10.1
+flake8==4.0.1
+nbqa==1.4.0
 

.github/workflows/linter/run_linter.sh

@@ -84,7 +84,7 @@ if [ ${#notebooks[@]} -gt 0 ]; then
                 # python3 -m nbqa black "$notebook" --check
                 # BLACK_RTN=$?
                 echo "Running pyupgrade..."
-                python3 -m nbqa pyupgrade --exit-zero-even-if-changed "$notebook"
+                python3 -m nbqa pyupgrade "$notebook"
                 PYUPGRADE_RTN=$?
                 echo "Running isort..."
                 python3 -m nbqa isort "$notebook" --check
@@ -97,7 +97,7 @@ if [ ${#notebooks[@]} -gt 0 ]; then
                 python3 -m nbqa black "$notebook"
                 BLACK_RTN=$?
                 echo "Running pyupgrade..."
-                python3 -m nbqa pyupgrade --exit-zero-even-if-changed "$notebook"
+                python3 -m nbqa pyupgrade "$notebook"
                 PYUPGRADE_RTN=$?
                 echo "Running isort..."
                 python3 -m nbqa isort "$notebook"

CONTRIBUTING.md

@@ -44,10 +44,12 @@ Finally, run this code block to check for errors. Each step will attempt to
 automatically fix any issues. If the fixes can't be performed automatically,
 then you will need to manually address them before submitting your PR.
 
-Note: For official, only submit one notebook per PR.
-
 ```shell
-docker run -v ${PWD}:/setup/app gcr.io/cloud-devrel-public-resources/notebook_linter:latest your_notebook
+nbqa black "$notebook"
+nbqa pyupgrade "$notebook"
+nbqa isort "$notebook"
+nbqa flake8 "$notebook" --extend-ignore=W391,E501,F821,E402,F404,W503,E203,E722,W293,W291
+python3 -m tensorflow_docs.tools.nbfmt --remove_outputs "$notebook"
 ```
 
 ## Code Reviews

community-content/CODEOWNERS

@@ -7,9 +7,3 @@
 /pluto_on_workbench @wkharold
 /cpr-examples @samthrasher
 /Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines @Ark-kun
-/pipeline_components @Ark-kun
-/pipeline_components/image_ml_model_training @lakeyk
-/prediction_featurestore_integration @googleapis/vertex-prediction-team
-/vertex_vision_model_garden/model_oss/util @weigary
-/vertex_vision_model_garden/model_oss/diffusers @weigary
-/vertex_vision_model_garden/model_oss/transformers @dstnluong-google

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_logistic_regression_model_using_Scikit_learn_and_import_to_Vertex_AI/pipeline.py

@@ -2,13 +2,13 @@
 from kfp import components
 
 # %% Loading components
-download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/storage/download/component.yaml")
-select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml")
-fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
-binarize_column_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Binarize_column/in_CSV_format/component.yaml")
-train_logistic_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/1f5cf6e06409b704064b2086c0a705e4e6b4fcde/community-content/pipeline_components/ML_frameworks/Scikit_learn/Train_logistic_regression_model/from_CSV/component.yaml")
-upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/component.yaml")
-deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
+download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml")
+select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml")
+fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
+binarize_column_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml")
+train_logistic_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/cb44b75c9c062fcc40c2b905b2024b4493dbc62b/components/ML_frameworks/Scikit_learn/Train_logistic_regression_model/from_CSV/component.yaml")
+upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
+deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/workaround_for_buggy_KFPv2_compiler/component.yaml")
 
 # %% Pipeline definition
 def train_tabular_classification_logistic_regression_model_using_Scikit_learn_pipeline():

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_model_using_PyTorch_and_import_to_Vertex_AI/pipeline.py

@@ -2,15 +2,15 @@
 from kfp import components
 
 # %% Loading components
-download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/storage/download/component.yaml")
-select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml")
-fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
-binarize_column_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Binarize_column/in_CSV_format/component.yaml")
-create_fully_connected_pytorch_network_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Create_fully_connected_network/component.yaml")
-train_pytorch_model_from_csv_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml")
-create_pytorch_model_archive_with_base_handler_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml")
-upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/component.yaml")
-deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
+download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml")
+select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml")
+fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
+binarize_column_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml")
+create_fully_connected_pytorch_network_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/1a2ef3eeb77bc278f33cad0dd29008ea2431e191/components/PyTorch/Create_fully_connected_network/component.yaml")
+train_pytorch_model_from_csv_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml")
+create_pytorch_model_archive_with_base_handler_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/46d51383e6554b7f3ab4fd8cf614d8c2b422fb22/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml")
+upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/workaround_for_buggy_KFPv2_compiler/component.yaml")
+deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/workaround_for_buggy_KFPv2_compiler/component.yaml")
 
 # %% Pipeline definition
 def train_tabular_classification_model_using_PyTorch_pipeline():

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_model_using_TensorFlow_and_import_to_Vertex_AI/pipeline.py

@@ -2,16 +2,16 @@
 from kfp import components
 
 # %% Loading components
-download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/storage/download/component.yaml")
-select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml")
-fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
-binarize_column_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Binarize_column/in_CSV_format/component.yaml")
-split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
-create_fully_connected_tensorflow_network_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Create_fully_connected_network/component.yaml")
-train_model_using_Keras_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml")
-predict_with_TensorFlow_model_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Predict/on_CSV/component.yaml")
-upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/component.yaml")
-deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
+download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml")
+select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml")
+fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
+binarize_column_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml")
+split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
+create_fully_connected_tensorflow_network_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/9ca0f9eecf5f896f65b8538bbd809747052617d1/components/tensorflow/Create_fully_connected_network/component.yaml")
+train_model_using_Keras_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c504a4010348c50eaaf6d4337586ccc008f4dcef/components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml")
+predict_with_TensorFlow_model_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/59c759ce6f543184e30db6817d2a703879bc0f39/components/tensorflow/Predict/on_CSV/component.yaml")
+upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
+deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/workaround_for_buggy_KFPv2_compiler/component.yaml")
 
 # %% Pipeline definition
 def train_tabular_classification_model_using_TensorFlow_pipeline():

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_model_using_XGBoost_and_import_to_Vertex_AI/pipeline.py

@@ -2,15 +2,15 @@
 from kfp import components
 
 # %% Loading components
-download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/storage/download/component.yaml")
-select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml")
-fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
-binarize_column_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Binarize_column/in_CSV_format/component.yaml")
-split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
-train_XGBoost_model_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Train/component.yaml")
-xgboost_predict_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Predict/component.yaml")
-upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/component.yaml")
-deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
+download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml")
+select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml")
+fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
+binarize_column_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml")
+split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
+train_XGBoost_model_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/58d3a47f904f32a64af8403330ba7e2134cae46d/components/XGBoost/Train/component.yaml")
+xgboost_predict_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/4694ec97baccf59284c2a1db4aa2250c22291eab/components/XGBoost/Predict/component.yaml")
+upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
+deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/workaround_for_buggy_KFPv2_compiler/component.yaml")
 
 # %% Pipeline definition
 def train_tabular_classification_model_using_XGBoost_pipeline():

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_model_using_all_frameworks_and_import_to_Vertex_AI/pipeline.py

@@ -2,36 +2,36 @@
 from kfp import components
 
 # %% Loading components
-download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/storage/download/component.yaml")
-select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml")
-fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
-binarize_column_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Binarize_column/in_CSV_format/component.yaml")
-split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
+download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml")
+select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml")
+fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
+binarize_column_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml")
+split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
 
 # TensorFlow
-create_fully_connected_tensorflow_network_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Create_fully_connected_network/component.yaml")
-train_model_using_Keras_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml")
-predict_with_TensorFlow_model_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Predict/on_CSV/component.yaml")
-upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/component.yaml")
+create_fully_connected_tensorflow_network_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/9ca0f9eecf5f896f65b8538bbd809747052617d1/components/tensorflow/Create_fully_connected_network/component.yaml")
+train_model_using_Keras_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c504a4010348c50eaaf6d4337586ccc008f4dcef/components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml")
+predict_with_TensorFlow_model_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/59c759ce6f543184e30db6817d2a703879bc0f39/components/tensorflow/Predict/on_CSV/component.yaml")
+upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
 
 # PyTorch
-create_fully_connected_pytorch_network_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Create_fully_connected_network/component.yaml")
-train_pytorch_model_from_csv_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml")
-create_pytorch_model_archive_with_base_handler_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml")
-upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/component.yaml")
+create_fully_connected_pytorch_network_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/1a2ef3eeb77bc278f33cad0dd29008ea2431e191/components/PyTorch/Create_fully_connected_network/component.yaml")
+train_pytorch_model_from_csv_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml")
+create_pytorch_model_archive_with_base_handler_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/46d51383e6554b7f3ab4fd8cf614d8c2b422fb22/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml")
+upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/workaround_for_buggy_KFPv2_compiler/component.yaml")
 
 # XGBoost
-train_XGBoost_model_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Train/component.yaml")
-xgboost_predict_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Predict/component.yaml")
-upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/component.yaml")
+train_XGBoost_model_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/58d3a47f904f32a64af8403330ba7e2134cae46d/components/XGBoost/Train/component.yaml")
+xgboost_predict_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/4694ec97baccf59284c2a1db4aa2250c22291eab/components/XGBoost/Predict/component.yaml")
+upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
 
 # Scikit-learn
-#train_linear_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/1f5cf6e06409b704064b2086c0a705e4e6b4fcde/community-content/pipeline_components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml")
-train_logistic_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/1f5cf6e06409b704064b2086c0a705e4e6b4fcde/community-content/pipeline_components/ML_frameworks/Scikit_learn/Train_logistic_regression_model/from_CSV/component.yaml")
-upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/component.yaml")
+#train_linear_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/f807e02b54d4886c65a05f40848fd51c72407f40/components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml")
+train_logistic_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/cb44b75c9c062fcc40c2b905b2024b4493dbc62b/components/ML_frameworks/Scikit_learn/Train_logistic_regression_model/from_CSV/component.yaml")
+upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
 
 # Vertex AI
-deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
+deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/workaround_for_buggy_KFPv2_compiler/component.yaml")
 
 # %% Pipeline definition
 def train_tabular_classification_model_using_all_frameworks_pipeline():

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_regression_linear_model_using_Scikit_learn_and_import_to_Vertex_AI/pipeline.py

@@ -2,12 +2,12 @@
 from kfp import components
 
 # %% Loading components
-download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/storage/download/component.yaml")
-select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml")
-fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
-train_linear_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/1f5cf6e06409b704064b2086c0a705e4e6b4fcde/community-content/pipeline_components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml")
-upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/component.yaml")
-deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
+download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml")
+select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml")
+fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
+train_linear_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/f807e02b54d4886c65a05f40848fd51c72407f40/components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml")
+upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
+deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/workaround_for_buggy_KFPv2_compiler/component.yaml")
 
 # %% Pipeline definition
 def train_tabular_regression_linear_model_using_Scikit_learn_pipeline():

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_regression_model_using_PyTorch_and_import_to_Vertex_AI/pipeline.py

@@ -2,14 +2,14 @@
 from kfp import components
 
 # %% Loading components
-download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/storage/download/component.yaml")
-select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml")
-fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
-create_fully_connected_pytorch_network_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Create_fully_connected_network/component.yaml")
-train_pytorch_model_from_csv_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml")
-create_pytorch_model_archive_with_base_handler_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml")
-upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/component.yaml")
-deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
+download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml")
+select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml")
+fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
+create_fully_connected_pytorch_network_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/1a2ef3eeb77bc278f33cad0dd29008ea2431e191/components/PyTorch/Create_fully_connected_network/component.yaml")
+train_pytorch_model_from_csv_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml")
+create_pytorch_model_archive_with_base_handler_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/46d51383e6554b7f3ab4fd8cf614d8c2b422fb22/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml")
+upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/workaround_for_buggy_KFPv2_compiler/component.yaml")
+deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/workaround_for_buggy_KFPv2_compiler/component.yaml")
 
 # %% Pipeline definition
 def train_tabular_regression_model_using_PyTorch_pipeline():

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_regression_model_using_TensorFlow_and_import_to_Vertex_AI/pipeline.py

@@ -2,15 +2,15 @@
 from kfp import components
 
 # %% Loading components
-download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/storage/download/component.yaml")
-select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml")
-fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
-split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
-create_fully_connected_tensorflow_network_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Create_fully_connected_network/component.yaml")
-train_model_using_Keras_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml")
-predict_with_TensorFlow_model_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Predict/on_CSV/component.yaml")
-upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/component.yaml")
-deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
+download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml")
+select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml")
+fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
+split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
+create_fully_connected_tensorflow_network_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/9ca0f9eecf5f896f65b8538bbd809747052617d1/components/tensorflow/Create_fully_connected_network/component.yaml")
+train_model_using_Keras_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c504a4010348c50eaaf6d4337586ccc008f4dcef/components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml")
+predict_with_TensorFlow_model_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/59c759ce6f543184e30db6817d2a703879bc0f39/components/tensorflow/Predict/on_CSV/component.yaml")
+upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
+deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/workaround_for_buggy_KFPv2_compiler/component.yaml")
 
 # %% Pipeline definition
 def train_tabular_regression_model_using_Tensorflow_pipeline():

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_regression_model_using_XGBoost_and_import_to_Vertex_AI/pipeline.py

@@ -2,14 +2,14 @@
 from kfp import components
 
 # %% Loading components
-download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/storage/download/component.yaml")
-select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml")
-fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
-split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
-train_XGBoost_model_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Train/component.yaml")
-xgboost_predict_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Predict/component.yaml")
-upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/component.yaml")
-deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
+download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml")
+select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml")
+fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
+split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
+train_XGBoost_model_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/58d3a47f904f32a64af8403330ba7e2134cae46d/components/XGBoost/Train/component.yaml")
+xgboost_predict_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/4694ec97baccf59284c2a1db4aa2250c22291eab/components/XGBoost/Predict/component.yaml")
+upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
+deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/workaround_for_buggy_KFPv2_compiler/component.yaml")
 
 # %% Pipeline definition
 def train_tabular_regression_model_using_XGBoost_pipeline():

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_regression_model_using_all_frameworks_and_import_to_Vertex_AI/pipeline.py

@@ -2,34 +2,34 @@
 from kfp import components
 
 # %% Loading components
-download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/storage/download/component.yaml")
-select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml")
-fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
-split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
+download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml")
+select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml")
+fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
+split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
 
 # TensorFlow
-create_fully_connected_tensorflow_network_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Create_fully_connected_network/component.yaml")
-train_model_using_Keras_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml")
-predict_with_TensorFlow_model_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Predict/on_CSV/component.yaml")
-upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/component.yaml")
+create_fully_connected_tensorflow_network_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/9ca0f9eecf5f896f65b8538bbd809747052617d1/components/tensorflow/Create_fully_connected_network/component.yaml")
+train_model_using_Keras_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c504a4010348c50eaaf6d4337586ccc008f4dcef/components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml")
+predict_with_TensorFlow_model_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/59c759ce6f543184e30db6817d2a703879bc0f39/components/tensorflow/Predict/on_CSV/component.yaml")
+upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
 
 # PyTorch
-create_fully_connected_pytorch_network_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Create_fully_connected_network/component.yaml")
-train_pytorch_model_from_csv_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml")
-create_pytorch_model_archive_with_base_handler_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml")
-upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/component.yaml")
+create_fully_connected_pytorch_network_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/1a2ef3eeb77bc278f33cad0dd29008ea2431e191/components/PyTorch/Create_fully_connected_network/component.yaml")
+train_pytorch_model_from_csv_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml")
+create_pytorch_model_archive_with_base_handler_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/46d51383e6554b7f3ab4fd8cf614d8c2b422fb22/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml")
+upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/workaround_for_buggy_KFPv2_compiler/component.yaml")
 
 # XGBoost
-train_XGBoost_model_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Train/component.yaml")
-xgboost_predict_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Predict/component.yaml")
-upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/component.yaml")
+train_XGBoost_model_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/58d3a47f904f32a64af8403330ba7e2134cae46d/components/XGBoost/Train/component.yaml")
+xgboost_predict_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/4694ec97baccf59284c2a1db4aa2250c22291eab/components/XGBoost/Predict/component.yaml")
+upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
 
 # Scikit-learn
-train_linear_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/1f5cf6e06409b704064b2086c0a705e4e6b4fcde/community-content/pipeline_components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml")
-upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/component.yaml")
+train_linear_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/f807e02b54d4886c65a05f40848fd51c72407f40/components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml")
+upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
 
 # Vertex AI
-deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
+deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/workaround_for_buggy_KFPv2_compiler/component.yaml")
 
 # %% Pipeline definition
 def train_tabular_regression_model_using_all_frameworks_pipeline():

community-content/pipeline_components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml

@@ -1,64 +0,0 @@
-name: Train linear regression model using scikit learn from CSV
-metadata:
-  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml'}
-inputs:
-- {name: dataset, type: CSV}
-- {name: label_column_name, type: String}
-outputs:
-- {name: model, type: ScikitLearnPickleModel}
-implementation:
-  container:
-    image: python:3.9
-    command:
-    - sh
-    - -c
-    - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
-      'scikit-learn==1.0.2' 'pandas==1.4.3' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3
-      -m pip install --quiet --no-warn-script-location 'scikit-learn==1.0.2' 'pandas==1.4.3'
-      --user) && "$0" "$@"
-    - sh
-    - -ec
-    - |
-      program_path=$(mktemp)
-      printf "%s" "$0" > "$program_path"
-      python3 -u "$program_path" "$@"
-    - |
-      def _make_parent_dirs_and_return_path(file_path: str):
-          import os
-          os.makedirs(os.path.dirname(file_path), exist_ok=True)
-          return file_path
-
-      def train_linear_regression_model_using_scikit_learn_from_CSV(
-          dataset_path,
-          model_path,
-          label_column_name,
-      ):
-          import pandas
-          import pickle
-          from sklearn import linear_model
-
-          df = pandas.read_csv(dataset_path)
-          model = linear_model.LinearRegression()
-          model.fit(
-              X=df.drop(columns=label_column_name),
-              y=df[label_column_name],
-          )
-
-          with open(model_path, "wb") as f:
-              pickle.dump(model, f)
-
-      import argparse
-      _parser = argparse.ArgumentParser(prog='Train linear regression model using scikit learn from CSV', description='')
-      _parser.add_argument("--dataset", dest="dataset_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--label-column-name", dest="label_column_name", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--model", dest="model_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
-      _parsed_args = vars(_parser.parse_args())
-
-      _outputs = train_linear_regression_model_using_scikit_learn_from_CSV(**_parsed_args)
-    args:
-    - --dataset
-    - {inputPath: dataset}
-    - --label-column-name
-    - {inputValue: label_column_name}
-    - --model
-    - {outputPath: model}

community-content/pipeline_components/ML_frameworks/Scikit_learn/Train_logistic_regression_model/from_CSV/component.yaml

@@ -1,163 +0,0 @@
-name: Train logistic regression model using scikit learn from CSV
-description: Train logistic regression model using Scikit-learn
-metadata:
-  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/ML_frameworks/Scikit_learn/Train_logistic_regression_model/from_CSV/component.yaml'}
-inputs:
-- {name: dataset, type: CSV}
-- {name: label_column_name, type: String}
-- {name: penalty, type: String, default: l2, optional: true}
-- {name: solver, type: String, default: lbfgs, optional: true}
-- {name: max_iterations, type: Integer, default: '100', optional: true}
-- {name: multi_class_mode, type: String, default: auto, optional: true}
-- {name: random_seed, type: Integer, default: '0', optional: true}
-outputs:
-- {name: model, type: ScikitLearnPickleModel}
-- {name: model_parameters, type: JsonObject}
-implementation:
-  container:
-    image: python:3.9
-    command:
-    - sh
-    - -c
-    - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
-      'scikit-learn==1.0.2' 'pandas==1.4.3' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3
-      -m pip install --quiet --no-warn-script-location 'scikit-learn==1.0.2' 'pandas==1.4.3'
-      --user) && "$0" "$@"
-    - sh
-    - -ec
-    - |
-      program_path=$(mktemp)
-      printf "%s" "$0" > "$program_path"
-      python3 -u "$program_path" "$@"
-    - |
-      def _make_parent_dirs_and_return_path(file_path: str):
-          import os
-          os.makedirs(os.path.dirname(file_path), exist_ok=True)
-          return file_path
-
-      def train_logistic_regression_model_using_scikit_learn_from_CSV(
-          dataset_path,
-          model_path,
-          label_column_name,
-          penalty = "l2", # l1, l2, elasticnet, none
-          solver = "lbfgs", # newton-cg, lbfgs, liblinear, sag, saga
-          max_iterations = 100,
-          multi_class_mode = "auto", # auto, ovr, multinomial
-          random_seed = 0,
-      ):
-          """Train logistic regression model using Scikit-learn
-
-          See https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html
-          """
-          import json
-          import pandas
-          import pickle
-          from sklearn import linear_model
-
-          df = pandas.read_csv(dataset_path)
-          model = linear_model.LogisticRegression(
-              penalty=penalty,
-              #dual=False,
-              #tol=1e-4,
-              #C=1.0,
-              #fit_intercept=True,
-              #intercept_scaling=1,
-              #class_weight=None,
-              random_state=random_seed,
-              solver=solver,
-              max_iter=max_iterations,
-              multi_class=multi_class_mode,
-              #l1_ratio=None,
-              verbose=1,
-          )
-
-          model_parameters = model.get_params()
-          model_parameters_json = json.dumps(model_parameters, indent=2)
-          print("Model parameters:")
-          print(model_parameters_json)
-          print()
-
-          model.fit(
-              X=df.drop(columns=label_column_name),
-              y=df[label_column_name],
-          )
-
-          with open(model_path, "wb") as f:
-              pickle.dump(model, f)
-
-          return (model_parameters_json,)
-
-      def _serialize_json(obj) -> str:
-          if isinstance(obj, str):
-              return obj
-          import json
-          def default_serializer(obj):
-              if hasattr(obj, 'to_struct'):
-                  return obj.to_struct()
-              else:
-                  raise TypeError("Object of type '%s' is not JSON serializable and does not have .to_struct() method." % obj.__class__.__name__)
-          return json.dumps(obj, default=default_serializer, sort_keys=True)
-
-      import argparse
-      _parser = argparse.ArgumentParser(prog='Train logistic regression model using scikit learn from CSV', description='Train logistic regression model using Scikit-learn')
-      _parser.add_argument("--dataset", dest="dataset_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--label-column-name", dest="label_column_name", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--penalty", dest="penalty", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--solver", dest="solver", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--max-iterations", dest="max_iterations", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--multi-class-mode", dest="multi_class_mode", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--random-seed", dest="random_seed", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--model", dest="model_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("----output-paths", dest="_output_paths", type=str, nargs=1)
-      _parsed_args = vars(_parser.parse_args())
-      _output_files = _parsed_args.pop("_output_paths", [])
-
-      _outputs = train_logistic_regression_model_using_scikit_learn_from_CSV(**_parsed_args)
-
-      _output_serializers = [
-          _serialize_json,
-
-      ]
-
-      import os
-      for idx, output_file in enumerate(_output_files):
-          try:
-              os.makedirs(os.path.dirname(output_file))
-          except OSError:
-              pass
-          with open(output_file, 'w') as f:
-              f.write(_output_serializers[idx](_outputs[idx]))
-    args:
-    - --dataset
-    - {inputPath: dataset}
-    - --label-column-name
-    - {inputValue: label_column_name}
-    - if:
-        cond: {isPresent: penalty}
-        then:
-        - --penalty
-        - {inputValue: penalty}
-    - if:
-        cond: {isPresent: solver}
-        then:
-        - --solver
-        - {inputValue: solver}
-    - if:
-        cond: {isPresent: max_iterations}
-        then:
-        - --max-iterations
-        - {inputValue: max_iterations}
-    - if:
-        cond: {isPresent: multi_class_mode}
-        then:
-        - --multi-class-mode
-        - {inputValue: multi_class_mode}
-    - if:
-        cond: {isPresent: random_seed}
-        then:
-        - --random-seed
-        - {inputValue: random_seed}
-    - --model
-    - {outputPath: model}
-    - '----output-paths'
-    - {outputPath: model_parameters}

community-content/pipeline_components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml

@@ -1,41 +0,0 @@
-name: Create PyTorch Model Archive with base handler
-inputs:
-- {name: Model, type: PyTorchScriptModule}
-- {name: Model name, type: String, default: model}
-- {name: Model version, type: String, default: "1.0"}
-outputs:
-- {name: Model archive, type: PyTorchModelArchive}
-metadata:
-  annotations:
-    author: Alexey Volkov <alexey.volkov@ark-kun.com>
-    canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml'
-implementation:
-  container:
-    image: pytorch/torchserve:0.6.0-cpu
-    command:
-    - bash
-    - -exc
-    - |
-      model_path=$0
-      model_name=$1
-      model_version=$2
-      output_model_archive_path=$3
-
-      mkdir -p "$(dirname "$output_model_archive_path")"
-
-      # TODO: Use the built-in base_handler once my fix is merged: https://github.com/pytorch/serve/pull/1682
-      echo '
-      from ts.torch_handler import base_handler
-      class BaseHandler(base_handler.BaseHandler):
-          pass
-      ' > base_handler.py  # torch-model-archiver needs the handler to have .py extension
-      torch-model-archiver --model-name "$model_name" --version "$model_version" --serialized-file "$model_path" --handler base_handler.py
-
-      # torch-model-archiver does not allow specifying the output path, but always writes to "${model_name}.<format>"
-      expected_model_archive_path="${model_name}.mar"
-      mv "$expected_model_archive_path" "$output_model_archive_path"
-
-    - {inputPath: Model}
-    - {inputValue: Model name}
-    - {inputValue: Model version}
-    - {outputPath: Model archive}

community-content/pipeline_components/PyTorch/Create_fully_connected_network/component.yaml

@@ -1,117 +0,0 @@
-name: Create fully connected pytorch network
-description: Creates fully-connected network in PyTorch ScriptModule format
-metadata:
-  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/PyTorch/Create_fully_connected_network/component.yaml'}
-inputs:
-- {name: input_size, type: Integer}
-- {name: hidden_layer_sizes, type: JsonArray, default: '[]', optional: true}
-- {name: output_size, type: Integer, default: '1', optional: true}
-- {name: activation_name, type: String, default: relu, optional: true}
-- {name: output_activation_name, type: String, optional: true}
-- {name: random_seed, type: Integer, default: '0', optional: true}
-outputs:
-- {name: model, type: PyTorchScriptModule}
-implementation:
-  container:
-    image: pytorch/pytorch:1.7.1-cuda11.0-cudnn8-runtime
-    command:
-    - sh
-    - -ec
-    - |
-      program_path=$(mktemp)
-      printf "%s" "$0" > "$program_path"
-      python3 -u "$program_path" "$@"
-    - |
-      def _make_parent_dirs_and_return_path(file_path: str):
-          import os
-          os.makedirs(os.path.dirname(file_path), exist_ok=True)
-          return file_path
-
-      def create_fully_connected_pytorch_network(
-          input_size,
-          model_path,
-          hidden_layer_sizes = [],
-          output_size = 1,
-          activation_name = 'relu',
-          output_activation_name = None,
-          random_seed = 0,
-      ):
-          '''Creates fully-connected network in PyTorch ScriptModule format'''
-          import torch
-          torch.manual_seed(random_seed)
-
-          activation = getattr(torch, activation_name, None) or getattr(torch.nn.functional, activation_name, None)
-          if not activation:
-              raise ValueError(f'Activation "{activation_name}" was not found.')
-
-          class ActivationLayer(torch.nn.Module):
-              def forward(self, input):
-                  return activation(input)
-
-          layers = []
-          prev_layer_size = input_size
-          for layer_size in hidden_layer_sizes:
-              layer = torch.nn.Linear(prev_layer_size, layer_size)
-              prev_layer_size = layer_size
-              layers.append(layer)
-              layers.append(ActivationLayer())
-
-          # Adding the output layer
-          layers.append(torch.nn.Linear(prev_layer_size, output_size))
-
-          # Adding the optional activation after the output layer
-          if output_activation_name:
-              output_activation = getattr(torch, output_activation_name, None) or getattr(torch.nn.functional, output_activation_name, None)
-              class OutputActivationLayer(torch.nn.Module):
-                  def forward(self, input):
-                      return output_activation(input)
-              layers.append(OutputActivationLayer())
-
-          network = torch.nn.Sequential(*layers)
-          script_module = torch.jit.script(network)
-          print(script_module)
-          script_module.save(model_path)
-
-      import json
-      import argparse
-      _parser = argparse.ArgumentParser(prog='Create fully connected pytorch network', description='Creates fully-connected network in PyTorch ScriptModule format')
-      _parser.add_argument("--input-size", dest="input_size", type=int, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--hidden-layer-sizes", dest="hidden_layer_sizes", type=json.loads, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--output-size", dest="output_size", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--activation-name", dest="activation_name", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--output-activation-name", dest="output_activation_name", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--random-seed", dest="random_seed", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--model", dest="model_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
-      _parsed_args = vars(_parser.parse_args())
-
-      _outputs = create_fully_connected_pytorch_network(**_parsed_args)
-    args:
-    - --input-size
-    - {inputValue: input_size}
-    - if:
-        cond: {isPresent: hidden_layer_sizes}
-        then:
-        - --hidden-layer-sizes
-        - {inputValue: hidden_layer_sizes}
-    - if:
-        cond: {isPresent: output_size}
-        then:
-        - --output-size
-        - {inputValue: output_size}
-    - if:
-        cond: {isPresent: activation_name}
-        then:
-        - --activation-name
-        - {inputValue: activation_name}
-    - if:
-        cond: {isPresent: output_activation_name}
-        then:
-        - --output-activation-name
-        - {inputValue: output_activation_name}
-    - if:
-        cond: {isPresent: random_seed}
-        then:
-        - --random-seed
-        - {inputValue: random_seed}
-    - --model
-    - {outputPath: model}

community-content/pipeline_components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml

@@ -1,209 +0,0 @@
-name: Train pytorch model from csv
-description: Trains PyTorch model
-metadata:
-  annotations:
-    author: Alexey Volkov <alexey.volkov@ark-kun.com>
-    canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml'
-inputs:
-- {name: model, type: PyTorchScriptModule}
-- {name: training_data, type: CSV}
-- {name: label_column_name, type: String}
-- {name: loss_function_name, type: String, default: mse_loss, optional: true}
-- {name: number_of_epochs, type: Integer, default: '1', optional: true}
-- {name: learning_rate, type: Float, default: '0.1', optional: true}
-- {name: optimizer_name, type: String, default: Adadelta, optional: true}
-- {name: optimizer_parameters, type: JsonObject, optional: true}
-- {name: batch_size, type: Integer, default: '32', optional: true}
-- {name: batch_log_interval, type: Integer, default: '100', optional: true}
-- {name: random_seed, type: Integer, default: '0', optional: true}
-outputs:
-- {name: trained_model, type: PyTorchScriptModule}
-implementation:
-  container:
-    image: pytorch/pytorch:1.7.1-cuda11.0-cudnn8-runtime
-    command:
-    - sh
-    - -c
-    - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
-      'pandas==1.4.3' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet
-      --no-warn-script-location 'pandas==1.4.3' --user) && "$0" "$@"
-    - sh
-    - -ec
-    - |
-      program_path=$(mktemp)
-      printf "%s" "$0" > "$program_path"
-      python3 -u "$program_path" "$@"
-    - |
-      def _make_parent_dirs_and_return_path(file_path: str):
-          import os
-          os.makedirs(os.path.dirname(file_path), exist_ok=True)
-          return file_path
-
-      def train_pytorch_model_from_csv(
-          model_path,
-          training_data_path,
-          trained_model_path,
-          label_column_name,
-          loss_function_name = 'mse_loss',
-          number_of_epochs = 1,
-          learning_rate = 0.1,
-          optimizer_name = 'Adadelta',
-          optimizer_parameters = None,
-          batch_size = 32,
-          batch_log_interval = 100,
-          random_seed = 0,
-      ):
-          '''Trains PyTorch model'''
-          import pandas
-          import torch
-
-          torch.manual_seed(random_seed)
-
-          use_cuda = torch.cuda.is_available()
-          device = torch.device("cuda" if use_cuda else "cpu")
-
-          model = torch.jit.load(model_path)
-          model.to(device)
-          model.train()
-
-          optimizer_class = getattr(torch.optim, optimizer_name, None)
-          if not optimizer_class:
-              raise ValueError(f'Optimizer "{optimizer_name}" was not found.')
-
-          optimizer_parameters = optimizer_parameters or {}
-          optimizer_parameters['lr'] = learning_rate
-          optimizer = optimizer_class(model.parameters(), **optimizer_parameters)
-
-          loss_function = getattr(torch, loss_function_name, None) or getattr(torch.nn, loss_function_name, None) or getattr(torch.nn.functional, loss_function_name, None)
-          if not loss_function:
-              raise ValueError(f'Loss function "{loss_function_name}" was not found.')
-
-          class CsvDataset(torch.utils.data.Dataset):
-
-              def __init__(self, file_path, label_column_name, drop_nan_columns_or_rows = 'columns'):
-                  dataframe = pandas.read_csv(file_path).convert_dtypes()
-                  # Preventing error: default_collate: batch must contain tensors, numpy arrays, numbers, dicts or lists; found object
-                  if drop_nan_columns_or_rows == 'columns':
-                      non_nan_data = dataframe.dropna(axis='columns')
-                      removed_columns = set(dataframe.columns) - set(non_nan_data.columns)
-                      if removed_columns:
-                          print('Skipping columns with NaNs: ' + str(removed_columns))
-                      dataframe = non_nan_data
-                  if drop_nan_columns_or_rows == 'rows':
-                      non_nan_data = dataframe.dropna(axis='index')
-                      number_of_removed_rows = len(dataframe) - len(non_nan_data)
-                      if number_of_removed_rows:
-                          print(f'Skipped {number_of_removed_rows} rows with NaNs.')
-                      dataframe = non_nan_data
-                  numerical_data = dataframe.select_dtypes(include='number')
-                  non_numerical_data = dataframe.select_dtypes(exclude='number')
-                  if not non_numerical_data.empty:
-                      print('Skipping non-number columns:')
-                      print(non_numerical_data.dtypes)
-                  self._dataframe = dataframe
-                  self.labels = numerical_data[[label_column_name]]
-                  self.features = numerical_data.drop(columns=[label_column_name])
-
-              def __len__(self):
-                  return len(self._dataframe)
-
-              def __getitem__(self, index):
-                  return [self.features.loc[index].to_numpy(dtype='float32'), self.labels.loc[index].to_numpy(dtype='float32')]
-
-          dataset = CsvDataset(
-              file_path=training_data_path,
-              label_column_name=label_column_name,
-          )
-          train_loader = torch.utils.data.DataLoader(
-              dataset=dataset,
-              batch_size=batch_size,
-              shuffle=True,
-          )
-
-          last_full_batch_loss = None
-          for epoch in range(1, number_of_epochs + 1):
-              for batch_idx, (data, target) in enumerate(train_loader):
-                  data, target = data.to(device), target.to(device)
-                  optimizer.zero_grad()
-                  output = model(data)
-                  loss = loss_function(output, target)
-                  loss.backward()
-                  optimizer.step()
-                  if len(data) == batch_size:
-                      last_full_batch_loss = loss.item()
-                  if batch_idx % batch_log_interval == 0:
-                      print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
-                          epoch, batch_idx * len(data), len(train_loader.dataset),
-                          100. * batch_idx / len(train_loader), loss.item()))
-              print(f'Training epoch {epoch} completed. Last full batch loss: {last_full_batch_loss:.6f}')
-
-          # print(optimizer.state_dict())
-          model.save(trained_model_path)
-
-      import json
-      import argparse
-      _parser = argparse.ArgumentParser(prog='Train pytorch model from csv', description='Trains PyTorch model')
-      _parser.add_argument("--model", dest="model_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--training-data", dest="training_data_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--label-column-name", dest="label_column_name", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--loss-function-name", dest="loss_function_name", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--number-of-epochs", dest="number_of_epochs", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--learning-rate", dest="learning_rate", type=float, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--optimizer-name", dest="optimizer_name", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--optimizer-parameters", dest="optimizer_parameters", type=json.loads, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--batch-size", dest="batch_size", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--batch-log-interval", dest="batch_log_interval", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--random-seed", dest="random_seed", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--trained-model", dest="trained_model_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
-      _parsed_args = vars(_parser.parse_args())
-
-      _outputs = train_pytorch_model_from_csv(**_parsed_args)
-    args:
-    - --model
-    - {inputPath: model}
-    - --training-data
-    - {inputPath: training_data}
-    - --label-column-name
-    - {inputValue: label_column_name}
-    - if:
-        cond: {isPresent: loss_function_name}
-        then:
-        - --loss-function-name
-        - {inputValue: loss_function_name}
-    - if:
-        cond: {isPresent: number_of_epochs}
-        then:
-        - --number-of-epochs
-        - {inputValue: number_of_epochs}
-    - if:
-        cond: {isPresent: learning_rate}
-        then:
-        - --learning-rate
-        - {inputValue: learning_rate}
-    - if:
-        cond: {isPresent: optimizer_name}
-        then:
-        - --optimizer-name
-        - {inputValue: optimizer_name}
-    - if:
-        cond: {isPresent: optimizer_parameters}
-        then:
-        - --optimizer-parameters
-        - {inputValue: optimizer_parameters}
-    - if:
-        cond: {isPresent: batch_size}
-        then:
-        - --batch-size
-        - {inputValue: batch_size}
-    - if:
-        cond: {isPresent: batch_log_interval}
-        then:
-        - --batch-log-interval
-        - {inputValue: batch_log_interval}
-    - if:
-        cond: {isPresent: random_seed}
-        then:
-        - --random-seed
-        - {inputValue: random_seed}
-    - --trained-model
-    - {outputPath: trained_model}

community-content/pipeline_components/XGBoost/Predict/component.yaml

@@ -1,110 +0,0 @@
-name: Xgboost predict on CSV
-description: Makes predictions using a trained XGBoost model.
-metadata:
-  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/XGBoost/Predict/component.yaml'}
-inputs:
-- {name: data, type: CSV, description: Feature data in Apache Parquet format.}
-- {name: model, type: XGBoostModel, description: Trained model in binary XGBoost format.}
-- {name: label_column_name, type: String, description: Optional. Name of the column
-    containing the label data that is excluded during the prediction., optional: true}
-outputs:
-- {name: predictions, description: Model predictions.}
-implementation:
-  container:
-    image: python:3.10
-    command:
-    - sh
-    - -c
-    - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
-      'xgboost==1.6.1' 'pandas==1.4.3' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3
-      -m pip install --quiet --no-warn-script-location 'xgboost==1.6.1' 'pandas==1.4.3'
-      --user) && "$0" "$@"
-    - sh
-    - -ec
-    - |
-      program_path=$(mktemp)
-      printf "%s" "$0" > "$program_path"
-      python3 -u "$program_path" "$@"
-    - |
-      def _make_parent_dirs_and_return_path(file_path: str):
-          import os
-          os.makedirs(os.path.dirname(file_path), exist_ok=True)
-          return file_path
-
-      def xgboost_predict_on_CSV(
-          data_path,
-          model_path,
-          predictions_path,
-          label_column_name = None,
-      ):
-          """Makes predictions using a trained XGBoost model.
-
-          Args:
-              data_path: Feature data in Apache Parquet format.
-              model_path: Trained model in binary XGBoost format.
-              predictions_path: Model predictions.
-              label_column_name: Optional. Name of the column containing the label data that is excluded during the prediction.
-
-          Annotations:
-              author: Alexey Volkov <alexey.volkov@ark-kun.com>
-          """
-          from pathlib import Path
-
-          import numpy
-          import pandas
-          import xgboost
-
-          df = pandas.read_csv(
-              data_path,
-          ).convert_dtypes()
-          print("Evaluation data information:")
-          df.info(verbose=True)
-          # Converting column types that XGBoost does not support
-          for column_name, dtype in df.dtypes.items():
-              if dtype in ["string", "object"]:
-                  print(f"Treating the {dtype.name} column '{column_name}' as categorical.")
-                  df[column_name] = df[column_name].astype("category")
-                  print(f"Inferred {len(df[column_name].cat.categories)} categories for the '{column_name}' column.")
-              # Working around the XGBoost issue with nullable floats: https://github.com/dmlc/xgboost/issues/8213
-              if pandas.api.types.is_float_dtype(dtype):
-                  # Converting from "Float64" to "float64"
-                  df[column_name] = df[column_name].astype(dtype.name.lower())
-          print("Final evaluation data information:")
-          df.info(verbose=True)
-
-          if label_column_name is not None:
-              df = df.drop(columns=[label_column_name])
-
-          testing_data = xgboost.DMatrix(
-              data=df,
-              enable_categorical=True,
-          )
-
-          model = xgboost.Booster(model_file=model_path)
-
-          predictions = model.predict(testing_data)
-
-          Path(predictions_path).parent.mkdir(parents=True, exist_ok=True)
-          numpy.savetxt(predictions_path, predictions)
-
-      import argparse
-      _parser = argparse.ArgumentParser(prog='Xgboost predict on CSV', description='Makes predictions using a trained XGBoost model.')
-      _parser.add_argument("--data", dest="data_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--model", dest="model_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--label-column-name", dest="label_column_name", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--predictions", dest="predictions_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
-      _parsed_args = vars(_parser.parse_args())
-
-      _outputs = xgboost_predict_on_CSV(**_parsed_args)
-    args:
-    - --data
-    - {inputPath: data}
-    - --model
-    - {inputPath: model}
-    - if:
-        cond: {isPresent: label_column_name}
-        then:
-        - --label-column-name
-        - {inputValue: label_column_name}
-    - --predictions
-    - {outputPath: predictions}

community-content/pipeline_components/XGBoost/Train/component.yaml

@@ -1,241 +0,0 @@
-name: Train XGBoost model on CSV
-description: Trains an XGBoost model.
-metadata:
-  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/XGBoost/Train/component.yaml'}
-inputs:
-- {name: training_data, type: CSV, description: Training data in CSV format.}
-- {name: label_column_name, type: String, description: Name of the column containing
-    the label data.}
-- {name: starting_model, type: XGBoostModel, description: Existing trained model to
-    start from (in the binary XGBoost format)., optional: true}
-- {name: num_iterations, type: Integer, description: Number of boosting iterations.,
-  default: '10', optional: true}
-- name: objective
-  type: String
-  description: |-
-    The learning task and the corresponding learning objective.
-    See https://xgboost.readthedocs.io/en/latest/parameter.html#learning-task-parameters
-    The most common values are:
-    "reg:squarederror" - Regression with squared loss (default).
-    "reg:logistic" - Logistic regression.
-    "binary:logistic" - Logistic regression for binary classification, output probability.
-    "binary:logitraw" - Logistic regression for binary classification, output score before logistic transformation
-    "rank:pairwise" - Use LambdaMART to perform pairwise ranking where the pairwise loss is minimized
-    "rank:ndcg" - Use LambdaMART to perform list-wise ranking where Normalized Discounted Cumulative Gain (NDCG) is maximized
-  default: reg:squarederror
-  optional: true
-- {name: booster, type: String, description: 'The booster to use. Can be `gbtree`,
-    `gblinear` or `dart`; `gbtree` and `dart` use tree based models while `gblinear`
-    uses linear functions.', default: gbtree, optional: true}
-- {name: learning_rate, type: Float, description: 'Step size shrinkage used in update
-    to prevents overfitting. Range: [0,1].', default: '0.3', optional: true}
-- name: min_split_loss
-  type: Float
-  description: |-
-    Minimum loss reduction required to make a further partition on a leaf node of the tree.
-    The larger `min_split_loss` is, the more conservative the algorithm will be. Range: [0,Inf].
-  default: '0'
-  optional: true
-- name: max_depth
-  type: Integer
-  description: |-
-    Maximum depth of a tree. Increasing this value will make the model more complex and more likely to overfit.
-    0 indicates no limit on depth. Range: [0,Inf].
-  default: '6'
-  optional: true
-- {name: booster_params, type: JsonObject, description: 'Parameters for the booster.
-    See https://xgboost.readthedocs.io/en/latest/parameter.html', optional: true}
-outputs:
-- {name: model, type: XGBoostModel, description: Trained model in the binary XGBoost
-    format.}
-- {name: model_config, type: XGBoostModelConfig, description: The internal parameter
-    configuration of Booster as a JSON string.}
-implementation:
-  container:
-    image: python:3.10
-    command:
-    - sh
-    - -c
-    - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
-      'xgboost==1.6.1' 'pandas==1.4.3' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3
-      -m pip install --quiet --no-warn-script-location 'xgboost==1.6.1' 'pandas==1.4.3'
-      --user) && "$0" "$@"
-    - sh
-    - -ec
-    - |
-      program_path=$(mktemp)
-      printf "%s" "$0" > "$program_path"
-      python3 -u "$program_path" "$@"
-    - |
-      def _make_parent_dirs_and_return_path(file_path: str):
-          import os
-          os.makedirs(os.path.dirname(file_path), exist_ok=True)
-          return file_path
-
-      def train_XGBoost_model_on_CSV(
-          training_data_path,
-          model_path,
-          model_config_path,
-          label_column_name,
-          starting_model_path = None,
-          num_iterations = 10,
-          # Booster parameters
-          objective = "reg:squarederror",
-          booster = "gbtree",
-          learning_rate = 0.3,
-          min_split_loss = 0,
-          max_depth = 6,
-          booster_params = None,
-      ):
-          """Trains an XGBoost model.
-
-          Args:
-              training_data_path: Training data in CSV format.
-              model_path: Trained model in the binary XGBoost format.
-              model_config_path: The internal parameter configuration of Booster as a JSON string.
-              starting_model_path: Existing trained model to start from (in the binary XGBoost format).
-              label_column_name: Name of the column containing the label data.
-              num_iterations: Number of boosting iterations.
-              booster_params: Parameters for the booster. See https://xgboost.readthedocs.io/en/latest/parameter.html
-              objective: The learning task and the corresponding learning objective.
-                  See https://xgboost.readthedocs.io/en/latest/parameter.html#learning-task-parameters
-                  The most common values are:
-                  "reg:squarederror" - Regression with squared loss (default).
-                  "reg:logistic" - Logistic regression.
-                  "binary:logistic" - Logistic regression for binary classification, output probability.
-                  "binary:logitraw" - Logistic regression for binary classification, output score before logistic transformation
-                  "rank:pairwise" - Use LambdaMART to perform pairwise ranking where the pairwise loss is minimized
-                  "rank:ndcg" - Use LambdaMART to perform list-wise ranking where Normalized Discounted Cumulative Gain (NDCG) is maximized
-              booster: The booster to use. Can be `gbtree`, `gblinear` or `dart`; `gbtree` and `dart` use tree based models while `gblinear` uses linear functions.
-              learning_rate: Step size shrinkage used in update to prevents overfitting. Range: [0,1].
-              min_split_loss: Minimum loss reduction required to make a further partition on a leaf node of the tree.
-                  The larger `min_split_loss` is, the more conservative the algorithm will be. Range: [0,Inf].
-              max_depth: Maximum depth of a tree. Increasing this value will make the model more complex and more likely to overfit.
-                  0 indicates no limit on depth. Range: [0,Inf].
-
-          Annotations:
-              author: Alexey Volkov <alexey.volkov@ark-kun.com>
-          """
-          import pandas
-          import xgboost
-
-          df = pandas.read_csv(
-              training_data_path,
-          ).convert_dtypes()
-          print("Training data information:")
-          df.info(verbose=True)
-          # Converting column types that XGBoost does not support
-          for column_name, dtype in df.dtypes.items():
-              if dtype in ["string", "object"]:
-                  print(f"Treating the {dtype.name} column '{column_name}' as categorical.")
-                  df[column_name] = df[column_name].astype("category")
-                  print(f"Inferred {len(df[column_name].cat.categories)} categories for the '{column_name}' column.")
-              # Working around the XGBoost issue with nullable floats: https://github.com/dmlc/xgboost/issues/8213
-              if pandas.api.types.is_float_dtype(dtype):
-                  # Converting from "Float64" to "float64"
-                  df[column_name] = df[column_name].astype(dtype.name.lower())
-          print()
-          print("Final training data information:")
-          df.info(verbose=True)
-
-          training_data = xgboost.DMatrix(
-              data=df.drop(columns=[label_column_name]),
-              label=df[[label_column_name]],
-              enable_categorical=True,
-          )
-
-          booster_params = booster_params or {}
-          booster_params.setdefault("objective", objective)
-          booster_params.setdefault("booster", booster)
-          booster_params.setdefault("learning_rate", learning_rate)
-          booster_params.setdefault("min_split_loss", min_split_loss)
-          booster_params.setdefault("max_depth", max_depth)
-
-          starting_model = None
-          if starting_model_path:
-              starting_model = xgboost.Booster(model_file=starting_model_path)
-
-          print()
-          print("Training the model:")
-          model = xgboost.train(
-              params=booster_params,
-              dtrain=training_data,
-              num_boost_round=num_iterations,
-              xgb_model=starting_model,
-              evals=[(training_data, "training_data")],
-          )
-
-          # Saving the model in binary format
-          model.save_model(model_path)
-
-          model_config_str = model.save_config()
-          with open(model_config_path, "w") as model_config_file:
-              model_config_file.write(model_config_str)
-
-      import json
-      import argparse
-      _parser = argparse.ArgumentParser(prog='Train XGBoost model on CSV', description='Trains an XGBoost model.')
-      _parser.add_argument("--training-data", dest="training_data_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--label-column-name", dest="label_column_name", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--starting-model", dest="starting_model_path", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--num-iterations", dest="num_iterations", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--objective", dest="objective", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--booster", dest="booster", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--learning-rate", dest="learning_rate", type=float, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--min-split-loss", dest="min_split_loss", type=float, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--max-depth", dest="max_depth", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--booster-params", dest="booster_params", type=json.loads, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--model", dest="model_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--model-config", dest="model_config_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
-      _parsed_args = vars(_parser.parse_args())
-
-      _outputs = train_XGBoost_model_on_CSV(**_parsed_args)
-    args:
-    - --training-data
-    - {inputPath: training_data}
-    - --label-column-name
-    - {inputValue: label_column_name}
-    - if:
-        cond: {isPresent: starting_model}
-        then:
-        - --starting-model
-        - {inputPath: starting_model}
-    - if:
-        cond: {isPresent: num_iterations}
-        then:
-        - --num-iterations
-        - {inputValue: num_iterations}
-    - if:
-        cond: {isPresent: objective}
-        then:
-        - --objective
-        - {inputValue: objective}
-    - if:
-        cond: {isPresent: booster}
-        then:
-        - --booster
-        - {inputValue: booster}
-    - if:
-        cond: {isPresent: learning_rate}
-        then:
-        - --learning-rate
-        - {inputValue: learning_rate}
-    - if:
-        cond: {isPresent: min_split_loss}
-        then:
-        - --min-split-loss
-        - {inputValue: min_split_loss}
-    - if:
-        cond: {isPresent: max_depth}
-        then:
-        - --max-depth
-        - {inputValue: max_depth}
-    - if:
-        cond: {isPresent: booster_params}
-        then:
-        - --booster-params
-        - {inputValue: booster_params}
-    - --model
-    - {outputPath: model}
-    - --model-config
-    - {outputPath: model_config}

community-content/pipeline_components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml

@@ -1,204 +0,0 @@
-name: Split rows into subsets
-description: Splits the data table according to the split fractions.
-metadata:
-  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml'}
-inputs:
-- {name: table, type: CSV}
-- {name: fraction_1, type: Float, description: 'The proportion of the lines to put
-    into the 1st split. Range: [0, 1]'}
-- name: fraction_2
-  type: Float
-  description: |-
-    The proportion of the lines to put into the 2nd split. Range: [0, 1]
-    If fraction_2 is not specified, then fraction_2 = 1 - fraction_1.
-    The remaining lines go to the 3rd split (if any).
-  optional: true
-- {name: random_seed, type: Integer, default: '0', optional: true}
-outputs:
-- {name: split_1, type: CSV}
-- {name: split_2, type: CSV}
-- {name: split_3, type: CSV}
-- {name: split_1_count, type: Integer}
-- {name: split_2_count, type: Integer}
-- {name: split_3_count, type: Integer}
-implementation:
-  container:
-    image: python:3.9
-    command:
-    - sh
-    - -ec
-    - |
-      program_path=$(mktemp)
-      printf "%s" "$0" > "$program_path"
-      python3 -u "$program_path" "$@"
-    - |
-      def _make_parent_dirs_and_return_path(file_path: str):
-          import os
-          os.makedirs(os.path.dirname(file_path), exist_ok=True)
-          return file_path
-
-      def split_rows_into_subsets(
-          table_path,
-          split_1_path,
-          split_2_path,
-          split_3_path,
-          fraction_1,
-          fraction_2 = None,
-          random_seed = 0,
-      ):
-          """Splits the data table according to the split fractions.
-
-          Args:
-              fraction_1: The proportion of the lines to put into the 1st split. Range: [0, 1]
-              fraction_2: The proportion of the lines to put into the 2nd split. Range: [0, 1]
-                  If fraction_2 is not specified, then fraction_2 = 1 - fraction_1.
-                  The remaining lines go to the 3rd split (if any).
-          """
-          import random
-
-          random.seed(random_seed)
-
-          SHUFFLE_BUFFER_SIZE = 10000
-
-          num_splits = 3
-
-          if fraction_1 < 0 or fraction_1 > 1:
-              raise ValueError("fraction_1 must be in between 0 and 1.")
-
-          if fraction_2 is None:
-              fraction_2 = 1 - fraction_1
-          if fraction_2 < 0 or fraction_2 > 1:
-              raise ValueError("fraction_2 must be in between 0 and 1.")
-
-          fraction_3 = 1 - fraction_1 - fraction_2
-
-          fractions = [
-              fraction_1,
-              fraction_2,
-              fraction_3,
-          ]
-
-          assert sum(fractions) == 1
-
-          written_line_counts = [0] * num_splits
-
-          output_files = [
-              open(split_1_path, "wb"),
-              open(split_2_path, "wb"),
-              open(split_3_path, "wb"),
-          ]
-
-          with open(table_path, "rb") as input_file:
-              # Writing the headers
-              header_line = input_file.readline()
-              for output_file in output_files:
-                  output_file.write(header_line)
-
-              while True:
-                  line_buffer = []
-                  for i in range(SHUFFLE_BUFFER_SIZE):
-                      line = input_file.readline()
-                      if not line:
-                          break
-                      line_buffer.append(line)
-
-                  # We need to exactly partition the lines between the output files
-                  # To overcome possible systematic bias, we could calculate the total numbers
-                  # of lines written to each file and take that into account.
-                  num_read_lines = len(line_buffer)
-                  number_of_lines_for_files = [0] * num_splits
-                  # List that will have the index of the destination file for each line
-                  file_index_for_line = []
-                  remaining_lines = num_read_lines
-                  remaining_fraction = 1
-                  for i in range(num_splits):
-                      number_of_lines_for_file = (
-                          round(remaining_lines * (fractions[i] / remaining_fraction))
-                          if remaining_fraction > 0
-                          else 0
-                      )
-                      number_of_lines_for_files[i] = number_of_lines_for_file
-                      remaining_lines -= number_of_lines_for_file
-                      remaining_fraction -= fractions[i]
-                      file_index_for_line.extend([i] * number_of_lines_for_file)
-
-                  assert remaining_lines == 0, f"{remaining_lines}"
-                  assert len(file_index_for_line) == num_read_lines
-
-                  random.shuffle(file_index_for_line)
-
-                  for i in range(num_read_lines):
-                      output_files[file_index_for_line[i]].write(line_buffer[i])
-                      written_line_counts[file_index_for_line[i]] += 1
-
-                  # Exit if the file ended before we were able to fully fill the buffer
-                  if len(line_buffer) != SHUFFLE_BUFFER_SIZE:
-                      break
-
-          for output_file in output_files:
-              output_file.close()
-
-          return written_line_counts
-
-      def _serialize_int(int_value: int) -> str:
-          if isinstance(int_value, str):
-              return int_value
-          if not isinstance(int_value, int):
-              raise TypeError('Value "{}" has type "{}" instead of int.'.format(str(int_value), str(type(int_value))))
-          return str(int_value)
-
-      import argparse
-      _parser = argparse.ArgumentParser(prog='Split rows into subsets', description='Splits the data table according to the split fractions.')
-      _parser.add_argument("--table", dest="table_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--fraction-1", dest="fraction_1", type=float, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--fraction-2", dest="fraction_2", type=float, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--random-seed", dest="random_seed", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--split-1", dest="split_1_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--split-2", dest="split_2_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--split-3", dest="split_3_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("----output-paths", dest="_output_paths", type=str, nargs=3)
-      _parsed_args = vars(_parser.parse_args())
-      _output_files = _parsed_args.pop("_output_paths", [])
-
-      _outputs = split_rows_into_subsets(**_parsed_args)
-
-      _output_serializers = [
-          _serialize_int,
-          _serialize_int,
-          _serialize_int,
-
-      ]
-
-      import os
-      for idx, output_file in enumerate(_output_files):
-          try:
-              os.makedirs(os.path.dirname(output_file))
-          except OSError:
-              pass
-          with open(output_file, 'w') as f:
-              f.write(_output_serializers[idx](_outputs[idx]))
-    args:
-    - --table
-    - {inputPath: table}
-    - --fraction-1
-    - {inputValue: fraction_1}
-    - if:
-        cond: {isPresent: fraction_2}
-        then:
-        - --fraction-2
-        - {inputValue: fraction_2}
-    - if:
-        cond: {isPresent: random_seed}
-        then:
-        - --random-seed
-        - {inputValue: random_seed}
-    - --split-1
-    - {outputPath: split_1}
-    - --split-2
-    - {outputPath: split_2}
-    - --split-3
-    - {outputPath: split_3}
-    - '----output-paths'
-    - {outputPath: split_1_count}
-    - {outputPath: split_2_count}
-    - {outputPath: split_3_count}

community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml

@@ -1,241 +0,0 @@
-name: Deploy model to endpoint for Google Cloud Vertex AI Model
-description: Deploys Google Cloud Vertex AI Model to a Google Cloud Vertex AI Endpoint.
-metadata:
-  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/KFPv2_hell/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/workaround_for_buggy_KFPv2_compiler/component.yaml'}
-inputs:
-- {name: model_name, type: String, description: Full resource name of a Google Cloud
-    Vertex AI Model}
-- name: endpoint_name
-  type: String
-  description: |-
-    Optional. Full name of Google Cloud Vertex Endpoint. A new
-    endpoint is created if the name is not passed.
-  optional: true
-- name: machine_type
-  type: String
-  description: |-
-    The type of the machine. See the [list of machine types
-    supported for prediction
-    ](https://cloud.google.com/vertex-ai/docs/predictions/configure-compute#machine-types).
-    Defaults to "n1-standard-2"
-  default: n1-standard-2
-  optional: true
-- name: min_replica_count
-  type: Integer
-  description: |-
-    Optional. The minimum number of machine replicas this deployed
-    model will be always deployed on. If traffic against it increases,
-    it may dynamically be deployed onto more replicas, and as traffic
-    decreases, some of these extra replicas may be freed.
-  default: '1'
-  optional: true
-- name: max_replica_count
-  type: Integer
-  description: |-
-    Optional. The maximum number of replicas this deployed model may
-    be deployed on when the traffic against it increases. If requested
-    value is too large, the deployment will error, but if deployment
-    succeeds then the ability to scale the model to that many replicas
-    is guaranteed (barring service outages). If traffic against the
-    deployed model increases beyond what its replicas at maximum may
-    handle, a portion of the traffic will be dropped. If this value
-    is not provided, the smaller value of min_replica_count or 1 will
-    be used.
-  default: '1'
-  optional: true
-- name: accelerator_type
-  type: String
-  description: |-
-    Optional. Hardware accelerator type. Must also set accelerator_count if used.
-    One of ACCELERATOR_TYPE_UNSPECIFIED, NVIDIA_TESLA_K80, NVIDIA_TESLA_P100,
-    NVIDIA_TESLA_V100, NVIDIA_TESLA_P4, NVIDIA_TESLA_T4
-  optional: true
-- {name: accelerator_count, type: Integer, description: Optional. The number of accelerators
-    to attach to a worker replica., optional: true}
-outputs:
-- {name: endpoint_name, type: String}
-- {name: endpoint_dict, type: JsonObject}
-implementation:
-  container:
-    image: python:3.9
-    command:
-    - sh
-    - -c
-    - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
-      'google-cloud-aiplatform==1.7.0' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3
-      -m pip install --quiet --no-warn-script-location 'google-cloud-aiplatform==1.7.0'
-      --user) && "$0" "$@"
-    - sh
-    - -ec
-    - |
-      program_path=$(mktemp)
-      printf "%s" "$0" > "$program_path"
-      python3 -u "$program_path" "$@"
-    - |
-      def deploy_model_to_endpoint_for_Google_Cloud_Vertex_AI_Model(
-          model_name,
-          endpoint_name = None,
-          machine_type = "n1-standard-2",
-          min_replica_count = 1,
-          max_replica_count = 1,
-          accelerator_type = None,
-          accelerator_count = None,
-          #
-          # Uncomment when anyone requests these:
-          # deployed_model_display_name: str = None,
-          # traffic_percentage: int = 0,
-          # traffic_split: dict = None,
-          # service_account: str = None,
-          # explanation_metadata: "google.cloud.aiplatform_v1.types.explanation_metadata.ExplanationMetadata" = None,
-          # explanation_parameters: "google.cloud.aiplatform_v1.types.explanation.ExplanationParameters" = None,
-          #
-          # encryption_spec_key_name: str = None,
-      ):
-          """Deploys Google Cloud Vertex AI Model to a Google Cloud Vertex AI Endpoint.
-
-          Args:
-              model_name: Full resource name of a Google Cloud Vertex AI Model
-              endpoint_name: Optional. Full name of Google Cloud Vertex Endpoint. A new
-                  endpoint is created if the name is not passed.
-              machine_type: The type of the machine. See the [list of machine types
-                  supported for prediction
-                  ](https://cloud.google.com/vertex-ai/docs/predictions/configure-compute#machine-types).
-                  Defaults to "n1-standard-2"
-              min_replica_count (int):
-                  Optional. The minimum number of machine replicas this deployed
-                  model will be always deployed on. If traffic against it increases,
-                  it may dynamically be deployed onto more replicas, and as traffic
-                  decreases, some of these extra replicas may be freed.
-              max_replica_count (int):
-                  Optional. The maximum number of replicas this deployed model may
-                  be deployed on when the traffic against it increases. If requested
-                  value is too large, the deployment will error, but if deployment
-                  succeeds then the ability to scale the model to that many replicas
-                  is guaranteed (barring service outages). If traffic against the
-                  deployed model increases beyond what its replicas at maximum may
-                  handle, a portion of the traffic will be dropped. If this value
-                  is not provided, the smaller value of min_replica_count or 1 will
-                  be used.
-              accelerator_type (str):
-                  Optional. Hardware accelerator type. Must also set accelerator_count if used.
-                  One of ACCELERATOR_TYPE_UNSPECIFIED, NVIDIA_TESLA_K80, NVIDIA_TESLA_P100,
-                  NVIDIA_TESLA_V100, NVIDIA_TESLA_P4, NVIDIA_TESLA_T4
-              accelerator_count (int):
-                  Optional. The number of accelerators to attach to a worker replica.
-          """
-          import json
-          from google.cloud import aiplatform
-
-          model = aiplatform.Model(model_name=model_name)
-
-          if endpoint_name:
-              endpoint = aiplatform.Endpoint(endpoint_name=endpoint_name)
-          else:
-              endpoint_display_name = model.display_name[:118] + "_endpoint"
-              endpoint = aiplatform.Endpoint.create(
-                  display_name=endpoint_display_name,
-                  project=model.project,
-                  location=model.location,
-                  # encryption_spec_key_name=encryption_spec_key_name,
-                  labels={"component-source": "github-com-ark-kun-pipeline-components"},
-              )
-
-          endpoint = model.deploy(
-              endpoint=endpoint,
-              # deployed_model_display_name=deployed_model_display_name,
-              machine_type=machine_type,
-              min_replica_count=min_replica_count,
-              max_replica_count=max_replica_count,
-              accelerator_type=accelerator_type,
-              accelerator_count=accelerator_count,
-              # service_account=service_account,
-              # explanation_metadata=explanation_metadata,
-              # explanation_parameters=explanation_parameters,
-              # encryption_spec_key_name=encryption_spec_key_name,
-          )
-
-          endpoint_json = json.dumps(endpoint.to_dict(), indent=2)
-          print(endpoint_json)
-          return (endpoint.resource_name, endpoint_json)
-
-      def _serialize_json(obj) -> str:
-          if isinstance(obj, str):
-              return obj
-          import json
-          def default_serializer(obj):
-              if hasattr(obj, 'to_struct'):
-                  return obj.to_struct()
-              else:
-                  raise TypeError("Object of type '%s' is not JSON serializable and does not have .to_struct() method." % obj.__class__.__name__)
-          return json.dumps(obj, default=default_serializer, sort_keys=True)
-
-      def _serialize_str(str_value: str) -> str:
-          if not isinstance(str_value, str):
-              raise TypeError('Value "{}" has type "{}" instead of str.'.format(str(str_value), str(type(str_value))))
-          return str_value
-
-      import argparse
-      _parser = argparse.ArgumentParser(prog='Deploy model to endpoint for Google Cloud Vertex AI Model', description='Deploys Google Cloud Vertex AI Model to a Google Cloud Vertex AI Endpoint.')
-      _parser.add_argument("--model-name", dest="model_name", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--endpoint-name", dest="endpoint_name", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--machine-type", dest="machine_type", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--min-replica-count", dest="min_replica_count", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--max-replica-count", dest="max_replica_count", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--accelerator-type", dest="accelerator_type", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--accelerator-count", dest="accelerator_count", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("----output-paths", dest="_output_paths", type=str, nargs=2)
-      _parsed_args = vars(_parser.parse_args())
-      _output_files = _parsed_args.pop("_output_paths", [])
-
-      _outputs = deploy_model_to_endpoint_for_Google_Cloud_Vertex_AI_Model(**_parsed_args)
-
-      _output_serializers = [
-          _serialize_str,
-          _serialize_json,
-
-      ]
-
-      import os
-      for idx, output_file in enumerate(_output_files):
-          try:
-              os.makedirs(os.path.dirname(output_file))
-          except OSError:
-              pass
-          with open(output_file, 'w') as f:
-              f.write(_output_serializers[idx](_outputs[idx]))
-    args:
-    - --model-name
-    - {inputValue: model_name}
-    - if:
-        cond: {isPresent: endpoint_name}
-        then:
-        - --endpoint-name
-        - {inputValue: endpoint_name}
-    - if:
-        cond: {isPresent: machine_type}
-        then:
-        - --machine-type
-        - {inputValue: machine_type}
-    - if:
-        cond: {isPresent: min_replica_count}
-        then:
-        - --min-replica-count
-        - {inputValue: min_replica_count}
-    - if:
-        cond: {isPresent: max_replica_count}
-        then:
-        - --max-replica-count
-        - {inputValue: max_replica_count}
-    - if:
-        cond: {isPresent: accelerator_type}
-        then:
-        - --accelerator-type
-        - {inputValue: accelerator_type}
-    - if:
-        cond: {isPresent: accelerator_count}
-        then:
-        - --accelerator-count
-        - {inputValue: accelerator_count}
-    - '----output-paths'
-    - {outputPath: endpoint_name}
-    - {outputPath: endpoint_dict}

community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/component.yaml

@@ -1,297 +0,0 @@
-name: Upload PyTorch model archive to Google Cloud Vertex AI
-metadata:
-  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/KFPv2_hell/components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/workaround_for_buggy_KFPv2_compiler/component.yaml'}
-inputs:
-- {name: model_archive, type: PyTorchModelArchive}
-- {name: torchserve_version, type: String, default: 0.6.0, optional: true}
-- name: use_gpu
-  type: Boolean
-  default: "False"
-  optional: true
-- {name: display_name, type: String, optional: true}
-- {name: description, type: String, optional: true}
-- {name: project, type: String, optional: true}
-- {name: location, type: String, optional: true}
-- {name: labels, type: JsonObject, optional: true}
-- {name: staging_bucket, type: String, optional: true}
-outputs:
-- {name: model_name, type: String}
-- {name: model_dict, type: JsonObject}
-implementation:
-  container:
-    image: python:3.9
-    command:
-    - sh
-    - -c
-    - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
-      'google-cloud-aiplatform==1.13.1' 'google-cloud-build==3.8.3' || PIP_DISABLE_PIP_VERSION_CHECK=1
-      python3 -m pip install --quiet --no-warn-script-location 'google-cloud-aiplatform==1.13.1'
-      'google-cloud-build==3.8.3' --user) && "$0" "$@"
-    - sh
-    - -ec
-    - |
-      program_path=$(mktemp)
-      printf "%s" "$0" > "$program_path"
-      python3 -u "$program_path" "$@"
-    - |
-      def upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI(
-          model_archive_path,
-          torchserve_version = "0.6.0",
-          use_gpu = False,
-
-          display_name = None,
-          description = None,
-
-          # Uncomment when anyone requests these:
-          # instance_schema_uri: str = None,
-          # parameters_schema_uri: str = None,
-          # prediction_schema_uri: str = None,
-          # explanation_metadata: "google.cloud.aiplatform_v1.types.explanation_metadata.ExplanationMetadata" = None,
-          # explanation_parameters: "google.cloud.aiplatform_v1.types.explanation.ExplanationParameters" = None,
-
-          project = None,
-          location = None,
-          labels = None,
-          # encryption_spec_key_name: str = None,
-          staging_bucket = None,
-      ):
-          import json
-          import os
-          from google.cloud import aiplatform
-
-          if not location:
-              location = os.environ.get("CLOUD_ML_REGION")
-
-          if not labels:
-              labels = {}
-          labels["component-source"] = "github-com-ark-kun-pipeline-components"
-
-          container_image_tag = torchserve_version + "-" + ("gpu" if use_gpu else "cpu")
-          container_image_uri = f"pytorch/torchserve:{container_image_tag}"
-
-          # Vertex Endpoints refuse to support non-Google container registries.
-          # We have to work around this to reduce user frustration
-          # TODO: Remove this code when Vertex Endpoints service starts supporting other container registries.
-          def copy_container_image(
-              src_container_image_uri,
-              dst_container_image_uri,
-              project_id,
-          ):
-              from google.cloud.devtools import cloudbuild
-              from google import protobuf
-              build_client = cloudbuild.CloudBuildClient()
-              build_config = cloudbuild.Build(
-                  images=[dst_container_image_uri],
-                  steps=[
-                      cloudbuild.BuildStep(
-                          name="gcr.io/cloud-builders/docker",
-                          entrypoint="bash",
-                          args=[
-                              "-exc",
-                              'docker pull --quiet "$0" && docker tag "$0" "$1"',
-                              src_container_image_uri,
-                              dst_container_image_uri,
-                          ],
-                      ),
-                  ],
-                  timeout=protobuf.duration_pb2.Duration(
-                      seconds=1800,
-                  ),
-              )
-              build_operation = build_client.create_build(
-                  project_id=project_id,
-                  build=build_config,
-              )
-              try:
-                  result = build_operation.result()
-              except:
-                  print(f"Logs are available at [{build_operation.metadata.build.log_url}].")
-                  raise
-              return result
-
-          project_id = aiplatform.initializer.global_config.project
-          mirrored_container_uri = f"gcr.io/{project_id}/container_mirror/{container_image_uri}"
-          # FIX: Only mirror when image does not exist
-          # docker does is unable to get the registry data from inside container (it cannot connecto to docker socket):
-          # docker.errors.DockerException: Error while fetching server API version: ('Connection aborted.', FileNotFoundError(2, 'No such file or directory'))
-          # import docker
-          # try:
-          #     docker_client = docker.from_env()
-          #     docker_client.images.get_registry_data(mirrored_container_uri)
-          # except docker.errors.NotFound:
-          if True:
-              print(f"Mirroring {container_image_uri} to {mirrored_container_uri}")
-              copy_container_image(
-                  src_container_image_uri=container_image_uri,
-                  dst_container_image_uri=mirrored_container_uri,
-                  project_id=project_id,
-              )
-          container_image_uri = mirrored_container_uri
-          # End of container image mirroring code
-
-          model_archive_file_name = os.path.basename(model_archive_path)
-          model_archive_dir = os.path.dirname(model_archive_path)
-
-          model = aiplatform.Model.upload(
-              # FIX: Use public image or mirror the official image
-              #serving_container_image_uri="gcr.io/avolkov-31337/mirror/pytorch/torchserve",
-              serving_container_image_uri=container_image_uri,
-              artifact_uri=model_archive_dir,
-              serving_container_command=[
-                  "bash",
-                  "-exc",
-                  '''
-      model_archive_uri="$0"
-      #model_archive_local_path=$(mktemp --suffix ".mar")
-      # For some reason the model must already be inside the model-store directory.
-      model_archive_local_path=./model-store/model.mar
-
-      # Downloading the model archive from GCS
-      # TODO: Fix gsutil bugs (requires project ID, has auth issues) and use gsutil instead.
-      # gsutil cp "$model_archive_uri" "$model_archive_local_path"
-      pip install google-cloud-storage
-      python -c '
-      import sys
-      from google.cloud import storage
-
-      model_archive_uri = sys.argv[1]
-      model_archive_local_path = sys.argv[2]
-
-      storage_client = storage.Client()
-      blob = storage.Blob.from_string(uri=model_archive_uri, client=storage_client)
-      blob.download_to_filename(filename=model_archive_local_path)
-      ' "$model_archive_uri" "$model_archive_local_path"
-
-      #Note: config.properties is owned by root. Our user is not root.
-      echo "
-      service_envelope=json
-      # Needed for external access
-      inference_address=http://0.0.0.0:8080
-      management_address=http://0.0.0.0:8081
-      " > config2.properties
-      torchserve --start --foreground --no-config-snapshots --models main-model="$model_archive_local_path" --model-store ./model-store/ --ts-config config2.properties
-                  ''',
-                  "$(AIP_STORAGE_URI)/" + model_archive_file_name,
-              ],
-              serving_container_predict_route="/predictions/main-model",
-              #serving_container_predict_route="/v1/models/main-model:predict",
-              serving_container_health_route="/ping",
-              serving_container_ports=[8080],
-
-              display_name=display_name,
-              description=description,
-
-              # instance_schema_uri=instance_schema_uri,
-              # parameters_schema_uri=parameters_schema_uri,
-              # prediction_schema_uri=prediction_schema_uri,
-              # explanation_metadata=explanation_metadata,
-              # explanation_parameters=explanation_parameters,
-
-              project=project,
-              location=location,
-              labels=labels,
-              # encryption_spec_key_name=encryption_spec_key_name,
-              staging_bucket=staging_bucket,
-          )
-          model_json = json.dumps(model.to_dict(), indent=2)
-          print(model_json)
-          return (model.resource_name, model_json)
-
-      def _deserialize_bool(s) -> bool:
-          from distutils.util import strtobool
-          return strtobool(s) == 1
-
-      def _serialize_json(obj) -> str:
-          if isinstance(obj, str):
-              return obj
-          import json
-          def default_serializer(obj):
-              if hasattr(obj, 'to_struct'):
-                  return obj.to_struct()
-              else:
-                  raise TypeError("Object of type '%s' is not JSON serializable and does not have .to_struct() method." % obj.__class__.__name__)
-          return json.dumps(obj, default=default_serializer, sort_keys=True)
-
-      def _serialize_str(str_value: str) -> str:
-          if not isinstance(str_value, str):
-              raise TypeError('Value "{}" has type "{}" instead of str.'.format(str(str_value), str(type(str_value))))
-          return str_value
-
-      import json
-      import argparse
-      _parser = argparse.ArgumentParser(prog='Upload PyTorch model archive to Google Cloud Vertex AI', description='')
-      _parser.add_argument("--model-archive", dest="model_archive_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--torchserve-version", dest="torchserve_version", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--use-gpu", dest="use_gpu", type=_deserialize_bool, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--display-name", dest="display_name", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--description", dest="description", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--project", dest="project", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--location", dest="location", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--labels", dest="labels", type=json.loads, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--staging-bucket", dest="staging_bucket", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("----output-paths", dest="_output_paths", type=str, nargs=2)
-      _parsed_args = vars(_parser.parse_args())
-      _output_files = _parsed_args.pop("_output_paths", [])
-
-      _outputs = upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI(**_parsed_args)
-
-      _output_serializers = [
-          _serialize_str,
-          _serialize_json,
-
-      ]
-
-      import os
-      for idx, output_file in enumerate(_output_files):
-          try:
-              os.makedirs(os.path.dirname(output_file))
-          except OSError:
-              pass
-          with open(output_file, 'w') as f:
-              f.write(_output_serializers[idx](_outputs[idx]))
-    args:
-    - --model-archive
-    - {inputPath: model_archive}
-    - if:
-        cond: {isPresent: torchserve_version}
-        then:
-        - --torchserve-version
-        - {inputValue: torchserve_version}
-    - if:
-        cond: {isPresent: use_gpu}
-        then:
-        - --use-gpu
-        - {inputValue: use_gpu}
-    - if:
-        cond: {isPresent: display_name}
-        then:
-        - --display-name
-        - {inputValue: display_name}
-    - if:
-        cond: {isPresent: description}
-        then:
-        - --description
-        - {inputValue: description}
-    - if:
-        cond: {isPresent: project}
-        then:
-        - --project
-        - {inputValue: project}
-    - if:
-        cond: {isPresent: location}
-        then:
-        - --location
-        - {inputValue: location}
-    - if:
-        cond: {isPresent: labels}
-        then:
-        - --labels
-        - {inputValue: labels}
-    - if:
-        cond: {isPresent: staging_bucket}
-        then:
-        - --staging-bucket
-        - {inputValue: staging_bucket}
-    - '----output-paths'
-    - {outputPath: model_name}
-    - {outputPath: model_dict}

community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/component.yaml

@@ -1,181 +0,0 @@
-name: Upload Scikit learn pickle model to Google Cloud Vertex AI
-metadata:
-  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/KFPv2_hell/components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/workaround_for_buggy_KFPv2_compiler/component.yaml'}
-inputs:
-- {name: model, type: ScikitLearnPickleModel}
-- {name: sklearn_version, type: String, optional: true}
-- {name: display_name, type: String, optional: true}
-- {name: description, type: String, optional: true}
-- {name: project, type: String, optional: true}
-- {name: location, type: String, optional: true}
-- {name: labels, type: JsonObject, optional: true}
-- {name: staging_bucket, type: String, optional: true}
-outputs:
-- {name: model_name, type: String}
-- {name: model_dict, type: JsonObject}
-implementation:
-  container:
-    image: python:3.9
-    command:
-    - sh
-    - -c
-    - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
-      'google-cloud-aiplatform==1.16.0' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3
-      -m pip install --quiet --no-warn-script-location 'google-cloud-aiplatform==1.16.0'
-      --user) && "$0" "$@"
-    - sh
-    - -ec
-    - |
-      program_path=$(mktemp)
-      printf "%s" "$0" > "$program_path"
-      python3 -u "$program_path" "$@"
-    - |
-      def upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI(
-          model_path,
-          sklearn_version = None,
-
-          display_name = None,
-          description = None,
-
-          # Uncomment when anyone requests these:
-          # instance_schema_uri: str = None,
-          # parameters_schema_uri: str = None,
-          # prediction_schema_uri: str = None,
-          # explanation_metadata: "google.cloud.aiplatform_v1.types.explanation_metadata.ExplanationMetadata" = None,
-          # explanation_parameters: "google.cloud.aiplatform_v1.types.explanation.ExplanationParameters" = None,
-
-          project = None,
-          location = None,
-          labels = None,
-          # encryption_spec_key_name: str = None,
-          staging_bucket = None,
-      ):
-          import json
-          import os
-          import shutil
-          import tempfile
-          from google.cloud import aiplatform
-
-          if not location:
-              location = os.environ.get("CLOUD_ML_REGION")
-
-          if not labels:
-              labels = {}
-          labels["component-source"] = "github-com-ark-kun-pipeline-components"
-
-          # The serving container decides the model type based on the model file extension.
-          # So we need to rename the mode file (e.g. /tmp/inputs/model/data) to *.pkl
-          _, renamed_model_path = tempfile.mkstemp(suffix=".pkl")
-          shutil.copyfile(src=model_path, dst=renamed_model_path)
-
-          model = aiplatform.Model.upload_scikit_learn_model_file(
-              model_file_path=renamed_model_path,
-              sklearn_version=sklearn_version,
-
-              display_name=display_name,
-              description=description,
-
-              # instance_schema_uri=instance_schema_uri,
-              # parameters_schema_uri=parameters_schema_uri,
-              # prediction_schema_uri=prediction_schema_uri,
-              # explanation_metadata=explanation_metadata,
-              # explanation_parameters=explanation_parameters,
-
-              project=project,
-              location=location,
-              labels=labels,
-              # encryption_spec_key_name=encryption_spec_key_name,
-              staging_bucket=staging_bucket,
-          )
-          model_json = json.dumps(model.to_dict(), indent=2)
-          print(model_json)
-          return (model.resource_name, model_json)
-
-      def _serialize_json(obj) -> str:
-          if isinstance(obj, str):
-              return obj
-          import json
-          def default_serializer(obj):
-              if hasattr(obj, 'to_struct'):
-                  return obj.to_struct()
-              else:
-                  raise TypeError("Object of type '%s' is not JSON serializable and does not have .to_struct() method." % obj.__class__.__name__)
-          return json.dumps(obj, default=default_serializer, sort_keys=True)
-
-      def _serialize_str(str_value: str) -> str:
-          if not isinstance(str_value, str):
-              raise TypeError('Value "{}" has type "{}" instead of str.'.format(str(str_value), str(type(str_value))))
-          return str_value
-
-      import json
-      import argparse
-      _parser = argparse.ArgumentParser(prog='Upload Scikit learn pickle model to Google Cloud Vertex AI', description='')
-      _parser.add_argument("--model", dest="model_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--sklearn-version", dest="sklearn_version", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--display-name", dest="display_name", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--description", dest="description", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--project", dest="project", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--location", dest="location", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--labels", dest="labels", type=json.loads, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--staging-bucket", dest="staging_bucket", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("----output-paths", dest="_output_paths", type=str, nargs=2)
-      _parsed_args = vars(_parser.parse_args())
-      _output_files = _parsed_args.pop("_output_paths", [])
-
-      _outputs = upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI(**_parsed_args)
-
-      _output_serializers = [
-          _serialize_str,
-          _serialize_json,
-
-      ]
-
-      import os
-      for idx, output_file in enumerate(_output_files):
-          try:
-              os.makedirs(os.path.dirname(output_file))
-          except OSError:
-              pass
-          with open(output_file, 'w') as f:
-              f.write(_output_serializers[idx](_outputs[idx]))
-    args:
-    - --model
-    - {inputPath: model}
-    - if:
-        cond: {isPresent: sklearn_version}
-        then:
-        - --sklearn-version
-        - {inputValue: sklearn_version}
-    - if:
-        cond: {isPresent: display_name}
-        then:
-        - --display-name
-        - {inputValue: display_name}
-    - if:
-        cond: {isPresent: description}
-        then:
-        - --description
-        - {inputValue: description}
-    - if:
-        cond: {isPresent: project}
-        then:
-        - --project
-        - {inputValue: project}
-    - if:
-        cond: {isPresent: location}
-        then:
-        - --location
-        - {inputValue: location}
-    - if:
-        cond: {isPresent: labels}
-        then:
-        - --labels
-        - {inputValue: labels}
-    - if:
-        cond: {isPresent: staging_bucket}
-        then:
-        - --staging-bucket
-        - {inputValue: staging_bucket}
-    - '----output-paths'
-    - {outputPath: model_name}
-    - {outputPath: model_dict}

community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/component.yaml

@@ -1,190 +0,0 @@
-name: Upload Tensorflow model to Google Cloud Vertex AI
-metadata:
-  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/KFPv2_hell/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/workaround_for_buggy_KFPv2_compiler/component.yaml'}
-inputs:
-- {name: model, type: TensorflowSavedModel}
-- {name: tensorflow_version, type: String, optional: true}
-- name: use_gpu
-  type: Boolean
-  default: "False"
-  optional: true
-- {name: display_name, type: String, optional: true}
-- {name: description, type: String, optional: true}
-- {name: project, type: String, optional: true}
-- {name: location, type: String, optional: true}
-- {name: labels, type: JsonObject, optional: true}
-- {name: staging_bucket, type: String, optional: true}
-outputs:
-- {name: model_name, type: String}
-- {name: model_dict, type: JsonObject}
-implementation:
-  container:
-    image: python:3.9
-    command:
-    - sh
-    - -c
-    - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
-      'google-cloud-aiplatform==1.16.0' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3
-      -m pip install --quiet --no-warn-script-location 'google-cloud-aiplatform==1.16.0'
-      --user) && "$0" "$@"
-    - sh
-    - -ec
-    - |
-      program_path=$(mktemp)
-      printf "%s" "$0" > "$program_path"
-      python3 -u "$program_path" "$@"
-    - |
-      def upload_Tensorflow_model_to_Google_Cloud_Vertex_AI(
-          model_path,
-          tensorflow_version = None,
-          use_gpu = False,
-
-          display_name = None,
-          description = None,
-
-          # Uncomment when anyone requests these:
-          # instance_schema_uri: str = None,
-          # parameters_schema_uri: str = None,
-          # prediction_schema_uri: str = None,
-          # explanation_metadata: "google.cloud.aiplatform_v1.types.explanation_metadata.ExplanationMetadata" = None,
-          # explanation_parameters: "google.cloud.aiplatform_v1.types.explanation.ExplanationParameters" = None,
-
-          project = None,
-          location = None,
-          labels = None,
-          # encryption_spec_key_name: str = None,
-          staging_bucket = None,
-      ):
-          import json
-          import os
-          from google.cloud import aiplatform
-
-          if not location:
-              location = os.environ.get("CLOUD_ML_REGION")
-
-          if not labels:
-              labels = {}
-          labels["component-source"] = "github-com-ark-kun-pipeline-components"
-
-          model = aiplatform.Model.upload_tensorflow_saved_model(
-              saved_model_dir=model_path,
-              tensorflow_version=tensorflow_version,
-              use_gpu=use_gpu,
-
-              display_name=display_name,
-              description=description,
-
-              # instance_schema_uri=instance_schema_uri,
-              # parameters_schema_uri=parameters_schema_uri,
-              # prediction_schema_uri=prediction_schema_uri,
-              # explanation_metadata=explanation_metadata,
-              # explanation_parameters=explanation_parameters,
-
-              project=project,
-              location=location,
-              labels=labels,
-              # encryption_spec_key_name=encryption_spec_key_name,
-              staging_bucket=staging_bucket,
-          )
-          model_json = json.dumps(model.to_dict(), indent=2)
-          print(model_json)
-          return (model.resource_name, model_json)
-
-      def _deserialize_bool(s) -> bool:
-          from distutils.util import strtobool
-          return strtobool(s) == 1
-
-      def _serialize_json(obj) -> str:
-          if isinstance(obj, str):
-              return obj
-          import json
-          def default_serializer(obj):
-              if hasattr(obj, 'to_struct'):
-                  return obj.to_struct()
-              else:
-                  raise TypeError("Object of type '%s' is not JSON serializable and does not have .to_struct() method." % obj.__class__.__name__)
-          return json.dumps(obj, default=default_serializer, sort_keys=True)
-
-      def _serialize_str(str_value: str) -> str:
-          if not isinstance(str_value, str):
-              raise TypeError('Value "{}" has type "{}" instead of str.'.format(str(str_value), str(type(str_value))))
-          return str_value
-
-      import json
-      import argparse
-      _parser = argparse.ArgumentParser(prog='Upload Tensorflow model to Google Cloud Vertex AI', description='')
-      _parser.add_argument("--model", dest="model_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--tensorflow-version", dest="tensorflow_version", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--use-gpu", dest="use_gpu", type=_deserialize_bool, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--display-name", dest="display_name", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--description", dest="description", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--project", dest="project", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--location", dest="location", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--labels", dest="labels", type=json.loads, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--staging-bucket", dest="staging_bucket", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("----output-paths", dest="_output_paths", type=str, nargs=2)
-      _parsed_args = vars(_parser.parse_args())
-      _output_files = _parsed_args.pop("_output_paths", [])
-
-      _outputs = upload_Tensorflow_model_to_Google_Cloud_Vertex_AI(**_parsed_args)
-
-      _output_serializers = [
-          _serialize_str,
-          _serialize_json,
-
-      ]
-
-      import os
-      for idx, output_file in enumerate(_output_files):
-          try:
-              os.makedirs(os.path.dirname(output_file))
-          except OSError:
-              pass
-          with open(output_file, 'w') as f:
-              f.write(_output_serializers[idx](_outputs[idx]))
-    args:
-    - --model
-    - {inputPath: model}
-    - if:
-        cond: {isPresent: tensorflow_version}
-        then:
-        - --tensorflow-version
-        - {inputValue: tensorflow_version}
-    - if:
-        cond: {isPresent: use_gpu}
-        then:
-        - --use-gpu
-        - {inputValue: use_gpu}
-    - if:
-        cond: {isPresent: display_name}
-        then:
-        - --display-name
-        - {inputValue: display_name}
-    - if:
-        cond: {isPresent: description}
-        then:
-        - --description
-        - {inputValue: description}
-    - if:
-        cond: {isPresent: project}
-        then:
-        - --project
-        - {inputValue: project}
-    - if:
-        cond: {isPresent: location}
-        then:
-        - --location
-        - {inputValue: location}
-    - if:
-        cond: {isPresent: labels}
-        then:
-        - --labels
-        - {inputValue: labels}
-    - if:
-        cond: {isPresent: staging_bucket}
-        then:
-        - --staging-bucket
-        - {inputValue: staging_bucket}
-    - '----output-paths'
-    - {outputPath: model_name}
-    - {outputPath: model_dict}

community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/component.yaml

@@ -1,181 +0,0 @@
-name: Upload XGBoost model to Google Cloud Vertex AI
-metadata:
-  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/workaround_for_buggy_KFPv2_compiler/component.yaml'}
-inputs:
-- {name: model, type: XGBoostModel}
-- {name: xgboost_version, type: String, optional: true}
-- {name: display_name, type: String, optional: true}
-- {name: description, type: String, optional: true}
-- {name: project, type: String, optional: true}
-- {name: location, type: String, optional: true}
-- {name: labels, type: JsonObject, optional: true}
-- {name: staging_bucket, type: String, optional: true}
-outputs:
-- {name: model_name, type: String}
-- {name: model_dict, type: JsonObject}
-implementation:
-  container:
-    image: python:3.9
-    command:
-    - sh
-    - -c
-    - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
-      'google-cloud-aiplatform==1.16.0' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3
-      -m pip install --quiet --no-warn-script-location 'google-cloud-aiplatform==1.16.0'
-      --user) && "$0" "$@"
-    - sh
-    - -ec
-    - |
-      program_path=$(mktemp)
-      printf "%s" "$0" > "$program_path"
-      python3 -u "$program_path" "$@"
-    - |
-      def upload_XGBoost_model_to_Google_Cloud_Vertex_AI(
-          model_path,
-          xgboost_version = None,
-
-          display_name = None,
-          description = None,
-
-          # Uncomment when anyone requests these:
-          # instance_schema_uri: str = None,
-          # parameters_schema_uri: str = None,
-          # prediction_schema_uri: str = None,
-          # explanation_metadata: "google.cloud.aiplatform_v1.types.explanation_metadata.ExplanationMetadata" = None,
-          # explanation_parameters: "google.cloud.aiplatform_v1.types.explanation.ExplanationParameters" = None,
-
-          project = None,
-          location = None,
-          labels = None,
-          # encryption_spec_key_name: str = None,
-          staging_bucket = None,
-      ):
-          import json
-          import os
-          import shutil
-          import tempfile
-          from google.cloud import aiplatform
-
-          if not location:
-              location = os.environ.get("CLOUD_ML_REGION")
-
-          if not labels:
-              labels = {}
-          labels["component-source"] = "github-com-ark-kun-pipeline-components"
-
-          # The serving container decides the model type based on the model file extension.
-          # So we need to rename the mode file (e.g. /tmp/inputs/model/data) to *.pkl
-          _, renamed_model_path = tempfile.mkstemp(suffix=".pkl")
-          shutil.copyfile(src=model_path, dst=renamed_model_path)
-
-          model = aiplatform.Model.upload_xgboost_model_file(
-              model_file_path=renamed_model_path,
-              xgboost_version=xgboost_version,
-
-              display_name=display_name,
-              description=description,
-
-              # instance_schema_uri=instance_schema_uri,
-              # parameters_schema_uri=parameters_schema_uri,
-              # prediction_schema_uri=prediction_schema_uri,
-              # explanation_metadata=explanation_metadata,
-              # explanation_parameters=explanation_parameters,
-
-              project=project,
-              location=location,
-              labels=labels,
-              # encryption_spec_key_name=encryption_spec_key_name,
-              staging_bucket=staging_bucket,
-          )
-          model_json = json.dumps(model.to_dict(), indent=2)
-          print(model_json)
-          return (model.resource_name, model_json)
-
-      def _serialize_json(obj) -> str:
-          if isinstance(obj, str):
-              return obj
-          import json
-          def default_serializer(obj):
-              if hasattr(obj, 'to_struct'):
-                  return obj.to_struct()
-              else:
-                  raise TypeError("Object of type '%s' is not JSON serializable and does not have .to_struct() method." % obj.__class__.__name__)
-          return json.dumps(obj, default=default_serializer, sort_keys=True)
-
-      def _serialize_str(str_value: str) -> str:
-          if not isinstance(str_value, str):
-              raise TypeError('Value "{}" has type "{}" instead of str.'.format(str(str_value), str(type(str_value))))
-          return str_value
-
-      import json
-      import argparse
-      _parser = argparse.ArgumentParser(prog='Upload XGBoost model to Google Cloud Vertex AI', description='')
-      _parser.add_argument("--model", dest="model_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--xgboost-version", dest="xgboost_version", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--display-name", dest="display_name", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--description", dest="description", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--project", dest="project", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--location", dest="location", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--labels", dest="labels", type=json.loads, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--staging-bucket", dest="staging_bucket", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("----output-paths", dest="_output_paths", type=str, nargs=2)
-      _parsed_args = vars(_parser.parse_args())
-      _output_files = _parsed_args.pop("_output_paths", [])
-
-      _outputs = upload_XGBoost_model_to_Google_Cloud_Vertex_AI(**_parsed_args)
-
-      _output_serializers = [
-          _serialize_str,
-          _serialize_json,
-
-      ]
-
-      import os
-      for idx, output_file in enumerate(_output_files):
-          try:
-              os.makedirs(os.path.dirname(output_file))
-          except OSError:
-              pass
-          with open(output_file, 'w') as f:
-              f.write(_output_serializers[idx](_outputs[idx]))
-    args:
-    - --model
-    - {inputPath: model}
-    - if:
-        cond: {isPresent: xgboost_version}
-        then:
-        - --xgboost-version
-        - {inputValue: xgboost_version}
-    - if:
-        cond: {isPresent: display_name}
-        then:
-        - --display-name
-        - {inputValue: display_name}
-    - if:
-        cond: {isPresent: description}
-        then:
-        - --description
-        - {inputValue: description}
-    - if:
-        cond: {isPresent: project}
-        then:
-        - --project
-        - {inputValue: project}
-    - if:
-        cond: {isPresent: location}
-        then:
-        - --location
-        - {inputValue: location}
-    - if:
-        cond: {isPresent: labels}
-        then:
-        - --labels
-        - {inputValue: labels}
-    - if:
-        cond: {isPresent: staging_bucket}
-        then:
-        - --staging-bucket
-        - {inputValue: staging_bucket}
-    - '----output-paths'
-    - {outputPath: model_name}
-    - {outputPath: model_dict}

community-content/pipeline_components/google-cloud/storage/download/component.yaml

@@ -1,35 +0,0 @@
-name: Download from GCS
-inputs:
-- {name: GCS path, type: String}
-outputs:
-- {name: Data}
-metadata:
-  annotations:
-    author: Alexey Volkov <alexey.volkov@ark-kun.com>
-    canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml'
-implementation:
-    container:
-        image: google/cloud-sdk
-        command:
-        - bash # Pattern comparison only works in Bash
-        - -ex
-        - -c
-        - |
-            if [ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then
-                gcloud auth activate-service-account --key-file="${GOOGLE_APPLICATION_CREDENTIALS}"
-            fi
-
-            uri="$0"
-            output_path="$1"
-
-            # Checking whether the URI points to a single blob, a directory or a URI pattern
-            # URI points to a blob when that URI does not end with slash and listing that URI only yields the same URI
-            if [[ "$uri" != */ ]] && (gsutil ls "$uri" | grep --fixed-strings --line-regexp "$uri"); then
-                mkdir -p "$(dirname "$output_path")"
-                gsutil -m cp -r "$uri" "$output_path"
-            else
-                mkdir -p "$output_path" # When source path is a directory, gsutil requires the destination to also be a directory
-                gsutil -m rsync -r "$uri" "$output_path" # gsutil cp has different path handling than Linux cp. It always puts the source directory (name) inside the destination directory. gsutil rsync does not have that problem.
-            fi
-        - inputValue: GCS path
-        - outputPath: Data

community-content/pipeline_components/image_ml_model_training/load_image_classification_model/component.yaml

@@ -1,112 +0,0 @@
-name: Load image classification model from tfhub
-description: |
-  Loads specified model from TFHub, creates layer to receive additional (3 channel) imagery data.
-  Args:
-    class_names (Sequence[str]):
-        Sequence of strings of categories for classification corresponding to input data.
-    loaded_model_path (str):
-        Output path for the loaded model.
-    image_size_path (str):
-        Output path for the model expected image size.
-    model_name (Optional[str]):
-        Name of the pre-trained image classification model to load from TFHub.
-        Eligible model_name:
-        - efficientnetv2-s
-        - efficientnetv2-m
-        - efficientnetv2-l
-        - efficientnetv2-s-21k
-        - efficientnetv2-m-21k
-        - efficientnetv2-l-21k
-        - efficientnetv2-xl-21k
-        - efficientnetv2-b0-21k
-        - efficientnetv2-b1-21k
-        - efficientnetv2-b2-21k
-        - efficientnetv2-b3-21k
-        - efficientnetv2-s-21k-ft1k
-        - efficientnetv2-m-21k-ft1k
-        - efficientnetv2-l-21k-ft1k
-        - efficientnetv2-xl-21k-ft1k
-        - efficientnetv2-b0-21k-ft1k
-        - efficientnetv2-b1-21k-ft1k
-        - efficientnetv2-b2-21k-ft1k
-        - efficientnetv2-b3-21k-ft1k
-        - efficientnetv2-b0
-        - efficientnetv2-b1
-        - efficientnetv2-b2
-        - efficientnetv2-b3
-        - efficientnet_b0
-        - efficientnet_b1
-        - efficientnet_b2
-        - efficientnet_b3
-        - efficientnet_b4
-        - efficientnet_b5
-        - efficientnet_b6
-        - efficientnet_b7
-        - bit_s-r50x1
-        - inception_v3
-        - inception_resnet_v2
-        - resnet_v1_50
-        - resnet_v1_101
-        - resnet_v1_152
-        - resnet_v2_50
-        - resnet_v2_101
-        - resnet_v2_152
-        - nasnet_large
-        - nasnet_mobile
-        - pnasnet_large
-        - mobilenet_v2_100_224
-        - mobilenet_v2_130_224
-        - mobilenet_v2_140_224
-        - mobilenet_v3_small_100_224
-        - mobilenet_v3_small_075_224
-        - mobilenet_v3_large_100_224
-        - mobilenet_v3_large_075_224
-    dropout_rate (Optional[float]):
-        Fraction of input units to drop in the last layer. Value should be between 0.0 and 1.0.
-    trainable (Optional[bool]):
-        If true fine tuning will be performed on entire Hub model. If false only additional
-        layers will be trained.
-    l2_regularization_penalty (Optional[float]):
-        l2 regularization penalty.
-inputs:
-- {name: class_names, type: 'typing.List[str]', description: List of class names corresponding
-    to the input image data}
-- {name: model_name, type: String, description: Name of the TFHub model to load, default: efficientnetv2-xl-21k,
-  optional: true}
-- {name: dropout_rate, type: Float, description: Dropout rate, default: '0.2', optional: true}
-- name: trainable
-  type: Boolean
-  description: True if fine tuning should be performed
-  default: "True"
-  optional: true
-- {name: l2_regularization_penalty, type: Float, description: Regularization penalty,
-  default: '0.0001', optional: true}
-outputs:
-- {name: loaded_model_path, type: TensorflowSavedModel, description: Output path for
-    the loaded model}
-- {name: image_size_path, type: HeightWidth}
-implementation:
-  container:
-    image: us-docker.pkg.dev/vertex-ai/ready-to-go-image-classification/image-components:v0.1
-    # command is a list of strings (command-line arguments).
-    # The YAML language has two syntaxes for lists and you can use either of them.
-    # Here we use the "flow syntax" - comma-separated strings inside square brackets.
-    command: [
-      python3,
-      # Path of the program inside the container
-      /pipelines/component/src/loading_component.py,
-      --loaded-model-path,
-      {outputPath: loaded_model_path},
-      --class-names,
-      {inputValue: class_names},
-      --model-name,
-      {inputValue: model_name},
-      --dropout-rate,
-      {inputValue: dropout_rate},
-      --trainable,
-      {inputValue: trainable},
-      --l2-regularization-penalty,
-      {inputValue: l2_regularization_penalty},
-      --image-size-path,
-      {outputPath: image_size_path},
-    ]

community-content/pipeline_components/image_ml_model_training/pipeline.py

@@ -1,62 +0,0 @@
-# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
-from kfp import components
-from kfp.v2 import dsl
-
-# %% Loading components
-upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url('https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/component.yaml')
-deploy_model_to_endpoint_op = components.load_component_from_url('https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml')
-transcode_imagedataset_tfrecord_from_csv_op = components.load_component_from_url('https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/community-content/pipeline_components/image_ml_model_training/transcode_tfrecord_image_dataset_from_csv/component.yaml')
-load_image_classification_model_from_tfhub_op = components.load_component_from_url('https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/b5b65198a6c2ffe8c0fa2aa70127e3325752df68/community-content/pipeline_components/image_ml_model_training/load_image_classification_model/component.yaml')
-preprocess_image_data_op = components.load_component_from_url('https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/community-content/pipeline_components/image_ml_model_training/preprocess_image_data/component.yaml')
-train_tensorflow_image_classification_model_op = components.load_component_from_url('https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/community-content/pipeline_components/image_ml_model_training/train_image_classification_model/component.yaml')
-
-
-# %% Pipeline definition
-def image_classification_pipeline():
-    class_names = ['daisy', 'dandelion', 'roses', 'sunflowers', 'tulips']
-    csv_image_data_path = 'gs://cloud-samples-data/ai-platform/flowers/flowers.csv'
-    deploy_model = False
-
-    image_data = dsl.importer(
-        artifact_uri=csv_image_data_path, artifact_class=dsl.Dataset).output
-
-    image_tfrecord_data = transcode_imagedataset_tfrecord_from_csv_op(
-        csv_image_data_path=image_data,
-        class_names=class_names
-    ).outputs['tfrecord_image_data_path']
-
-    loaded_model_outputs = load_image_classification_model_from_tfhub_op(
-        class_names=class_names,
-    ).outputs
-
-    preprocessed_data = preprocess_image_data_op(
-        image_tfrecord_data,
-        height_width_path=loaded_model_outputs['image_size_path'],
-    ).outputs
-
-    trained_model = (train_tensorflow_image_classification_model_op(
-        preprocessed_training_data_path = preprocessed_data['preprocessed_training_data_path'],
-        preprocessed_validation_data_path = preprocessed_data['preprocessed_validation_data_path'],
-        model_path=loaded_model_outputs['loaded_model_path']).
-                   set_cpu_limit('96').
-                   set_memory_limit('128G').
-                   add_node_selector_constraint('cloud.google.com/gke-accelerator', 'NVIDIA_TESLA_A100').
-                   set_gpu_limit('8').
-                   outputs['trained_model_path'])
-
-    vertex_model_name = upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op(
-        model=trained_model,
-    ).outputs['model_name']
-
-    # Deploying the model might incur additional costs over time
-    if deploy_model:
-        vertex_endpoint_name = deploy_model_to_endpoint_op(
-            model_name=vertex_model_name,
-        ).outputs['endpoint_name']
-
-pipeline_func = image_classification_pipeline
-
-# %% Pipeline submission
-if __name__ == '__main__':
-    from google.cloud import aiplatform
-    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

community-content/pipeline_components/image_ml_model_training/preprocess_image_data/component.yaml

@@ -1,57 +0,0 @@
-name: Preprocess image data
-description: |
-  Preprocess the image data and split between train and validation.
-  Args:
-    input_data_path (str):
-        Input path for the TFRecord image data. Data will be formatted as 'label' (encoded image
-        label), and 'image_raw' (the binary string of the image data).
-    height_width_path (str):
-        Path to square height and width to resize images to. File should contain single float value.
-        Value is dependent on training model.
-    preprocessed_training_data_path (str):
-        Output path for the TFRecord training data. Data will be formatted as 'label' (encoded image
-        label), and 'image_raw' (the binary string of the image data).
-    preprocessed_validation_data_path (str):
-        Output path for the TFRecord validation data. Data will be formatted as 'label' (encoded
-        image label), and 'image_raw' (the binary string of the image data).
-    validation_split (Optional[float]):
-        Fraction of data that will make up validation dataset. Value should be between 0.0 and 1.0.
-    seed (Optional[int]):
-        The global random seed to ensure the system gets a unique random sequence
-        that is deterministic (https://www.tensorflow.org/api_docs/python/tf/random/set_seed).
-inputs:
-- {name: input_data_path, type: ImageDatasetTFRecord, description: 'Input path for
-    the TFRecord image data,'}
-- {name: height_width_path, type: HeightWidth, description: 'Path to square height and width to
-    resize images to,'}
-- {name: validation_split, type: Float, description: 'Fraction of data that will make
-    up validation dataset,', default: '0.2', optional: true}
-- {name: seed, type: Integer, description: Random seed, default: '0', optional: true}
-outputs:
-- {name: preprocessed_training_data_path, type: ImageDatasetTFRecord, description: 'Output
-    path for the training data,'}
-- {name: preprocessed_validation_data_path, type: ImageDatasetTFRecord, description: 'Output
-    path for the validation data,'}
-implementation:
-  container:
-    image: us-docker.pkg.dev/vertex-ai/ready-to-go-image-classification/image-components:v0.1
-    # command is a list of strings (command-line arguments).
-    # The YAML language has two syntaxes for lists and you can use either of them.
-    # Here we use the "flow syntax" - comma-separated strings inside square brackets.
-    command: [
-      python3,
-      # Path of the program inside the container
-      /pipelines/component/src/preprocessing_component.py,
-      --input-data-path,
-      {inputPath: input_data_path},
-      --height-width-path,
-      {inputPath: height_width_path},
-      --validation-split,
-      {inputValue: validation_split},
-      --seed,
-      {inputValue: seed},
-      --preprocessed-training-data-path,
-      {outputPath: preprocessed_training_data_path},
-      --preprocessed-validation-data-path,
-      {outputPath: preprocessed_validation_data_path},
-    ]

community-content/pipeline_components/image_ml_model_training/train_image_classification_model/component.yaml

@@ -1,90 +0,0 @@
-name: Train tensorflow image classification model
-description: |
-  Creates a trained image classification TensorFlow model.
-  Args:
-    preprocessed_training_data_path (str):
-        Input path to the TFRecord training data. Data will be formatted as 'label' (encoded image
-        label), and 'image_raw' (the binary string of the image data).
-    preprocessed_validation_data_path (str):
-        Input path to the TFRecord validation data. Data will be formatted as 'label' (encoded
-        image label), and 'image_raw' (the binary string of the image data).
-    model_path (str):
-        Input path to the loaded pre-trained model.
-    trained_model_path (str):
-        Output path to save the trained model to.
-    optimizer_name (Optional[str]):
-        Name of the tf.keras optimizer. Available optimizers are listed at
-        https://keras.io/api/optimizers/
-    optimizer_parameters (Optional[Dict[str, str]]):
-        Optimizer parameters.
-    loss_function_name (Optional[str]):
-        Name of the loss function.
-    loss_function_parameters (Optional[Dict[str, str]]):
-        Loss function parameters.
-    number_of_epochs (Optional[int]):
-        Number of training iterations over data.
-    metric_names (Optional[Sequence[str]]):
-        List of tf.keras.metrics to be evaluated by the model during training and testing. Available
-        metrics are listed at https://keras.io/api/metrics/.
-    seed Optional(int):
-        The global random seed to ensure the system gets a unique random sequence
-        that is deterministic (https://www.tensorflow.org/api_docs/python/tf/random/set_seed).
-inputs:
-- {name: preprocessed_training_data_path, type: ImageDatasetTFRecord, description: 'Input
-    path for the training data,'}
-- {name: preprocessed_validation_data_path, type: ImageDatasetTFRecord, description: 'Input
-    path for the validation data,'}
-- {name: model_path, type: TensorflowSavedModel, description: 'Input path for the
-    model,'}
-- {name: optimizer_name, type: String, description: 'Name of the optimizer,', default: SGD,
-  optional: true}
-- {name: optimizer_parameters, type: 'typing.Dict[str, str]', description: 'Optimizer
-    parameters,', default: '{}', optional: true}
-- {name: loss_function_name, type: String, description: 'Name of the loss function,',
-  default: CategoricalCrossentropy, optional: true}
-- {name: loss_function_parameters, type: 'typing.Dict[str, str]', description: 'Loss
-    function parameters,', default: '{}', optional: true}
-- {name: number_of_epochs, type: Integer, description: 'Number of epochs,', default: '10',
-  optional: true}
-- {name: metric_names, type: 'typing.List[str]', description: 'List of metrics to
-    use,', default: '["accuracy"]', optional: true}
-- {name: seed, type: Integer, description: 'Random seed,', default: '0', optional: true}
-- {name: batch_size, type: Integer, description: Batch size, default: '16', optional: true}
-outputs:
-- {name: trained_model_path, type: TensorflowSavedModel, description: 'Output path
-    for the saved model,'}
-implementation:
-  container:
-    image: us-docker.pkg.dev/vertex-ai/ready-to-go-image-classification/image-components:v0.1
-    # command is a list of strings (command-line arguments).
-    # The YAML language has two syntaxes for lists and you can use either of them.
-    # Here we use the "flow syntax" - comma-separated strings inside square brackets.
-    command: [
-      python3,
-      # Path of the program inside the container
-      /pipelines/component/src/training_component.py,
-      --preprocessed-training-data-path,
-      {inputPath: preprocessed_training_data_path},
-      --preprocessed-validation-data-path,
-      {inputPath: preprocessed_validation_data_path},
-      --model-path,
-      {inputPath: model_path},
-      --trained-model-path,
-      {outputPath: trained_model_path},
-      --optimizer-name,
-      {inputValue: optimizer_name},
-      --loss-function-name,
-      {inputValue: loss_function_name},
-      --number-of-epochs,
-      {inputValue: number_of_epochs},
-      --seed,
-      {inputValue: seed},
-      --batch-size,
-      {inputValue: batch_size},
-      --metric-names,
-      {inputValue: metric_names},
-      --optimizer-parameters,
-      {inputValue: optimizer_parameters},
-      --loss-function-parameters,
-      {inputValue: loss_function_parameters},
-    ]

community-content/pipeline_components/image_ml_model_training/transcode_tfrecord_image_dataset_from_csv/component.yaml

@@ -1,37 +0,0 @@
-name: Transcode imagedataset tfrecord from csv
-description: |
-  Transcodes CSV Data into TFRecord file of TFExamples.
-  Args:
-    csv_image_data_path (str):
-        Path to the CSV image data. Data must include 'image_filepath' (Path to image file) and
-        'image_label' (output for a prediction) fields.
-    class_names (Sequence[str]):
-        Sequence of strings of categories for classification corresponding to input data.
-    tfrecord_image_data_path (str):
-        Output path for the TFRecord image data. Data will be formatted as 'label' (encoded image
-        label), and 'image_raw' (the binary string of the image data).
-inputs:
-- {name: csv_image_data_path, type: ImageDatasetCSV, description: Input path for the
-    CSV image data}
-- {name: class_names, type: 'typing.List[str]', description: List of class names corresponding
-    to the input image data}
-outputs:
-- {name: tfrecord_image_data_path, type: ImageDatasetTFRecord, description: Output
-    path for the TFRecord image data}
-implementation:
-  container:
-    image: us-docker.pkg.dev/vertex-ai/ready-to-go-image-classification/image-components:v0.1
-    # command is a list of strings (command-line arguments).
-    # The YAML language has two syntaxes for lists and you can use either of them.
-    # Here we use the "flow syntax" - comma-separated strings inside square brackets.
-    command: [
-      python3,
-      # Path of the program inside the container
-      /pipelines/component/src/transcoding_csv_component.py,
-      --csv-image-data-path,
-      {inputPath: csv_image_data_path},
-      --tfrecord-image-data-path,
-      {outputPath: tfrecord_image_data_path},
-      --class-names,
-      {inputValue: class_names},
-    ]

community-content/pipeline_components/image_ml_model_training/transcode_tfrecord_image_dataset_from_jsonl/component.yaml

@@ -1,39 +0,0 @@
-name: Transcode imagedataset tfrecord from jsonlines
-description: |
-  Transcodes JSONL Data into TFRecord file of TFExamples.
-  Args:
-    jsonl_image_data_path (str):
-        Input path for the JSONL image data
-        Path to the JSONL image data. Each line corresponds to a JSON input describing an image.
-        Schema follows AutoML image classification JSONL format
-        https://cloud.google.com/vertex-ai/docs/image-data/classification/prepare-data#json-lines.
-    class_names (Sequence[str]):
-        Sequence of strings of categories for classification corresponding to input data.
-    tfrecord_image_data_path (str):
-        Output path for the TFRecord image data. Data will be formatted as 'label' (encoded image
-        label), and 'image_raw' (the binary string of the image data).
-inputs:
-- {name: jsonl_image_data_path, type: ImageDatasetJsonLines, description: Input path
-    for the JSONL image data}
-- {name: class_names, type: 'typing.List[str]', description: List of class names corresponding
-    to the input image data}
-outputs:
-- {name: tfrecord_image_data_path, type: ImageDatasetTFRecord, description: Output
-    path for the TFRecord image data}
-implementation:
-  container:
-    image: us-docker.pkg.dev/vertex-ai/ready-to-go-image-classification/image-components:v0.1
-    # command is a list of strings (command-line arguments).
-    # The YAML language has two syntaxes for lists and you can use either of them.
-    # Here we use the "flow syntax" - comma-separated strings inside square brackets.
-    command: [
-      python3,
-      # Path of the program inside the container
-      /pipelines/component/src/transcoding_jsonl_component.py,
-      --jsonl-image-data-path,
-      {inputPath: jsonl_image_data_path},
-      --tfrecord-image-data-path,
-      {outputPath: tfrecord_image_data_path},
-      --class-names,
-      {inputValue: class_names},
-    ]

community-content/pipeline_components/pandas/Binarize_column/in_CSV_format/component.yaml

@@ -1,113 +0,0 @@
-name: Binarize column using Pandas on CSV data
-metadata:
-  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/pandas/Binarize_column/in_CSV_format/component.yaml'}
-inputs:
-- {name: table, type: CSV}
-- {name: column_name, type: String}
-- {name: predicate, type: String, default: '> 0', optional: true}
-- {name: new_column_name, type: String, optional: true}
-- name: keep_original_column
-  type: Boolean
-  default: "False"
-  optional: true
-outputs:
-- {name: transformed_table, type: CSV}
-implementation:
-  container:
-    image: python:3.9
-    command:
-    - sh
-    - -c
-    - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
-      'pandas==1.4.3' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet
-      --no-warn-script-location 'pandas==1.4.3' --user) && "$0" "$@"
-    - sh
-    - -ec
-    - |
-      program_path=$(mktemp)
-      printf "%s" "$0" > "$program_path"
-      python3 -u "$program_path" "$@"
-    - |
-      def _make_parent_dirs_and_return_path(file_path: str):
-          import os
-          os.makedirs(os.path.dirname(file_path), exist_ok=True)
-          return file_path
-
-      def binarize_column_using_Pandas_on_CSV_data(
-          table_path,
-          transformed_table_path,
-          column_name,
-          predicate = "> 0",
-          new_column_name = None,
-          keep_original_column = False,
-      ):
-          import pandas
-
-          df = pandas.read_csv(table_path).convert_dtypes()
-          original_series = df[column_name]
-
-          # Dynamically executing the predicate code
-          # Variable namespace for code execution
-          namespace = dict(x=original_series)
-          # I though that there should be no space before `predicate` so that "dot" predicate methods like ".between(min, max)" work.
-          # However Python allows spaces before dot: `df .isna()`.
-          # So having a space is not a problem
-          transform_code = f"""new_series_boolean = x {predicate}"""
-          # Note: exec() takes no keyword arguments
-          # exec(__source=transform_code, __globals=namespace)
-          exec(transform_code, namespace)
-          new_series_boolean = namespace["new_series_boolean"]
-
-          # There are multiple ways to convert boolean column to integer.
-          # .apply(int) might be faster. https://stackoverflow.com/a/49804868/1497385
-          # TODO: Do a proper benchmark.
-          new_series = new_series_boolean.apply(int)
-          # new_series = new_series_boolean.astype(int)
-          # new_series = new_series_boolean.replace({False: 0, True: 1})
-
-          if new_column_name:
-              df.insert(loc=0, column=new_column_name, value=new_series)
-              if not keep_original_column:
-                  df = df.drop(columns=[column_name])
-          else:
-              df[column_name] = new_series
-
-          df.to_csv(transformed_table_path, index=False)
-
-      def _deserialize_bool(s) -> bool:
-          from distutils.util import strtobool
-          return strtobool(s) == 1
-
-      import argparse
-      _parser = argparse.ArgumentParser(prog='Binarize column using Pandas on CSV data', description='')
-      _parser.add_argument("--table", dest="table_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--column-name", dest="column_name", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--predicate", dest="predicate", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--new-column-name", dest="new_column_name", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--keep-original-column", dest="keep_original_column", type=_deserialize_bool, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--transformed-table", dest="transformed_table_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
-      _parsed_args = vars(_parser.parse_args())
-
-      _outputs = binarize_column_using_Pandas_on_CSV_data(**_parsed_args)
-    args:
-    - --table
-    - {inputPath: table}
-    - --column-name
-    - {inputValue: column_name}
-    - if:
-        cond: {isPresent: predicate}
-        then:
-        - --predicate
-        - {inputValue: predicate}
-    - if:
-        cond: {isPresent: new_column_name}
-        then:
-        - --new-column-name
-        - {inputValue: new_column_name}
-    - if:
-        cond: {isPresent: keep_original_column}
-        then:
-        - --keep-original-column
-        - {inputValue: keep_original_column}
-    - --transformed-table
-    - {outputPath: transformed_table}

community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml

@@ -1,75 +0,0 @@
-name: Fill all missing values using Pandas on CSV data
-metadata:
-  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml'}
-inputs:
-- {name: table, type: CSV}
-- {name: replacement_value, type: String, default: '0', optional: true}
-- {name: column_names, type: JsonArray, optional: true}
-outputs:
-- {name: transformed_table, type: CSV}
-implementation:
-  container:
-    image: python:3.9
-    command:
-    - sh
-    - -c
-    - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
-      'pandas==1.4.1' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet
-      --no-warn-script-location 'pandas==1.4.1' --user) && "$0" "$@"
-    - sh
-    - -ec
-    - |
-      program_path=$(mktemp)
-      printf "%s" "$0" > "$program_path"
-      python3 -u "$program_path" "$@"
-    - |
-      def _make_parent_dirs_and_return_path(file_path: str):
-          import os
-          os.makedirs(os.path.dirname(file_path), exist_ok=True)
-          return file_path
-
-      def fill_all_missing_values_using_Pandas_on_CSV_data(
-          table_path,
-          transformed_table_path,
-          replacement_value = "0",
-          column_names = None,
-      ):
-          import pandas
-
-          df = pandas.read_csv(
-              table_path,
-              dtype="string",
-          )
-
-          for column_name in column_names or df.columns:
-              df[column_name] = df[column_name].fillna(value=replacement_value)
-
-          df.to_csv(
-              transformed_table_path, index=False,
-          )
-
-      import json
-      import argparse
-      _parser = argparse.ArgumentParser(prog='Fill all missing values using Pandas on CSV data', description='')
-      _parser.add_argument("--table", dest="table_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--replacement-value", dest="replacement_value", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--column-names", dest="column_names", type=json.loads, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--transformed-table", dest="transformed_table_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
-      _parsed_args = vars(_parser.parse_args())
-
-      _outputs = fill_all_missing_values_using_Pandas_on_CSV_data(**_parsed_args)
-    args:
-    - --table
-    - {inputPath: table}
-    - if:
-        cond: {isPresent: replacement_value}
-        then:
-        - --replacement-value
-        - {inputValue: replacement_value}
-    - if:
-        cond: {isPresent: column_names}
-        then:
-        - --column-names
-        - {inputValue: column_names}
-    - --transformed-table
-    - {outputPath: transformed_table}

community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml

@@ -1,59 +0,0 @@
-name: Select columns using Pandas on CSV data
-metadata:
-  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/pandas/Select_columns/in_CSV_format/component.yaml'}
-inputs:
-- {name: table, type: CSV}
-- {name: column_names, type: JsonArray}
-outputs:
-- {name: transformed_table, type: CSV}
-implementation:
-  container:
-    image: python:3.9
-    command:
-    - sh
-    - -c
-    - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
-      'pandas==1.4.2' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet
-      --no-warn-script-location 'pandas==1.4.2' --user) && "$0" "$@"
-    - sh
-    - -ec
-    - |
-      program_path=$(mktemp)
-      printf "%s" "$0" > "$program_path"
-      python3 -u "$program_path" "$@"
-    - |
-      def _make_parent_dirs_and_return_path(file_path: str):
-          import os
-          os.makedirs(os.path.dirname(file_path), exist_ok=True)
-          return file_path
-
-      def select_columns_using_Pandas_on_CSV_data(
-          table_path,
-          transformed_table_path,
-          column_names,
-      ):
-          import pandas
-
-          df = pandas.read_csv(
-              table_path,
-              dtype="string",
-          )
-          df = df[column_names]
-          df.to_csv(transformed_table_path, index=False)
-
-      import json
-      import argparse
-      _parser = argparse.ArgumentParser(prog='Select columns using Pandas on CSV data', description='')
-      _parser.add_argument("--table", dest="table_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--column-names", dest="column_names", type=json.loads, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--transformed-table", dest="transformed_table_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
-      _parsed_args = vars(_parser.parse_args())
-
-      _outputs = select_columns_using_Pandas_on_CSV_data(**_parsed_args)
-    args:
-    - --table
-    - {inputPath: table}
-    - --column-names
-    - {inputValue: column_names}
-    - --transformed-table
-    - {outputPath: transformed_table}

community-content/pipeline_components/tensorflow/Create_fully_connected_network/component.yaml

@@ -1,102 +0,0 @@
-name: Create fully connected tensorflow network
-description: Creates fully-connected network in Tensorflow SavedModel format
-metadata:
-  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/tensorflow/Create_fully_connected_network/component.yaml'}
-inputs:
-- {name: input_size, type: Integer}
-- {name: hidden_layer_sizes, type: JsonArray, default: '[]', optional: true}
-- {name: output_size, type: Integer, default: '1', optional: true}
-- {name: activation_name, type: String, default: relu, optional: true}
-- {name: output_activation_name, type: String, optional: true}
-- {name: random_seed, type: Integer, default: '0', optional: true}
-outputs:
-- {name: model, type: TensorflowSavedModel}
-implementation:
-  container:
-    image: tensorflow/tensorflow:2.7.0
-    command:
-    - sh
-    - -ec
-    - |
-      program_path=$(mktemp)
-      printf "%s" "$0" > "$program_path"
-      python3 -u "$program_path" "$@"
-    - |
-      def _make_parent_dirs_and_return_path(file_path: str):
-          import os
-          os.makedirs(os.path.dirname(file_path), exist_ok=True)
-          return file_path
-
-      def create_fully_connected_tensorflow_network(
-          input_size,
-          model_path,
-          hidden_layer_sizes = [],
-          output_size = 1,
-          activation_name = "relu",
-          output_activation_name = None,
-          random_seed = 0,
-      ):
-          """Creates fully-connected network in Tensorflow SavedModel format"""
-          import tensorflow as tf
-          tf.random.set_seed(seed=random_seed)
-
-          model = tf.keras.models.Sequential()
-          model.add(tf.keras.Input(shape=(input_size,)))
-          for layer_size in hidden_layer_sizes:
-              model.add(tf.keras.layers.Dense(units=layer_size, activation=activation_name))
-          # The last layer is left without activation
-          model.add(tf.keras.layers.Dense(units=output_size, activation=output_activation_name))
-
-          print(model.summary())
-
-          # Using tf.keras.models.save_model instead of tf.saved_model.save to prevent downstream error:
-          #tf.saved_model.save(model, model_path)
-          # ValueError: Unable to create a Keras model from this SavedModel.
-          # This SavedModel was created with `tf.saved_model.save`, and lacks the Keras metadata.
-          # Please save your Keras model by calling `model.save`or `tf.keras.models.save_model`.
-          # See https://github.com/keras-team/keras/issues/16451
-          tf.keras.models.save_model(model, model_path)
-
-      import json
-      import argparse
-      _parser = argparse.ArgumentParser(prog='Create fully connected tensorflow network', description='Creates fully-connected network in Tensorflow SavedModel format')
-      _parser.add_argument("--input-size", dest="input_size", type=int, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--hidden-layer-sizes", dest="hidden_layer_sizes", type=json.loads, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--output-size", dest="output_size", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--activation-name", dest="activation_name", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--output-activation-name", dest="output_activation_name", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--random-seed", dest="random_seed", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--model", dest="model_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
-      _parsed_args = vars(_parser.parse_args())
-
-      _outputs = create_fully_connected_tensorflow_network(**_parsed_args)
-    args:
-    - --input-size
-    - {inputValue: input_size}
-    - if:
-        cond: {isPresent: hidden_layer_sizes}
-        then:
-        - --hidden-layer-sizes
-        - {inputValue: hidden_layer_sizes}
-    - if:
-        cond: {isPresent: output_size}
-        then:
-        - --output-size
-        - {inputValue: output_size}
-    - if:
-        cond: {isPresent: activation_name}
-        then:
-        - --activation-name
-        - {inputValue: activation_name}
-    - if:
-        cond: {isPresent: output_activation_name}
-        then:
-        - --output-activation-name
-        - {inputValue: output_activation_name}
-    - if:
-        cond: {isPresent: random_seed}
-        then:
-        - --random-seed
-        - {inputValue: random_seed}
-    - --model
-    - {outputPath: model}

community-content/pipeline_components/tensorflow/Predict/on_CSV/component.yaml

@@ -1,100 +0,0 @@
-name: Predict with TensorFlow model on CSV data
-metadata:
-  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/tensorflow/Predict/on_CSV/component.yaml'}
-inputs:
-- {name: dataset, type: CSV}
-- {name: model, type: TensorflowSavedModel}
-- {name: label_column_name, type: String, optional: true}
-- {name: batch_size, type: Integer, default: '1000', optional: true}
-outputs:
-- {name: predictions}
-implementation:
-  container:
-    image: tensorflow/tensorflow:2.9.1
-    command:
-    - sh
-    - -ec
-    - |
-      program_path=$(mktemp)
-      printf "%s" "$0" > "$program_path"
-      python3 -u "$program_path" "$@"
-    - |
-      def _make_parent_dirs_and_return_path(file_path: str):
-          import os
-          os.makedirs(os.path.dirname(file_path), exist_ok=True)
-          return file_path
-
-      def predict_with_TensorFlow_model_on_CSV_data(
-          dataset_path,
-          model_path,
-          predictions_path,
-          label_column_name = None,
-          batch_size = 1000,
-      ):
-          import numpy
-          import tensorflow as tf
-
-          model = tf.saved_model.load(export_dir=model_path)
-
-          dataset = tf.data.experimental.make_csv_dataset(
-              file_pattern=dataset_path,
-              batch_size=batch_size,
-              label_name=label_column_name,
-              header=True,
-              num_epochs=1,
-              shuffle=False,
-              ignore_errors=False,
-          )
-
-          def stack_feature_batches(features_batch):
-              # Need to stack individual feature columns to create a single feature tensor
-              # Need to cast all column tensor types to float to prevent errors.
-              list_of_feature_batches = list(
-                  tf.cast(x=feature_batch, dtype=tf.float32)
-                  for feature_batch in features_batch.values()
-              )
-              return tf.stack(list_of_feature_batches, axis=-1)
-
-          def transform_features_and_drop_labels(features_batch, labels_batch):
-              return stack_feature_batches(features_batch)
-
-          dataset_map_fn = (
-              transform_features_and_drop_labels
-              if label_column_name
-              else stack_feature_batches
-          )
-
-          dataset = dataset.map(dataset_map_fn)
-
-          with open(predictions_path, "w") as predictions_file:
-              for features_batch in dataset:
-                  predictions_tensor = model(features_batch)
-                  numpy.savetxt(predictions_file, predictions_tensor.numpy())
-
-      import argparse
-      _parser = argparse.ArgumentParser(prog='Predict with TensorFlow model on CSV data', description='')
-      _parser.add_argument("--dataset", dest="dataset_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--model", dest="model_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--label-column-name", dest="label_column_name", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--batch-size", dest="batch_size", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--predictions", dest="predictions_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
-      _parsed_args = vars(_parser.parse_args())
-
-      _outputs = predict_with_TensorFlow_model_on_CSV_data(**_parsed_args)
-    args:
-    - --dataset
-    - {inputPath: dataset}
-    - --model
-    - {inputPath: model}
-    - if:
-        cond: {isPresent: label_column_name}
-        then:
-        - --label-column-name
-        - {inputValue: label_column_name}
-    - if:
-        cond: {isPresent: batch_size}
-        then:
-        - --batch-size
-        - {inputValue: batch_size}
-    - --predictions
-    - {outputPath: predictions}

community-content/pipeline_components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml

@@ -1,170 +0,0 @@
-name: Train model using Keras on CSV
-metadata:
-  annotations: {author: Alexey Volkov <alexey.volkov@ark-kun.com>, canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml'}
-inputs:
-- {name: training_data, type: CSV}
-- {name: model, type: TensorflowSavedModel}
-- {name: label_column_name, type: String}
-- {name: loss_function_name, type: String, default: mean_squared_error, optional: true}
-- {name: number_of_epochs, type: Integer, default: '1', optional: true}
-- {name: learning_rate, type: Float, default: '0.1', optional: true}
-- {name: optimizer_name, type: String, default: Adadelta, optional: true}
-- {name: optimizer_parameters, type: JsonObject, optional: true}
-- {name: batch_size, type: Integer, default: '32', optional: true}
-- {name: metric_names, type: JsonArray, optional: true}
-- {name: random_seed, type: Integer, default: '0', optional: true}
-outputs:
-- {name: trained_model, type: TensorflowSavedModel}
-implementation:
-  container:
-    image: tensorflow/tensorflow:2.8.0
-    command:
-    - sh
-    - -ec
-    - |
-      program_path=$(mktemp)
-      printf "%s" "$0" > "$program_path"
-      python3 -u "$program_path" "$@"
-    - |
-      def _make_parent_dirs_and_return_path(file_path: str):
-          import os
-          os.makedirs(os.path.dirname(file_path), exist_ok=True)
-          return file_path
-
-      def train_model_using_Keras_on_CSV(
-          training_data_path,
-          model_path,
-          trained_model_path,
-          label_column_name,
-          loss_function_name = "mean_squared_error",
-          number_of_epochs = 1,
-          learning_rate = 0.1,
-          optimizer_name = "Adadelta",
-          optimizer_parameters = None,
-          batch_size = 32,
-          metric_names = None,
-          random_seed = 0,
-      ):
-          import tensorflow as tf
-          tf.random.set_seed(seed=random_seed)
-
-          # Loading model using Keras. Model loaded using TensorFlow does not have .fit.
-          #model = tf.saved_model.load(export_dir=model_path)
-          keras_model = tf.keras.models.load_model(filepath=model_path)
-
-          optimizer_parameters = optimizer_parameters or {}
-          optimizer_parameters["learning_rate"] = learning_rate
-          optimizer_config = {
-              "class_name": optimizer_name,
-              "config": optimizer_parameters,
-          }
-          optimizer = tf.keras.optimizers.get(optimizer_config)
-          loss = tf.keras.losses.get(loss_function_name)
-
-          training_dataset = tf.data.experimental.make_csv_dataset(
-              file_pattern=training_data_path,
-              batch_size=batch_size,
-              label_name=label_column_name,
-              header=True,
-              # Need to specify num_epochs=1 otherwise the training becomes infinite
-              num_epochs=1,
-              shuffle=True,
-              shuffle_seed=random_seed,
-              ignore_errors=True,
-          )
-          def stack_feature_batches(features_batch, labels_batch):
-              # Need to stack individual feature columns to create a single feature tensor
-              # Need to cast all column tensor types to float to prevent error:
-              # TypeError: Tensors in list passed to 'values' of 'Pack' Op have types [int32, float32, float32, int32, int32] that don't all match.
-              list_of_feature_batches = list(tf.cast(x=feature_batch, dtype=tf.float32) for feature_batch in features_batch.values())
-              return tf.stack(list_of_feature_batches, axis=-1), labels_batch
-
-          training_dataset = training_dataset.map(stack_feature_batches)
-
-          # Need to compile the model to prevent error:
-          # ValueError: No gradients provided for any variable: [..., ...].
-          keras_model.compile(
-              optimizer=optimizer,
-              loss=loss,
-              metrics=metric_names,
-          )
-          keras_model.fit(
-              training_dataset,
-              epochs=number_of_epochs,
-          )
-
-          # Using tf.keras.models.save_model instead of tf.saved_model.save to prevent downstream error:
-          #tf.saved_model.save(keras_model, trained_model_path)
-          # ValueError: Unable to create a Keras model from this SavedModel.
-          # This SavedModel was created with `tf.saved_model.save`, and lacks the Keras metadata.
-          # Please save your Keras model by calling `model.save`or `tf.keras.models.save_model`.
-          # See https://github.com/keras-team/keras/issues/16451
-          tf.keras.models.save_model(keras_model, trained_model_path)
-
-      import json
-      import argparse
-      _parser = argparse.ArgumentParser(prog='Train model using Keras on CSV', description='')
-      _parser.add_argument("--training-data", dest="training_data_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--model", dest="model_path", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--label-column-name", dest="label_column_name", type=str, required=True, default=argparse.SUPPRESS)
-      _parser.add_argument("--loss-function-name", dest="loss_function_name", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--number-of-epochs", dest="number_of_epochs", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--learning-rate", dest="learning_rate", type=float, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--optimizer-name", dest="optimizer_name", type=str, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--optimizer-parameters", dest="optimizer_parameters", type=json.loads, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--batch-size", dest="batch_size", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--metric-names", dest="metric_names", type=json.loads, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--random-seed", dest="random_seed", type=int, required=False, default=argparse.SUPPRESS)
-      _parser.add_argument("--trained-model", dest="trained_model_path", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)
-      _parsed_args = vars(_parser.parse_args())
-
-      _outputs = train_model_using_Keras_on_CSV(**_parsed_args)
-    args:
-    - --training-data
-    - {inputPath: training_data}
-    - --model
-    - {inputPath: model}
-    - --label-column-name
-    - {inputValue: label_column_name}
-    - if:
-        cond: {isPresent: loss_function_name}
-        then:
-        - --loss-function-name
-        - {inputValue: loss_function_name}
-    - if:
-        cond: {isPresent: number_of_epochs}
-        then:
-        - --number-of-epochs
-        - {inputValue: number_of_epochs}
-    - if:
-        cond: {isPresent: learning_rate}
-        then:
-        - --learning-rate
-        - {inputValue: learning_rate}
-    - if:
-        cond: {isPresent: optimizer_name}
-        then:
-        - --optimizer-name
-        - {inputValue: optimizer_name}
-    - if:
-        cond: {isPresent: optimizer_parameters}
-        then:
-        - --optimizer-parameters
-        - {inputValue: optimizer_parameters}
-    - if:
-        cond: {isPresent: batch_size}
-        then:
-        - --batch-size
-        - {inputValue: batch_size}
-    - if:
-        cond: {isPresent: metric_names}
-        then:
-        - --metric-names
-        - {inputValue: metric_names}
-    - if:
-        cond: {isPresent: random_seed}
-        then:
-        - --random-seed
-        - {inputValue: random_seed}
-    - --trained-model
-    - {outputPath: trained_model}

community-content/pytorch_efficient_training/README.md

@@ -1,33 +0,0 @@
-# PyTorch Efficient Training Examples
-
-This folder provides PyTorch efficient training examples using ResNet-50 and ImageNet data.
-
-## Requirements
-
-```shell
-pip install --upgrade pip
-pip install -r requirements.txt
-```
-
-## Description
-
-* resnet.py - Train ResNet-50 on single GPU.
-* resnet_dp.py - Train ResNet-50 on single node multiple GPUs with `DataParallel` strategy.
-* resnet_ddp.py - Train ResNet-50 on single node multiple GPUs with `DistributedDataParallel` strategy.
-* resnet_ddp_wds.py - Train ResNet-50 on single node multiple GPUs with `DistributedDataParallel` strategy and `Webdataset`.
-* resnet_fsdp.py - Train ResNet-50 on single node multiple GPUs with `FullyShardedDataParallel` strategy.
-* resnet_fsdp_wds.py - Train ResNet-50 on single node multiple GPUs with `FullyShardedDataParallel` strategy and `Webdataset`.
-* shard_imagenet.py - Shard ImagNet individual files into `tar` files.
-
-## Benchmark
-
-When run the benchmark on Nvidia T4 GPUs using ImageNet validation dataset, you can get the result like:
-Strategy               | Seconds/Epoch - Local Data | Seconds/Epoch - Cloud Data
----------------------- | -------------------------- | --------------------------
-On 1 GPU               | 489                        | 804 (2x slower)
-On 4 GPUs (DP)         | 157                        | 738 (5x slower)
-On 4 GPUs (DDP)        | 134                        | 432 (3x slower)
-On 4 GPUs (DDP + WDS)  | 131                        | 133 (same performance)
-On 4 GPUs (FSDP)       | 139                        | 353 (3x slower)
-On 4 GPUs (FSDP + WDS) | 138                        | 135 (same performance)
-

community-content/pytorch_efficient_training/requirements.txt

@@ -1 +0,0 @@
-webdataset == 0.2.26

community-content/pytorch_efficient_training/resnet.py

@@ -1,197 +0,0 @@
-# Copyright 2022 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the \"License\");
-# you may not use this file except in compliance with the License.\n",
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an \"AS IS\" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Train resnet on single GPU."""
-
-import argparse
-import time
-
-from PIL import Image
-import torch
-from torch import nn
-import torchmetrics
-import torchvision
-from torchvision.models import resnet50
-
-
-class ImageFolder(torchvision.datasets.ImageFolder):
-  """Class for loading imagenet."""
-
-  def __init__(self, image_list_file, transform=None, target_transform=None):
-    self.samples = self._make_dataset(image_list_file)
-    self.loader = self._loader
-
-    self.imgs = self.samples
-    self.targets = [s[1] for s in self.samples]
-
-    self.transform = transform
-    self.target_transform = target_transform
-
-  def _make_dataset(self, image_list_file):
-    items = []
-    with open(image_list_file, 'r') as f:
-      for line in f:
-        item = line.strip().split(' ')
-        items.append((item[0], int(item[1])))
-    return items
-
-  def _loader(self, image_path):
-    with open(image_path, 'rb') as f:
-      img = Image.open(f)
-      img = img.convert('RGB')
-      return img
-
-
-def train(model, device, dataloader, optimizer):
-  model.train()
-  for image, target in dataloader:
-    image, target = image.to(device), target.to(device)
-    pred = model(image)
-    # pred.shape (N, C), target.shape (N)
-    loss = nn.functional.cross_entropy(pred, target)
-    optimizer.zero_grad()
-    loss.backward()
-    optimizer.step()
-  return loss
-
-
-def evaluate(model, device, dataloader, metric):
-  model.eval()
-  with torch.no_grad():
-    for image, target in dataloader:
-      image, target = image.to(device), target.to(device)
-      pred = model(image)
-      metric.update(pred, target)
-  accuracy = metric.compute()
-  metric.reset()
-  return accuracy
-
-
-def run_training(args):
-  """Run training and evaluation."""
-  # Create model.
-  model = resnet50(weights=None)
-  model = model.to(args.device)
-
-  # Create train dataloader.
-  train_dataset = ImageFolder(
-      image_list_file=args.train_data_path,
-      transform=torchvision.transforms.Compose([
-          torchvision.transforms.RandomResizedCrop(224),
-          torchvision.transforms.RandomHorizontalFlip(),
-          torchvision.transforms.ToTensor(),
-          torchvision.transforms.Normalize(
-              mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
-      ]))
-  train_dataloader = torch.utils.data.DataLoader(
-      dataset=train_dataset,
-      batch_size=args.train_batch_size,
-      shuffle=True,
-      num_workers=args.dataloader_num_workers,
-      pin_memory=True)
-  print(f'Train dataloader | samples: {len(train_dataloader.dataset)}, '
-        f'num workers: {train_dataloader.num_workers}, '
-        f'batch size: {args.train_batch_size}, '
-        f'batches/epoch: {len(train_dataloader)}')
-
-  # Create eval dataloader.
-  eval_dataset = ImageFolder(
-      image_list_file=args.eval_data_path,
-      transform=torchvision.transforms.Compose([
-          torchvision.transforms.Resize(256),
-          torchvision.transforms.CenterCrop(224),
-          torchvision.transforms.ToTensor(),
-          torchvision.transforms.Normalize(
-              mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
-      ]))
-  eval_dataloader = torch.utils.data.DataLoader(
-      dataset=eval_dataset,
-      batch_size=args.eval_batch_size,
-      shuffle=False,
-      num_workers=args.dataloader_num_workers,
-      pin_memory=True,
-      drop_last=True)
-  print(f'Eval dataloader | samples: {len(eval_dataloader.dataset)}, '
-        f'num workers: {eval_dataloader.num_workers}, '
-        f'batch size: {args.eval_batch_size}, '
-        f'batches/epoch: {len(eval_dataloader)}')
-
-  # Optimizer.
-  optimizer = torch.optim.SGD(model.parameters(), 0.1)
-
-  # Main loop.
-  metric = torchmetrics.classification.Accuracy(top_k=1).to(args.device)
-  for epoch in range(1, args.epochs + 1):
-    print(f'Running epoch {epoch}')
-
-    start = time.time()
-    train(model, args.device, train_dataloader, optimizer)
-    end = time.time()
-    print(f'Training finished in {(end - start):>0.3f} seconds')
-
-    start = time.time()
-    evaluate(model, args.device, eval_dataloader, metric)
-    end = time.time()
-    print(f'Evaluation finished in {(end - start):>0.3f} seconds')
-  print('Done')
-
-
-def create_args():
-  """Create main args."""
-  parser = argparse.ArgumentParser(
-      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-  parser.add_argument(
-      '--epochs',
-      default=1,
-      type=int,
-      help='number of total epochs to run')
-  parser.add_argument(
-      '--dataloader_num_workers',
-      default=2,
-      type=int,
-      help='number of workders for dataloader')
-  parser.add_argument(
-      '--train_data_path',
-      default='',
-      type=str,
-      help='path to training data')
-  parser.add_argument(
-      '--train_batch_size',
-      default=32,
-      type=int,
-      help='batch size for training')
-  parser.add_argument(
-      '--eval_data_path',
-      default='',
-      type=str,
-      help='path to evaluation data')
-  parser.add_argument(
-      '--eval_batch_size',
-      default=32,
-      type=int,
-      help='batch size for evaluation')
-  args = parser.parse_args()
-  return args
-
-
-def main():
-  args = create_args()
-  args.device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
-
-  print('Launch job on 1 GPU')
-  run_training(args)
-
-
-if __name__ == '__main__':
-  main()

community-content/pytorch_efficient_training/resnet_ddp.py

@@ -1,234 +0,0 @@
-# Copyright 2022 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the \"License\");
-# you may not use this file except in compliance with the License.\n",
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an \"AS IS\" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Train resnet on multiple GPUs with DDP."""
-
-import argparse
-import os
-import time
-
-from PIL import Image
-import torch
-from torch import nn
-import torch.distributed as dist
-import torch.multiprocessing as mp
-import torchmetrics
-import torchvision
-from torchvision.models import resnet50
-
-
-class ImageFolder(torchvision.datasets.ImageFolder):
-  """Class for loading imagenet."""
-
-  def __init__(self, image_list_file, transform=None, target_transform=None):
-    self.samples = self._make_dataset(image_list_file)
-    self.loader = self._loader
-
-    self.imgs = self.samples
-    self.targets = [s[1] for s in self.samples]
-
-    self.transform = transform
-    self.target_transform = target_transform
-
-  def _make_dataset(self, image_list_file):
-    items = []
-    with open(image_list_file, 'r') as f:
-      for line in f:
-        item = line.strip().split(' ')
-        items.append((item[0], int(item[1])))
-    return items
-
-  def _loader(self, image_path):
-    with open(image_path, 'rb') as f:
-      img = Image.open(f)
-      img = img.convert('RGB')
-      return img
-
-
-def train(model, device, dataloader, optimizer):
-  model.train()
-  for image, target in dataloader:
-    image = image.to(device, non_blocking=True)
-    target = target.to(device, non_blocking=True)
-    pred = model(image)
-    # pred.shape (N, C), target.shape (N)
-    loss = nn.functional.cross_entropy(pred, target)
-    optimizer.zero_grad()
-    loss.backward()
-    optimizer.step()
-  return loss
-
-
-def evaluate(model, device, dataloader, metric):
-  model.eval()
-  with torch.no_grad():
-    for image, target in dataloader:
-      image = image.to(device, non_blocking=True)
-      target = target.to(device, non_blocking=True)
-      pred = model(image)
-      metric.update(pred, target)
-  accuracy = metric.compute()
-  metric.reset()
-  return accuracy
-
-
-def worker(gpu, args):
-  """Run training and evaluation."""
-  # Init process group.
-  print(f'Initiating process {gpu}')
-  dist.init_process_group(
-      backend='nccl',
-      init_method='env://',
-      world_size=args.gpus,
-      rank=gpu)
-
-  # Create model.
-  model = resnet50(weights=None)
-  torch.cuda.set_device(gpu)
-  model.to(args.device)
-  model = torch.nn.SyncBatchNorm.convert_sync_batchnorm(model)
-  model = nn.parallel.DistributedDataParallel(model, device_ids=[gpu])
-
-  # Create train dataloader.
-  train_dataset = ImageFolder(
-      image_list_file=args.train_data_path,
-      transform=torchvision.transforms.Compose([
-          torchvision.transforms.RandomResizedCrop(224),
-          torchvision.transforms.RandomHorizontalFlip(),
-          torchvision.transforms.ToTensor(),
-          torchvision.transforms.Normalize(
-              mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
-      ]))
-  train_sampler = torch.utils.data.distributed.DistributedSampler(
-      train_dataset, num_replicas=args.gpus, rank=gpu)
-  train_dataloader = torch.utils.data.DataLoader(
-      dataset=train_dataset,
-      batch_size=args.train_batch_size,
-      shuffle=False,
-      num_workers=args.dataloader_num_workers,
-      pin_memory=True,
-      sampler=train_sampler)
-  if gpu == 0:
-    print(f'Train dataloader | samples: {len(train_dataloader.dataset)}, '
-          f'num workers: {train_dataloader.num_workers}, '
-          f'global batch size: {args.train_batch_size * args.gpus}, '
-          f'batches/epoch: {len(train_dataloader)}')
-
-  # Create eval dataloader.
-  eval_dataset = ImageFolder(
-      image_list_file=args.eval_data_path,
-      transform=torchvision.transforms.Compose([
-          torchvision.transforms.Resize(256),
-          torchvision.transforms.CenterCrop(224),
-          torchvision.transforms.ToTensor(),
-          torchvision.transforms.Normalize(
-              mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
-      ]))
-  eval_sampler = torch.utils.data.distributed.DistributedSampler(
-      eval_dataset, num_replicas=args.gpus, rank=gpu)
-  eval_dataloader = torch.utils.data.DataLoader(
-      dataset=eval_dataset,
-      batch_size=args.eval_batch_size,
-      shuffle=False,
-      num_workers=args.dataloader_num_workers,
-      pin_memory=True,
-      drop_last=True,
-      sampler=eval_sampler)
-  if gpu == 0:
-    print(f'Eval dataloader | samples: {len(eval_dataloader.dataset)}, '
-          f'num workers: {eval_dataloader.num_workers}, '
-          f'batch size: {args.eval_batch_size}, '
-          f'batches/epoch: {len(eval_dataloader)}')
-
-  # Optimizer.
-  optimizer = torch.optim.SGD(model.parameters(), 0.1)
-
-  # Main loop.
-  metric = torchmetrics.classification.Accuracy(top_k=1).to(args.device)
-  for epoch in range(1, args.epochs + 1):
-    if gpu == 0:
-      print(f'Running epoch {epoch}')
-    train_sampler.set_epoch(epoch)
-
-    start = time.time()
-    train(model, args.device, train_dataloader, optimizer)
-    end = time.time()
-    if gpu == 0:
-      print(f'Training finished in {(end - start):>0.3f} seconds')
-
-    start = time.time()
-    evaluate(model, args.device, eval_dataloader, metric)
-    end = time.time()
-    if gpu == 0:
-      print(f'Evaluation finished in {(end - start):>0.3f} seconds')
-
-  if gpu == 0:
-    print('Done')
-
-
-def create_args():
-  """Create main args."""
-  parser = argparse.ArgumentParser(
-      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-  parser.add_argument(
-      '--gpus',
-      default=4,
-      type=int,
-      help='number of gpus to use')
-  parser.add_argument(
-      '--epochs',
-      default=1,
-      type=int,
-      help='number of total epochs to run')
-  parser.add_argument(
-      '--dataloader_num_workers',
-      default=2,
-      type=int,
-      help='number of workders for dataloader')
-  parser.add_argument(
-      '--train_data_path',
-      default='',
-      type=str,
-      help='path to training data')
-  parser.add_argument(
-      '--train_batch_size',
-      default=32,
-      type=int,
-      help='batch size for training per gpu')
-  parser.add_argument(
-      '--eval_data_path',
-      default='',
-      type=str,
-      help='path to evaluation data')
-  parser.add_argument(
-      '--eval_batch_size',
-      default=32,
-      type=int,
-      help='batch size for evaluation per gpu')
-  args = parser.parse_args()
-  return args
-
-
-def main():
-  args = create_args()
-  os.environ['MASTER_ADDR'] = 'localhost'
-  os.environ['MASTER_PORT'] = '8888'
-
-  args.device = 'cuda' if torch.cuda.is_available() else 'cpu'
-  print(f'Launch job on {args.gpus} GPUs with DDP')
-  mp.spawn(worker, nprocs=args.gpus, args=(args,))
-
-
-if __name__ == '__main__':
-  main()

community-content/pytorch_efficient_training/resnet_ddp_wds.py

@@ -1,249 +0,0 @@
-# Copyright 2022 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the \"License\");
-# you may not use this file except in compliance with the License.\n",
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an \"AS IS\" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Train resnet on multiple GPUs with DDP."""
-
-import argparse
-import functools
-import itertools
-import math
-import os
-import time
-
-import torch
-from torch import nn
-import torch.distributed as dist
-import torch.multiprocessing as mp
-import torchmetrics
-from torchvision.models import resnet50
-from torchvision.transforms import transforms
-import webdataset as wds
-
-
-def wds_split(src, rank, world_size):
-  """Shards split function for webdataset."""
-  # The context of caller of this function is within multiple processes
-  # (by DDP world_size) and multiple workers (by dataloader_num_workers).
-  # So we totally have (world_size * num_workers) workers for processing data.
-  # NOTE: Raw data should be sharded to enough shards to make sure one process
-  # can handle at least one shard, otherwise the process may hang.
-  worker_id = 0
-  num_workers = 1
-  worker_info = torch.utils.data.get_worker_info()
-  if worker_info:
-    worker_id = worker_info.id
-    num_workers = worker_info.num_workers
-  for s in itertools.islice(src, rank * num_workers + worker_id, None,
-                            world_size * num_workers):
-    yield s
-
-
-def identity(x):
-  return x
-
-
-def create_wds_dataloader(rank, args, mode):
-  """Create webdataset dataset and dataloader."""
-  if mode == 'train':
-    transform = transforms.Compose([
-        transforms.RandomResizedCrop(224),
-        transforms.RandomHorizontalFlip(),
-        transforms.ToTensor(),
-        transforms.Normalize(
-            mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
-    ])
-    data_path = args.train_data_path
-    data_size = args.train_data_size
-    batch_size_local = args.train_batch_size
-    batch_size_global = args.train_batch_size * args.gpus
-    # Since webdataset disallows partial batch, we pad the last batch for train.
-    batches = int(math.ceil(data_size / batch_size_global))
-  else:
-    transform = transforms.Compose([
-        transforms.Resize(256),
-        transforms.CenterCrop(224),
-        transforms.ToTensor(),
-        transforms.Normalize(
-            mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
-    ])
-    data_path = args.eval_data_path
-    data_size = args.eval_data_size
-    batch_size_local = args.eval_batch_size
-    batch_size_global = args.eval_batch_size * args.gpus
-    # Since webdataset disallows partial batch, we drop the last batch for eval.
-    batches = int(data_size / batch_size_global)
-
-  dataset = wds.DataPipeline(
-      wds.SimpleShardList(data_path),
-      functools.partial(wds_split, rank=rank, world_size=args.gpus),
-      wds.tarfile_to_samples(),
-      wds.decode('pil'),
-      wds.to_tuple('jpg;png;jpeg cls'),
-      wds.map_tuple(transform, identity),
-      wds.batched(batch_size_local, partial=False),
-  )
-  num_workers = args.dataloader_num_workers
-  dataloader = wds.WebLoader(
-      dataset=dataset,
-      batch_size=None,
-      shuffle=False,
-      num_workers=num_workers,
-      persistent_workers=True if num_workers > 0 else False,
-      pin_memory=True).repeat(nbatches=batches)
-  print(f'{mode} dataloader | samples: {data_size}, '
-        f'num_workers: {num_workers}, '
-        f'local batch size: {batch_size_local}, '
-        f'global batch size: {batch_size_global}, '
-        f'batches: {batches}')
-  return dataloader
-
-
-def train(model, device, dataloader, optimizer):
-  model.train()
-  for image, target in dataloader:
-    image = image.to(device, non_blocking=True)
-    target = target.to(device, non_blocking=True)
-    pred = model(image)
-    # pred.shape (N, C), target.shape (N)
-    loss = nn.functional.cross_entropy(pred, target)
-    optimizer.zero_grad()
-    loss.backward()
-    optimizer.step()
-  return loss
-
-
-def evaluate(model, device, dataloader, metric):
-  model.eval()
-  with torch.no_grad():
-    for image, target in dataloader:
-      image = image.to(device, non_blocking=True)
-      target = target.to(device, non_blocking=True)
-      pred = model(image)
-      metric.update(pred, target)
-  accuracy = metric.compute()
-  metric.reset()
-  return accuracy
-
-
-def worker(gpu, args):
-  """Run training and evaluation."""
-  # Init process group.
-  print(f'Initiating process {gpu}')
-  dist.init_process_group(
-      backend='nccl',
-      init_method='env://',
-      world_size=args.gpus,
-      rank=gpu)
-
-  # Create model.
-  model = resnet50(weights=None)
-  torch.cuda.set_device(gpu)
-  model.to(args.device)
-  model = torch.nn.SyncBatchNorm.convert_sync_batchnorm(model)
-  model = nn.parallel.DistributedDataParallel(model, device_ids=[gpu])
-
-  # Create dataloader.
-  train_dataloader = create_wds_dataloader(gpu, args, 'train')
-  eval_dataloader = create_wds_dataloader(gpu, args, 'eval')
-
-  # Optimizer.
-  optimizer = torch.optim.SGD(model.parameters(), 0.1)
-
-  # Main loop.
-  metric = torchmetrics.classification.Accuracy(top_k=1).to(args.device)
-  for epoch in range(1, args.epochs + 1):
-    if gpu == 0:
-      print(f'Running epoch {epoch}')
-
-    start = time.time()
-    train(model, args.device, train_dataloader, optimizer)
-    end = time.time()
-    if gpu == 0:
-      print(f'Training finished in {(end - start):>0.3f} seconds')
-
-    start = time.time()
-    evaluate(model, args.device, eval_dataloader, metric)
-    end = time.time()
-    if gpu == 0:
-      print(f'Evaluation finished in {(end - start):>0.3f} seconds')
-
-  if gpu == 0:
-    print('Done')
-
-
-def create_args():
-  """Create main args."""
-  parser = argparse.ArgumentParser(
-      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-  parser.add_argument(
-      '--gpus',
-      default=4,
-      type=int,
-      help='number of gpus to use')
-  parser.add_argument(
-      '--epochs',
-      default=1,
-      type=int,
-      help='number of total epochs to run')
-  parser.add_argument(
-      '--dataloader_num_workers',
-      default=2,
-      type=int,
-      help='number of workders for dataloader')
-  parser.add_argument(
-      '--train_data_path',
-      default='',
-      type=str,
-      help='path to training data')
-  parser.add_argument(
-      '--train_batch_size',
-      default=32,
-      type=int,
-      help='batch size for training per gpu')
-  parser.add_argument(
-      '--train_data_size',
-      default=50000,
-      type=int,
-      help='data size for training')
-  parser.add_argument(
-      '--eval_data_path',
-      default='',
-      type=str,
-      help='path to evaluation data')
-  parser.add_argument(
-      '--eval_batch_size',
-      default=32,
-      type=int,
-      help='batch size for evaluation per gpu')
-  parser.add_argument(
-      '--eval_data_size',
-      default=50000,
-      type=int,
-      help='data size for evaluation')
-  args = parser.parse_args()
-  return args
-
-
-def main():
-  args = create_args()
-  os.environ['MASTER_ADDR'] = 'localhost'
-  os.environ['MASTER_PORT'] = '8888'
-
-  args.device = 'cuda' if torch.cuda.is_available() else 'cpu'
-  print(f'Launch job on {args.gpus} GPUs with DDP')
-  mp.spawn(worker, nprocs=args.gpus, args=(args,))
-
-
-if __name__ == '__main__':
-  main()

community-content/pytorch_efficient_training/resnet_dp.py

@@ -1,207 +0,0 @@
-# Copyright 2022 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the \"License\");
-# you may not use this file except in compliance with the License.\n",
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an \"AS IS\" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Train resnet on multiple GPUs with DP."""
-
-import argparse
-import time
-
-from PIL import Image
-import torch
-from torch import nn
-import torchmetrics
-import torchvision
-from torchvision.models import resnet50
-
-
-class ImageFolder(torchvision.datasets.ImageFolder):
-  """Class for loading imagenet."""
-
-  def __init__(self, image_list_file, transform=None, target_transform=None):
-    self.samples = self._make_dataset(image_list_file)
-    self.loader = self._loader
-
-    self.imgs = self.samples
-    self.targets = [s[1] for s in self.samples]
-
-    self.transform = transform
-    self.target_transform = target_transform
-
-  def _make_dataset(self, image_list_file):
-    items = []
-    with open(image_list_file, 'r') as f:
-      for line in f:
-        item = line.strip().split(' ')
-        items.append((item[0], int(item[1])))
-    return items
-
-  def _loader(self, image_path):
-    with open(image_path, 'rb') as f:
-      img = Image.open(f)
-      img = img.convert('RGB')
-      return img
-
-
-def train(model, device, dataloader, optimizer):
-  model.train()
-  for image, target in dataloader:
-    image, target = image.to(device), target.to(device)
-    pred = model(image)
-    # pred.shape (N, C), target.shape (N)
-    loss = nn.functional.cross_entropy(pred, target)
-    optimizer.zero_grad()
-    loss.backward()
-    optimizer.step()
-  return loss
-
-
-def evaluate(model, device, dataloader, metric):
-  model.eval()
-  with torch.no_grad():
-    for image, target in dataloader:
-      image, target = image.to(device), target.to(device)
-      pred = model(image)
-      metric.update(pred, target)
-  accuracy = metric.compute()
-  metric.reset()
-  return accuracy
-
-
-def run_training(args):
-  """Run training and evaluation."""
-  # Create model.
-  model = resnet50(weights=None)
-  model = nn.DataParallel(model)
-  model = model.to(args.device)
-
-  # Create train dataloader.
-  train_dataset = ImageFolder(
-      image_list_file=args.train_data_path,
-      transform=torchvision.transforms.Compose([
-          torchvision.transforms.RandomResizedCrop(224),
-          torchvision.transforms.RandomHorizontalFlip(),
-          torchvision.transforms.ToTensor(),
-          torchvision.transforms.Normalize(
-              mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
-      ]))
-  train_dataloader = torch.utils.data.DataLoader(
-      dataset=train_dataset,
-      batch_size=args.train_batch_size,
-      shuffle=True,
-      num_workers=args.dataloader_num_workers,
-      pin_memory=True)
-  print(f'Train dataloader | samples: {len(train_dataloader.dataset)}, '
-        f'num workers: {train_dataloader.num_workers}, '
-        f'global batch size: {args.train_batch_size}, '
-        f'batches/epoch: {len(train_dataloader)}')
-
-  # Create eval dataloader.
-  eval_dataset = ImageFolder(
-      image_list_file=args.eval_data_path,
-      transform=torchvision.transforms.Compose([
-          torchvision.transforms.Resize(256),
-          torchvision.transforms.CenterCrop(224),
-          torchvision.transforms.ToTensor(),
-          torchvision.transforms.Normalize(
-              mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
-      ]))
-  eval_dataloader = torch.utils.data.DataLoader(
-      dataset=eval_dataset,
-      batch_size=args.eval_batch_size,
-      shuffle=False,
-      num_workers=args.dataloader_num_workers,
-      pin_memory=True,
-      drop_last=True)
-  print(f'Eval dataloader | samples: {len(eval_dataloader.dataset)}, '
-        f'num workers: {eval_dataloader.num_workers}, '
-        f'global batch size: {args.eval_batch_size}, '
-        f'batches/epoch: {len(eval_dataloader)}')
-
-  # Optimizer.
-  optimizer = torch.optim.SGD(model.parameters(), 0.1)
-
-  # Main loop.
-  metric = torchmetrics.classification.Accuracy(top_k=1).to(args.device)
-  for epoch in range(1, args.epochs + 1):
-    print(f'Running epoch {epoch}')
-
-    start = time.time()
-    train(model, args.device, train_dataloader, optimizer)
-    end = time.time()
-    print(f'Training finished in {(end - start):>0.3f} seconds')
-
-    start = time.time()
-    evaluate(model, args.device, eval_dataloader, metric)
-    end = time.time()
-    print(f'Evaluation finished in {(end - start):>0.3f} seconds')
-  print('Done')
-
-
-def create_args():
-  """Create main args."""
-  parser = argparse.ArgumentParser(
-      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-  parser.add_argument(
-      '--gpus',
-      default=4,
-      type=int,
-      help='number of gpus to use')
-  parser.add_argument(
-      '--epochs',
-      default=1,
-      type=int,
-      help='number of total epochs to run')
-  parser.add_argument(
-      '--dataloader_num_workers',
-      default=2,
-      type=int,
-      help='number of workders for dataloader')
-  parser.add_argument(
-      '--train_data_path',
-      default='',
-      type=str,
-      help='path to training data')
-  parser.add_argument(
-      '--train_batch_size',
-      default=32,
-      type=int,
-      help='batch size for training per gpu')
-  parser.add_argument(
-      '--eval_data_path',
-      default='',
-      type=str,
-      help='path to evaluation data')
-  parser.add_argument(
-      '--eval_batch_size',
-      default=32,
-      type=int,
-      help='batch size for evaluation per gpu')
-  args = parser.parse_args()
-  return args
-
-
-def main():
-  args = create_args()
-
-  args.device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
-  args.train_batch_size *= args.gpus
-  args.eval_batch_size *= args.gpus
-  args.dataloader_num_workers *= args.gpus
-
-  print(f'Launch job on {args.gpus} GPU with nn.DataParallel')
-  run_training(args)
-
-
-if __name__ == '__main__':
-  main()

community-content/pytorch_efficient_training/resnet_fsdp.py

@@ -1,242 +0,0 @@
-# Copyright 2022 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the \"License\");
-# you may not use this file except in compliance with the License.\n",
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an \"AS IS\" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Train resnet on multiple GPUs with FSDP."""
-
-import argparse
-import functools
-import os
-import time
-
-from PIL import Image
-import torch
-from torch import nn
-import torch.distributed as dist
-from torch.distributed.fsdp import FullyShardedDataParallel as FSDP
-from torch.distributed.fsdp.wrap import size_based_auto_wrap_policy
-import torch.multiprocessing as mp
-import torchmetrics
-import torchvision
-from torchvision.models import resnet50
-
-
-class ImageFolder(torchvision.datasets.ImageFolder):
-  """Class for loading imagenet."""
-
-  def __init__(self, image_list_file, transform=None, target_transform=None):
-    self.samples = self._make_dataset(image_list_file)
-    self.loader = self._loader
-
-    self.imgs = self.samples
-    self.targets = [s[1] for s in self.samples]
-
-    self.transform = transform
-    self.target_transform = target_transform
-
-  def _make_dataset(self, image_list_file):
-    items = []
-    with open(image_list_file, 'r') as f:
-      for line in f:
-        item = line.strip().split(' ')
-        items.append((item[0], int(item[1])))
-    return items
-
-  def _loader(self, image_path):
-    with open(image_path, 'rb') as f:
-      img = Image.open(f)
-      img = img.convert('RGB')
-      return img
-
-
-def train(model, device, dataloader, optimizer):
-  model.train()
-  for image, target in dataloader:
-    image = image.to(device, non_blocking=True)
-    target = target.to(device, non_blocking=True)
-    pred = model(image)
-    # pred.shape (N, C), target.shape (N)
-    loss = nn.functional.cross_entropy(pred, target)
-    optimizer.zero_grad()
-    loss.backward()
-    optimizer.step()
-  return loss
-
-
-def evaluate(model, device, dataloader, metric):
-  model.eval()
-  with torch.no_grad():
-    for image, target in dataloader:
-      image = image.to(device, non_blocking=True)
-      target = target.to(device, non_blocking=True)
-      pred = model(image)
-      metric.update(pred, target)
-  accuracy = metric.compute()
-  metric.reset()
-  return accuracy
-
-
-def worker(gpu, args):
-  """Run training and evaluation."""
-  # Init process group.
-  print(f'Initiating process {gpu}')
-  dist.init_process_group(
-      backend='nccl',
-      init_method='env://',
-      world_size=args.gpus,
-      rank=gpu)
-
-  # Create train dataloader.
-  train_dataset = ImageFolder(
-      image_list_file=args.train_data_path,
-      transform=torchvision.transforms.Compose([
-          torchvision.transforms.RandomResizedCrop(224),
-          torchvision.transforms.RandomHorizontalFlip(),
-          torchvision.transforms.ToTensor(),
-          torchvision.transforms.Normalize(
-              mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
-      ]))
-  train_sampler = torch.utils.data.distributed.DistributedSampler(
-      train_dataset, num_replicas=args.gpus, rank=gpu)
-  train_dataloader = torch.utils.data.DataLoader(
-      dataset=train_dataset,
-      batch_size=args.train_batch_size,
-      shuffle=False,
-      num_workers=args.dataloader_num_workers,
-      pin_memory=True,
-      sampler=train_sampler)
-  if gpu == 0:
-    print(f'Train dataloader | samples: {len(train_dataloader.dataset)}, '
-          f'num workers: {train_dataloader.num_workers}, '
-          f'global batch size: {args.train_batch_size * args.gpus}, '
-          f'batches/epoch: {len(train_dataloader)}')
-
-  # Create eval dataloader.
-  eval_dataset = ImageFolder(
-      image_list_file=args.eval_data_path,
-      transform=torchvision.transforms.Compose([
-          torchvision.transforms.Resize(256),
-          torchvision.transforms.CenterCrop(224),
-          torchvision.transforms.ToTensor(),
-          torchvision.transforms.Normalize(
-              mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
-      ]))
-  eval_sampler = torch.utils.data.distributed.DistributedSampler(
-      eval_dataset, num_replicas=args.gpus, rank=gpu)
-  eval_dataloader = torch.utils.data.DataLoader(
-      dataset=eval_dataset,
-      batch_size=args.eval_batch_size,
-      shuffle=False,
-      num_workers=args.dataloader_num_workers,
-      pin_memory=True,
-      drop_last=True,
-      sampler=eval_sampler)
-  if gpu == 0:
-    print(f'Eval dataloader | samples: {len(eval_dataloader.dataset)}, '
-          f'num workers: {eval_dataloader.num_workers}, '
-          f'batch size: {args.eval_batch_size}, '
-          f'batches/epoch: {len(eval_dataloader)}')
-
-  # Wrap policy.
-  my_auto_wrap_policy = functools.partial(
-      size_based_auto_wrap_policy, min_num_params=100)
-  torch.cuda.set_device(gpu)
-
-  # Create model.
-  model = resnet50(weights=None)
-  model.to(args.device)
-  model = FSDP(model, auto_wrap_policy=my_auto_wrap_policy)
-
-  # Optimizer.
-  optimizer = torch.optim.SGD(model.parameters(), 0.1)
-
-  # Main loop.
-  metric = torchmetrics.classification.Accuracy(top_k=1).to(args.device)
-  for epoch in range(1, args.epochs + 1):
-    if gpu == 0:
-      print(f'Running epoch {epoch}')
-    train_sampler.set_epoch(epoch)
-
-    start = time.time()
-    train(model, args.device, train_dataloader, optimizer)
-    end = time.time()
-    if gpu == 0:
-      print(f'Training finished in {(end - start):>0.3f} seconds')
-
-    start = time.time()
-    evaluate(model, args.device, eval_dataloader, metric)
-    end = time.time()
-    if gpu == 0:
-      print(f'Evaluation finished in {(end - start):>0.3f} seconds')
-
-  if gpu == 0:
-    print('Done')
-  dist.destroy_process_group()
-
-
-def create_args():
-  """Create main args."""
-  parser = argparse.ArgumentParser(
-      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-  parser.add_argument(
-      '--gpus',
-      default=4,
-      type=int,
-      help='number of gpus to use')
-  parser.add_argument(
-      '--epochs',
-      default=2,
-      type=int,
-      help='number of total epochs to run')
-  parser.add_argument(
-      '--dataloader_num_workers',
-      default=2,
-      type=int,
-      help='number of workders for dataloader')
-  parser.add_argument(
-      '--train_data_path',
-      default='',
-      type=str,
-      help='path to training data')
-  parser.add_argument(
-      '--train_batch_size',
-      default=32,
-      type=int,
-      help='batch size for training per gpu')
-  parser.add_argument(
-      '--eval_data_path',
-      default='',
-      type=str,
-      help='path to evaluation data')
-  parser.add_argument(
-      '--eval_batch_size',
-      default=32,
-      type=int,
-      help='batch size for evaluation per gpu')
-  args = parser.parse_args()
-  return args
-
-
-def main():
-  args = create_args()
-
-  os.environ['MASTER_ADDR'] = 'localhost'
-  os.environ['MASTER_PORT'] = '8888'
-
-  args.device = 'cuda' if torch.cuda.is_available() else 'cpu'
-  print(f'Launch job on {args.gpus} GPUs with FSDP')
-  mp.spawn(worker, nprocs=args.gpus, args=(args,))
-
-
-if __name__ == '__main__':
-  main()

community-content/pytorch_efficient_training/resnet_fsdp_wds.py

@@ -1,240 +0,0 @@
-"""Train resnet on multiple GPUs with DDP."""
-
-import argparse
-import functools
-import itertools
-import math
-import os
-import time
-
-import torch
-from torch import nn
-import torch.distributed as dist
-from torch.distributed.fsdp import FullyShardedDataParallel as FSDP
-from torch.distributed.fsdp.wrap import size_based_auto_wrap_policy
-import torch.multiprocessing as mp
-import torchmetrics
-from torchvision.models import resnet50
-from torchvision.transforms import transforms
-import webdataset as wds
-
-
-def wds_split(src, rank, world_size):
-  """Shards split function for webdataset."""
-  # The context of caller of this function is within multiple processes
-  # (by DDP world_size) and multiple workers (by dataloader_num_workers).
-  # So we totally have (world_size * num_workers) workers for processing data.
-  # NOTE: Raw data should be sharded to enough shards to make sure one process
-  # can handle at least one shard, otherwise the process may hang.
-  worker_id = 0
-  num_workers = 1
-  worker_info = torch.utils.data.get_worker_info()
-  if worker_info:
-    worker_id = worker_info.id
-    num_workers = worker_info.num_workers
-  for s in itertools.islice(src, rank * num_workers + worker_id, None,
-                            world_size * num_workers):
-    yield s
-
-
-def identity(x):
-  return x
-
-
-def create_wds_dataloader(rank, args, mode):
-  """Create webdataset dataset and dataloader."""
-  if mode == 'train':
-    transform = transforms.Compose([
-        transforms.RandomResizedCrop(224),
-        transforms.RandomHorizontalFlip(),
-        transforms.ToTensor(),
-        transforms.Normalize(
-            mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
-    ])
-    data_path = args.train_data_path
-    data_size = args.train_data_size
-    batch_size_local = args.train_batch_size
-    batch_size_global = args.train_batch_size * args.gpus
-    # Since webdataset disallows partial batch, we pad the last batch for train.
-    batches = int(math.ceil(data_size / batch_size_global))
-  else:
-    transform = transforms.Compose([
-        transforms.Resize(256),
-        transforms.CenterCrop(224),
-        transforms.ToTensor(),
-        transforms.Normalize(
-            mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
-    ])
-    data_path = args.eval_data_path
-    data_size = args.eval_data_size
-    batch_size_local = args.eval_batch_size
-    batch_size_global = args.eval_batch_size * args.gpus
-    # Since webdataset disallows partial batch, we drop the last batch for eval.
-    batches = int(data_size / batch_size_global)
-
-  dataset = wds.DataPipeline(
-      wds.SimpleShardList(data_path),
-      functools.partial(wds_split, rank=rank, world_size=args.gpus),
-      wds.tarfile_to_samples(),
-      wds.decode('pil'),
-      wds.to_tuple('jpg;png;jpeg cls'),
-      wds.map_tuple(transform, identity),
-      wds.batched(batch_size_local, partial=False),
-  )
-  num_workers = args.dataloader_num_workers
-  dataloader = wds.WebLoader(
-      dataset=dataset,
-      batch_size=None,
-      shuffle=False,
-      num_workers=num_workers,
-      persistent_workers=True if num_workers > 0 else False,
-      pin_memory=True).repeat(nbatches=batches)
-  print(f'{mode} dataloader | samples: {data_size}, '
-        f'num_workers: {num_workers}, '
-        f'local batch size: {batch_size_local}, '
-        f'global batch size: {batch_size_global}, '
-        f'batches: {batches}')
-  return dataloader
-
-
-def train(model, device, dataloader, optimizer):
-  model.train()
-  for image, target in dataloader:
-    image = image.to(device, non_blocking=True)
-    target = target.to(device, non_blocking=True)
-    pred = model(image)
-    # pred.shape (N, C), target.shape (N)
-    loss = nn.functional.cross_entropy(pred, target)
-    optimizer.zero_grad()
-    loss.backward()
-    optimizer.step()
-  return loss
-
-
-def evaluate(model, device, dataloader, metric):
-  model.eval()
-  with torch.no_grad():
-    for image, target in dataloader:
-      image = image.to(device, non_blocking=True)
-      target = target.to(device, non_blocking=True)
-      pred = model(image)
-      metric.update(pred, target)
-  accuracy = metric.compute()
-  metric.reset()
-  return accuracy
-
-
-def worker(gpu, args):
-  """Run training and evaluation."""
-  # Init process group.
-  print(f'Initiating process {gpu}')
-  dist.init_process_group(
-      backend='nccl',
-      init_method='env://',
-      world_size=args.gpus,
-      rank=gpu)
-
-  # Create dataloader.
-  train_dataloader = create_wds_dataloader(gpu, args, 'train')
-  eval_dataloader = create_wds_dataloader(gpu, args, 'eval')
-
-  # Wrap policy.
-  my_auto_wrap_policy = functools.partial(
-      size_based_auto_wrap_policy, min_num_params=100)
-  torch.cuda.set_device(gpu)
-
-  # Create model.
-  model = resnet50(weights=None)
-  model.to(args.device)
-  model = FSDP(model, auto_wrap_policy=my_auto_wrap_policy)
-
-  # Optimizer.
-  optimizer = torch.optim.SGD(model.parameters(), 0.1)
-
-  # Main loop.
-  metric = torchmetrics.classification.Accuracy(top_k=1).to(args.device)
-  for epoch in range(1, args.epochs + 1):
-    if gpu == 0:
-      print(f'Running epoch {epoch}')
-
-    start = time.time()
-    train(model, args.device, train_dataloader, optimizer)
-    end = time.time()
-    if gpu == 0:
-      print(f'Training finished in {(end - start):>0.3f} seconds')
-
-    start = time.time()
-    evaluate(model, args.device, eval_dataloader, metric)
-    end = time.time()
-    if gpu == 0:
-      print(f'Evaluation finished in {(end - start):>0.3f} seconds')
-
-  if gpu == 0:
-    print('Done')
-
-
-def create_args():
-  """Create main args."""
-  parser = argparse.ArgumentParser(
-      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-  parser.add_argument(
-      '--gpus',
-      default=4,
-      type=int,
-      help='number of gpus to use')
-  parser.add_argument(
-      '--epochs',
-      default=2,
-      type=int,
-      help='number of total epochs to run')
-  parser.add_argument(
-      '--dataloader_num_workers',
-      default=2,
-      type=int,
-      help='number of workders for dataloader')
-  parser.add_argument(
-      '--train_data_path',
-      default='',
-      type=str,
-      help='path to training data')
-  parser.add_argument(
-      '--train_batch_size',
-      default=32,
-      type=int,
-      help='batch size for training per gpu')
-  parser.add_argument(
-      '--train_data_size',
-      default=50000,
-      type=int,
-      help='data size for training')
-  parser.add_argument(
-      '--eval_data_path',
-      default='',
-      type=str,
-      help='path to evaluation data')
-  parser.add_argument(
-      '--eval_batch_size',
-      default=32,
-      type=int,
-      help='batch size for evaluation per gpu')
-  parser.add_argument(
-      '--eval_data_size',
-      default=50000,
-      type=int,
-      help='data size for evaluation')
-  args = parser.parse_args()
-  return args
-
-
-def main():
-  args = create_args()
-  os.environ['MASTER_ADDR'] = 'localhost'
-  os.environ['MASTER_PORT'] = '8888'
-
-  args.device = 'cuda' if torch.cuda.is_available() else 'cpu'
-  print(f'Launch job on {args.gpus} GPUs with FSDP')
-  mp.spawn(worker, nprocs=args.gpus, args=(args,))
-
-
-if __name__ == '__main__':
-  main()

community-content/pytorch_efficient_training/shard_imagenet.py

@@ -1,98 +0,0 @@
-# Copyright 2022 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the \"License\");
-# you may not use this file except in compliance with the License.\n",
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an \"AS IS\" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-r"""Main function to shard ImageNet dataset.
-
-Example usage:
-python3 -u shard_imagenet.py \
---image_list_file=/home/jupyter/data/imagenet/train_list.txt \
---output_pattern=/home/jupyter/data/imagenet/validation-%06d.tar
-"""
-
-import argparse
-import os
-import random
-import webdataset as wds  # version: 0.2.26
-
-
-# NOTE: only supports writing to local path,
-# need gcsfuse mounting if want to write to gcs bucket.
-def write_shards(args):
-  """Shard individual data files."""
-  output_dir = os.path.dirname(args.output_pattern)
-  if not os.path.isdir(output_dir):
-    os.makedirs(output_dir)
-
-  items = []
-  # Image list file is a text file, each line is a pair (image_path, label).
-  with open(args.image_list_file, 'r') as f:
-    for line in f:
-      item = line.strip().split(' ')
-      items.append((item[0], int(item[1])))
-  # Shuffle items to avoid any large sequences of a single class
-  # in the dataset.
-  random.shuffle(items)
-
-  def _read_image(image_path):
-    with open(image_path, 'rb') as f:
-      return f.read()
-
-  with wds.ShardWriter(pattern=args.output_pattern,
-                       maxcount=args.max_images_per_shard,
-                       maxsize=args.max_bytes_per_shard) as sink:
-    for i, (image_path, target) in enumerate(items):
-      key = str(i)
-      image = _read_image(image_path)
-      sample = {'__key__': key, 'jpg': image, 'cls': target}
-      sink.write(sample)
-    if len(items) != sink.total:
-      raise ValueError('Items read {} != items written {}'.format(
-          len(items), sink.total))
-
-
-def create_args():
-  """Creates arg parser."""
-  parser = argparse.ArgumentParser(
-      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-  parser.add_argument(
-      '--image_list_file',
-      default='',
-      type=str,
-      help='path to image list file')
-  parser.add_argument(
-      '--output_pattern',
-      default='',
-      type=str,
-      help='the pattern for output shards, like /path/to/train-%06d.tar')
-  parser.add_argument(
-      '--max_images_per_shard',
-      default=10 * 1024,
-      type=int,
-      help='max number of images per shard')
-  parser.add_argument(
-      '--max_bytes_per_shard',
-      default=300 * 1024 * 1024,
-      type=int,
-      help='max bytes per shard')
-  args = parser.parse_args()
-  return args
-
-
-def main():
-  args = create_args()
-  write_shards(args)
-
-
-if __name__ == '__main__':
-  main()

community-content/pytorch_image_classification_distributed_data_parallel_training_with_vertex_sdk/trainer/requirements.txt

@@ -1,3 +1,3 @@
-torch==1.13.1
+torch==1.8.1
 torchvision==0.9.1
 tensorboard==2.5.0

community-content/pytorch_image_classification_single_gpu_with_vertex_sdk_and_torchserve/trainer/requirements.txt

@@ -1,3 +1,3 @@
-torch==1.13.1
+torch==1.8.1
 torchvision==0.9.1
 tensorboard==2.5.0

community-content/pytorch_pre_built_images_deployment/pytorch-text-classification-vertex-ai-deploy.ipynb

@@ -31,7 +31,17 @@
       "source": [
         "# Deploying a PyTorch Text Classification Model on [Vertex AI](https://cloud.google.com/vertex-ai)\n",
         "\n",
-        "**Kindly reach out to Vertex AI before you run any scale tests or you have any questions.**\n"
+        "**This is an Experimental release**, covered by the Pre-GA Offerings Terms of your Google Cloud Platform [Terms of Service](https://cloud.google.com/terms).\n",
+        "\n",
+        "Experiments are focused on validating a prototype and are not guaranteed to be released. They are not intended for production use or covered by any SLA, support obligation, or deprecation policy and might be subject to backward-incompatible changes.\n",
+        "\n",
+        "**Kindly drop us a note before you run any scale tests.**\n",
+        "\n",
+        "**Do not hesitate to contact vertexai-prediction-preview-feedback@google.com if you have any questions or run into any issues.**\n",
+        "\n",
+        "The usage of the product is free during the Experimental release period: you will still incur charges for other GCP products usage, such as storage.\n",
+        "\n",
+        "The projects need to be added to the allowlist in order to deploy PyTorch models using Vertex AI Prediction pre-built PyTorch images. If you are interested in the feature, please send an email to vertexai-prediction-preview-feedback@google.com to provide your project numbers OR project ids."
       ]
     },
     {

community-content/vertex_vision_model_garden/.pylintrc

@@ -1,4 +0,0 @@
-[MASTER]
-
-generated-members=get_concrete_function,cv2.*
-ignored-modules=tensorflow,google.cloud

community-content/vertex_vision_model_garden/model_oss/diffusers/dockerfile/serve.Dockerfile

@@ -1,74 +0,0 @@
-# Dockerfile for Diffuser Serving.
-#
-# To build:
-# docker build -f model_oss/diffusers/dockerfile/serve.Dockerfile . -t ${YOUR_IMAGE_TAG}
-#
-# To push to gcr:
-# docker tag ${YOUR_IMAGE_TAG} gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
-# docker push gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
-
-FROM pytorch/torchserve:0.7.0-gpu
-
-USER root
-
-ENV infer_port=7080
-ENV mng_port=7081
-ENV model_name="diffusers_serving"
-ENV PATH="/home/model-server/:${PATH}"
-
-# Install libraries.
-ENV PIP_ROOT_USER_ACTION=ignore
-RUN python3 -m pip install --upgrade pip
-RUN pip install torch==1.13.1
-RUN pip install torchvision==0.14.1
-RUN pip install transformers==4.27.4
-RUN pip install datasets==2.9.0
-RUN pip install accelerate==0.17.0
-RUN pip install triton==2.0.0.dev20221120
-RUN pip install xformers==0.0.16
-RUN pip install google-cloud-storage==2.7.0
-RUN pip install imageio[ffmpeg]==2.31.0
-RUN pip install absl-py==1.4.0
-
-# Copy LICENSE file
-RUN apt-get update && apt-get install wget
-RUN wget https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/LICENSE
-
-# Install diffusers from main branch source code with a pinned commit.
-RUN git clone --depth 1 --branch v0.18.1 https://github.com/huggingface/diffusers.git
-WORKDIR diffusers
-RUN pip install -e .
-
-# Copy model artifacts.
-COPY model_oss/diffusers/handler.py /home/model-server/handler.py
-COPY model_oss/util/ /home/model-server/util/
-ENV PYTHONPATH /home/model-server/
-
-# Create torchserve configuration file.
-RUN echo \
-    "default_response_timeout=1800\n" \
-    "service_envelope=json\n" \
-    "inference_address=http://0.0.0.0:${infer_port}\n"  \
-    "management_address=http://0.0.0.0:${mng_port}" >> /home/model-server/config.properties
-
-# Expose ports.
-EXPOSE ${infer_port}
-EXPOSE ${mng_port}
-
-# Archive model artifacts and dependencies.
-# Do not set --model-file and --serialized-file because model and checkpoint
-# will be dynamically loaded in handler.py.
-RUN torch-model-archiver \
---model-name=${model_name} \
---version=1.0 \
---handler=/home/model-server/handler.py \
---runtime=python3 \
---export-path=/home/model-server/model-store \
---archive-format=default \
---force
-
-# Run Torchserve HTTP serve to respond to prediction requests.
-CMD ["torchserve", "--start", \
-     "--ts-config", "/home/model-server/config.properties", \
-     "--models", "${model_name}=${model_name}.mar", \
-     "--model-store", "/home/model-server/model-store"]

community-content/vertex_vision_model_garden/model_oss/diffusers/dockerfile/train.Dockerfile

@@ -1,47 +0,0 @@
-# Dockerfile for Diffuser Training.
-#
-# To build:
-# docker build -f model_oss/diffusers/dockerfile/train.Dockerfile . -t ${YOUR_IMAGE_TAG}
-#
-# To push to gcr:
-# docker tag ${YOUR_IMAGE_TAG} gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
-# docker push gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
-
-# Base on pytorch-cuda image.
-FROM pytorch/pytorch:1.13.0-cuda11.6-cudnn8-runtime
-
-# Install tools.
-RUN apt-get update && apt-get install -y --no-install-recommends \
-        curl \
-        wget \
-        git \
-        vim
-
-# Copy license.
-RUN wget https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/LICENSE
-
-# Install libraries.
-RUN pip install torchvision==0.14.1
-RUN pip install transformers==4.26.1
-RUN pip install datasets==2.9.0
-RUN pip install accelerate==0.17.0
-RUN pip install triton==2.0.0.dev20221120
-RUN pip install xformers==0.0.16
-RUN pip install Jinja2==3.1.2
-RUN pip install ftfy==6.1.1
-RUN pip install cloudml-hypertune==0.1.0.dev6
-RUN pip install tensorboard==2.12.0
-
-# Install diffusers from main branch source code with a pinned commit.
-RUN git clone --depth 1 --branch v0.18.1 https://github.com/huggingface/diffusers.git
-WORKDIR diffusers
-RUN pip install -e .
-
-# Switch to diffusers examples folder.
-WORKDIR examples
-
-# Config accelerate.
-COPY model_oss/diffusers/train.sh train.sh
-
-# Generate accelerate config at the beginning of docker run.
-ENTRYPOINT ["/bin/bash", "train.sh"]

community-content/vertex_vision_model_garden/model_oss/diffusers/handler.py

@@ -1,247 +0,0 @@
-"""Custom handler for huggingface/diffusers models."""
-
-# pylint: disable=g-importing-member
-# pylint: disable=logging-fstring-interpolation
-
-import base64
-import logging
-import os
-from typing import Any, List, Tuple
-
-from diffusers import ControlNetModel
-from diffusers import DiffusionPipeline
-from diffusers import DPMSolverMultistepScheduler
-from diffusers import EulerAncestralDiscreteScheduler
-from diffusers import StableDiffusionControlNetPipeline
-from diffusers import StableDiffusionImg2ImgPipeline
-from diffusers import StableDiffusionInpaintPipeline
-from diffusers import StableDiffusionInstructPix2PixPipeline
-from diffusers import StableDiffusionPipeline
-from diffusers import StableDiffusionUpscalePipeline
-from diffusers import TextToVideoZeroPipeline
-from diffusers import UniPCMultistepScheduler
-import numpy as np
-from PIL import Image
-import torch
-from ts.torch_handler.base_handler import BaseHandler
-
-from util import constants
-from util import fileutils
-from util import image_format_converter
-from video_util import video_format_converter
-
-STABLE_DIFFUSION_MODEL = "runwayml/stable-diffusion-v1-5"
-
-# Tasks
-TEXT_TO_IMAGE = "text-to-image"
-IMAGE_TO_IMAGE = "image-to-image"
-IMAGE_INPAINTING = "image-inpainting"
-INSTRUCT_PIX2PIX = "instruct-pix2pix"
-CONTROLNET = "controlnet"
-CONDITIONED_SUPER_RES = "conditioned-super-res"
-TEXT_TO_VIDEO_ZERO_SHOT = "text-to-video-zero-shot"
-TEXT_TO_VIDEO = "text-to-video"
-
-
-class DiffusersHandler(BaseHandler):
-  """Custom handler for TIMM models."""
-
-  def initialize(self, context: Any):
-    """Custom initialize."""
-
-    properties = context.system_properties
-    self.map_location = (
-        "cuda"
-        if torch.cuda.is_available() and properties.get("gpu_id") is not None
-        else "cpu"
-    )
-    self.device = torch.device(
-        self.map_location + ":" + str(properties.get("gpu_id"))
-        if torch.cuda.is_available() and properties.get("gpu_id") is not None
-        else self.map_location
-    )
-    self.manifest = context.manifest
-
-    self.model_id = os.environ["MODEL_ID"]
-    if self.model_id.startswith(constants.GCS_URI_PREFIX):
-      gcs_path = self.model_id[len(constants.GCS_URI_PREFIX) :]
-      local_model_dir = os.path.join(constants.LOCAL_MODEL_DIR, gcs_path)
-      logging.info(f"Download {self.model_id} to {local_model_dir}")
-      fileutils.download_gcs_dir_to_local(self.model_id, local_model_dir)
-      self.model_id = local_model_dir
-
-    self.task = os.environ.get("TASK", TEXT_TO_IMAGE)
-    logging.info(f"Using task:{self.task}, model:{self.model_id}")
-
-    if self.task == TEXT_TO_IMAGE:
-      pipeline = StableDiffusionPipeline.from_pretrained(
-          self.model_id, torch_dtype=torch.float16
-      )
-      pipeline.scheduler = EulerAncestralDiscreteScheduler.from_config(
-          pipeline.scheduler.config
-      )
-      pipeline = pipeline.to(self.map_location)
-      # Reduce memory footprint.
-      pipeline.enable_attention_slicing()
-    elif self.task == IMAGE_TO_IMAGE:
-      pipeline = StableDiffusionImg2ImgPipeline.from_pretrained(
-          self.model_id, torch_dtype=torch.float16
-      )
-      pipeline.scheduler = EulerAncestralDiscreteScheduler.from_config(
-          pipeline.scheduler.config
-      )
-      pipeline = pipeline.to(self.map_location)
-      # Reduce memory footprint.
-      pipeline.enable_attention_slicing()
-    elif self.task == IMAGE_INPAINTING:
-      pipeline = StableDiffusionInpaintPipeline.from_pretrained(
-          self.model_id, torch_dtype=torch.float16
-      )
-      pipeline.scheduler = EulerAncestralDiscreteScheduler.from_config(
-          pipeline.scheduler.config
-      )
-      pipeline = pipeline.to(self.map_location)
-      # Reduce memory footprint.
-      pipeline.enable_attention_slicing()
-    elif self.task == INSTRUCT_PIX2PIX:
-      pipeline = StableDiffusionInstructPix2PixPipeline.from_pretrained(
-          self.model_id, torch_dtype=torch.float16
-      )
-      pipeline.scheduler = EulerAncestralDiscreteScheduler.from_config(
-          pipeline.scheduler.config
-      )
-      pipeline = pipeline.to(self.map_location)
-      # Reduce memory footprint.
-      pipeline.enable_attention_slicing()
-    elif self.task == CONTROLNET:
-      controlnet = ControlNetModel.from_pretrained(
-          self.model_id, torch_dtype=torch.float16
-      )
-      pipeline = StableDiffusionControlNetPipeline.from_pretrained(
-          STABLE_DIFFUSION_MODEL,
-          controlnet=controlnet,
-          torch_dtype=torch.float16,
-      )
-      pipeline.scheduler = UniPCMultistepScheduler.from_config(
-          pipeline.scheduler.config
-      )
-      pipeline.enable_xformers_memory_efficient_attention()
-      pipeline.enable_model_cpu_offload()
-      pipeline = pipeline.to(self.map_location)
-      # Reduce memory footprint.
-      pipeline.enable_attention_slicing()
-    elif self.task == CONDITIONED_SUPER_RES:
-      pipeline = StableDiffusionUpscalePipeline.from_pretrained(
-          self.model_id, torch_dtype=torch.float16
-      )
-      pipeline.scheduler = EulerAncestralDiscreteScheduler.from_config(
-          pipeline.scheduler.config
-      )
-      # This is necessary to 4x upscale >=256x256 input images with V100.
-      logging.info("Enable xformers memory efficient attention for inference.")
-      pipeline.enable_xformers_memory_efficient_attention()
-      pipeline = pipeline.to(self.map_location)
-      # Reduce memory footprint.
-      pipeline.enable_attention_slicing()
-    elif self.task == TEXT_TO_VIDEO_ZERO_SHOT:
-      pipeline = TextToVideoZeroPipeline.from_pretrained(
-          STABLE_DIFFUSION_MODEL, torch_dtype=torch.float16
-      )
-      # Memory optimization.
-      pipeline.enable_xformers_memory_efficient_attention()
-      pipeline.enable_model_cpu_offload()
-      pipeline = pipeline.to(self.map_location)
-    elif self.task == TEXT_TO_VIDEO:
-      pipeline = DiffusionPipeline.from_pretrained(
-          self.model_id, torch_dtype=torch.float16, variant="fp16"
-      )
-      pipeline.enable_model_cpu_offload()
-      # Memory optimization.
-      pipeline.enable_vae_slicing()
-      pipeline.scheduler = DPMSolverMultistepScheduler.from_config(
-          pipeline.scheduler.config
-      )
-    else:
-      raise ValueError(f"Invalid TASK: {self.task}")
-
-    self.pipeline = pipeline
-    self.initialized = True
-    logging.info("Handler initialization done.")
-
-  def preprocess(self, data: Any) -> Tuple[Any, Any, Any]:
-    """Preprocess input data."""
-    prompts = [item["prompt"] for item in data]
-    images = None
-    mask_images = None
-
-    if "image" in data[0]:
-      images = [
-          image_format_converter.base64_to_image(item["image"]) for item in data
-      ]
-    if "mask_image" in data[0]:
-      mask_images = [
-          image_format_converter.base64_to_image(item["mask_image"])
-          for item in data
-      ]
-    return prompts, images, mask_images
-
-  def inference(self, data: Any, *args, **kwargs) -> List[Image.Image]:
-    """Run the inference."""
-    prompts, images, mask_images = data
-    if self.task == TEXT_TO_IMAGE:
-      predicted_images = self.pipeline(prompt=prompts).images
-    elif self.task == IMAGE_TO_IMAGE:
-      predicted_images = self.pipeline(prompt=prompts, image=images).images
-    elif self.task == IMAGE_INPAINTING:
-      predicted_images = self.pipeline(
-          prompt=prompts, image=images, mask_image=mask_images
-      ).images
-    elif self.task == INSTRUCT_PIX2PIX:
-      predicted_images = self.pipeline(prompt=prompts, image=images).images
-    elif self.task == CONTROLNET:
-      predicted_images = self.pipeline(
-          prompt=prompts, image=images, num_inference_steps=20
-      ).images
-    elif self.task == CONDITIONED_SUPER_RES:
-      predicted_images = self.pipeline(
-          prompt=prompts, image=images, num_inference_steps=20
-      ).images
-    elif self.task == TEXT_TO_VIDEO_ZERO_SHOT:
-      # For each given prompt, generate a short video.
-      # The pipeline doesn't support multiple prompts in one run yet.
-      videos = []
-      for prompt in prompts:
-        numpy_arrays = self.pipeline(prompt=prompt).images
-        numpy_arrays = [(i * 255).astype("uint8") for i in numpy_arrays]
-        videos.append(
-            video_format_converter.frames_to_video_bytes(numpy_arrays, fps=4)
-        )
-      return videos
-    elif self.task == TEXT_TO_VIDEO:
-      predicted_images = np.asarray(self.pipeline(prompt=prompts).frames)
-      # For multiple prompts, the model concatenates video frames, i.e. the
-      # output shape is (num_frames, height, width * len(prompts), channels).
-      # Therefore we need to split the output into different videos.
-      predicted_images = np.array_split(predicted_images, len(prompts), axis=2)
-      videos = [
-          video_format_converter.frames_to_video_bytes(images, fps=8)
-          for images in predicted_images
-      ]
-      return videos
-    else:
-      raise ValueError(f"Invalid TASK: {self.task}")
-    return predicted_images
-
-  def postprocess(self, data: Any) -> List[str]:
-    """Convert the images to base64 string."""
-    outputs = []
-    for prediction in data:
-      if isinstance(prediction, bytes):
-        # This is the video bytes.
-        outputs.append(base64.b64encode(prediction).decode("utf-8"))
-      else:
-        outputs.append(image_format_converter.image_to_base64(prediction))
-    return outputs
-
-
-# pylint: enable=logging-fstring-interpolation

community-content/vertex_vision_model_garden/model_oss/diffusers/train.sh

@@ -1,6 +0,0 @@
-#!/bin/bash
-
-# Setup accelerate config before running trainer.
-python -c "from accelerate.utils import write_basic_config; write_basic_config(mixed_precision='fp16')"
-
-accelerate launch "$@"

community-content/vertex_vision_model_garden/model_oss/transformers/dockerfile/serve.Dockerfile

@@ -1,79 +0,0 @@
-# Dockerfile for serving dockers for transformers.
-#
-# To build:
-# docker build -f model_oss/transformers/dockerfile/serve.Dockerfile . -t ${YOUR_IMAGE_TAG}
-#
-# To push to gcr:
-# docker tag ${YOUR_IMAGE_TAG} gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
-# docker push gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
-# Switch to this base image for gpu serve.
-
-FROM pytorch/torchserve:0.7.0-gpu
-
-USER root
-
-ENV infer_port=7080
-ENV mng_port=7081
-ENV model_name="transformers_serving"
-ENV PATH="/home/model-server/:${PATH}"
-
-# Install libraries.
-ENV PIP_ROOT_USER_ACTION=ignore
-RUN python3 -m pip install --upgrade pip
-RUN pip install torch==1.13.1
-RUN pip install torchvision==0.14.1
-RUN pip install transformers==4.27.4
-RUN pip install datasets==2.9.0
-RUN pip install accelerate==0.17.0
-RUN pip install triton==2.0.0.dev20221120
-RUN pip install xformers==0.0.16
-RUN pip install google-cloud-storage==2.7.0
-RUN pip install absl-py==1.4.0
-
-# Install libraries for document-question-answering.
-RUN apt-get update
-RUN apt-get install -y --no-install-recommends tesseract-ocr
-RUN pip install tesseract==0.1.3
-RUN pip install pytesseract==0.3.10
-
-# Install tools.
-RUN apt-get update && apt-get install -y --no-install-recommends \
-        curl \
-        wget \
-        vim
-
-# Copy license.
-RUN wget https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/LICENSE
-
-# Copy model artifacts.
-COPY model_oss/transformers/handler.py /home/model-server/handler.py
-COPY model_oss/util/ /home/model-server/util/
-ENV PYTHONPATH /home/model-server/
-
-# Create torchserve configuration file.
-RUN echo \
-    "default_response_timeout=1800\n" \
-    "service_envelope=json\n" \
-    "inference_address=http://0.0.0.0:${infer_port}\n"  \
-    "management_address=http://0.0.0.0:${mng_port}" >> /home/model-server/config.properties
-
-# Expose ports.
-EXPOSE ${infer_port}
-EXPOSE ${mng_port}
-
-# Archive model artifacts and dependencies.
-# Do not set --model-file and --serialized-file because model and checkpoint will be dynamically loaded in handler.py.
-RUN torch-model-archiver \
---model-name=${model_name} \
---version=1.0 \
---handler=/home/model-server/handler.py \
---runtime=python3 \
---export-path=/home/model-server/model-store \
---archive-format=default \
---force
-
-# Run Torchserve HTTP serve to respond to prediction requests.
-CMD ["torchserve", "--start", \
-     "--ts-config", "/home/model-server/config.properties", \
-     "--models", "${model_name}=${model_name}.mar", \
-     "--model-store", "/home/model-server/model-store"]

community-content/vertex_vision_model_garden/model_oss/transformers/handler.py

@@ -1,233 +0,0 @@
-"""Custom handler for huggingface/transformers models."""
-
-# pylint: disable=g-multiple-import
-# pylint: disable=g-importing-member
-
-import logging
-import os
-from typing import Any, List, Optional, Tuple
-
-from PIL import Image
-import torch
-from transformers import (
-    AutoProcessor,
-    AutoTokenizer,
-    Blip2ForConditionalGeneration,
-    Blip2Processor,
-    BlipForConditionalGeneration,
-    BlipForQuestionAnswering,
-    BlipProcessor,
-    CLIPModel,
-)
-from transformers import pipeline
-from ts.torch_handler.base_handler import BaseHandler
-
-from util import constants
-from util import fileutils
-from util import image_format_converter
-
-DEFAULT_MODEL_ID = "openai/clip-vit-base-patch32"
-SALESFORCE_BLIP = "Salesforce/blip"
-SALESFORCE_BLIP2 = "Salesforce/blip2"
-FLAN_T5 = "flan-t5"
-BART_LARGE_CNN = "facebook/bart-large-cnn"
-
-ZERO_CLASSIFICATION = "zero-shot-image-classification"
-FEATURE_EMBEDDING = "feature-embedding"
-ZERO_DETECTION = "zero-shot-object-detection"
-IMAGE_CAPTIONING = "image-to-text"
-VQA = "visual-question-answering"
-DQA = "document-question-answering"
-SUMMARIZATION = "summarization"
-SUMMARIZATION_TEMPLATE = (
-    "Summarize the following news article:\n{input}\nSummary:\n"
-)
-
-
-class TransformersHandler(BaseHandler):
-  """Custom handler for huggingface/transformers models."""
-
-  def initialize(self, context: Any):
-    """Custom initialize."""
-
-    properties = context.system_properties
-    self.map_location = (
-        "cuda"
-        if torch.cuda.is_available() and properties.get("gpu_id") is not None
-        else "cpu"
-    )
-    self.device = torch.device(
-        self.map_location + ":" + str(properties.get("gpu_id"))
-        if torch.cuda.is_available() and properties.get("gpu_id") is not None
-        else self.map_location
-    )
-
-    self.manifest = context.manifest
-    # The model id is can be either:
-    # 1) a huggingface model card id, like "Salesforce/blip", or
-    # 2) a GCS path to the model files, like "gs://foo/bar".
-    # If it's a model card id, the model will be loaded from huggingface.
-    self.model_id = (
-        DEFAULT_MODEL_ID
-        if os.environ.get("MODEL_ID") is None
-        else os.environ["MODEL_ID"]
-    )
-    # Else it will be downloaded from GCS to local first.
-    # Since the transformers from_pretrained API can't read from GCS.
-    if self.model_id.startswith(constants.GCS_URI_PREFIX):
-      gcs_path = self.model_id[len(constants.GCS_URI_PREFIX) :]
-      local_model_dir = os.path.join(constants.LOCAL_MODEL_DIR, gcs_path)
-      logging.info("Download %s to %s", self.model_id, local_model_dir)
-      fileutils.download_gcs_dir_to_local(self.model_id, local_model_dir)
-      self.model_id = local_model_dir
-
-    self.task = (
-        ZERO_CLASSIFICATION
-        if os.environ.get("TASK") is None
-        else os.environ["TASK"]
-    )
-    logging.info(
-        "Handler initializing task:%s, model:%s", self.task, self.model_id
-    )
-
-    if SALESFORCE_BLIP in self.model_id:
-      # pipeline() hasn't been ready for Salesforce/blip models.
-      self.salesforce_blip = True
-      self._create_blip_model()
-    else:
-      self.salesforce_blip = False
-      if self.task == FEATURE_EMBEDDING:
-        self.model = CLIPModel.from_pretrained(self.model_id).to(
-            self.map_location
-        )
-        self.tokenizer = AutoTokenizer.from_pretrained(self.model_id)
-        self.processor = AutoProcessor.from_pretrained(self.model_id)
-      elif self.task == SUMMARIZATION and FLAN_T5 in self.model_id:
-        self.pipeline = pipeline(
-            task=self.task,
-            model=self.model_id,
-            torch_dtype=torch.bfloat16,
-            device_map="auto",
-        )
-      else:
-        self.pipeline = pipeline(
-            task=self.task, model=self.model_id, device=self.device
-        )
-
-    self.initialized = True
-    logging.info("Handler initialization done.")
-
-  def _create_blip_model(self):
-    """A helper for creating BLIP and BLIP2 models."""
-    if SALESFORCE_BLIP2 in self.model_id:
-      self.torch_type = torch.float16
-      self.processor = Blip2Processor.from_pretrained(self.model_id)
-      self.model = Blip2ForConditionalGeneration.from_pretrained(
-          self.model_id, torch_dtype=self.torch_type
-      ).to(self.map_location)
-    else:
-      self.torch_type = torch.float32
-      self.processor = BlipProcessor.from_pretrained(self.model_id)
-      if self.task == IMAGE_CAPTIONING:
-        self.model = BlipForConditionalGeneration.from_pretrained(
-            self.model_id
-        ).to(self.map_location)
-      elif self.task == VQA:
-        self.model = BlipForQuestionAnswering.from_pretrained(self.model_id).to(
-            self.map_location
-        )
-
-  def _reformat_detection_result(self, data: List[Any]) -> List[Any]:
-    """Reformat zero-shot-object-detection output."""
-    if not data:
-      return [data]
-    boxes = {}
-    boxes["label"] = data[0]["label"]
-    boxes["boxes"] = []
-    for item in data:
-      box = {}
-      box["score"] = item["score"]
-      box.update(item["box"])
-      boxes["boxes"].append(box)
-    outputs = [boxes]
-    return outputs
-
-  def preprocess(
-      self, data: Any
-  ) -> Tuple[Optional[List[str]], Optional[List[Image.Image]]]:
-    """Preprocess input data."""
-    texts = None
-    images = None
-    if "text" in data[0]:
-      texts = [item["text"] for item in data]
-    if "image" in data[0]:
-      images = [
-          image_format_converter.base64_to_image(item["image"]) for item in data
-      ]
-    return texts, images
-
-  def inference(self, data: Any, *args, **kwargs) -> List[Any]:
-    """Run the inference."""
-    texts, images = data
-    preds = None
-    if self.task == ZERO_CLASSIFICATION:
-      preds = self.pipeline(images=images, candidate_labels=texts)
-    elif self.task == ZERO_DETECTION:
-      # The object detection pipeline doesn't support batch prediction.
-      preds = self.pipeline(image=images[0], candidate_labels=texts[0])
-    elif self.task == IMAGE_CAPTIONING:
-      if self.salesforce_blip:
-        inputs = self.processor(images[0], return_tensors="pt").to(
-            self.map_location, self.torch_type
-        )
-        preds = self.model.generate(**inputs)
-        preds = [
-            self.processor.decode(preds[0], skip_special_tokens=True).strip()
-        ]
-      else:
-        preds = self.pipeline(images=images)
-    elif self.task == VQA:
-      # The VQA pipelines doesn't support batch prediction.
-      if self.salesforce_blip:
-        inputs = self.processor(images[0], texts[0], return_tensors="pt").to(
-            self.map_location, self.torch_type
-        )
-        preds = self.model.generate(**inputs)
-        preds = [
-            self.processor.decode(preds[0], skip_special_tokens=True).strip()
-        ]
-      else:
-        preds = self.pipeline(image=images[0], question=texts[0])
-    elif self.task == DQA:
-      # The DQA pipelines doesn't support batch prediction.
-      preds = self.pipeline(image=images[0], question=texts[0])
-    elif self.task == FEATURE_EMBEDDING:
-      preds = {}
-      if texts:
-        inputs = self.tokenizer(
-            text=texts, padding=True, return_tensors="pt"
-        ).to(self.map_location)
-        text_features = self.model.get_text_features(**inputs)
-        preds["text_features"] = text_features.detach().cpu().numpy().tolist()
-      if images:
-        inputs = self.processor(images=images, return_tensors="pt").to(
-            self.map_location
-        )
-        image_features = self.model.get_image_features(**inputs)
-        preds["image_features"] = image_features.detach().cpu().numpy().tolist()
-      preds = [preds]
-    elif self.task == SUMMARIZATION and FLAN_T5 in self.model_id:
-      texts = [SUMMARIZATION_TEMPLATE.format(input=text) for text in texts]
-      preds = self.pipeline(texts, max_length=130)
-    elif self.task == SUMMARIZATION and self.model_id == BART_LARGE_CNN:
-      preds = self.pipeline(
-          texts[0], max_length=130, min_length=30, do_sample=False
-      )
-    else:
-      raise ValueError(f"Invalid TASK: {self.task}")
-    return preds
-
-  def postprocess(self, data: Any) -> List[Any]:
-    if self.task == ZERO_DETECTION:
-      data = self._reformat_detection_result(data)
-    return data

community-content/vertex_vision_model_garden/model_oss/util/constants.py

@@ -1,64 +0,0 @@
-"""Vertex vision model garden util constants."""
-
-# Objectives.
-OBJECTIVE_IMAGE_CLASSIFICATION = 'icn'
-OBJECTIVE_IMAGE_OBJECT_DETECTION = 'iod'
-OBJECTIVE_IMAGE_SEGMENTATION = 'isg'
-
-# Input file types.
-INPUT_FILE_TYPE_CSV = 'csv'
-INPUT_FILE_TYPE_JSONL = 'jsonl'
-INPUT_FILE_TYPE_COCO_JSON = 'coco_json'
-
-# Output file types.
-OUTPUT_FILE_TYPE_TFRECORD = 'tfrecord'
-OUTPUT_FILE_TYPE_COCO_JSON = 'coco_json'
-
-# Best evaluation metrics.
-IMAGE_CLASSIFICATION_SINGLE_LABEL_BEST_EVAL_METRIC = 'accuracy'
-IMAGE_CLASSIFICATION_MULTI_LABEL_BEST_EVAL_METRIC = 'meanPR-AUC'
-
-IMAGE_OBJECT_DETECTION_BEST_EVAL_METRIC = 'AP50'
-IMAGE_SEGMENTATION_BEST_EVAL_METRIC = 'mean_iou'
-
-VIDEO_CLASSIFICATION_BEST_EVAL_METRIC = 'accuracy'
-
-# Best checkpoints.
-BEST_CKPT_DIRNAME = 'best_ckpt'
-BEST_CKPT_EVAL_FILENAME = 'info.json'
-BEST_CKPT_STEP_NAME = 'best_ckpt_global_step'
-BEST_CKPT_METRIC_COMP = 'higher'
-
-# Reported hyperparameter tuning metric tag.
-HP_METRIC_TAG = 'model_performance'
-
-# HPT trial prefix.
-TRIAL_PREFIX = 'trial_'
-
-# ML uses from user input.
-ML_USE_TRAINING = 'training'
-ML_USE_VALIDATION = 'validation'
-ML_USE_TEST = 'test'
-
-# COCO json keys
-COCO_JSON_ANNOTATIONS = 'annotations'
-COCO_JSON_ANNOTATION_IMAGE_ID = 'image_id'
-COCO_JSON_ANNOTATION_CATEGORY_ID = 'category_id'
-COCO_JSON_CATEGORIES = 'categories'
-COCO_JSON_CATEGORY_ID = 'id'
-COCO_JSON_CATEGORY_NAME = 'name'
-COCO_JSON_FILE_NAME = 'file_name'
-COCO_JSON_IMAGES = 'images'
-COCO_JSON_IMAGE_ID = 'id'
-COCO_JSON_IMAGE_WIDTH = 'width'
-COCO_JSON_IMAGE_HEIGHT = 'height'
-COCO_JSON_IMAGE_COCO_URL = 'coco_url'
-COCO_ANNOTATION_BBOX = 'bbox'
-
-# GCS prefixes
-GCS_URI_PREFIX = 'gs://'
-GCSFUSE_URI_PREFIX = '/gcs/'
-
-LOCAL_EVALUATION_RESULT_DIR = '/tmp/evaluation_result_dir'
-LOCAL_MODEL_DIR = '/tmp/model_dir'
-LOCAL_DATA_DIR = '/tmp/data'

community-content/vertex_vision_model_garden/model_oss/util/fileutils.py

@@ -1,79 +0,0 @@
-"""Fileutil lib to copy files between gcs and local."""
-
-import glob
-import os
-
-from absl import logging
-from google.cloud import storage
-
-from util import constants
-
-
-def download_gcs_file_to_local(gcs_uri: str, local_path: str):
-  """Download a gcs file to a local path.
-
-  Args:
-    gcs_uri: A string of file path on GCS.
-    local_path: A string of local file path.
-  """
-  if not gcs_uri.startswith(constants.GCS_URI_PREFIX):
-    raise ValueError(
-        f'{gcs_uri} is not a GCS path starting with {constants.GCS_URI_PREFIX}.'
-    )
-  client = storage.Client()
-  os.makedirs(os.path.dirname(local_path), exist_ok=True)
-  with open(local_path, 'wb') as f:
-    client.download_blob_to_file(gcs_uri, f)
-
-
-def download_gcs_dir_to_local(gcs_dir: str, local_dir: str):
-  """Downloads files in a GCS directory to a local directory.
-
-  For example:
-    download_gcs_dir_to_local(gs://bucket/foo, /tmp/bar)
-    gs://bucket/foo/a -> /tmp/bar/a
-    gs://bucket/foo/b/c -> /tmp/bar/b/c
-
-  Arguments:
-    gcs_dir: A string of directory path on GCS.
-    local_dir: A string of local directory path.
-  """
-  bucket_name = gcs_dir.split('/')[2]
-  prefix = gcs_dir[len(constants.GCS_URI_PREFIX + bucket_name) :].strip('/')
-  client = storage.Client()
-  blobs = client.list_blobs(bucket_name, prefix=prefix)
-  for blob in blobs:
-    if blob.name[-1] == '/':
-      continue
-    file_path = blob.name[len(prefix) :].strip('/')
-    local_file_path = os.path.join(local_dir, file_path)
-    os.makedirs(os.path.dirname(local_file_path), exist_ok=True)
-    logging.info('Downloading %s to %s', file_path, local_file_path)
-    blob.download_to_filename(local_file_path)
-
-
-def upload_local_dir_to_gcs(local_dir: str, gcs_dir: str):
-  """Uploads local dir to gcs.
-
-  For example:
-    upload_local_dir_to_gcs(/tmp/bar, gs://bucket/foo)
-    gs://bucket/foo/a -> /tmp/bar/a
-    gs://bucket/foo/b/c -> /tmp/bar/b/c
-
-  Arguments:
-    local_dir: A string of local directory path.
-    gcs_dir: A string of directory path on GCS.
-  """
-  bucket_name = gcs_dir.split('/')[2]
-  blob_dir = '/'.join(gcs_dir.split('/')[3:])
-  client = storage.Client()
-  bucket = client.bucket(bucket_name)
-  for local_file in glob.glob(local_dir + '/**'):
-    if os.path.isfile(local_file):
-      logging.info(
-          'Uploading %s to %s',
-          local_file,
-          os.path.join(constants.GCS_URI_PREFIX, bucket_name, blob_dir),
-      )
-      blob = bucket.blob(os.path.join(blob_dir, os.path.basename(local_file)))
-      blob.upload_from_filename(local_file)

community-content/vertex_vision_model_garden/model_oss/util/hypertune_util.py

@@ -1,21 +0,0 @@
-"""Utility functions for Vertex Hyperparameter Tuning Jobs."""
-
-import os
-
-from absl import logging
-
-_ENVIRONMENT_VARIABLE_FOR_TRIAL_ID = 'CLOUD_ML_TRIAL_ID'
-
-
-def get_trial_id_from_environment() -> str:
-  """Gets the trial id from environment variable.
-
-  Returns:
-    The trial id from environement or '0' if not found.
-  """
-  if _ENVIRONMENT_VARIABLE_FOR_TRIAL_ID not in os.environ:
-    logging.warning(
-        'Environment variable %s not found, return 0 as default trial id.',
-        _ENVIRONMENT_VARIABLE_FOR_TRIAL_ID,
-    )
-  return os.environ.get(_ENVIRONMENT_VARIABLE_FOR_TRIAL_ID, '0')

community-content/vertex_vision_model_garden/model_oss/util/image_format_converter.py

@@ -1,20 +0,0 @@
-"""Image format converter util lib."""
-
-import base64
-import io
-
-from PIL import Image
-
-
-def image_to_base64(image: Image.Image) -> str:
-  """Convert a PIL image to a base64 string."""
-  buffer = io.BytesIO()
-  image.save(buffer, format="JPEG")
-  image_str = base64.b64encode(buffer.getvalue()).decode("utf-8")
-  return image_str
-
-
-def base64_to_image(image_str: str) -> Image.Image:
-  """Convert a base64 string to a PIL image."""
-  image = Image.open(io.BytesIO(base64.b64decode(image_str)))
-  return image

community-content/vertex_vision_model_garden/model_oss/video_util/video_format_converter.py

@@ -1,14 +0,0 @@
-"""Video format converter util lib."""
-
-import io
-from typing import Sequence
-import imageio
-import numpy as np
-from PIL import Image
-
-
-def frames_to_video_bytes(frames: Sequence[np.ndarray], fps: int) -> bytes:
-  images = [Image.fromarray(array) for array in frames]
-  io_obj = io.BytesIO()
-  imageio.mimsave(io_obj, images, format=".mp4", fps=fps)
-  return io_obj.getvalue()

notebooks/ci_notebook_ingestion.md

@@ -1,102 +0,0 @@
-# Administrative Howto notes on CI Notebook Ingestion
-
-
-This readme covers administrative actions that are performed on an as-needed basis.
-
-## Team: vertex-ai-owners
-
-Members of the vertex-ai-owners (git team) have administrative privileges. 
-
-
-### Viewing members
-
-1. Goto the repo
-2. From top-level menu, select: (Settings -> Collaborators and Teams)[https://github.com/GoogleCloudPlatform/vertex-ai-samples/settings/access]
-
-
-### Adding a new member
-
-If another member needs to be added:
-   - Have the new member make a request to join the team.
-   - vertex-ai-owners with the `Maintainer` tag may add the new member.
-
-   
-## Executing CI notebook ingestion checks on a PR
-   
-### Killing a stuck PR
-
-If the CI notebook ingestion test is stuck (not terminating), you can kill the process by:
-
-1. Goto the PR
-2. Under checks, find the entry: vertex-ai-notebook-execution-test (python-docs-samples-tests) In progress —> Summary
-3. Select Details
-4. At bottom of details page, select: View more details on Google Cloud Build
-5. In Cloud Build history page, select Cancel on the top menu bar.
-
-### Restart a PR test
-
-There are two ways to restart the CI notebook ingestion tests on an open PR.
-
-1. In Cloud Build history page, select Rebuild on the top menu bar.
-2. or, in a comment in the PR enter: /gcbrun
-
-## Bypassing CI notebook ingestion checks on a PR
-
-We strongly discourage this, unless there is a compelling reason that would impact the integrity of the quality process.
-
-There are two ways of doing this. In both cases, you do:
-
-1. Goto the repo
-2. From top-level menu, select: (Settings -> Branches)[https://github.com/GoogleCloudPlatform/vertex-ai-samples/settings/branches]
-3. Under Branch Protection Rules, select the `main` branch.
-
-### Allowing a member to disable requirements for merging
-
-Specific member(s) can be assigned the ability to override requirements and merge a PR, by:
-
-1. Select Edit for the `main` branch in Branch Protection Rules.
-2. Find the entry "Allow specified actors to bypass required pull requests".
-3. Under this entry, add the member's git LDAP.
-4. Select SAVE.
-5. The "Squash and Merge" button will now be enabled on all PRs viewed by that member.
-
-### Temporarily disable checks.
-
-You can disable requirement checks temporarily on all PRs. 
-
-1. Select Edit for the `main` branch in Branch Protection Rules.
-2. Uncheck:
-  - Require approvals
-  - Require review from Code Owners
-  - Require status checks to pass before merging
-3. Select SAVE
-4. Now all members will see a green "Squash and Merge" on all PRs viewed by that member.
-
-To reverse, recheck the settings you unchecked above.
-
-## Linting
-
-To execute the identical lint image locally, from the CI notebook ingestion checks, do:
-
-1. Goto the corresponding local folder in the repo.
-2. Run: `docker run -v ${PWD}:/setup/app gcr.io/python-docs-samples-tests/notebook_linter:latest <your_notebooks>`
-
-## Install dependency issues
-
-Some packages (and combinations) have dependencies that fail on the virgin VM image used for the CI notebook ingestion test.
-
-### TFDV
-
-If the notebook installs and uses tensorflow_data_validation, install as follows:
-
-! pip3 install -q {USER_FLAG} google-cloud-aiplatform \
-                              tensorflow-data-validation \
-                              protobuf==3.20.3
-
-! pip3 install -q {USER_FLAG} cachetools==5.2.0
-
-
-
-
-   
-   

notebooks/community/CODEOWNERS

@@ -12,65 +12,26 @@
 
 /managed_notebooks/
 /bigquery_ml/ @polong
+/sdk/SDK_FBProphet_Forecasting_Online.ipynb @brianchunkang
 /pipelines/google_cloud_pipeline_components_TPU_model_train_upload_deploy.ipynb @brianchunkang
 /explainable_ai/SDK_Custom_Container_XAI.ipynb @brianchunkang
 /matching_engine/sdk_matching_engine_for_indexing.ipynb @ivanmkc
 /matching_engine/matching_engine_for_indexing.ipynb @yinghsienwu
 /matching_engine/stream_update_for_matching_engine.ipynb @peterping666
 /sdk/pytorch_lightning_custom_container_training.ipynb @brianchunkang
-/sdk/sdk_pytorch_torchrun_custom_container_training_imagenet.ipynb @brianchunkang
 /tensorboard @yfang1
 /feature_store @nayaknishant @morgandu
 /prediction @googleapis/vertex-prediction-team
 /vertex_endpoints/tf_hub_obj_detection/deploy_tfhub_object_detection_on_vertex_endpoints.ipynb @entrpn
-/vertex_endpoints/find_ideal_machine_type/find_ideal_machine_type/find_ideal_machine_type.ipynb @entrpn
 /vertex_endpoints/nvidia-triton/nvidia-triton-custom-container-prediction.ipynb @RajeshThallam
 /vertex_endpoints/optimized_tensorflow_runtime @vlasenkoalexey
 /notebooks/community/ml_ops/stage2/get_started_with_visionapi_and_automl.ipynb @mansari
 /notebooks/community/neo4j/graph_paysim.ipynb @benofben @laeg
 /notebooks/community/ml_ops/stage1/get_started_with_visionapi_and_vertex_datasets.ipynb @mansari
 /notebooks/community/pipelines/google_cloud_pipeline_components_bqml_pipeline_demand_forecasting.ipynb @inardini
-/notebooks/community/cohere/cohere_embedding_with_matching_engine.ipynb @stewart-co
 /notebooks/community/ml_ops/stage2/get_started_vertex_hpt_r_kernel.ipynb @fhirschmann
 /notebooks/community/ml_ops/stage2/get_started_vertex_training_r_using_r_kernel.ipynb @fhirschmann
-/notebooks/community/ml_ops/stage3/get_started_with_dataflow_flex_template_component.ipynb @wintwoo
 /notebooks/community/vertex-ai-samples/notebooks/community/model_registry/vertex_ai_model_registry_bqml_custom_model_versioning.ipynb @inardini
 /notebooks/community/vertex-ai-samples/notebooks/community/model_registry/vertex_ai_model_registry_automl_model_versioning.ipynb @inardini
 /notebooks/community/vizier/conversions_vertex_vizier_and_open_source_vizier.ipynb @halio-g
-/notebooks/community/experiments/vertex_ai_model_experimentation.ipynb @inardini @asobran
-/notebooks/community/pipelines/google_cloud_pipeline_components_bqml_pipeline_anomaly_detection.ipynb @inardini
-/notebooks/community/pipelines/google_cloud_pipeline_components_cloud_natural_language_pipeline.ipynb @Narwhalprime
-/notebooks/community/pipelines/google_cloud_pipeline_components_ready_to_go_text_classification_pipeline.ipynb @Narwhalprime
-/notebooks/community/feature_store/get_started_vertex_feature_store.ipynb @junkourata
-/notebooks/community/model_garden/model_garden_huggingface_local_inference.ipynb @dstnluong-google
-/notebooks/community/model_garden/model_garden_mediapipe_image_classification.ipynb @schmidt-sebastian
-/notebooks/community/model_garden/model_garden_mediapipe_object_detection.ipynb @schmidt-sebastian
-/notebooks/community/model_garden/model_garden_proprietary_image_classification.ipynb @weigary
-/notebooks/community/model_garden/model_garden_proprietary_image_object_detection.ipynb @weigary
-/notebooks/community/model_garden/model_garden_tfvision_image_classification.ipynb @genquan9
-/notebooks/community/model_garden/model_garden_tfvision_image_object_detection.ipynb @genquan9
-/notebooks/community/model_garden/model_garden_tfvision_image_segmentation.ipynb @genquan9
-/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion.ipynb @xiangxu-google
-/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion_2_1.ipynb @bingatgoogle
-/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion_inpainting.ipynb @xiangxu-google
-/notebooks/community/model_garden/model_garden_pytorch_instructpix2pix.ipynb @xiangxu-google
-/notebooks/community/model_garden/model_garden_pytorch_controlnet.ipynb @xiangxu-google
-/notebooks/community/model_garden/model_garden_pytorch_blip_image_captioning.ipynb @xiangxu-google
-/notebooks/community/model_garden/model_garden_pytorch_blip_vqa.ipynb @xiangxu-google
-/notebooks/community/model_garden/model_garden_pytorch_vilt_vqa.ipynb @xiangxu-google
-/notebooks/community/model_garden/model_garden_pytorch_vit_gpt2_image_captioning.ipynb @xiangxu-google
-/notebooks/community/model_garden/model_garden_pytorch_clip.ipynb @xiangxu-google
-/notebooks/community/model_garden/model_garden_pytorch_owlvit.ipynb @xiangxu-google
-/notebooks/community/model_garden/model_garden_pytorch_layoutml_document_qa.ipynb @xiangxu-google
-/notebooks/community/model_garden/model_garden_pytorch_blip2.ipynb @xiangxu-google
-/notebooks/community/model_garden/model_garden_pytorch_detectron2.ipynb @lavraicse
-/notebooks/community/model_garden/model_garden_pytorch_dolly_v2.ipynb @lavraicse
-/notebooks/community/model_garden/model_garden_pytorch_bart_large_cnn.ipynb @lavraicse
-/notebooks/community/model_garden/model_garden_jax_vision_transformer.ipynb @lavraicse
-/notebooks/community/model_garden/model_garden_pytorch_text_to_video_zero_shot.ipynb @bingatgoogle
-/notebooks/community/model_garden/model_garden_pytorch_text_to_video.ipynb @KCFindstr
-/notebooks/community/generative_ai/text_embedding_api_semantic_search_with_scann.ipynb @henrytansetiawan
-/notebooks/community/bigquery_ml_inference/bq_ml_with_vision_translation_nlp.ipynb @deaconsmith
-/notebooks/community/model_garden/model_garden_keras_stable_diffusion.ipynb @genquan9
-/notebooks/community/model_garden/model_garden_pytorch_sam.ipynb @huguensjean
-/notebooks/community/model_garden/model_garden_pytorch_peft.ipynb @genquan9
+/notebooks/community/experiments/vertex_ai_model_experimentation.ipynb @inardini @asobran

notebooks/community/bigquery_ml/README.md

@@ -1,19 +0,0 @@
-
-[Unstructured data analytics with BigQuery ML and Vertex AI pre-trained models](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/bigquery_ml/bq_ml_with_vision_translation_nlp.ipynb)
-
-```
-Learn how to analyze unstructured data within BigQuery using BigQuery's inference engine. You will use BigQuery ML to connect to three pretrained Vertex AI APIs - Vision API, Translation API and Natural Language Processing API.
-
-The steps performed include:
-
-- Define pre-trained models for Vision AI, Translation AI and NLP AI in BigQuery ML
-- Call the Vision API (`ML.ANNOTATE_IMAGE`) to detect text in images stored in Cloud Storage
-  You will need to create an object table in BigQuery to do this
-- Call the Translation API (`ML.TRANSLATE`) to detect the language of text, and translate non-English movie titles to English
-- Call the Natural Language API (`ML.UNDERSTAND_TEXT`) to run sentiment analysis over movie reviews stored in BigQuery
-
-```
-
-   Check out the [blog for this notebook](https://cloud.google.com/blog/products/data-analytics/how-simplify-unstructured-data-analytics-using-bigquery-ml-and-vertex-ai).
-   Learn more about [BigQuery ML inference engine](https://cloud.google.com/bigquery/docs/reference/standard-sql/inference-overview).
-

notebooks/community/cohere/README.md

@@ -1,3 +0,0 @@
-# README
-
-These are notebooks [Cohere](https://cohere.ai/) built in collaboration with Google.  They demonstrate how to use Cohere's modeling API along with Vertex AI.

notebooks/community/generative_ai/text_embedding_api_semantic_search_with_scann.ipynb

@@ -1,373 +0,0 @@
-{
-  "cells": [
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "id": "KSP1duKDeaDR",
-      "metadata": {
-        "id": "KSP1duKDeaDR"
-      },
-      "outputs": [],
-      "source": [
-        "# Copyright 2023 Google LLC\n",
-        "#\n",
-        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
-        "# you may not use this file except in compliance with the License.\n",
-        "# You may obtain a copy of the License at\n",
-        "#\n",
-        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
-        "#\n",
-        "# Unless required by applicable law or agreed to in writing, software\n",
-        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
-        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
-        "# See the License for the specific language governing permissions and\n",
-        "# limitations under the License."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "id": "67b2c5cc-8fc6-4082-9052-69fa0377d770",
-      "metadata": {
-        "id": "67b2c5cc-8fc6-4082-9052-69fa0377d770"
-      },
-      "source": [
-        "# Semantic Search using Embeddings\n",
-        "\n",
-        "Semantic search is a type of search that uses the meaning of words and phrases to find relevant results.\n",
-        "\n",
-        "In this tutorial, we will demonstrate how to do semantic search with embeddings generated from the news text and using [Google ScaNN: Efficient Vector Similarity Search](https://ai.googleblog.com/2020/07/announcing-scann-efficient-vector.html) to retrieve the most relevant news semantically.\n",
-        "\n",
-        "## Pre-requisites:\n",
-        "- Vertex LLM SDK\n",
-        "- ScaNN [github](https://github.com/google-research/google-research/tree/master/scann)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "id": "FyyMdUeAJIVv",
-      "metadata": {
-        "id": "FyyMdUeAJIVv"
-      },
-      "source": [
-        "## Install Vertex LLM SDK\n",
-        "\n",
-        "DISCLAIMER: Text Embedding API is now in Experimental Preview. This release focuses on validating model prototypes and these models are not guaranteed to be released. Use of Text Embedding API is governed by the Google Cloud Terms of Service, the Pre-GA Offerings Terms of the GCP Service Specific Terms. The Acceptance Use Policy, and the Generative AI Prohibited Use Policy. Vertex Text Embedding API’s features may be unstable, change in backward-incompatible ways, and are not guaranteed to be released. There are no SLAs provided and no technical support obligations. GCP’s Cloud Data Processing Addendum does not apply to Pre-GA Offerings and customers should not use Text Embedding API to process personal data or other data subject to legal or regulatory compliance requirements. See description of launch stage for details.\n",
-        "\n",
-        "The information in this documentation is provided to the customer on an “as is” and “with all faults” basis without any warranty of any kind, either express or implied. Google does not warrant or guarantee the correctness, accuracy or reliability of the information in here. In no event will Google or its affiliates or licensors be liable for any damage or harm to customers from customer’s use of these materials."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "id": "h6KaE3XRJdxc",
-      "metadata": {
-        "id": "h6KaE3XRJdxc"
-      },
-      "outputs": [],
-      "source": [
-        "from google.colab import auth as google_auth\n",
-        "\n",
-        "google_auth.authenticate_user()"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "id": "snBUuUamoJPz",
-      "metadata": {
-        "id": "snBUuUamoJPz"
-      },
-      "outputs": [],
-      "source": [
-        "!pip3 install google-cloud-aiplatform>=1.25 \"shapely<2.0.0\""
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "id": "zgVQcE0ewO8W",
-      "metadata": {
-        "id": "zgVQcE0ewO8W"
-      },
-      "outputs": [],
-      "source": [
-        "PROJECT_ID = \"cloud-nl-llm-embedding\"  # @param {type:\"string\"}\n",
-        "LOCATION = \"us-central1\"  # @param {type:\"string\"}\n",
-        "\n",
-        "import vertexai\n",
-        "\n",
-        "vertexai.init(project=PROJECT_ID, location=LOCATION)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "id": "4xFzXmPbY7FC",
-      "metadata": {
-        "id": "4xFzXmPbY7FC"
-      },
-      "source": [
-        "**Attention**: you would need to restart runtime so that the right package is installed."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "id": "xnfG88OPZI18",
-      "metadata": {
-        "id": "xnfG88OPZI18"
-      },
-      "source": [
-        "## Import TextEmbeddingModel"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "id": "1e6e1b98-a632-44a2-afb8-fc212018ef4f",
-      "metadata": {
-        "id": "1e6e1b98-a632-44a2-afb8-fc212018ef4f"
-      },
-      "outputs": [],
-      "source": [
-        "from vertexai.preview.language_models import TextEmbeddingModel\n",
-        "\n",
-        "model = TextEmbeddingModel.from_pretrained(\"textembedding-gecko@001\")"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "id": "1suA-1HuaGj6",
-      "metadata": {
-        "id": "1suA-1HuaGj6"
-      },
-      "source": [
-        "## Install ScaNN Package"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "id": "770255d3-54dd-48c8-bbdd-fbc0be41f085",
-      "metadata": {
-        "id": "770255d3-54dd-48c8-bbdd-fbc0be41f085"
-      },
-      "outputs": [],
-      "source": [
-        "!pip install scann"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "id": "dae340cb-0583-4e7e-a562-6817ee4d7f6d",
-      "metadata": {
-        "id": "dae340cb-0583-4e7e-a562-6817ee4d7f6d"
-      },
-      "source": [
-        "## Imports packages"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "id": "412d00f1-08db-4880-8ced-52a9583757b8",
-      "metadata": {
-        "id": "412d00f1-08db-4880-8ced-52a9583757b8"
-      },
-      "outputs": [],
-      "source": [
-        "import json\n",
-        "import time\n",
-        "\n",
-        "import numpy as np\n",
-        "import pandas as pd\n",
-        "import scann"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "id": "f50f22f3-ec85-463e-b6fe-5c8e6b80b07b",
-      "metadata": {
-        "id": "f50f22f3-ec85-463e-b6fe-5c8e6b80b07b"
-      },
-      "source": [
-        "## Create Embedding Dataset.\n",
-        "\n",
-        "The dataset is solely to demonstrate the use of the Text Embedding API with a vector database. It is not intended to be used for any other purpose, such as evaluating models. The dataset is small and does not represent a comprehensive sample of all possible text."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "id": "2OUg-Qf8iFta",
-      "metadata": {
-        "id": "2OUg-Qf8iFta"
-      },
-      "outputs": [],
-      "source": [
-        "!gsutil cp gs://cloud-samples-data/vertex-ai/dataset-management/datasets/bert_finetuning/wide_and_deep_trainer_container_tests_input.jsonl ."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "id": "BNPapKXviHlE",
-      "metadata": {
-        "id": "BNPapKXviHlE"
-      },
-      "outputs": [],
-      "source": [
-        "records = []\n",
-        "with open(\"wide_and_deep_trainer_container_tests_input.jsonl\") as f:\n",
-        "    for line in f:\n",
-        "        record = json.loads(line)\n",
-        "        records.append(record)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "id": "Z1Y9Bx2miJba",
-      "metadata": {
-        "id": "Z1Y9Bx2miJba"
-      },
-      "outputs": [],
-      "source": [
-        "# Peek at the data.\n",
-        "df = pd.DataFrame(records)\n",
-        "df.head(50)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "id": "1v7zUwoWiPl-",
-      "metadata": {
-        "id": "1v7zUwoWiPl-"
-      },
-      "outputs": [],
-      "source": [
-        "def get_embedding(text):\n",
-        "    get_embedding.counter += 1\n",
-        "    try:\n",
-        "        if get_embedding.counter % 100 == 0:\n",
-        "            time.sleep(3)\n",
-        "        return model.get_embeddings([text])[0].values\n",
-        "    except:\n",
-        "        return []\n",
-        "\n",
-        "\n",
-        "get_embedding.counter = 0\n",
-        "\n",
-        "# This may take several minutes to complete.\n",
-        "df[\"embedding\"] = df[\"textContent\"].apply(lambda x: get_embedding(x))"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "id": "ba4f49b6-65e1-49ea-988b-c4e195219deb",
-      "metadata": {
-        "id": "ba4f49b6-65e1-49ea-988b-c4e195219deb"
-      },
-      "source": [
-        "## Create an Index"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "id": "245bc8cd-038b-484a-acb2-3a705d4cc4cf",
-      "metadata": {
-        "id": "245bc8cd-038b-484a-acb2-3a705d4cc4cf"
-      },
-      "outputs": [],
-      "source": [
-        "record_count = len(records)\n",
-        "dataset = np.empty((record_count, 768))\n",
-        "for i in range(record_count):\n",
-        "    dataset[i] = df.embedding[i]\n",
-        "\n",
-        "normalized_dataset = dataset / np.linalg.norm(dataset, axis=1)[:, np.newaxis]\n",
-        "# configure ScaNN as a tree - asymmetric hash hybrid with reordering\n",
-        "# anisotropic quantization as described in the paper; see README\n",
-        "\n",
-        "# use scann.scann_ops.build() to instead create a TensorFlow-compatible searcher\n",
-        "searcher = (\n",
-        "    scann.scann_ops_pybind.builder(normalized_dataset, 10, \"dot_product\")\n",
-        "    .tree(\n",
-        "        num_leaves=record_count,\n",
-        "        num_leaves_to_search=record_count,\n",
-        "        training_sample_size=record_count,\n",
-        "    )\n",
-        "    .score_ah(2, anisotropic_quantization_threshold=0.2)\n",
-        "    .reorder(100)\n",
-        "    .build()\n",
-        ")"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "id": "9f1689eb-c27c-4566-afc7-7fbc55552aad",
-      "metadata": {
-        "id": "9f1689eb-c27c-4566-afc7-7fbc55552aad"
-      },
-      "source": [
-        "## Queries the Index"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "id": "494079df-d8f6-4a6a-b26b-0b0477791adc",
-      "metadata": {
-        "id": "494079df-d8f6-4a6a-b26b-0b0477791adc"
-      },
-      "outputs": [],
-      "source": [
-        "def search(query):\n",
-        "    start = time.time()\n",
-        "    query = model.get_embeddings([query])[0].values\n",
-        "    neighbors, distances = searcher.search(query, final_num_neighbors=3)\n",
-        "    end = time.time()\n",
-        "\n",
-        "    for id, dist in zip(neighbors, distances):\n",
-        "        print(f\"[docid:{id}] [{dist}] -- {df.textContent[int(id)][:125]}...\")\n",
-        "    print(\"Latency (ms):\", 1000 * (end - start))"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "id": "50497681-6112-4147-a13e-afedb72b54f5",
-      "metadata": {
-        "id": "50497681-6112-4147-a13e-afedb72b54f5"
-      },
-      "outputs": [],
-      "source": [
-        "search(\"tell me about shark or animal\")"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "id": "c814f835-3e2a-4366-a334-ccc636f00b83",
-      "metadata": {
-        "id": "c814f835-3e2a-4366-a334-ccc636f00b83"
-      },
-      "outputs": [],
-      "source": [
-        "search(\"tell me about an important moment or event in your life\")"
-      ]
-    }
-  ],
-  "metadata": {
-    "colab": {
-      "name": "text_embedding_api_semantic_search_with_scann.ipynb",
-      "toc_visible": true
-    },
-    "kernelspec": {
-      "display_name": "Python 3",
-      "name": "python3"
-    }
-  },
-  "nbformat": 4,
-  "nbformat_minor": 0
-}

notebooks/community/matching_engine/stream_update_for_matching_engine.ipynb

@@ -33,7 +33,7 @@
         "\n",
         "  <td>\n",
         "    <a href=\"https://console.cloud.google.com/vertex-ai/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/matching_engine/stream_update_matching_engine.ipynb\">\n",
-        "      Run in Workbench AI Notebooks\n",
+        "      Run in Google Cloud Notebooks\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
@@ -53,7 +53,7 @@
       "source": [
         "## Overview\n",
         "\n",
-        "This example demonstrates how to use the Vertex AI Matching Engine Stream Update Service. \n",
+        "This example demonstrates how to use the GCP matching engine Stream Update Service. \n",
         "\n",
         "### Dataset\n",
         "\n",
@@ -150,7 +150,7 @@
       "source": [
         "### Installation\n",
         "\n",
-        "Download and install the latest (preview) version of the Vertex AI SDK for Python."
+        "Download and install the latest (preview) version of the Vertex SDK for Python."
       ]
     },
     {
@@ -442,7 +442,7 @@
         "id": "8292bcedab58"
       },
       "source": [
-        "## Prepare the data\n",
+        "## Prepare the Data\n",
         "\n",
         "The GloVe dataset consists of a set of pre-trained embeddings. The embeddings are split into a \"train\" split, and a \"test\" split.\n",
         "We will create a vector search index from the \"train\" split, and use the embedding vectors in the \"test\" split as query vectors to test the vector search index.\n",
@@ -525,7 +525,7 @@
         "        f.write('{\"id\":\"' + str(i) + '\",')\n",
         "        f.write('\"embedding\":[' + \",\".join(str(x) for x in train[i]) + \"],\")\n",
         "        f.write(\n",
-        "            '\"restricts\":[{\"namespace\": \"class\", \"allow\": [\"' + str(i) + '\"]}],'\n",
+        "            '\"restricts\":[{\"namespace\": \"class\", \"allow_list\": [\"' + str(i) + '\"]}],'\n",
         "        )\n",
         "        f.write('\"crowding_tag\":' + ('\"a\"' if i % 2 == 0 else '\"b\"') + \"}\")\n",
         "        f.write(\"\\n\")\n",
@@ -854,7 +854,7 @@
         "id": "00c606bc97b5"
       },
       "source": [
-        "## Create online queries\n",
+        "## Create Online Queries\n",
         "\n",
         "After you built your indexes, you may query against the deployed index through the online querying gRPC API (Match service) within the virtual machine instances from the same region (for example 'us-central1' in this tutorial).  \n",
         "\n",

notebooks/community/ml_ops/stage1/README.md

@@ -28,11 +28,9 @@ The first stage in MLOps is the collection and preparation for the purpose of de
 
 ### Get Started
 
+[Get started with Dataflow](community/ml_ops/stage1/get_started_dataflow.ipynb)
 
-[Get started with Dataflow](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_dataflow.ipynb)
-
-```
-Learn how to use `Dataflow` for training with `Vertex AI`.
+In this tutorial, you learn how to use `Dataflow` for training with `Vertex AI`.
 
 The steps performed include:
 
@@ -42,13 +40,10 @@ The steps performed include:
 - Upstream preprocessing of data:
     - tabular data
     - image data
-```
 
+[Get started with Vertex AI datasets](community/ml_ops/stage1/get_started_vertex_datasets.ipynb)
 
-[Get started with Vertex AI datasets](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_vertex_datasets.ipynb)
-
-```
-Learn how to use `Vertex AI Dataset` for training with `Vertex AI`.
+In this tutorial, you learn how to use `Vertex AI Dataset` for training with `Vertex AI`.
 
 The steps performed include:
 
@@ -66,13 +61,10 @@ The steps performed include:
 - Detect anomalies in new data using TensorFlow Data Validation.
 - Generate a TFRecord feature specification using TensorFlow Transform from the data schema.
 - Export a dataset and convert to TFRecords.
-```
 
+[Get started with BigQuery datasets](community/ml_ops/stage1/get_started_bq_datasets.ipynb)
 
-[Get started with BigQuery datasets](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_bq_datasets.ipynb)
-
-```
-Learn how to use `BigQuery` as a dataset for training with `Vertex AI`.
+In this tutorial, you learn how to use `BigQuery` as a dataset for training with `Vertex AI`.
 
 The steps performed include:
 
@@ -83,13 +75,10 @@ The steps performed include:
 - Select rows from extracted CSV files into a `tf.data.Dataset` -- compatible for custom training `TensorFlow` models.
 - Create a `BigQuery` dataset from CSV files.
 - Extract data from `BigQuery` table into a `DMatrix` -- compatible for custom training `XGBoost` models.
-```
 
+[Get started with Vertex AI Data Labeling](community/ml_ops/stage1/get_started_with_data_labeling.ipynb)
 
-[Get started with Vertex AI Data Labeling](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_with_data_labeling.ipynb)
-
-```
-Learn how to use the `Vertex AI Data Labeling` service/
+In this tutorial, you learn how to use the `Vertex AI Data Labeling` service.
 
 The steps performed include:
 
@@ -98,31 +87,28 @@ The steps performed include:
 - Submit the data labeling job.
 - List data labeling jobs.
 - Cancel a data labeling job.
-```
 
 
-[Create an unlabelled Vertex AI AutoML text entity extraction dataset from PDFs using Vision API](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_with_visionapi_and_vertex_datasets.ipynb)
 
-```
-Learn to use `Vision API` to extract text from PDF files stored on a Cloud Storage bucket.
+[Create an unlabelled Vertex AI AutoML text entity extraction dataset from PDFs using Vision API](community/ml_ops/stage1/get_started_with_visionapi_and_vertex_datasets.ipynb)
+
+In this tutorial, you learn to use `Vision API` to extract text from PDF files stored on a Cloud Storage bucket. You then process the results and create an unlabelled `Vertex AI Dataset`, compatible with `AutoML`, for text entity extraction.
 
 The steps performed include:
 
 1. Using `Vision API` to perform Optical Character Recognition (OCR) to extract text from PDF files.
 2. Processing the results and saving them to text files.
 3. Generating a `Vertex AI Dataset` import file.
-4. Cr
+4. Creating a new unlabelled text entity extraction `Vertex AI Dataset` resource in `Vertex AI`.
 
 
 ### E2E Stage Example
 
-[Data management](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/mlops_data_management.ipynb)
+[Stage 1: Data Management](mlops_data_management.ipynb)
+
 
 ```
-In this tutorial, you create a MLOps stage 1: data management process.
-
 The steps performed include:
-
 - Explore and visualize the data.
 - Create a Vertex AI `Dataset` resource from `BigQuery` table -- for AutoML training.
 - Extract a copy of the dataset to a CSV file in Cloud Storage.
@@ -131,4 +117,4 @@ The steps performed include:
 - Generate statistics and data schema using TensorFlow Data Validation from the samples in the dataframe.
 - Generate a TFRecord feature specification using TensorFlow Data Validation from the data schema.
 - Preprocess a portion of the BigQuery data using `Dataflow` -- for custom training.
-```
+```

notebooks/community/ml_ops/stage2/README.md

@@ -35,10 +35,9 @@ The second stage in MLOps is experimenting in developing one or more baseline mo
 
 ### Get Started
 
-[Get started with Vertex AI Training for R](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_r.ipynb)
+[Get started with Vertex AI Training for R](community/ml_ops/stage2/get_started_vertex_training_r.ipynb)
 
-```
-Learn how to use `Vertex AI Training` for training a R custom model.
+In this tutorial, you learn how to use `Vertex AI Training` for training a R custom model.
 
 The steps performed include:
 
@@ -52,26 +51,18 @@ The steps performed include:
 - Create a training image for training the model.
 - Train a R model using `Vertex AI Trainingh` service with the R-to-Python training package.
 
-```
+[Get started with Logging](community/ml_ops/stage2/get_started_with_logging.ipynb)
 
-
-[Get started with Logging](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_with_logging.ipynb)
-
-```
-Learn how to use Python and Cloud logging when training with `Vertex AI`.
+In this tutorial, you learn how to use Python and Cloud logging awhen training with `Vertex AI`.
 
 The steps performed include:
 
 - Use Python logging to log training configuration/results locally.
 - Use Google Cloud Logging to log training configuration/results in cloud storage.
 
-```
+[Get started with Vertex AI Hyperparameter Tuning for XGBoost] (community/ml_ops/stage2/get_started_vertex_hpt_xgboost.ipynb)
 
-
-[Get started with Vertex AI Hyperparameter Tuning for XGBoost](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_hpt_xgboost.ipynb)
-
-```
-Learn how to use `Vertex AI Hyperparameter Tuning` for training a XGBoost custom model.
+In this tutorial, you learn how to use `Vertex AI Hyperparameter Tuning` for training a XGBoost custom model.
 
 The steps performed include:
 
@@ -80,13 +71,9 @@ The steps performed include:
 - Save the model artifacts to Cloud Storage using GCSFuse.
 - Create a `Vertex AI Model` resource.
 
-```
+[Get started with Vertex AI Training for XGBoost](community/ml_ops/stage2/get_started_vertex_training_xgboost.ipynb)
 
-
-[Get started with Vertex AI Training for XGBoost](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_xgboost.ipynb)
-
-```
-Learn how to use `Vertex AI Training` for training a XGBoost custom model.
+In this tutorial, you learn how to use `Vertex AI Training` for training a XGBoost custom model.
 
 The steps performed include:
 
@@ -95,13 +82,9 @@ The steps performed include:
 - Save the model artifacts to Cloud Storage using GCSFuse.
 - Create a `Vertex AI Model` resource.
 
-```
+[Get started with TabNet builtin algorithm for training tabular models](community/ml_ops/stage2/get_started_with_tabnet.ipynb)
 
-
-[Get started with TabNet builtin algorithm for training tabular models](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_with_tabnet.ipynb)
-
-```
-Learn how to run `Vertex AI TabNet` built algorithm for training custom tabular models.
+In this notebook, you learn how to run `Vertex AI TabNet` built algorithm for training custom tabular models.
 
 The steps performed include:
 
@@ -114,13 +97,9 @@ The steps performed include:
 - Hyperparameter tuning the `Vertex AI TabNet` model.
 - Train the model using `Vertex AI Training` using BigQuery table.
 
-```
+[Get started with prebuilt TFHub models](community/ml_ops/stage2/get_started_with_tfhub_models.ipynb)
 
-
-[Get started with prebuilt TFHub models](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_with_tfhub_models.ipynb)
-
-```
-Learn how to use `Vertex AI Training` with prebuilt models from TensorFlow Hub.
+In this tutorial, you learn how to use `Vertex AI Training` with prebuilt models from TensorFlow Hub.
 
 The steps performed include:
 
@@ -133,31 +112,23 @@ The steps performed include:
     - Train then model
     - Save model artifacts and upload as Vertex AI Model resource.
 
-```
+[Get started with BigQuery ML Training](community/ml_ops/stage2/get_started_bqml_training.ipynb)
 
-
-[Get started with BigQuery ML Training](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_bqml_training.ipynb)
-
-```
-Learn how to use `BigQueryML` for training with `Vertex AI`.
+In this tutorial, you learn how to use `BigQueryML` (BQML) for training with `Vertex AI`.
 
 The steps performed include:
 
 - Create a local BigQuery table in your project
-- Train a BigQuery ML model
-- Evaluate the BigQuery ML model
-- Export the BigQuery ML model as a cloud model
+- Train a BQML model
+- Evaluate the BQML model
+- Export the BQML model as a cloud model
 - Upload the exported model as a `Vertex AI Model` resource
-- Hyperparameter tune a BigQuery ML model with `Vertex AI Vizier`
-- Automatically register a BigQuery ML model to `Vertex AI Model Registry`
+- Hyperparameter tune a BQML model with `Vertex AI Vizier`
+- Automatically register a BQML model to `Vertex AI Model Registry`
 
-```
+[Get started with Vertex AI Vizier](community/ml_ops/stage2/get_started_vertex_vizier.ipynb)
 
-
-[Get started with Vertex AI Vizier](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_vizier.ipynb)
-
-```
-Learn how to use `Vertex AI Vizier` for when training with `Vertex AI`.
+In this tutorial, you learn how to use `Vertex AI Vizier` for when training with `Vertex AI`.
 
 The steps performed include:
 
@@ -165,13 +136,9 @@ The steps performed include:
 - Hyperparameter tuning with Vizier (Bayesian) algorithm.
 - Suggesting trials and updating results for Vizier study
 
-```
+[Get started with distributed training using DASK](community/ml_ops/stage2/get_started_with_distributed_training_xgboost.ipynb)
 
-
-[Get started with distributed training using DASK](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_with_distributed_training_xgboost.ipynb)
-
-```
-Learn how to use `Vertex AI Training` for distributed training of XGBoost model using the OSS package DASK.
+In this tutorial, you learn how to use `Vertex AI Training` for distributed training of XGBoost model using the OSS package DASK. Additionally, you learn to construct and deploy a custom serving container using a Flask web server.
 
 The steps performed include:
 
@@ -185,13 +152,9 @@ The steps performed include:
 - Deploy the `Vertex AI Model` resource to `Vertex AI Endpoint` resource.
 - Make a prediction.
 
-```
+[Get started with Vertex AI TensorBoard](community/ml_ops/stage2/get_started_vertex_tensorboard.ipynb)
 
-
-[Get started with Vertex AI TensorBoard](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_tensorboard.ipynb)
-
-```
-Learn how to use `Vertex AI TensorBoard` when training with `Vertex AI`.
+In this tutorial, you learn how to use `Vertex AI TensorBoard` when training with `Vertex AI`.
 
 The steps performed include:
 
@@ -199,13 +162,9 @@ The steps performed include:
 - Using TensorBoard with locally trained model.
 - Using Vertex AI TensorBoard with Vertex AI Training.
 
-```
+[Get started with Vertex AI Training for R using R Kernel](community/ml_ops/stage2/get_started_vertex_training_r_using_r_kernel.ipynb)
 
-
-[Get started with Vertex AI Training for R using R Kernel](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_r_using_r_kernel.ipynb)
-
-```
-Learn how to use `Vertex AI`, using an R kernel, for training and deploying an R custom model.
+In this tutorial, you learn how to use `Vertex AI`, using an R kernel, for training and deploying an R custom model.
 
 The steps performed include:
 
@@ -217,13 +176,10 @@ The steps performed include:
 - Deploy the `Model` resource (trained R model) to the `Endpoint` resource.
 - Make an online prediction.
 
-```
 
+[Get started Vision API test preprocessing and AutoML text model generation](community/ml_ops/stage2/get_started_with_visionapi_and_automl.ipynb)
 
-[Get started Vision API test preprocessing and AutoML text model generation](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_with_visionapi_and_automl.ipynb)
-
-```
-In this tutorial, you create an `AutoML` text entity extraction model pre-existing extracted data by generating a custom import file.
+In this tutorial, you create an `AutoML` text entity extraction model pre-existing extracted data by generating a custom import file. You deploy this mode for online prediction from a Python script using the `BigQuery`, `Vision AI`, Cloud Storage and `Vertex AI SDK` for Python. 
 
 The steps performed include:
 
@@ -236,13 +192,9 @@ The steps performed include:
 - Make a prediction.
 - Undeploy the `Model`.
 
-```
+[Get started with Vertex AI Experiments](community/ml_ops/stage2/get_started_vertex_experiments.ipynb)
 
-
-[Get started with Vertex AI Experiments](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_experiments.ipynb)
-
-```
-Learn how to use `Vertex AI Experiments` when training with `Vertex AI`.
+In this tutorial, you learn how to use `Vertex AI Experiments` when training with `Vertex AI`.
 
 The steps performed include:
 
@@ -263,13 +215,9 @@ The steps performed include:
     - Execute the custom job
     - Visualize the experiment results
 
-```
+[AutoML Image Classfication Training with Customer Managed Encryption Keys (CMEK)](community/ml_ops/stage2/get_started_with_cmek_training.ipynb)
 
-
-[AutoML Image Classfication Training with Customer Managed Encryption Keys (CMEK)](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_with_cmek_training.ipynb)
-
-```
-Learn how to use a customer managed encryption key (CMEK) for `Vertex AI AutoML` training.
+In this tutorial, you learn how to use a customer managed encryption key (CMEK) for `Vertex AI AutoML` training.
 
 The steps performed include:
 
@@ -277,13 +225,9 @@ The steps performed include:
 - Creating an image dataset with CMEK encryption.
 - Train an AutoML model with CMEK encryption.
 
-```
+[Get started with Vertex AI Feature Store](community/ml_ops/stage2/get_started_vertex_feature_store.ipynb)
 
-
-[Get started with Vertex AI Feature Store](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_feature_store.ipynb)
-
-```
-Learn how to use `Vertex AI Feature Store` when training and predicting with `Vertex AI`.
+In this tutorial, you learn how to use `Vertex AI Feature Store` when training and predicting with `Vertex AI`.
 
 The steps performed include:
 
@@ -296,13 +240,9 @@ The steps performed include:
 - Perform online serving from a `Featurestore` resource.
 - Perform batch serving from a `Featurestore` resource.
 
-```
+[Get started with AutoML Training](community/ml_ops/stage2/get_started_automl_training.ipynb)
 
-
-[Get started with AutoML Training](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_automl_training.ipynb)
-
-```
-Learn how to use `AutoML` for training with `Vertex AI`.
+In this tutorial, you learn how to use `AutoML` for training with `Vertex AI`.
 
 The steps performed include:
 
@@ -313,29 +253,9 @@ The steps performed include:
 - Train a text model
 - Train a video model
 
-```
+[Get started with Vertex AI Training for LightGBM](community/ml_ops/stage2/get_started_vertex_training_lightgbm.ipynb)
 
-
-[Get started with autologging using Vertex AI Experiments for XGBoost models](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_with_vertex_experiments_autologging_xgboost.ipynb)
-
-```
-Learn how to create an experiment for training an XGBoost model, and automatically log parameters and metrics using the enclosed do-it-yourself (DIY) code.
-
-The steps performed include:
-
-- Construct the DIY autologging code.
-- Construct training package with call to autologging.
-- Train a model.
-- View the experiment
-- Delete the experiment.
-
-```
-
-
-[Get started with Vertex AI Training for LightGBM](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_lightgbm.ipynb)
-
-```
-Learn how to use `Vertex AI Training` for training a LightGBM custom model.
+In this tutorial, you learn how to use `Vertex AI Training` for training a LightGBM custom model.
 
 The steps performed include:
 
@@ -346,26 +266,9 @@ The steps performed include:
 - Test the deployment image locally.
 - Create a `Vertex AI Model` resource.
 
-```
+[Get started with Vertex AI Training for Scikit-Learn](community/ml_ops/stage2/get_started_vertex_training_sklearn.ipynb)
 
-
-[Vertex AI Hyperparameter Tuning with R kernel](None)
-
-```
-Learn how to use `Vertex AI`, using an R kernel, for tuning hyperparameters of a R custom model.
-
-The steps performed include:
-
-- Create a custom R training script
-- Create a custom R deployment container.
-- Perform hyperparameter tuning using `Vertex AI`.
-
-```
-
-[Get started with Vertex AI Training for Scikit-Learn](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_sklearn.ipynb)
-
-```
-Learn how to use `Vertex AI Training` for training a Scikit-Learn custom model.
+In this tutorial, you learn how to use `Vertex AI Training` for training a Scikit-Learn custom model.
 
 The steps performed include:
 
@@ -374,13 +277,9 @@ The steps performed include:
 - Save the model artifacts to Cloud Storage using GCSFuse.
 - Create a `Vertex AI Model` resource.
 
-```
+[Get started with Vertex AI Training](community/ml_ops/stage2/get_started_vertex_training.ipynb)
 
-
-[Get started with Vertex AI Training](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training.ipynb)
-
-```
-Learn how to use `Vertex AI Training` for custom models when training with `Vertex AI`.
+In this tutorial, you learn how to use `Vertex AI Training` for custom models when training with `Vertex AI`.
 
 The steps performed include:
 
@@ -389,13 +288,10 @@ The steps performed include:
 - Training using a custom training image.
 - Laying out a training package.
 
-```
 
+[Get started with Vertex AI Training for Pytorch](community/ml_ops/stage2/get_started_vertex_training_pytorch.ipynb)
 
-[Get started with Vertex AI Training for PyTorch](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_pytorch.ipynb)
-
-```
-Learn how to use `Vertex AI Training` for training a PyTorch custom model.
+In this tutorial, you learn how to use `Vertex AI Training` for training a Pytorch custom model.
 
 The steps performed include:
 
@@ -404,31 +300,9 @@ The steps performed include:
 - Save the model artifacts to Cloud Storage using GCSFuse.
 - Create a `Vertex AI Model` resource.
 
-```
+[Get started with Vertex AI Distributed Training](community/ml_ops/stage2/get_started_vertex_distributed_training.ipynb)
 
-
-[Get started with autologging using Vertex AI Experiments for TensorFlow models](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_with_vertex_experiments_autologging_tf.ipynb)
-
-```
-Learn how to create an experiment for training a TensorFlow model, and automatically log parameters and metrics using the enclosed do-it-yourself (DIY) code.
-
-The steps performed include:
-
-- Construct the DIY autologging code.
-- Construct training package for TensorFlow Sequential model with call to autologging.
-- Train a model.
-- View the experiment
-- Construct training package for TensorFlow Functional model with call to autologging.
-- Compare the experiment runs.
-- Delete the experiment.
-
-```
-
-
-[Get started with Vertex AI Distributed Training](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_distributed_training.ipynb)
-
-```
-Learn how to use `Vertex AI Distributed Training` for when training with `Vertex AI`.
+In this tutorial, you learn how to use `Vertex AI Distributed Training` for when training with `Vertex AI`.
 
 The steps performed include:
 
@@ -438,17 +312,12 @@ The steps performed include:
 - `ReductionServer`: Train on multiple VMS and sync updates across VMS with `Vertex AI Reduction Server`.
 - `TPUTraining`: Train with multiple Cloud TPUs.
 
-```
-
 ### E2E Stage Example
 
-[Experimentation](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/mlops_experimentation.ipynb)
+[Stage 2: Experimentation](mlops_experimentation.ipynb)
 
 ```
-In this tutorial, you create a MLOps stage 2: experimentation process.
-
 The steps performed include:
-
 - Review the `Dataset` resource created during stage 1.
 - Train an AutoML tabular binary classifier model in the background.
 - Build the experimental model architecture.
@@ -465,5 +334,4 @@ The steps performed include:
 - Set the evaluation results of the AutoML model as the baseline.
 - If the evaluation of the custom model is below baseline, continue to experiment with the custom model.
 - If the evaluation of the custom model is above baseline, save the model as the first best model.
-
 ```

notebooks/community/ml_ops/stage2/get_started_vertex_hpt_r_kernel.ipynb

@@ -48,7 +48,7 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "<a href=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_hpt_r_kernel.ipynb target='_blank'>",
+        "    <a href=\"https://github.com/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_hpt_r_kernel.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",

notebooks/community/ml_ops/stage2/get_started_with_vertex_experiments_autologging_tf.ipynb

@@ -82,11 +82,9 @@
         "The steps performed include:\n",
         "\n",
         "- Construct the DIY autologging code.\n",
-        "- Construct training package for TensorFlow Sequential model with call to autologging.\n",
+        "- Construct training package with call to autologging.\n",
         "- Train a model.\n",
         "- View the experiment\n",
-        "- Construct training package for TensorFlow Functional model with call to autologging.\n",
-        "- Compare the experiment runs.\n",
         "- Delete the experiment."
       ]
     },
@@ -96,9 +94,9 @@
         "id": "2739272aae1b"
       },
       "source": [
-        "### Dataset\n",
+        "### Model\n",
         "\n",
-        "The dataset used in this tutorial is the [Boston Housing Prices dataset](https://www.cs.toronto.edu/~delve/data/boston/bostonDetail.html). The version of the dataset this tutorial is built into TensorFlow. The trained model predicts the median price of a house in units of 1K USD."
+        "The model used for this tutorial is a pretrain TensorFlow model that was trained on the [Boston Housing Prices dataset](https://www.cs.toronto.edu/~delve/data/boston/bostonDetail.html). The version of the dataset this tutorial is built into TensorFlow. The trained model predicts the median price of a house in units of 1K USD."
       ]
     },
     {
@@ -483,8 +481,7 @@
         "    - `compile()`: overridden method of super class. Automatically logs specified hyperparameters and calls the underlying `compile()` method.\n",
         "    - `fit()`: overridden method of super class. Automatically logs specified hyperparameters, calls the underlying `fit()` method, and logs the resulting metrics.\n",
         "    - `evaluate()`: overridden method of super class. Calls the underlying `evaluate()` method, and logs the resulting metrics.\n",
-        "- `VertexTFModel`: A subclass of the tf.keras.Model class.\n",
-        "- `VertexTFHelper`: A class for common logging methods for both Sequential and Functional models."
+        "- `VertexTFModel`: A subclass of the tf.keras.Model class."
       ]
     },
     {
@@ -633,8 +630,6 @@
         "            use_multiprocessing=use_multiprocessing,\n",
         "        )\n",
         "\n",
-        "        TFHelper().model_size(self)\n",
-        "\n",
         "        for key, val in history.history.items():\n",
         "            aiplatform.log_metrics({f\"train.{key}\": val[-1]})\n",
         "        return history\n",
@@ -762,8 +757,6 @@
         "            use_multiprocessing=use_multiprocessing,\n",
         "        )\n",
         "\n",
-        "        TFHelper().model_size(self)\n",
-        "\n",
         "        for key, val in history.history.items():\n",
         "            aiplatform.log_metrics({f\"train.{key}\": val[-1]})\n",
         "        return history\n",
@@ -800,29 +793,7 @@
         "        aiplatform.log_metrics({\"eval.loss\": metrics[0]})\n",
         "        for _ in range(1, len(metrics)):\n",
         "            aiplatform.log_metrics({\"eval.metric\": metrics[_]})\n",
-        "        return metrics\n",
-        "\n",
-        "\n",
-        "class TFHelper(object):\n",
-        "    def model_size(self, model):\n",
-        "        \"\"\"\n",
-        "        Get the memory footprint as measured by the number of weights\n",
-        "        \"\"\"\n",
-        "\n",
-        "        def get_size(weights) -> int:\n",
-        "            n = 0\n",
-        "            for weight in weights:\n",
-        "                try:\n",
-        "                    n += len(weight)\n",
-        "                    n += get_size(weight)\n",
-        "                except:\n",
-        "                    pass\n",
-        "\n",
-        "            return n\n",
-        "\n",
-        "        n = get_size(model.get_weights())\n",
-        "        aiplatform.log_metrics({\"n_weights\": n})\n",
-        "        return n"
+        "        return metrics"
       ]
     },
     {
@@ -831,9 +802,9 @@
         "id": "ce76826902c0"
       },
       "source": [
-        "### Train a TensorFlow Sequential model with Vertex AI Experiments\n",
+        "### Train the model with Vertex AI Experiments\n",
         "\n",
-        "In the following code, you build, train and evaluate a TensorFlow Sequential tabular model. The Python script includes the following calls to integrate `Vertex AI Experiments`:\n",
+        "In the following code, you build, train and evaluate a TensorFlow tabular model. The Python script includes the following calls to integrate `Vertex AI Experiments`:\n",
         "\n",
         "- command-line arguments: The arguments `experiment` and `run` are used to pass in the experiment and run names for the experiment.\n",
         "- `autologging()`: Initializes the experiment and does the heap injection.\n",
@@ -937,116 +908,6 @@
         "experiment_df.T"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "ce76826902c0"
-      },
-      "source": [
-        "### Train a TensorFlow Functional model with Vertex AI Experiments\n",
-        "\n",
-        "In the following code, you build, train and evaluate a TensorFlow Functional tabular model. The Python script includes the following calls to integrate `Vertex AI Experiments`:\n",
-        "\n",
-        "- command-line arguments: The arguments `experiment` and `run` are used to pass in the experiment and run names for the experiment.\n",
-        "- `autologging()`: Initializes the experiment and does the heap injection.\n",
-        "- `aiplatform.start_execution()`: Initializes a context for linking artifacts.\n",
-        "- `aiplatform.end_run()`: Ends the experiment.\n",
-        "\n",
-        "*Note:* The initializer `Model` will be redirected to `VertexTFModel` by heap injection. When subsequent calls are made to the compile(), fit() and evaluate() methods, they will be executed as the corresponding `VertexTFModel` methods."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "427846783ed6"
-      },
-      "outputs": [],
-      "source": [
-        "RUN_NAME = \"run-2\"\n",
-        "\n",
-        "\n",
-        "def make_dataset():\n",
-        "\n",
-        "    # Scaling Boston Housing data features\n",
-        "    def scale(feature):\n",
-        "        max = np.max(feature)\n",
-        "        feature = (feature / max).astype(np.float)\n",
-        "        return feature, max\n",
-        "\n",
-        "    (x_train, y_train), (x_test, y_test) = tf.keras.datasets.boston_housing.load_data(\n",
-        "        path=\"boston_housing.npz\", test_split=0.2, seed=113\n",
-        "    )\n",
-        "    params = []\n",
-        "\n",
-        "    for _ in range(13):\n",
-        "        x_train[_], max = scale(x_train[_])\n",
-        "        x_test[_], _ = scale(x_test[_])\n",
-        "        params.append(max)\n",
-        "\n",
-        "    return (x_train, y_train), (x_test, y_test)\n",
-        "\n",
-        "\n",
-        "# Build the Keras model\n",
-        "def build_and_compile_dnn_model(lr):\n",
-        "    inputs = tf.keras.Input(shape=(13,))\n",
-        "    x = tf.keras.layers.Dense(128, activation=\"relu\")(inputs)\n",
-        "    x = tf.keras.layers.Dense(128, activation=\"relu\")(x)\n",
-        "    outputs = tf.keras.layers.Dense(1, activation=\"linear\")(x)\n",
-        "\n",
-        "    model = tf.keras.Model(inputs, outputs)\n",
-        "\n",
-        "    model.compile(\n",
-        "        loss=\"mse\",\n",
-        "        optimizer=tf.keras.optimizers.RMSprop(learning_rate=lr),\n",
-        "        metrics=[tf.keras.metrics.RootMeanSquaredError()],\n",
-        "    )\n",
-        "    return model\n",
-        "\n",
-        "\n",
-        "# autologging\n",
-        "autolog(experiment=EXPERIMENT_NAME, run=RUN_NAME)\n",
-        "\n",
-        "with aiplatform.start_execution(\n",
-        "    schema_title=\"system.ContainerExecution\", display_name=\"example_training\"\n",
-        ") as execution:\n",
-        "    BATCH_SIZE = 16\n",
-        "\n",
-        "    model = build_and_compile_dnn_model(lr=0.01)\n",
-        "\n",
-        "    # Train the model\n",
-        "    (x_train, y_train), (x_test, y_test) = make_dataset()\n",
-        "    model.fit(x_train, y_train, epochs=10, batch_size=BATCH_SIZE)\n",
-        "\n",
-        "    model.evaluate(x_test, y_test)\n",
-        "\n",
-        "aiplatform.end_run()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "5f40912e6500"
-      },
-      "source": [
-        "#### Get the experiment results\n",
-        "\n",
-        "Next, you use the experiment name as a parameter to the method `get_experiment_df()` to get the results of the experiment as a pandas dataframe."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "7e9671712230"
-      },
-      "outputs": [],
-      "source": [
-        "experiment_df = aiplatform.get_experiment_df()\n",
-        "experiment_df = experiment_df[experiment_df.experiment_name == EXPERIMENT_NAME]\n",
-        "experiment_df.T"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {

notebooks/community/ml_ops/stage2/get_started_with_vertex_experiments_autologging_xgboost.ipynb

@@ -1,929 +0,0 @@
-{
-  "cells": [
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "copyright"
-      },
-      "outputs": [],
-      "source": [
-        "# Copyright 2022 Google LLC\n",
-        "#\n",
-        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
-        "# you may not use this file except in compliance with the License.\n",
-        "# You may obtain a copy of the License at\n",
-        "#\n",
-        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
-        "#\n",
-        "# Unless required by applicable law or agreed to in writing, software\n",
-        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
-        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
-        "# See the License for the specific language governing permissions and\n",
-        "# limitations under the License."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "title"
-      },
-      "source": [
-        "# E2E ML on GCP: MLOps stage 2 : Get started with autologging using Vertex AI Experiments for XGBoost models\n",
-        "\n",
-        "<table align=\"left\">\n",
-        "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_with_vertex_experiments_autologging_xgboost.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
-        "    </a>\n",
-        "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_with_vertex_experiments_autologging_xgboost.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
-        "      View on GitHub\n",
-        "    </a>\n",
-        "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage2/get_started_with_vertex_experiments_autologging_xgboost.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
-        "    </a>\n",
-        "  </td>\n",
-        "</table>\n",
-        "<br/><br/><br/>"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "overview:automl"
-      },
-      "source": [
-        "## Overview\n",
-        "\n",
-        "\n",
-        "This tutorial demonstrates how to use the `Vertex AI Experiments` with DIY code to implement automatic logging of parameters and metrics for experiments."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "objective:automl,training,batch_prediction"
-      },
-      "source": [
-        "### Objective\n",
-        "\n",
-        "In this tutorial, you learn how to create an experiment for training an XGBoost model, and automatically log parameters and metrics using the enclosed do-it-yourself (DIY) code.\n",
-        "\n",
-        "This tutorial uses the following Google Cloud ML services and resources:\n",
-        "\n",
-        "- `Vertex AI Experiments`\n",
-        "\n",
-        "The steps performed include:\n",
-        "\n",
-        "- Construct the DIY autologging code.\n",
-        "- Construct training package with call to autologging.\n",
-        "- Train a model.\n",
-        "- View the experiment\n",
-        "- Delete the experiment."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "dataset:custom,boston,lrg"
-      },
-      "source": [
-        "### Dataset\n",
-        "\n",
-        "The dataset used for this tutorial is the [Iris dataset](https://www.tensorflow.org/datasets/catalog/iris) from [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/overview). This dataset does not require any feature engineering. The version of the dataset in this tutorial is stored in a public Cloud Storage bucket. The trained model predicts the type of Iris flower species from a class of three species: setosa, virginica, or versicolor."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "costs"
-      },
-      "source": [
-        "### Costs\n",
-        "\n",
-        "This tutorial uses billable components of Google Cloud:\n",
-        "\n",
-        "* Vertex AI\n",
-        "* Cloud Storage\n",
-        "\n",
-        "Learn about [Vertex AI\n",
-        "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
-        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
-        "Calculator](https://cloud.google.com/products/calculator/)\n",
-        "to generate a cost estimate based on your projected usage."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "setup_local"
-      },
-      "source": [
-        "### Set up your local development environment\n",
-        "\n",
-        "If you are using Colab or Vertex Workbench AI Notebooks, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
-        "\n",
-        "Otherwise, make sure your environment meets this notebook's requirements. You need the following:\n",
-        "\n",
-        "- The Cloud Storage SDK\n",
-        "- Git\n",
-        "- Python 3\n",
-        "- virtualenv\n",
-        "- Jupyter notebook running in a virtual environment with Python 3\n",
-        "\n",
-        "The Cloud Storage guide to [Setting up a Python development environment](https://cloud.google.com/python/setup) and the [Jupyter installation guide](https://jupyter.org/install) provide detailed instructions for meeting these requirements. The following steps provide a condensed set of instructions:\n",
-        "\n",
-        "1. [Install and initialize the SDK](https://cloud.google.com/sdk/docs/).\n",
-        "\n",
-        "2. [Install Python 3](https://cloud.google.com/python/setup#installing_python).\n",
-        "\n",
-        "3. [Install virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv) and create a virtual environment that uses Python 3.  Activate the virtual environment.\n",
-        "\n",
-        "4. To install Jupyter, run `pip3 install jupyter` on the command-line in a terminal shell.\n",
-        "\n",
-        "5. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
-        "\n",
-        "6. Open this notebook in the Jupyter Notebook Dashboard.\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "install_aip:mbsdk"
-      },
-      "source": [
-        "## Installation\n",
-        "\n",
-        "Install the following packages to execute this notebook."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "install_aip:mbsdk"
-      },
-      "outputs": [],
-      "source": [
-        "import os\n",
-        "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
-        "! pip3 install --upgrade --quiet {USER_FLAG} google-cloud-aiplatform \\\n",
-        "                                             xgboost \\\n",
-        "                                             scikit-learn \\\n",
-        "                                             numpy"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "restart"
-      },
-      "source": [
-        "### Restart the kernel\n",
-        "\n",
-        "After you install the additional packages, you need to restart the notebook kernel so it can find the packages."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "restart"
-      },
-      "outputs": [],
-      "source": [
-        "import os\n",
-        "\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    # Automatically restart kernel after installs\n",
-        "    import IPython\n",
-        "\n",
-        "    app = IPython.Application.instance()\n",
-        "    app.kernel.do_shutdown(True)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "before_you_begin:nogpu"
-      },
-      "source": [
-        "## Before you begin\n",
-        "\n",
-        "### GPU runtime\n",
-        "\n",
-        "This tutorial does not require a GPU runtime.\n",
-        "\n",
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "2. [Make sure that billing is enabled for your project.](https://cloud.google.com/billing/docs/how-to/modify-project)\n",
-        "\n",
-        "3. [Enable the following APIs: Vertex AI APIs, Compute Engine APIs, and Cloud Storage.](https://console.cloud.google.com/flows/enableapi?apiid=ml.googleapis.com,compute_component,storage-component.googleapis.com)\n",
-        "\n",
-        "4. If you are running this notebook locally, you need to install the [Cloud SDK]((https://cloud.google.com/sdk)).\n",
-        "\n",
-        "5. Enter your project ID in the cell below. Then run the  cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$`."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "project_id"
-      },
-      "source": [
-        "#### Set your project ID\n",
-        "\n",
-        "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "set_project_id"
-      },
-      "outputs": [],
-      "source": [
-        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "autoset_project_id"
-      },
-      "outputs": [],
-      "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
-        "    # Get your GCP project id from gcloud\n",
-        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
-        "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID:\", PROJECT_ID)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "set_gcloud_project_id"
-      },
-      "outputs": [],
-      "source": [
-        "! gcloud config set project $PROJECT_ID"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "region"
-      },
-      "source": [
-        "#### Region\n",
-        "\n",
-        "You can also change the `REGION` variable, which is used for operations\n",
-        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
-        "\n",
-        "- Americas: `us-central1`\n",
-        "- Europe: `europe-west4`\n",
-        "- Asia Pacific: `asia-east1`\n",
-        "\n",
-        "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
-        "\n",
-        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "region"
-      },
-      "outputs": [],
-      "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "timestamp"
-      },
-      "source": [
-        "#### UUID\n",
-        "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "timestamp"
-      },
-      "outputs": [],
-      "source": [
-        "import random\n",
-        "import string\n",
-        "\n",
-        "\n",
-        "# Generate a uuid of a specifed length(default=8)\n",
-        "def generate_uuid(length: int = 8) -> str:\n",
-        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
-        "\n",
-        "\n",
-        "UUID = generate_uuid()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "gcp_authenticate"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. \n",
-        "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
-        "\n",
-        "**Click Create service account**.\n",
-        "\n",
-        "In the **Service account name** field, enter a name, and click **Create**.\n",
-        "\n",
-        "In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex\" into the filter box, and select **Vertex Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "Click Create. A JSON file that contains your key downloads to your local environment.\n",
-        "\n",
-        "Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "gcp_authenticate"
-      },
-      "outputs": [],
-      "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = \"google.colab\" in sys.modules\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "setup_vars"
-      },
-      "source": [
-        "### Import libraries and define constants"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "59963fb7178f"
-      },
-      "outputs": [],
-      "source": [
-        "import google.cloud.aiplatform as aiplatform\n",
-        "import numpy as np\n",
-        "import xgboost as xgb\n",
-        "from sklearn.metrics import accuracy_score, precision_score, recall_score\n",
-        "\n",
-        "# to suppress lint message (unused)\n",
-        "precision_score, recall_score"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "init_aip:mbsdk"
-      },
-      "source": [
-        "## Initialize Vertex AI SDK for Python\n",
-        "\n",
-        "Initialize the Vertex AI SDK for Python for your project and corresponding bucket."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "init_aip:mbsdk"
-      },
-      "outputs": [],
-      "source": [
-        "aiplatform.init(project=PROJECT_ID, location=REGION)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "ae8f31c8c617"
-      },
-      "source": [
-        "## DIY code for autologging XGBoost models\n",
-        "\n",
-        "The code below implements autologging for XGBoost models.\n",
-        "\n",
-        "- `autologging()`: Initializes the experiment and uses heap injection to replace `xgboost.train()` symbols on the heap with the redirect wrapper function `VertexXGBtrain`.\n",
-        "\n",
-        "- `VertexXGBtrain`: A wrapper function for XGBoost train() function. Automatically logs hyperparameters and calls the underlyig function.\n",
-        "\n",
-        "- `VertexSKLaccuracy_score`: A wrapper function for scikit-learn accuracy_score() function. Automatically calls underlying function and logs the metrics results."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "8eb012e5d7ef"
-      },
-      "outputs": [],
-      "source": [
-        "def autolog(\n",
-        "    project: str = None,\n",
-        "    location: str = None,\n",
-        "    staging_bucket: str = None,\n",
-        "    experiment: str = None,\n",
-        "    run: str = None,\n",
-        "    framework: str = \"tf\",\n",
-        "):\n",
-        "    \"\"\"\n",
-        "    Enable automatic logging of parameters and metrics in Vertex AI Experiments,\n",
-        "    for corresponding framework.\n",
-        "\n",
-        "        project: The project ID\n",
-        "        location : The region\n",
-        "        staging_bucket: temporary bucket\n",
-        "        experiment: The name of the experiment\n",
-        "        run: The name of the run within the experiment\n",
-        "        framework: The ML framework for which a model is being trained.\n",
-        "    \"\"\"\n",
-        "    # autologging\n",
-        "    if framework == \"tf\":\n",
-        "        try:\n",
-        "            globals()[\"Sequential\"] = VertexTFSequential\n",
-        "            if \"tf\" in globals():\n",
-        "                tf.keras.Sequential = VertexTFSequential\n",
-        "            if \"tensorflow\" in globals():\n",
-        "                tensorflow.keras.Sequential = VertexTFSequential\n",
-        "        except:\n",
-        "            pass\n",
-        "\n",
-        "        try:\n",
-        "            globals()[\"Model\"] = VertexTFModel\n",
-        "            if \"tf\" in globals():\n",
-        "                tf.keras.Model = VertexTFModel\n",
-        "            if \"tensorflow\" in globals():\n",
-        "                tensorflow.keras.Model = VertexTFModel\n",
-        "        except:\n",
-        "            pass\n",
-        "    elif framework == \"xgb\":\n",
-        "        global real_xgb_train\n",
-        "        global real_accuracy_score, real_precision_score, real_recall_score\n",
-        "        import sklearn\n",
-        "\n",
-        "        try:\n",
-        "            if \"xgboost\" in globals():\n",
-        "                real_xgb_train = xgboost.train\n",
-        "                xgboost.train = VertexXGBtrain\n",
-        "        except:\n",
-        "            pass\n",
-        "\n",
-        "        try:\n",
-        "            if \"xgb\" in globals():\n",
-        "                real_xgb_train = xgb.train\n",
-        "                xgb.train = VertexXGBtrain\n",
-        "        except:\n",
-        "            pass\n",
-        "\n",
-        "        try:\n",
-        "            global accuracy_score, precision_score, recall_score\n",
-        "            if \"accuracy_score\" in globals():\n",
-        "                real_accuracy_score = sklearn.metrics.accuracy_score\n",
-        "                sklearn.metrics.accuracy_score = VertexSKLaccuracy_score\n",
-        "                accuracy_score = VertexSKLaccuracy_score\n",
-        "            if \"precision_score\" in globals():\n",
-        "                real_precision_score = sklearn.metrics.precision_score\n",
-        "                sklearn.metrics.precision_score = VertexSKLprecision_score\n",
-        "                precision_score = VertexSKLprecision_score\n",
-        "            if \"recall_score\" in globals():\n",
-        "                real_recall_score = sklearn.metrics.recall_score\n",
-        "                sklearn.metrics.recall_score = VertexSKLrecall_score\n",
-        "                recall_score = VertexSKLrecall_score\n",
-        "        except:\n",
-        "            pass\n",
-        "\n",
-        "    if project:\n",
-        "        aiplatform.init(\n",
-        "            project=project, location=location, staging_bucket=staging_bucket\n",
-        "        )\n",
-        "\n",
-        "    if experiment:\n",
-        "        aiplatform.init(experiment=experiment)\n",
-        "    if run:\n",
-        "        aiplatform.start_run(run)\n",
-        "\n",
-        "\n",
-        "def VertexXGBtrain(\n",
-        "    params,\n",
-        "    dtrain,\n",
-        "    num_boost_round=10,\n",
-        "    evals=None,\n",
-        "    obj=None,\n",
-        "    maximize=None,\n",
-        "    early_stopping_rounds=None,\n",
-        "    evals_result=None,\n",
-        "    verbose_eval=True,\n",
-        "    callbacks=None,\n",
-        "    custom_metric=None,\n",
-        "):\n",
-        "    \"\"\"\n",
-        "    Wrapper function for autologging training parameters with Vertex AI Experiments\n",
-        "    Args:\n",
-        "        same as underlying xgb.train() method\n",
-        "    \"\"\"\n",
-        "    global real_xgb_train\n",
-        "\n",
-        "    aiplatform.log_params({\"train.num_boost_round\": int(num_boost_round)})\n",
-        "\n",
-        "    if params:\n",
-        "        if \"booster\" in params:\n",
-        "            aiplatform.log_params({\"train.booster\": int(params[\"booster\"])})\n",
-        "\n",
-        "        # booster parameters\n",
-        "        if \"eta\" in params:\n",
-        "            aiplatform.log_params({\"train.eta\": int(params[\"eta\"])})\n",
-        "        if \"max_depth\" in params:\n",
-        "            aiplatform.log_params({\"train.max_depth\": int(params[\"max_depth\"])})\n",
-        "        if \"max_leaf_nodes\" in params:\n",
-        "            aiplatform.log_params(\n",
-        "                {\"train.max_leaf_nodes\": int(params[\"max_leaf_nodes\"])}\n",
-        "            )\n",
-        "        if \"gamma\" in params:\n",
-        "            aiplatform.log_params({\"train.gamma\": int(params[\"gamma\"])})\n",
-        "        if \"alpha\" in params:\n",
-        "            aiplatform.log_params({\"train.alpha\": int(params[\"alpha\"])})\n",
-        "\n",
-        "    return real_xgb_train(\n",
-        "        params=params,\n",
-        "        dtrain=dtrain,\n",
-        "        num_boost_round=num_boost_round,\n",
-        "        evals=evals,\n",
-        "        obj=obj,\n",
-        "        maximize=maximize,\n",
-        "        early_stopping_rounds=early_stopping_rounds,\n",
-        "        evals_result=evals_result,\n",
-        "        verbose_eval=verbose_eval,\n",
-        "        callbacks=callbacks,\n",
-        "        custom_metric=custom_metric,\n",
-        "    )\n",
-        "\n",
-        "\n",
-        "def VertexSKLaccuracy_score(labels, predictions):\n",
-        "    \"\"\"\n",
-        "    Wrapper function for autologging training metrics with Vertex AI Experiments\n",
-        "    Args:\n",
-        "        same as underlying accuracy_score function\n",
-        "    \"\"\"\n",
-        "    global real_accuracy_score\n",
-        "    accuracy = real_accuracy_score(labels, predictions)\n",
-        "    aiplatform.log_metrics({\"accuracy\": accuracy})\n",
-        "    return accuracy\n",
-        "\n",
-        "\n",
-        "def VertexSKLprecision_score(\n",
-        "    y_true,\n",
-        "    y_pred,\n",
-        "    *,\n",
-        "    labels=None,\n",
-        "    pos_label=1,\n",
-        "    average=\"binary\",\n",
-        "    sample_weight=None,\n",
-        "    zero_division=\"warn\",\n",
-        "):\n",
-        "    \"\"\"\n",
-        "    Wrapper function for autologging training metrics with Vertex AI Experiments\n",
-        "    Args:\n",
-        "        same as underlying precision_score function\n",
-        "    \"\"\"\n",
-        "    global real_precision_score\n",
-        "    precision = real_precision_score(\n",
-        "        y_true,\n",
-        "        y_pred,\n",
-        "        labels=labels,\n",
-        "        pos_label=pos_label,\n",
-        "        average=average,\n",
-        "        sample_weight=sample_weight,\n",
-        "        zero_division=zero_division,\n",
-        "    )\n",
-        "    aiplatform.log_metrics({\"precision\": precision})\n",
-        "    return precision\n",
-        "\n",
-        "\n",
-        "def VertexSKLrecall_score(\n",
-        "    y_true,\n",
-        "    y_pred,\n",
-        "    *,\n",
-        "    labels=None,\n",
-        "    pos_label=1,\n",
-        "    average=\"binary\",\n",
-        "    sample_weight=None,\n",
-        "    zero_division=\"warn\",\n",
-        "):\n",
-        "    \"\"\"\n",
-        "    Wrapper function for autologging training metrics with Vertex AI Experiments\n",
-        "    Args:\n",
-        "        same as underlying recall_score function\n",
-        "    \"\"\"\n",
-        "    global real_recall_score\n",
-        "    recall = real_recall_score(\n",
-        "        y_true,\n",
-        "        y_pred,\n",
-        "        labels=labels,\n",
-        "        pos_label=pos_label,\n",
-        "        average=average,\n",
-        "        sample_weight=sample_weight,\n",
-        "        zero_division=zero_division,\n",
-        "    )\n",
-        "    aiplatform.log_metrics({\"recall\": recall})\n",
-        "    return recall\n",
-        "\n",
-        "\n",
-        "class VertexXGBBooster(xgb.Booster):\n",
-        "    \"\"\"\n",
-        "    WIP\n",
-        "    \"\"\"\n",
-        "\n",
-        "    def __init__(self, params=None, cache=None, model_file=None):\n",
-        "        super().__init__(params, cache, model_file)\n",
-        "\n",
-        "    def boost(\n",
-        "        self, dtrain: xgb.core.DMatrix, grad: np.ndarray, hess: np.ndarray\n",
-        "    ) -> None:\n",
-        "        return super().boost(dtrain, grad, hess)\n",
-        "\n",
-        "    def eval(\n",
-        "        self, data: xgb.core.DMatrix, name: str = \"eval\", iteration: int = 0\n",
-        "    ) -> str:\n",
-        "        return super().eval(data, name, iteration)\n",
-        "\n",
-        "    def update(self, dtrain: xgb.core.DMatrix, iteration: int, fobj=None) -> None:\n",
-        "        return super().update(dtrain, iteration, fobj)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "ce76826902c0"
-      },
-      "source": [
-        "### Train the XGBoost model with Vertex AI Experiments\n",
-        "\n",
-        "In the following code, you build, train and evaluate an XGBoost tabular model. The Python script includes the following calls to integrate `Vertex AI Experiments`:\n",
-        "\n",
-        "- command-line arguments: The arguments `experiment` and `run` are used to pass in the experiment and run names for the experiment.\n",
-        "- `autologging()`: Initializes the experiment and does the heap injection.\n",
-        "- `aiplatform.start_execution()`: Initializes a context for linking artifacts.\n",
-        "- `aiplatform.end_run()`: Ends the experiment.\n",
-        "\n",
-        "*Note:* The functions `xgb.train` and `accuracy_score` will be redirected to `VertexXGBtrain` and VertexSKLaccuracy_score, respectively, by heap injection. When subsequent calls are made to the `train()` and `accuracy()` function,s they will be executed as the corresponding `VertexXGBtrain` and `VertexSKLaccuracy_score` functions."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "WiSnFuDoox9W"
-      },
-      "outputs": [],
-      "source": [
-        "EXPERIMENT_NAME = f\"myexperiment{UUID}\"\n",
-        "RUN_NAME = \"run-1\"\n",
-        "\n",
-        "DATASET_DIR = \"gs://cloud-samples-data/ai-platform/iris\"\n",
-        "DATASET_DATA_URL = DATASET_DIR + \"/iris_data.csv\"\n",
-        "DATASET_LABELS_URL = DATASET_DIR + \"/iris_target.csv\"\n",
-        "\n",
-        "BOOSTED_ROUNDS = 20\n",
-        "\n",
-        "import logging\n",
-        "import os\n",
-        "import subprocess\n",
-        "import sys\n",
-        "\n",
-        "import hypertune\n",
-        "import numpy as np\n",
-        "import pandas as pd\n",
-        "import xgboost as xgb\n",
-        "from sklearn.model_selection import train_test_split\n",
-        "\n",
-        "\n",
-        "def get_data():\n",
-        "    # gsutil outputs everything to stderr so we need to divert it to stdout.\n",
-        "    subprocess.check_call(\n",
-        "        [\"gsutil\", \"cp\", DATASET_DATA_URL, \"data.csv\"], stderr=sys.stdout\n",
-        "    )\n",
-        "    # gsutil outputs everything to stderr so we need to divert it to stdout.\n",
-        "    subprocess.check_call(\n",
-        "        [\"gsutil\", \"cp\", DATASET_LABELS_URL, \"labels.csv\"], stderr=sys.stdout\n",
-        "    )\n",
-        "\n",
-        "    # Load data into pandas, then use `.values` to get NumPy arrays\n",
-        "    data = pd.read_csv(\"data.csv\").values\n",
-        "    labels = pd.read_csv(\"labels.csv\").values\n",
-        "\n",
-        "    # Convert one-column 2D array into 1D array for use with XGBoost\n",
-        "    labels = labels.reshape((labels.size,))\n",
-        "\n",
-        "    train_data, test_data, train_labels, test_labels = train_test_split(\n",
-        "        data, labels, test_size=0.2, random_state=7\n",
-        "    )\n",
-        "\n",
-        "    # Load data into DMatrix object\n",
-        "    dtrain = xgb.DMatrix(train_data, label=train_labels)\n",
-        "    return dtrain, test_data, test_labels\n",
-        "\n",
-        "\n",
-        "def train_model(dtrain):\n",
-        "    logging.info(\"Start training ...\")\n",
-        "    # Train XGBoost model\n",
-        "    params = {\"max_depth\": 3, \"objective\": \"multi:softmax\", \"num_class\": 3}\n",
-        "    model = xgb.train(params=params, dtrain=dtrain, num_boost_round=BOOSTED_ROUNDS)\n",
-        "    logging.info(\"Training completed\")\n",
-        "    return model\n",
-        "\n",
-        "\n",
-        "def evaluate_model(model, test_data, test_labels):\n",
-        "    dtest = xgb.DMatrix(test_data)\n",
-        "    pred = model.predict(dtest)\n",
-        "    predictions = [round(value) for value in pred]\n",
-        "    # evaluate predictions\n",
-        "    accuracy = accuracy_score(test_labels, predictions)\n",
-        "\n",
-        "    logging.info(f\"Evaluation completed with model accuracy: {accuracy}\")\n",
-        "\n",
-        "    # report metric for hyperparameter tuning\n",
-        "    hpt = hypertune.HyperTune()\n",
-        "    hpt.report_hyperparameter_tuning_metric(\n",
-        "        hyperparameter_metric_tag=\"accuracy\", metric_value=accuracy\n",
-        "    )\n",
-        "    return accuracy\n",
-        "\n",
-        "\n",
-        "# autologging\n",
-        "autolog(experiment=EXPERIMENT_NAME, run=RUN_NAME, framework=\"xgb\")\n",
-        "\n",
-        "with aiplatform.start_execution(\n",
-        "    schema_title=\"system.ContainerExecution\", display_name=\"example_training\"\n",
-        ") as execution:\n",
-        "    dtrain, test_data, test_labels = get_data()\n",
-        "    model = train_model(dtrain)\n",
-        "    accuracy = evaluate_model(model, test_data, test_labels)\n",
-        "\n",
-        "aiplatform.end_run()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "5f40912e6500"
-      },
-      "source": [
-        "#### Get the experiment results\n",
-        "\n",
-        "Next, you use the experiment name as a parameter to the method `get_experiment_df()` to get the results of the experiment as a pandas dataframe."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "7e9671712230"
-      },
-      "outputs": [],
-      "source": [
-        "experiment_df = aiplatform.get_experiment_df()\n",
-        "experiment_df = experiment_df[experiment_df.experiment_name == EXPERIMENT_NAME]\n",
-        "experiment_df.T"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "e508c159d712"
-      },
-      "source": [
-        "#### Delete the experiment\n",
-        "\n",
-        "Since the experiment was created within a training script, to delete the experiment you use the `list()` method to obtain all the experiments for the project, and then filter on the experiment name."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "1a1b5fcbfde0"
-      },
-      "outputs": [],
-      "source": [
-        "experiments = aiplatform.Experiment.list()\n",
-        "for experiment in experiments:\n",
-        "    if experiment.name == EXPERIMENT_NAME:\n",
-        "        experiment.delete()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "cleanup:mbsdk"
-      },
-      "source": [
-        "# Cleaning up\n",
-        "\n",
-        "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
-        "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
-        "\n",
-        "Otherwise, you can delete the individual resources you created in this tutorial."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "9eb897e0e728"
-      },
-      "outputs": [],
-      "source": [
-        "# There are no resources to cleanup"
-      ]
-    }
-  ],
-  "metadata": {
-    "colab": {
-      "name": "get_started_with_vertex_experiments_autologging_xgboost.ipynb",
-      "toc_visible": true
-    },
-    "kernelspec": {
-      "display_name": "Python 3",
-      "name": "python3"
-    }
-  },
-  "nbformat": 4,
-  "nbformat_minor": 0
-}

notebooks/community/ml_ops/stage3/README.md

@@ -34,10 +34,9 @@ The third stage in MLOps is formalization to develop an automated pipeline proce
 ### Get Started
 
 
-[Get started with Vertex AI Model Registry](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_model_registry.ipynb)
+[Get started with Vertex AI Model Registry](community/ml_ops/stage3/get_started_with_model_registry.ipynb)
 
-```
-Learn how to use `Vertex AI Model Registry` to create and register multiple versions of a model.
+In this tutorial, you learn how to use `Vertex AI Model Registry` to create and register multiple versions of a model.
 
 The steps performed include:
 
@@ -47,13 +46,9 @@ The steps performed include:
 - Deleting a model version.
 - Retraining the next model version.
 
-```
+[Get started with Dataflow pipeline components](community/ml_ops/stage3/get_started_with_dataflow_pipeline_components.ipynb)
 
-
-[Get started with Dataflow pipeline components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_dataflow_pipeline_components.ipynb)
-
-```
-Learn how to use prebuilt `Google Cloud Pipeline Components` for `Dataflow`.
+In this tutorial, you learn how to use prebuilt `Google Cloud Pipeline Components` for `Dataflow`.
 
 The steps performed include:
 
@@ -61,13 +56,9 @@ The steps performed include:
 - Encapsulate the Apache Beam data pipeline with a Dataflow component in a Vertex AI pipeline.
 - Execute a Vertex AI pipeline.
 
-```
+[Get started with Apache Airflow and Vertex AI Pipelines](community/ml_ops/stage3/get_started_with_airflow_and_vertex_pipelines.ipynb)
 
-
-[Get started with Apache Airflow and Vertex AI Pipelines](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_airflow_and_vertex_pipelines.ipynb)
-
-```
-Learn how to use Apache Airflow with `Vertex AI Pipelines`.
+In this tutorial, you learn how to use Apache Airflow with `Vertex AI Pipelines`.
 
 The steps performed include:
 
@@ -76,13 +67,9 @@ The steps performed include:
 - Create a `Vertex AI Pipeline` that triggers the Airflow DAG.
 - Execute the `Vertex AI Pipeline`.
 
-```
+[Get started with Kubeflow Pipelines](community/ml_ops/stage3/get_started_with_kubeflow_pipelines.ipynb)
 
-
-[Get started with Kubeflow Pipelines](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_kubeflow_pipelines.ipynb)
-
-```
-Learn how to use `Kubeflow Pipelines`(KFP).
+In this tutorial, you learn how to use `Kubeflow Pipelines`(KFP).
 
 The steps performed include:
 
@@ -93,13 +80,9 @@ The steps performed include:
 - Building sequential, parallel, multiple output components.
 - Building control flow into pipelines.
 
-```
+[Get started with Vertex AI custom training pipeline components](community/ml_ops/stage3/get_started_with_custom_training_pipeline_components.ipynb)
 
-
-[Get started with Vertex AI custom training pipeline components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_custom_training_pipeline_components.ipynb)
-
-```
-Learn how to use prebuilt `Google Cloud Pipeline Components` for `Vertex AI Training`.
+In this tutorial, you learn how to use prebuilt `Google Cloud Pipeline Components` for `Vertex AI Training`.
 
 The steps performed include:
 
@@ -115,13 +98,11 @@ The steps performed include:
     - Deploying a Vertex AI custom trained model.
 - Execute a Vertex AI pipeline.
 
-```
+[Get started with Dataproc Serverless pipeline components](community/ml_ops/stage3/get_started_with_dataproc_serverless_pipeline_components.ipynb)
 
 
-[Get started with Dataproc Serverless pipeline components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_dataproc_serverless_pipeline_components.ipynb)
+In this tutorial, you learn how to use prebuilt `Google Cloud Pipeline Components` for `Dataproc Serverless` service. 
 
-```
-Learn how to use prebuilt `Google Cloud Pipeline Components` for `Dataproc Serverless` service.
 
 The steps performed include:
 
@@ -130,13 +111,9 @@ The steps performed include:
 - `DataprocSparkSqlBatchOp` for running Spark SQL batch workloads.
 - `DataprocSparkRBatchOp` for running SparkR batch workloads.
 
-```
+[Get started with Vertex AI Hyperparameter Tuning pipeline components](community/ml_ops/stage3/get_started_with_hpt_pipeline_components.ipynb)
 
-
-[Get started with Vertex AI Hyperparameter Tuning pipeline components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_hpt_pipeline_components.ipynb)
-
-```
-Learn how to use prebuilt `Google Cloud Pipeline Components` for `Vertex AI Hyperparameter Tuning`.
+In this tutorial, you learn how to use prebuilt `Google Cloud Pipeline Components` for `Vertex AI Hyperparameter Tuning`.
 
 The steps performed include:
 
@@ -148,28 +125,23 @@ The steps performed include:
       - Upload the model artifacts to a `Vertex AI Model` resource.
 - Execute a Vertex AI pipeline.
 
-```
+[Get started with machine management for Vertex AI Pipelines](community/ml_ops/stage3/get_started_with_machine_management.ipynb)
 
+In this tutorial, you convert a self-contained custom training component into a `Vertex AI CustomJob`, whereby:
 
-[Get started with machine management for Vertex AI Pipelines](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_machine_management.ipynb)
-
-```
-Learn how to convert a self-contained custom training component into a `Vertex AI CustomJob`, whereby:
+    - The training job and artifacts are trackable.
+    - Set machine resources, such as machine-type, cpu/gpu, memory, disk, etc.
 
 The steps performed in this tutorial include:
 
 - Create a custom component with a self-contained training job.
 - Execute pipeline using component-level settings for machine resources
 - Convert the self-contained training component into a `Vertex AI CustomJob`.
-- Execute pipeline using customjob-level settings for machine resources
+- Execute pipeline using customjob-level settings for machine resources 
 
-```
+[Get started with TFX pipelines](community/ml_ops/stage3/get_started_with_tfx_pipeline.ipynb)
 
-
-[Get started with TFX pipelines](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_tfx_pipeline.ipynb)
-
-```
-Learn how to use TensorFlow Extended (TFX) with `Vertex AI Pipelines`.
+In this tutorial, you learn how to use TensorFlow Extended (TFX) with `Vertex AI Pipelines`.
 
 The steps performed include:
 
@@ -178,28 +150,9 @@ The steps performed include:
 - Execute the pipeline on Google Cloud using `Vertex AI Training`
 - Execute the pipeline using `Vertex AI Pipelines`.
 
-```
+[Get started with BigQuery ML pipeline components](community/ml_ops/stage3/get_started_with_bqml_pipeline_components.ipynb)
 
-
-[Orchestrating a workflow to train and deploy an scikit-learn model using Vertex AI Pipelines with online prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_vertex_pipelines_sklearn_with_prediction.ipynb)
-
-```
-Learn how to use prebuilt components in `Vertex AI Pipelines` for training and deploying a scikit-Learn custom model, and then using `Vertex AI Prediction` to make an online prediction.
-
-The steps performed include:
-
-- Construct a scikit-learn training package.
-- Construct a pipeline to train and deploy a scikit-learn model.
-- Execute the pipeline.
-- Make an online prediction.
-
-```
-
-
-[Get started with BigQuery ML pipeline components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_bqml_pipeline_components.ipynb)
-
-```
-Learn how to use prebuilt `Google Cloud Pipeline Components` for `BigQuery ML`.
+In this tutorial, you learn how to use prebuilt `Google Cloud Pipeline Components` for `BigQuery ML`.
 
 The steps performed include:
 
@@ -212,28 +165,9 @@ The steps performed include:
 - Execute a Vertex AI pipeline.
 - Make a prediction with the deployed Vertex AI model.
 
-```
+[Get started with AutoML tabular pipeline workflows](community/ml_ops/stage3/get_started_with_automl_tabular_pipeline_workflow.ipynb)
 
-
-[Orchestrating a workflow to train and deploy an XGBoost model using Vertex AI Pipelines with Vertex AI Experiments](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_vertex_pipelines_xgboost_with_experiments.ipynb)
-
-```
-Learn how to use prebuilt components in `Vertex AI Pipelines` for training and deploying a XGBoost custom model, and using `Vertex AI Experiments` to log the corresponding training parameters and metrics, from within the training package.
-
-The steps performed include:
-
-- Construct a XGBoost training package.
-  - Add tracking the experiment
-- Construct a pipeline to train and deploy a XGBoost model.
-- Execute the pipeline.
-
-```
-
-
-[Get started with AutoML tabular pipeline workflows](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_automl_tabular_pipeline_workflow.ipynb)
-
-```
-Learn how to use `AutoML Tabular Pipeline Template` for training, exporting and tuning an AutoML tabular model.
+In this tutorial, you learn how to use `AutoML Tabular Pipeline Template` for training, exporting and tuning an AutoML tabular model.
 
 The steps performed include:
 
@@ -249,13 +183,9 @@ The steps performed include:
 - Deploy exported OSS TF model.
 - Make a prediction.
 
-```
+[Get started with rapid prototyping with AutoML and BigQuery ML](community/ml_ops/stage3/get_started_with_rapid_prototyping_bqml_automl.ipynb)
 
-
-[Get started with rapid prototyping with AutoML and BigQuery ML](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_rapid_prototyping_bqml_automl.ipynb)
-
-```
-Learn how to use `Vertex AI Predictions` for rapid prototyping a model.
+In this tutorial, you learn how to use `Vertex AI Predictions` for rapid prototyping a model.
 
 The steps performed include:
 
@@ -266,13 +196,9 @@ The steps performed include:
 - Deploying the best trained model.
 - Testing the deployed model infrastructure.
 
-```
+[Get started with AutoML pipeline components](community/ml_ops/stage3/get_started_with_automl_pipeline_components.ipynb)
 
-
-[Get started with AutoML pipeline components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_automl_pipeline_components.ipynb)
-
-```
-Learn how to use prebuilt `Google Cloud Pipeline Components` for `Vertex AI AutoML`.
+In this tutorial, you learn how to use prebuilt `Google Cloud Pipeline Components` for `Vertex AI AutoML`.
 
 The steps performed include:
 
@@ -282,28 +208,10 @@ The steps performed include:
     - Deploying a Vertex AI AutoML trained model.
 - Execute a Vertex AI pipeline.
 
-```
 
+[Get started with BigQuery and TFDV pipeline components](community/ml_ops/stage3/get_started_with_bq_tfdv_pipeline_components.ipynb)
 
-[Orchestrating a workflow to train and deploy an XGBoost model using Vertex AI Pipelines with online prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_vertex_pipelines_xgboost_with_prediction.ipynb)
-
-```
-Learn how to use prebuilt components in `Vertex AI Pipelines` for training and deploying a XGBoost custom model, and then using `Vertex AI Prediction` to make an online prediction.
-
-The steps performed include:
-
-- Construct a XGBoost training package.
-- Construct a pipeline to train and deploy a XGBoost model.
-- Execute the pipeline.
-- Make an online prediction.
-
-```
-
-
-[Get started with BigQuery and TFDV pipeline components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_bq_tfdv_pipeline_components.ipynb)
-
-```
-Learn how to use build lightweight Python components for BigQuery and TensorFlow Data Validation.
+In this tutorial, you learn how to use build lightweight Python components for BigQuery and TensorFlow Data Validation.
 
 The steps performed include:
 
@@ -311,28 +219,22 @@ The steps performed include:
 - Build and execute a pipeline component for generating TFDV statistics and schema from a Vertex AI Tabular Dataset.
 - Execute a Vertex AI pipeline.
 
-```
-
 ### E2E Stage Example
 
-[Formalization](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/mlops_formalization.ipynb)
+[Stage 3: Formalization](mlops_formalization.ipynb)
 
 ```
-In this tutorial, you create a MLOps stage 3: formalization process.
-
 The steps performed include:
-
 - Obtain resources from the experimentation stage.
     - Baseline model.
     - Dataset schema/statistics for baseline model.
 - Formalize a data preprocessing pipeline.
     - Extract columns/rows from BigQuery table to local BigQuery table.
-    - Use TensorFlow Data Validation library to determine statistics, schema, and features.
+    - Use Tensorflow Data Validation library to determine statistics, schema, and features.
     - Use Dataflow to preprocess the data.
     - Create a Vertex AI Dataset.
 - Formalize a build model architecture pipeline.
     - Create the Vertex AI Model base model.
 - Formalize a training pipeline.
-
 ```
 

notebooks/community/ml_ops/stage3/get_started_vertex_pipelines_sklearn_with_prediction.ipynb

@@ -560,30 +560,6 @@
         "    print(\"Service Account:\", SERVICE_ACCOUNT)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "set_service_account:pipelines"
-      },
-      "source": [
-        "#### Set service account access for Vertex AI Pipelines\n",
-        "\n",
-        "Run the following commands to grant your service account access to read and write pipeline artifacts in the bucket that you created in the previous step -- you only need to run these once per service account."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "set_service_account:pipelines"
-      },
-      "outputs": [],
-      "source": [
-        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
-        "\n",
-        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {

notebooks/community/ml_ops/stage3/get_started_vertex_pipelines_xgboost_with_experiments.ipynb

@@ -557,30 +557,6 @@
         "    print(\"Service Account:\", SERVICE_ACCOUNT)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "set_service_account:pipelines"
-      },
-      "source": [
-        "#### Set service account access for Vertex AI Pipelines\n",
-        "\n",
-        "Run the following commands to grant your service account access to read and write pipeline artifacts in the bucket that you created in the previous step -- you only need to run these once per service account."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 13,
-      "metadata": {
-        "id": "set_service_account:pipelines"
-      },
-      "outputs": [],
-      "source": [
-        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
-        "\n",
-        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {

notebooks/community/ml_ops/stage3/get_started_vertex_pipelines_xgboost_with_prediction.ipynb

@@ -196,7 +196,7 @@
         "    USER_FLAG = \"--user\"\n",
         "\n",
         "! pip3 install {USER_FLAG} --upgrade --quiet google-cloud-aiplatform \\\n",
-        "                                             google-cloud-pipeline-components==1.0.25 \\\n",
+        "                                             google-cloud-pipeline-components \\\n",
         "                                             kfp "
       ]
     },
@@ -557,30 +557,6 @@
         "    print(\"Service Account:\", SERVICE_ACCOUNT)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "set_service_account:pipelines"
-      },
-      "source": [
-        "#### Set service account access for Vertex AI Pipelines\n",
-        "\n",
-        "Run the following commands to grant your service account access to read and write pipeline artifacts in the bucket that you created in the previous step -- you only need to run these once per service account."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 13,
-      "metadata": {
-        "id": "set_service_account:pipelines"
-      },
-      "outputs": [],
-      "source": [
-        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
-        "\n",
-        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {

notebooks/community/ml_ops/stage3/get_started_with_dataflow_flex_template_component.ipynb

@@ -1,832 +0,0 @@
-{
-  "cells": [
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "copyright"
-      },
-      "outputs": [],
-      "source": [
-        "# Copyright 2023 Google LLC\n",
-        "#\n",
-        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
-        "# you may not use this file except in compliance with the License.\n",
-        "# You may obtain a copy of the License at\n",
-        "#\n",
-        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
-        "#\n",
-        "# Unless required by applicable law or agreed to in writing, software\n",
-        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
-        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
-        "# See the License for the specific language governing permissions and\n",
-        "# limitations under the License."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "title:generic,gcp"
-      },
-      "source": [
-        "# E2E ML on GCP: MLOps stage 3 : formalization: get started with the Dataflow Flex Template component\n",
-        "\n",
-        "<table align=\"left\">\n",
-        "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_dataflow_pipeline_components.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
-        "      View on GitHub\n",
-        "    </a>\n",
-        "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_dataflow_pipeline_components.ipynb\">\n",
-        "<img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> \n",
-        "        Run in Colab\n",
-        "    </a>\n",
-        "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage3/get_started_with_dataflow_pipeline_components.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
-        "    </a>\n",
-        "  </td>   \n",
-        "    \n",
-        "</table>\n",
-        "<br/><br/><br/>"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "overview:mlops"
-      },
-      "source": [
-        "## Overview\n",
-        "\n",
-        "\n",
-        "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 3 : formalization: get started with the Dataflow Flex Template component."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "objective:mlops,stage3,get_started_dataflow_pipeline_components"
-      },
-      "source": [
-        "### Objective\n",
-        "\n",
-        "In this tutorial, you learn how to use prebuilt `Google Cloud Pipeline Components` to execute `Dataflow` [Flex Template](https://cloud.google.com/dataflow/docs/guides/templates/using-flex-templates) jobs.\n",
-        "\n",
-        "This tutorial uses the following Google Cloud ML services:\n",
-        "\n",
-        "- `Vertex AI Pipelines`\n",
-        "- `Google Cloud Pipeline Components`\n",
-        "- `Dataflow`\n",
-        "\n",
-        "The steps performed include:\n",
-        "\n",
-        "- Defining a pipeline step to execute a Dataflow Flex Template job within a Vertex AI pipeline.\n",
-        "- Execute a Vertex AI pipeline."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "dataset:gsod,lrg"
-      },
-      "source": [
-        "### Dataset\n",
-        "\n",
-        "The dataset used for this tutorial is the sample data from the [Apache Beam Mobile Gaming Pipeline example](https://beam.apache.org/get-started/mobile-gaming-example/)."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "0c997d8d92ce"
-      },
-      "source": [
-        "### Costs\n",
-        "\n",
-        "This tutorial uses billable components of Google Cloud:\n",
-        "\n",
-        "* Vertex AI\n",
-        "* Cloud Storage\n",
-        "* Dataflow\n",
-        "\n",
-        "Learn about [Vertex AI\n",
-        "pricing](https://cloud.google.com/vertex-ai/pricing), [Cloud Storage\n",
-        "pricing](https://cloud.google.com/storage/pricing), and [Dataflow pricing](https://cloud.google.com/dataflow/pricing)\n",
-        "and use the [Pricing\n",
-        "Calculator](https://cloud.google.com/products/calculator/)\n",
-        "to generate a cost estimate based on your projected usage."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "install_mlops"
-      },
-      "source": [
-        "## Installations\n",
-        "\n",
-        "Install the required packages for executing the notebook."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "install_mlops"
-      },
-      "outputs": [],
-      "source": [
-        "import os\n",
-        "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
-        "! pip3 install --pre --upgrade google-cloud-pipeline-components $USER_FLAG -q"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "restart"
-      },
-      "source": [
-        "### Restart the kernel\n",
-        "\n",
-        "Once you've installed the additional packages, you need to restart the notebook kernel so it can find the packages."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "restart"
-      },
-      "outputs": [],
-      "source": [
-        "import os\n",
-        "\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    # Automatically restart kernel after installs\n",
-        "    import IPython\n",
-        "\n",
-        "    app = IPython.Application.instance()\n",
-        "    app.kernel.do_shutdown(True)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "BF1j6f9HApxa"
-      },
-      "source": [
-        "## Before you begin\n",
-        "\n",
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI API and Dataflow API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,dataflow.googleapis.com).\n",
-        "\n",
-        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "project_id"
-      },
-      "source": [
-        "#### Set your project ID\n",
-        "\n",
-        "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "set_project_id"
-      },
-      "outputs": [],
-      "source": [
-        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "autoset_project_id"
-      },
-      "outputs": [],
-      "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
-        "    # Get your GCP project id from gcloud\n",
-        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
-        "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID:\", PROJECT_ID)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "250cb8c648d5"
-      },
-      "outputs": [],
-      "source": [
-        "! gcloud config set project $PROJECT_ID"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "region"
-      },
-      "source": [
-        "#### Region\n",
-        "\n",
-        "You can also change the `REGION` variable, which is used for operations\n",
-        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
-        "\n",
-        "- Americas: `us-central1`\n",
-        "- Europe: `europe-west4`\n",
-        "- Asia Pacific: `asia-east1`\n",
-        "\n",
-        "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
-        "\n",
-        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "region"
-      },
-      "outputs": [],
-      "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "timestamp"
-      },
-      "source": [
-        "#### Timestamp\n",
-        "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "timestamp"
-      },
-      "outputs": [],
-      "source": [
-        "from datetime import datetime\n",
-        "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "927085b84a07"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
-        "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
-        "\n",
-        "**Click Create service account**.\n",
-        "\n",
-        "In the **Service account name** field, enter a name, and click **Create**.\n",
-        "\n",
-        "In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex\" into the filter box, and select **Vertex Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "Click Create. A JSON file that contains your key downloads to your local environment.\n",
-        "\n",
-        "Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "89788a802687"
-      },
-      "outputs": [],
-      "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = False\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        IS_COLAB = True\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "40ed98f5cc48"
-      },
-      "source": [
-        "#### If you are using Colab Notebooks, set the project using gcloud config."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "fde1a355f1e9"
-      },
-      "outputs": [],
-      "source": [
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        ! gcloud config set project $PROJECT_ID"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "bucket:mbsdk"
-      },
-      "source": [
-        "### Create a Cloud Storage bucket\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "When you initialize the Vertex SDK for Python, you specify a Cloud Storage staging bucket. The staging bucket is where all the data associated with your dataset and model resources are retained across sessions.\n",
-        "\n",
-        "Set the name of your Cloud Storage bucket below. Bucket names must be globally unique across all Google Cloud projects, including those outside of your organization."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "bucket"
-      },
-      "outputs": [],
-      "source": [
-        "BUCKET_URI = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "autoset_bucket"
-      },
-      "outputs": [],
-      "source": [
-        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "create_bucket"
-      },
-      "source": [
-        "**Only if your bucket doesn't already exist**: Run the following cell to create your Cloud Storage bucket."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "create_bucket"
-      },
-      "outputs": [],
-      "source": [
-        "! gsutil mb -l $REGION $BUCKET_URI"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "validate_bucket"
-      },
-      "source": [
-        "Finally, validate access to your Cloud Storage bucket by examining its contents:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "validate_bucket"
-      },
-      "outputs": [],
-      "source": [
-        "! gsutil ls -al $BUCKET_URI"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "set_service_account"
-      },
-      "source": [
-        "#### Service Account\n",
-        "\n",
-        "**If you don't know your service account**, try to get your service account using `gcloud` command by executing the second cell below."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "set_service_account"
-      },
-      "outputs": [],
-      "source": [
-        "SERVICE_ACCOUNT = \"[your-service-account]\"  # @param {type:\"string\"}"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "autoset_service_account"
-      },
-      "outputs": [],
-      "source": [
-        "if (\n",
-        "    SERVICE_ACCOUNT == \"\"\n",
-        "    or SERVICE_ACCOUNT is None\n",
-        "    or SERVICE_ACCOUNT == \"[your-service-account]\"\n",
-        "):\n",
-        "    # Get your service account from gcloud\n",
-        "    if not IS_COLAB:\n",
-        "        shell_output = !gcloud auth list 2>/dev/null\n",
-        "        SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
-        "\n",
-        "    if IS_COLAB:\n",
-        "        shell_output = ! gcloud projects describe  $PROJECT_ID\n",
-        "        project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n",
-        "        SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n",
-        "\n",
-        "    print(\"Service Account:\", SERVICE_ACCOUNT)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "set_service_account:pipelines"
-      },
-      "source": [
-        "#### Set service account access for Vertex AI Pipelines\n",
-        "\n",
-        "Run the following commands to grant your service account access to read and write pipeline artifacts in the bucket that you created in the previous step -- you only need to run these once per service account."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "set_service_account:pipelines"
-      },
-      "outputs": [],
-      "source": [
-        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
-        "\n",
-        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "setup_vars"
-      },
-      "source": [
-        "### Set up variables\n",
-        "\n",
-        "Next, set up some variables used throughout the tutorial.\n",
-        "### Import libraries and define constants"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "import_aip:mbsdk"
-      },
-      "outputs": [],
-      "source": [
-        "import google.cloud.aiplatform as aip\n",
-        "from google_cloud_pipeline_components.experimental.dataflow import \\\n",
-        "    DataflowFlexTemplateJobOp\n",
-        "from google_cloud_pipeline_components.v1.wait_gcp_resources import \\\n",
-        "    WaitGcpResourcesOp\n",
-        "from kfp import dsl\n",
-        "from kfp.v2 import compiler"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "init_aip:mbsdk"
-      },
-      "source": [
-        "### Initialize Vertex AI SDK for Python\n",
-        "\n",
-        "Initialize the Vertex AI SDK for Python for your project and corresponding bucket."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "init_aip:mbsdk"
-      },
-      "outputs": [],
-      "source": [
-        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "writefile:wc.py"
-      },
-      "source": [
-        "### Prepare a Flex Template job to convert a CSV file to Parquet\n",
-        "\n",
-        "In this tutorial, you use the Google-provided [File Format Conversion template](https://cloud.google.com/dataflow/docs/guides/templates/provided/file-format-conversion) to convert a CSV file to Parquet format."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "writefile:requirements,wc"
-      },
-      "source": [
-        "#### Create the Avro schema file\n",
-        "\n",
-        "First, create an Avro schema file that describes the example data."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "writefile:requirements,wc"
-      },
-      "outputs": [],
-      "source": [
-        "%%writefile gaming_schema.avsc\n",
-        "\n",
-        "{\n",
-        "   \"type\" : \"record\",\n",
-        "   \"name\" : \"user_score\",\n",
-        "   \"fields\" : [\n",
-        "      { \"name\" : \"user\" , \"type\" : \"string\" },\n",
-        "      { \"name\" : \"team\" , \"type\" : \"string\" },\n",
-        "      { \"name\" : \"score\" , \"type\" : \"int\" },\n",
-        "      { \"name\" : \"ts_epoch\" , \"type\" : \"long\" },\n",
-        "      { \"name\" : \"ts_str\" , \"type\" : \"string\" }\n",
-        "   ]\n",
-        "}"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "copy_to_gcs:wc"
-      },
-      "source": [
-        "#### Copy the Avro schema file to Cloud Storage\n",
-        "\n",
-        "Next, you copy the Avro schema file to your Cloud Storage bucket.\n",
-        "\n",
-        "Additional, you set the following:\n",
-        "\n",
-        "- The Cloud Storage location of the Flex Template definition file. In this tutorial, you use the location of the File Conversion template.\n",
-        "- A `Dict` containing the user parameters required by the Flex Template. These parameters include:\n",
-        "\n",
-        "    - `inputFileFormat`: The file format of the input files.\n",
-        "    - `outputFileFormat`: The file format of the output files.\n",
-        "    - `inputFileSpec`: The input filepattern to read from.\n",
-        "    - `outputBucket`: The Cloud Storage path to write the output files.\n",
-        "    - `schema`: The Cloud Storage path top the avro schema file used for the conversion.\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "copy_to_gcs:wc"
-      },
-      "outputs": [],
-      "source": [
-        "GCS_AVRO_SCHEMA = BUCKET_URI + \"/gaming_schema.avsc\"\n",
-        "! gsutil cp gaming_schema.avsc $GCS_AVRO_SCHEMA\n",
-        "\n",
-        "GCS_FLEX_TEMPLATE_PATH = \"gs://dataflow-templates/latest/flex/File_Format_Conversion\"\n",
-        "GCS_CONVERT_IN = \"gs://dataflow-samples/game/5000_gaming_data.csv\"\n",
-        "GCS_CONVERT_OUT = BUCKET_URI + \"/parquet_out/\"\n",
-        "\n",
-        "TEMPLATE_PARAMETERS = {\n",
-        "    \"inputFileFormat\": \"csv\",\n",
-        "    \"outputFileFormat\": \"parquet\",\n",
-        "    \"inputFileSpec\": GCS_CONVERT_IN,\n",
-        "    \"outputBucket\": GCS_CONVERT_OUT,\n",
-        "    \"schema\": GCS_AVRO_SCHEMA,\n",
-        "}"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "create_dataflow_pipeline:wc"
-      },
-      "source": [
-        "### Create and execute the pipeline job\n",
-        "\n",
-        "In this example, the `DataflowFlexTemplateJobOp` component takes the following parameters:\n",
-        "\n",
-        "- `project_id`: The project ID.\n",
-        "- `location`: The region.\n",
-        "- `container_spec_gcs_path`: The Cloud Storage path to a file that contains the Flex Template definition. This file contains a json serialized `ContainerSpec` as content.\n",
-        "- `temp_location`: The Cloud Storage path to use for temporary files.\n",
-        "- `parameters`: The parameters for the Flex Template.\n",
-        "\n",
-        "Learn more about [Google Cloud Pipeline Components for Dataflow](https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-2.0.0b2/google_cloud_pipeline_components.experimental.dataflow.html)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "create_dataflow_pipeline:wc"
-      },
-      "outputs": [],
-      "source": [
-        "PIPELINE_ROOT = \"{}/pipeline_root/dataflow_file_conversion\".format(BUCKET_URI)\n",
-        "\n",
-        "\n",
-        "@dsl.pipeline(\n",
-        "    name=\"dataflow-file-conversion\", description=\"Dataflow file format conversion\"\n",
-        ")\n",
-        "def pipeline(\n",
-        "    project_id: str = PROJECT_ID,\n",
-        "    location: str = REGION,\n",
-        "    container_spec_gcs_path: str = GCS_FLEX_TEMPLATE_PATH,\n",
-        "    temp_location: str = PIPELINE_ROOT,\n",
-        "    parameters: dict = TEMPLATE_PARAMETERS,\n",
-        "):\n",
-        "    flex_template_op = DataflowFlexTemplateJobOp(\n",
-        "        project=project_id,\n",
-        "        location=location,\n",
-        "        container_spec_gcs_path=container_spec_gcs_path,\n",
-        "        temp_location=temp_location,\n",
-        "        parameters=parameters,\n",
-        "    )\n",
-        "\n",
-        "    _ = WaitGcpResourcesOp(gcp_resources=flex_template_op.outputs[\"gcp_resources\"])\n",
-        "\n",
-        "\n",
-        "compiler.Compiler().compile(\n",
-        "    pipeline_func=pipeline, package_path=\"dataflow_file_conversion.yaml\"\n",
-        ")\n",
-        "\n",
-        "pipeline = aip.PipelineJob(\n",
-        "    display_name=\"dataflow_file_conversion\",\n",
-        "    template_path=\"dataflow_file_conversion.yaml\",\n",
-        "    pipeline_root=PIPELINE_ROOT,\n",
-        "    enable_caching=False,\n",
-        ")\n",
-        "\n",
-        "pipeline.run()\n",
-        "\n",
-        "! gsutil ls $GCS_CONVERT_OUT\n",
-        "\n",
-        "! rm -f dataflow_file_conversion.yaml gaming_schema.avsc"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "delete_pipeline"
-      },
-      "source": [
-        "### Delete a pipeline job\n",
-        "\n",
-        "After a pipeline job is completed, you can delete the pipeline job with the method `delete()`.  Prior to completion, a pipeline job can be canceled with the method `cancel()`."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "delete_pipeline"
-      },
-      "outputs": [],
-      "source": [
-        "pipeline.delete()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "cleanup:mbsdk"
-      },
-      "source": [
-        "# Cleaning up\n",
-        "\n",
-        "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
-        "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
-        "\n",
-        "Otherwise, you can delete the individual resources you created in this tutorial:\n",
-        "\n",
-        "- Dataset\n",
-        "- Cloud Storage Bucket"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "cleanup:mbsdk"
-      },
-      "outputs": [],
-      "source": [
-        "# Warning: Setting this to true will delete everything in your bucket\n",
-        "delete_bucket = False\n",
-        "\n",
-        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
-        "    ! gsutil rm -r $BUCKET_URI"
-      ]
-    }
-  ],
-  "metadata": {
-    "colab": {
-      "name": "get_started_with_dataflow_flex_template_component.ipynb",
-      "toc_visible": true
-    },
-    "kernelspec": {
-      "display_name": "Python 3",
-      "name": "python3"
-    }
-  },
-  "nbformat": 4,
-  "nbformat_minor": 0
-}

notebooks/community/ml_ops/stage3/get_started_with_dataproc_serverless_pipeline_components.ipynb

@@ -72,7 +72,7 @@
       "source": [
         "### Objective\n",
         "\n",
-        "In this tutorial, you learn how to use prebuilt `Google Cloud Pipeline Components` for `Dataproc Serverless` service. The documentation for the components can be found [here](https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-1.0.26/google_cloud_pipeline_components.v1.dataproc.html).\n",
+        "In this tutorial, you learn how to use prebuilt `Google Cloud Pipeline Components` for `Dataproc Serverless` service. The documentation for the components can be found [here](https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-1.0.0/google_cloud_pipeline_components.experimental.dataproc.html).\n",
         "\n",
         "This tutorial uses the following Google Cloud ML services:\n",
         "\n",
@@ -759,14 +759,14 @@
         "\n",
         "In this example, the `DataprocPySparkBatchOp` component takes the following parameters:\n",
         "\n",
+        "- `batch_id`: The batch ID to use for the Dataproc Batch workload.\n",
         "- `project_id`: The project ID.\n",
         "- `location`: The region.\n",
         "- `main_python_file_uri`: The URI of the main Python file.\n",
         "- `service_account`: The service account that runs the workload.\n",
         "- `args`: The arguments to pass to the PySpark program.\n",
-        "- `batch_id`: (Optional) The batch ID to use for the Dataproc Batch workload.\n",
         "\n",
-        "Learn more about the [Dataproc Serverless PySpark batch component](https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-1.0.26/google_cloud_pipeline_components.v1.dataproc.html#google_cloud_pipeline_components.v1.dataproc.DataprocPySparkBatchOp)."
+        "Learn more about the [Dataproc Serverless PySpark batch component](https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-1.0.0/google_cloud_pipeline_components.experimental.dataproc.html#google_cloud_pipeline_components.experimental.dataproc.DataprocPySparkBatchOp)."
       ]
     },
     {
@@ -799,16 +799,16 @@
         "    service_account: str = SERVICE_ACCOUNT,\n",
         "    args: list = ARGS,\n",
         "):\n",
-        "    from google_cloud_pipeline_components.v1.dataproc import \\\n",
+        "    from google_cloud_pipeline_components.experimental.dataproc import \\\n",
         "        DataprocPySparkBatchOp\n",
         "\n",
         "    _ = DataprocPySparkBatchOp(\n",
         "        project=project_id,\n",
         "        location=location,\n",
+        "        batch_id=batch_id,\n",
         "        main_python_file_uri=main_python_file_uri,\n",
         "        service_account=service_account,\n",
         "        args=args,\n",
-        "        batch_id=batch_id,  # `batch_id` is optional\n",
         "    )\n",
         "\n",
         "\n",
@@ -979,15 +979,15 @@
         "\n",
         "In this example, the `DataprocSparkBatchOp` component takes the following parameters:\n",
         "\n",
+        "- `batch_id`: The batch ID to use for the Dataproc Batch workload.\n",
         "- `project_id`: The project ID.\n",
         "- `location`: The region.\n",
         "- `main_class`: The main class.\n",
         "- `jar_file_uris`: The URIs of any required JARs to include in the executor and driver CLASSPATH.\n",
         "- `service_account`: The service account that runs the workload.\n",
         "- `args`: The arguments to pass to the Spark program.\n",
-        "- `batch_id`: (Optional) The batch ID to use for the Dataproc Batch workload.\n",
         "\n",
-        "Learn more about the [Dataproc Serverless Spark batch component](https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-1.0.26/google_cloud_pipeline_components.v1.dataproc.html#google_cloud_pipeline_components.v1.dataproc.DataprocSparkBatchOp)."
+        "Learn more about the [Dataproc Serverless Spark batch component](https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-1.0.0/google_cloud_pipeline_components.experimental.dataproc.html#google_cloud_pipeline_components.experimental.dataproc.DataprocSparkBatchOp)."
       ]
     },
     {
@@ -1019,17 +1019,17 @@
         "    service_account: str = SERVICE_ACCOUNT,\n",
         "    args: list = ARGS,\n",
         "):\n",
-        "    from google_cloud_pipeline_components.v1.dataproc import \\\n",
+        "    from google_cloud_pipeline_components.experimental.dataproc import \\\n",
         "        DataprocSparkBatchOp\n",
         "\n",
         "    _ = DataprocSparkBatchOp(\n",
         "        project=project_id,\n",
         "        location=location,\n",
+        "        batch_id=batch_id,\n",
         "        main_class=main_class,\n",
         "        jar_file_uris=jar_file_uris,\n",
         "        service_account=service_account,\n",
         "        args=args,\n",
-        "        batch_id=batch_id,  # `batch_id` is optional\n",
         "    )\n",
         "\n",
         "\n",
@@ -1281,14 +1281,14 @@
         "\n",
         "In this example, the `DataprocSparkSqlBatchOp` component takes the following parameters:\n",
         "\n",
+        "- `batch_id`: The batch ID to use for the Dataproc Batch workload.\n",
         "- `project_id`: The project ID.\n",
         "- `location`: The region.\n",
         "- `query_file_uri`: The URI of the file containing the SQL queries.\n",
         "- `query_variables`: The mapping of query variable names to values (equivalent to the Spark SQL command  `SET name=\"value\";`).\n",
         "- `service_account`: The service account that runs the workload.\n",
-        "- `batch_id`: (Optional) The batch ID to use for the Dataproc Batch workload.\n",
         "\n",
-        "Learn more about the [Dataproc Serverless Spark SQL batch component](https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-1.0.26/google_cloud_pipeline_components.v1.dataproc.html#google_cloud_pipeline_components.v1.dataproc.DataprocSparkSqlBatchOp)."
+        "Learn more about the [Dataproc Serverless Spark SQL batch component](https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-1.0.0/google_cloud_pipeline_components.experimental.dataproc.html#google_cloud_pipeline_components.experimental.dataproc.DataprocSparkSqlBatchOp)."
       ]
     },
     {
@@ -1326,16 +1326,16 @@
         "    query_variables: dict = QUERY_VARIABLES,\n",
         "    service_account: str = SERVICE_ACCOUNT,\n",
         "):\n",
-        "    from google_cloud_pipeline_components.v1.dataproc import \\\n",
+        "    from google_cloud_pipeline_components.experimental.dataproc import \\\n",
         "        DataprocSparkSqlBatchOp\n",
         "\n",
         "    _ = DataprocSparkSqlBatchOp(\n",
         "        project=project_id,\n",
         "        location=location,\n",
+        "        batch_id=batch_id,\n",
         "        query_file_uri=query_file_uri,\n",
         "        query_variables=query_variables,\n",
         "        service_account=service_account,\n",
-        "        batch_id=batch_id,  # `batch_id` is optional\n",
         "    )\n",
         "\n",
         "\n",
@@ -1502,14 +1502,14 @@
         "\n",
         "In this example, the `DataprocSparkRBatchOp` component takes the following parameters:\n",
         "\n",
+        "- `batch_id`: The batch ID to use for the Dataproc Batch workload.\n",
         "- `project_id`: The project ID.\n",
         "- `location`: The region.\n",
         "- `main_r_file_uri`: The URI of the main R file.\n",
         "- `service_account`: The service account that runs the workload.\n",
         "- `args`: The arguments to pass to the Spark program.\n",
-        "- `batch_id`: (Optional) The batch ID to use for the Dataproc Batch workload.\n",
         "\n",
-        "Learn more about the [Dataproc Serverless SparkR batch component](https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-1.0.26/google_cloud_pipeline_components.v1.dataproc.html#google_cloud_pipeline_components.v1.dataproc.DataprocSparkRBatchOp)."
+        "Learn more about the [Dataproc Serverless SparkR batch component](https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-1.0.0/google_cloud_pipeline_components.experimental.dataproc.html#google_cloud_pipeline_components.experimental.dataproc.DataprocSparkRBatchOp)."
       ]
     },
     {
@@ -1539,15 +1539,15 @@
         "    service_account: str = SERVICE_ACCOUNT,\n",
         "    args: list = ARGS,\n",
         "):\n",
-        "    from google_cloud_pipeline_components.v1.dataproc import \\\n",
+        "    from google_cloud_pipeline_components.experimental.dataproc import \\\n",
         "        DataprocSparkRBatchOp\n",
         "\n",
         "    _ = DataprocSparkRBatchOp(\n",
         "        project=project_id,\n",
         "        location=location,\n",
+        "        batch_id=batch_id,\n",
         "        main_r_file_uri=main_r_file_uri,\n",
         "        args=args,\n",
-        "        batch_id=batch_id,  # `batch_id` is optional\n",
         "    )\n",
         "\n",
         "\n",