remove preview models in description

Bumps [tqdm](https://github.com/tqdm/tqdm) from 4.66.1 to 4.66.3.
- [Release notes](https://github.com/tqdm/tqdm/releases)
- [Commits](https://github.com/tqdm/tqdm/compare/v4.66.1...v4.66.3)

---
updated-dependencies:
- dependency-name: tqdm
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

.cloud-build/cleanup/cleanup.py

@@ -1,5 +1,14 @@
 from typing import List
 from ratemate import RateLimit
+import argparse
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--dry_run",
+                    type=bool,
+                    default=False)
+args = parser.parse_args()
+
+
 from resource_cleanup_manager import (
     DatasetResourceCleanupManager,
     ModelResourceCleanupManager,
@@ -7,6 +16,15 @@ from resource_cleanup_manager import (
     ResourceCleanupManager,
     MatchingEngineIndexEndpointResourceCleanupManager,
     MatchingEngineIndexResourceCleanupManager,
+    FeatureStoreLegacyCleanupManager,
+    FeatureStoreCleanupManager,
+    PipelineJobCleanupManager,
+    TrainingJobCleanupManager,
+    HyperparameterTuningCleanupManager,
+    BatchPredictionJobCleanupManager,
+    ExperimentCleanupManager,
+    BucketCleanupManager,
+    ArtifactRegistryCleanupManager
 )
 
 rate_limit = RateLimit(max_count=25, per=60, greedy=False)
@@ -18,12 +36,14 @@ def run_cleanup_managers(managers: List[ResourceCleanupManager], is_dry_run: boo
 
         print(f"Fetching {type_name}'s...")
         resources = manager.list()
-        print(f"Found {len(resources)} {type_name}'s")
+        try:
+            print(f"Found {len(resources)} {type_name}'s")
+        except Exception as e:
+            print(f"{type_name} {e}")
         for resource in resources:
             try:
                 if not manager.is_deletable(resource):
                     continue
-
                 if is_dry_run:
                     resource_name = manager.resource_name(resource)
                     print(f"Will delete '{type_name}': {resource_name}")
@@ -36,9 +56,7 @@ def run_cleanup_managers(managers: List[ResourceCleanupManager], is_dry_run: boo
         print("")
 
 
-is_dry_run = False
-
-if is_dry_run:
+if args.dry_run:
     print("Starting cleanup in dry run mode...")
 
 # List of all cleanup managers
@@ -48,6 +66,15 @@ managers: List[ResourceCleanupManager] = [
     ModelResourceCleanupManager(),  # ModelResourceCleanupManager must follow EndpointResourceCleanupManager due to deployed models blocking model deletion.
     MatchingEngineIndexEndpointResourceCleanupManager(),
     MatchingEngineIndexResourceCleanupManager(),
+    FeatureStoreLegacyCleanupManager(),
+    FeatureStoreCleanupManager(),
+    PipelineJobCleanupManager(),
+    TrainingJobCleanupManager(),
+    HyperparameterTuningCleanupManager(),
+    BatchPredictionJobCleanupManager(),
+    ExperimentCleanupManager(), # Experiment missing _resource_noun
+    BucketCleanupManager(),
+    ArtifactRegistryCleanupManager()
 ]
 
-run_cleanup_managers(managers=managers, is_dry_run=is_dry_run)
+run_cleanup_managers(managers=managers, is_dry_run=args.dry_run)

.cloud-build/cleanup/resource_cleanup_manager.py

@@ -1,10 +1,26 @@
+'''
+READ FIRST BEFORE MAKING CHANGES
+-  Create a convention for resources created from vertex-ai-samples GH. We already have one IIRC
+- Only delete those objects as part of our clean-up script.
+- Don't run any tests on python-docs-samples-tests project, especially ones that affect resources created outside of our purview
+- Add --dry-run option to the clean-up script. This option will just output the list of resources the script will delete instead of actually deleting the resources.
+- Have a larger conversation in DEE before touching any resources that were not created as part of vertex-ai-samples
+'''
+import os
 import abc
 from typing import Any, Type
 
 from google.cloud import aiplatform
 from google.cloud.aiplatform import base
+from google.cloud.aiplatform_v1beta1 import (FeatureOnlineStoreAdminServiceClient,
+                                             FeatureOnlineStore)
+from google.cloud import storage
 from proto.datetime_helpers import DatetimeWithNanoseconds
 
+PROJECT_ID = "python-docs-samples-tests"
+REGION = "us-central1"
+API_ENDPOINT = f"{REGION}-aiplatform.googleapis.com"
+
 # If a resource was updated within this number of seconds, do not delete.
 RESOURCE_UPDATE_BUFFER_IN_SECONDS = 60 * 60 * 8
 
@@ -69,7 +85,7 @@ class VertexAIResourceCleanupManager(ResourceCleanupManager):
     def delete(self, resource):
         resource.delete()
 
-    def get_seconds_since_modification(self, resource: Any) -> bool:
+    def get_seconds_since_modification(self, resource: Any) -> float:
         update_time = resource.update_time
         current_time = DatetimeWithNanoseconds.now(tz=update_time.tzinfo)
         return (current_time - update_time).total_seconds()
@@ -97,13 +113,10 @@ class EndpointResourceCleanupManager(VertexAIResourceCleanupManager):
     vertex_ai_resource = aiplatform.Endpoint
 
     def delete(self, resource):
-        # TODO: Remove this once https://github.com/googleapis/python-aiplatform/issues/1441 is fixed
-        resource._sync_gca_resource()
         for deployed_model_id in [
             models.id for models in resource._gca_resource.deployed_models
         ]:
             resource._undeploy(deployed_model_id=deployed_model_id)
-
         resource.delete(force=True)
 
 
@@ -117,3 +130,176 @@ class MatchingEngineIndexResourceCleanupManager(VertexAIResourceCleanupManager):
 
 class MatchingEngineIndexEndpointResourceCleanupManager(VertexAIResourceCleanupManager):
     vertex_ai_resource = aiplatform.MatchingEngineIndexEndpoint
+
+    def delete(self, resource):
+        resource.undeploy_all()
+        resource.delete(force=True)
+
+class FeatureStoreLegacyCleanupManager(VertexAIResourceCleanupManager):
+    # TODO: only deleting legacy
+    #    not deleting ingestions jobs
+    #       ingest_from_xxx methods do not return a job ID, there is no list command, aka no python way to delete
+    #    not deleting batch serving jobs
+    #       batch_serve_to_xxx methods do not return a job ID, there is no list command, aka no python way to delete
+    vertex_ai_resource = aiplatform.Featurestore
+
+    def resource_name(self, resource: Any) -> str:
+        return resource.name
+
+    def delete(self, resource):
+        resource.delete(force=True)
+
+
+class FeatureStoreCleanupManager(VertexAIResourceCleanupManager):
+    # for FS 2.0
+    # TODO: use _v1beta1, and gapic clients
+    #    delete features, feature groups, feature views, feature online stores
+    vertex_ai_resource = FeatureOnlineStore
+
+    admin_client = FeatureOnlineStoreAdminServiceClient(
+        client_options={"api_endpoint": API_ENDPOINT}
+    )
+
+    def resource_name(self, resource: Any) -> str:
+        return resource.name
+
+    def type_name(self) -> str:
+        return "FeatureOnlineStore"
+
+    def list(self) -> Any:
+        try:
+            return self.admin_client.list_feature_online_stores(parent=f"projects/{PROJECT_ID}/locations/{REGION}")
+        except Exception as e:
+            print(e)
+            return []
+
+    def delete(self, resource):
+        try:
+            self.admin_client.delete_feature_online_store(name=resource.name, force=True)
+        except Exception as e:
+            print(e)
+
+
+class PipelineJobCleanupManager(VertexAIResourceCleanupManager):
+    vertex_ai_resource = aiplatform.PipelineJob
+
+class TrainingJobCleanupManager(VertexAIResourceCleanupManager):
+    vertex_ai_resource = aiplatform.training_jobs._CustomTrainingJob
+
+    job_types = [
+            aiplatform.AutoMLImageTrainingJob,
+            aiplatform.AutoMLTextTrainingJob,
+            aiplatform.AutoMLTabularTrainingJob,
+            aiplatform.AutoMLVideoTrainingJob,
+            aiplatform.AutoMLForecastingTrainingJob,
+            aiplatform.CustomJob,
+            aiplatform.CustomTrainingJob,
+            aiplatform.CustomContainerTrainingJob,
+            aiplatform.CustomPythonPackageTrainingJob
+    ]
+
+    def list(self) -> Any:
+        return [
+            job
+            for job_type in self.job_types
+            for job in job_type.list()
+        ]
+
+class HyperparameterTuningCleanupManager(VertexAIResourceCleanupManager):
+    vertex_ai_resource = aiplatform.HyperparameterTuningJob
+
+
+class BatchPredictionJobCleanupManager(VertexAIResourceCleanupManager):
+    vertex_ai_resource = aiplatform.BatchPredictionJob
+
+class ExperimentCleanupManager(VertexAIResourceCleanupManager):
+    vertex_ai_resource = aiplatform.Experiment
+
+    @property
+    def type_name(self) -> str:
+        return "Experiment"
+
+    def resource_name(self, resource: Any) -> str:
+        return resource.name
+
+    def get_seconds_since_modification(self, resource: Any) -> float:
+        update_time = resource._metadata_context.update_time
+        current_time = DatetimeWithNanoseconds.now()
+        return float(current_time.timestamp() - update_time.timestamp())
+
+class BucketCleanupManager(ResourceCleanupManager):
+    vertex_ai_resource = storage.bucket.Bucket
+
+    def list(self) -> Any:
+        storage_client = storage.Client()
+        return list(storage_client.list_buckets())
+
+    def delete(self, resource):
+        try:
+            resource.delete(force=True)
+        except Exception as e:
+            print(e)
+
+    @property
+    def type_name(self) -> str:
+        return "Bucket"
+
+    def get_seconds_since_modification(self, resource: Any) -> float:
+        # Bucket has no last_update property, only time created
+        created_time = resource.time_created
+        current_time = DatetimeWithNanoseconds.now()
+        return float(current_time.timestamp() - created_time.timestamp())
+
+    def resource_name(self, resource: Any) -> str:
+        return resource.name
+
+    def is_deletable(self, resource: Any) -> bool:
+        time_difference = self.get_seconds_since_modification(resource)
+
+        if not self.resource_name(resource).startswith('your-bucket-name'):
+            print(f"Skipping '{resource}' not a Vertex AI notebook bucket")
+            return False
+
+        # Check that it wasn't created too recently, to prevent race conditions
+        if time_difference <= RESOURCE_UPDATE_BUFFER_IN_SECONDS:
+            print(
+                f"Skipping '{resource}' due to update_time being '{time_difference}', which is less than '{RESOURCE_UPDATE_BUFFER_IN_SECONDS}'."
+            )
+            return False
+        return True
+
+class ArtifactRegistryCleanupManager(ResourceCleanupManager):
+    vertex_ai_resource = "Artifact Registry"
+
+    def list(self) -> Any:
+        import subprocess
+
+        result = subprocess.run(["gcloud artifacts repositories list --location=us-central1"], 
+                                shell=True, capture_output=True, text=True)
+
+        ret = []
+        lines = result.stdout.split('\n')[2:]
+        for line in lines:
+            repo = line.split(' ')[0]
+            if repo.startswith("my-docker-repo"):
+                ret.append(repo)
+
+        return ret
+
+    def delete(self, resource):
+        os.system(f"! gcloud artifacts repositories delete {resource} --location=us-central1")
+
+    @property
+    def type_name(self) -> str:
+        return "ArtifactRepository"
+
+    def resource_name(self, resource: Any) -> str:
+        return resource
+
+    # delete repository regardless of age
+    def get_seconds_since_modification(self, resource: Any) -> float:
+        return RESOURCE_UPDATE_BUFFER_IN_SECONDS + 1
+    
+    def is_deleteable(self, resource: Any) -> bool:
+        return True
+

.cloud-build/execute_changed_notebooks_cli.py

@@ -17,6 +17,8 @@
 
 import argparse
 import pathlib
+import os
+import csv
 
 import execute_changed_notebooks_helper
 
@@ -36,9 +38,22 @@ parser = argparse.ArgumentParser(description="Run changed notebooks.")
 parser.add_argument(
     "--test_paths_file",
     type=pathlib.Path,
-    help="The path to the file that has newline-limited folders of notebooks that should be tested.",
+    help="The path to the file that has newline-delimited folders of notebooks that should be tested.",
     required=True,
 )
+parser.add_argument(
+    "--test_percent",
+    type=int,
+    help="The percent of notebooks to be tested (between 1 and 100).",
+    required=False,
+    default=100,
+)
+parser.add_argument(
+    "--build_id",
+    type=str,
+    help="The build id (which may be a Cloud Build job specific or user explicit.",
+    required=True
+)
 parser.add_argument(
     "--base_branch",
     help="The base git branch to diff against to find changed files.",
@@ -107,24 +122,98 @@ parser.add_argument(
     default=True,
     help="Should run notebooks in parallel.",
 )
+parser.add_argument(
+    "--concurrent_notebooks",
+    type=int,
+    help="Maximum number of parallel notebook executions per minute",
+    default=10,
+    required=False,
+)
+parser.add_argument(
+    "--run_first_file",
+    type=pathlib.Path,
+    help="The path to the file that has newline-delimited of notebooks to run in the first batch",
+    default=None,
+    required=False,
+)
+parser.add_argument(
+    "--aiplatform_whl",
+    type=str,
+    help="The GCS path to a whl version google-cloud-aiplatform",
+    default=None,
+    required=False,
+)
+parser.add_argument(
+    "--dry_run",
+    type=str2bool,
+    default=False,
+    help="Dry run for testing - no execution",
+)
 
 args = parser.parse_args()
 
-notebooks = execute_changed_notebooks_helper.get_changed_notebooks(
+changed_notebooks = execute_changed_notebooks_helper.get_changed_notebooks(
     test_paths_file=args.test_paths_file,
     base_branch=args.base_branch,
 )
 
-execute_changed_notebooks_helper.process_and_execute_notebooks(
-    notebooks=notebooks,
-    container_uri=args.container_uri,
-    staging_bucket=args.staging_bucket,
-    artifacts_bucket=args.artifacts_bucket,
-    should_parallelize=args.should_parallelize,
-    timeout=args.timeout,
-    variable_project_id=args.variable_project_id,
-    variable_region=args.variable_region,
-    variable_service_account=args.variable_service_account,
-    variable_vpc_network=args.variable_vpc_network,
-    private_pool_id=args.private_pool_id,
+
+results_bucket = f"{args.artifacts_bucket}"
+# artifacts_bucket may get set by trigger to a full gs:// folder path
+if results_bucket.startswith("gs://"):
+    results_bucket = results_bucket[5:]
+results_bucket = results_bucket.split('/')[0]
+results_file = f"build_results/{args.build_id}.json"
+
+if args.test_percent == 100:
+    notebooks = changed_notebooks
+    accumulative_results = {}
+else:
+    accumulative_results = execute_changed_notebooks_helper.load_results(results_bucket, results_file)
+
+    notebooks = [changed_notebook for changed_notebook in changed_notebooks if execute_changed_notebooks_helper.select_notebook(changed_notebook, accumulative_results, args.test_percent)]
+    # cap the number of notebooks to the specified percentage
+    max_notebooks = int((len(changed_notebooks) * (args.test_percent/100)))
+    if (len(notebooks) > max_notebooks):
+        notebooks = notebooks[:max_notebooks]
+
+run_first = []
+if args.run_first_file:
+    if not os.path.isfile(args.run_first_file):
+        print("Error: file does not exist", args.run_first_file)
+    else:
+        with open(args.run_first_file, 'r') as csvfile:
+            reader = csv.reader(csvfile)
+            for row in reader:
+                notebook = row[0]
+                run_first.append(notebook)
+
+    for notebook in run_first:
+        if notebook in notebooks:
+            # remove from existing list
+            notebooks.remove(notebook)
+            # add back to the front of the list
+            notebooks.insert(0, notebook)
+            print(f"Run first: {notebook}")
+
+if args.dry_run:
+    print("Dry run ...\n")
+    for notebook in notebooks:
+        print(f"Would execute: {notebook}")
+else:
+    execute_changed_notebooks_helper.process_and_execute_notebooks(
+        notebooks=notebooks,
+        container_uri=args.container_uri,
+        staging_bucket=args.staging_bucket,
+        artifacts_bucket=args.artifacts_bucket,
+        results_file=results_file,
+        should_parallelize=args.should_parallelize,
+        timeout=args.timeout,
+        variable_project_id=args.variable_project_id,
+        variable_region=args.variable_region,
+        variable_service_account=args.variable_service_account,
+        variable_vpc_network=args.variable_vpc_network,
+        private_pool_id=args.private_pool_id,
+        concurrent_notebooks=args.concurrent_notebooks,
+        aiplatform_whl=args.aiplatform_whl
 )

.cloud-build/execute_changed_notebooks_helper.py

@@ -21,25 +21,34 @@ import json
 import git
 import operator
 import os
+import io
+import json
 import pathlib
 import re
 import subprocess
+import random
+from google.cloud import storage
 import utils
-from typing import List, Optional
+from typing import List, Optional, Dict, Any
 from utils import util
 
 import execute_notebook_helper
 import execute_notebook_remote
 import nbformat
 from google.cloud.devtools.cloudbuild_v1.types import BuildOperationMetadata
-from ratemate import RateLimit
 from tabulate import tabulate
 from utils import NotebookProcessors, util
 
 # A buffer so that workers finish before the orchestrating job
 WORKER_TIMEOUT_BUFFER_IN_SECONDS: int = 60 * 60
+
 PYTHON_VERSION = "3.9"  # Set default python version
 
+# rolling time window for accumulating build results for selecting notebooks
+MAX_RESULTS_AGE_SECONDS: int = (60 * 60) * 24 * 60  # 60 days
+# maximum time since last run to force a run on the current build
+MAX_AGE_BEFORE_FORCE_RUN: int = (60 * 60) * 24 * 30
+
 
 def format_timedelta(delta: datetime.timedelta) -> str:
     """Formats a timedelta duration to [N days] %H:%M:%S format"""
@@ -65,7 +74,9 @@ def format_timedelta(delta: datetime.timedelta) -> str:
 @dataclasses.dataclass
 class NotebookExecutionResult:
     name: str
+    path: str
     duration: datetime.timedelta
+    start_time: datetime.datetime
     is_pass: bool
     log_url: str
     output_uri: str
@@ -81,6 +92,97 @@ class NotebookExecutionResult:
             return None
 
 
+def load_results(results_bucket: str,
+                 results_file: str) -> Dict[str, Any]:
+    '''
+    Load accumulated notebook test results
+    '''
+
+    print("Loading existing accumulative results ...")
+    accumulative_results = {}
+    try:
+        client = storage.Client()
+        bucket = client.bucket(results_bucket)
+
+        build_results_dir = os.path.dirname(results_file)
+        blobs = client.list_blobs(results_bucket, prefix=build_results_dir)
+        for blob in blobs:
+            time_created = blob.time_created.replace(tzinfo=None)
+            if (datetime.datetime.now().replace(tzinfo=None) - time_created).total_seconds() > MAX_RESULTS_AGE_SECONDS:
+                continue
+
+            content = util.download_blob_into_memory(results_bucket, blob.name, download_as_text=True)
+
+            try:
+                build_results = json.loads(content)
+            except:
+                continue  # skip corrupted build results files
+            for notebook in build_results:
+                if notebook in accumulative_results:
+                    accumulative_results[notebook]['passed'] += build_results[notebook]['passed']
+                    accumulative_results[notebook]['failed'] += build_results[notebook]['failed']
+                    if accumulative_results[notebook]['last_time_ran'] < time_created:
+                        accumulative_results[notebook]['last_time_ran'] = time_created
+                else:
+                    accumulative_results[notebook] = build_results[notebook]
+                    accumulative_results[notebook]['failed_on_latest_run'] = build_results[notebook]['failed']
+                    accumulative_results[notebook]['last_time_ran'] = time_created
+
+        print(accumulative_results)
+    except Exception as e:
+        print(e)
+
+    # If there are no accumulative results, an empty dict is returned
+    return accumulative_results
+
+def select_notebook(changed_notebook: str,
+                    accumulative_results: Dict[str, Any],
+                    test_percent: int) -> bool:
+    '''
+    Algorithm to randomly select a notebook, but weight the propbability of selected based on past failures
+    '''
+
+    if changed_notebook in accumulative_results:
+        pass_count = accumulative_results[changed_notebook]['passed']
+        fail_count = accumulative_results[changed_notebook]['failed']
+        failed_on_latest_run = accumulative_results[changed_notebook]['failed_on_latest_run']
+        last_time_ran = accumulative_results[changed_notebook]['last_time_ran']
+    else:
+        pass_count = 1
+        fail_count = 0
+        failed_on_latest_run = 0
+        last_time_ran = datetime.datetime.now().replace(tzinfo=None)
+
+    # If notebook has not been ran in a long time, force running it
+    if (datetime.datetime.now().replace(tzinfo=None) - last_time_ran).total_seconds() > MAX_AGE_BEFORE_FORCE_RUN:
+        should_test_do_to_age = True
+    else:
+        should_test_do_to_age  = False
+
+
+    # if failed on the last time it was ran, select the notebook
+    if failed_on_latest_run:
+        inferred_failure_rate = 1
+    # otherwise, calculate the frequency of failure
+    else:
+        inferred_failure_rate = fail_count / (pass_count + fail_count)
+
+    # If failure rate is high, the chance of testing should be higher
+    should_test_due_to_failure = random.uniform(0, 1) <= inferred_failure_rate
+
+    #if accumulative_resultsi[changed_notebook]['latest_date_ran']
+
+    # Additionally, only test a percentage of these
+    should_test_due_to_random_subset = random.uniform(0, 1) <= (test_percent / 100)
+
+    if should_test_due_to_failure or should_test_due_to_random_subset or should_test_do_to_age:
+        print(f"Selected: {changed_notebook}, {should_test_due_to_failure}, {should_test_due_to_random_subset}")
+        return True
+    else:
+        print(f"Not Selected: {changed_notebook}, pass {pass_count}, fail {fail_count}")
+        return False
+
+
 def _process_notebook(
     notebook_path: str,
     variable_project_id: str,
@@ -136,7 +238,7 @@ def _get_notebook_python_version(notebook_path: str) -> str:
 
             # Look for the python version specification pattern
             re_match = re.search(
-                "python version = (\d\.\d)", markdown, flags=re.IGNORECASE
+                "python version = (\d+\.\d+)", markdown, flags=re.IGNORECASE
             )
             if re_match:
                 # get the version number
@@ -156,7 +258,6 @@ def _create_tag(filepath: str) -> str:
     return tag
 
 
-rate_limit = RateLimit(max_count=50, per=60, greedy=True)
 
 
 def process_and_execute_notebook(
@@ -170,9 +271,8 @@ def process_and_execute_notebook(
     private_pool_id: Optional[str],
     deadline: datetime.datetime,
     notebook: str,
-    should_get_tail_logs: bool = False,
+    should_get_tail_logs: bool = True,
 ) -> NotebookExecutionResult:
-    rate_limit.wait()  # wait before creating the task
 
     print(f"Running notebook: {notebook}")
 
@@ -191,7 +291,9 @@ def process_and_execute_notebook(
 
     result = NotebookExecutionResult(
         name=tag,
+        path=notebook,
         duration=datetime.timedelta(seconds=0),
+        start_time=datetime.datetime.now(),
         is_pass=False,
         output_uri=notebook_output_uri,
         log_url="",
@@ -201,7 +303,6 @@ def process_and_execute_notebook(
     )
 
     # TODO: Handle cases where multiple notebooks have the same name
-    time_start = datetime.datetime.now()
     operation = None
     try:
         # Get the python version for running the notebook if specified
@@ -247,9 +348,10 @@ def process_and_execute_notebook(
         # Block and wait for the result
         operation_result = operation.result(timeout=timeout_in_seconds)
 
-        result.duration = datetime.datetime.now() - time_start
+        result.duration = datetime.datetime.now() - result.start_time
         result.is_pass = True
         print(f"{notebook} PASSED in {format_timedelta(result.duration)}.")
+
     except Exception as error:
         result.error_message = str(error)
 
@@ -268,7 +370,7 @@ def process_and_execute_notebook(
             except Exception as error:
                 result.error_message = str(error)
 
-        result.duration = datetime.datetime.now() - time_start
+        result.duration = datetime.datetime.now() - result.start_time
         result.is_pass = False
 
         print(
@@ -336,12 +438,68 @@ def get_changed_notebooks(
 
     return notebooks
 
+def _save_results(results: List[NotebookExecutionResult],
+                  artifacts_bucket: str,
+                  results_file: str):
+
+    artifacts_bucket = artifacts_bucket.replace("gs://", "").split('/')[0]
+
+    print("Updating build results ...")
+    build_results = {}
+    for result in results:
+        if result.is_pass:
+            pass_count = 1
+            fail_count = 0
+        else:
+            pass_count = 0
+            fail_count = 1
+        if result.error_message is None:
+            error_type = ''
+        elif '500 Internal' in result.error_message or 'INTERNAL' in result.error_message or 'internal error' in result.error_message:
+            error_type = 'INTERNAL'
+        elif 'context deadline exceeded' in result.error_message or 'TIMEOUT' in result.error_message:
+            error_type = 'TIMEOUT'
+        elif 'Quota' in result.error_message or 'quotas are exceeded' in result.error_message:
+            error_type = 'QUOTA'
+        elif 'ServiceUnavailable' in result.error_message:
+            error_type = 'SERVICEUNAVAILABLE'
+        elif 'ModuleNotFoundError' in result.error_message:
+            error_type = 'IMPORT'
+        elif result.is_pass:
+            error_type = ''
+        else:
+            error_type = 'undetermined'
+
+        if error_type != '':
+            log_url = result.log_url
+        else:
+            log_url = ''
+
+        build_results[result.path] = {
+                'duration': result.duration.total_seconds(),
+                'start_time': str(result.start_time),
+                'passed': pass_count,
+                'failed': fail_count,
+                'error_type': error_type,
+                'log_url': log_url
+        }
+        print(f"adding {result.path}")
+
+    print(f"Saving accumulative results to {results_file}, nentries {len(build_results)}")
+    content = json.dumps(build_results)
+
+    client = storage.Client()
+    bucket = client.get_bucket(artifacts_bucket)
+    bucket.blob(str(results_file)).upload_from_string(content, 'text/json')
+
+
 
 def process_and_execute_notebooks(
     notebooks: List[str],
     container_uri: str,
     staging_bucket: str,
     artifacts_bucket: str,
+    results_file: str,
     should_parallelize: bool,
     timeout: int,
     variable_project_id: str,
@@ -349,6 +507,8 @@ def process_and_execute_notebooks(
     variable_service_account: str,
     variable_vpc_network: Optional[str] = None,
     private_pool_id: Optional[str] = None,
+    concurrent_notebooks: Optional[int] = 10,
+    aiplatform_whl: Optional[str] = None,
 ):
     """
     Run the notebooks that exist under the folders defined in the test_paths_file.
@@ -369,6 +529,8 @@ def process_and_execute_notebooks(
             Required. The GCS staging bucket to write source code to.
         artifacts_bucket (str):
             Required. The GCS staging bucket to write executed notebooks to.
+        results_file (str):
+            Required: The path to the artifacts bucket to save results 
         variable_project_id (str):
             Required. The value for PROJECT_ID to inject into notebooks.
         variable_region (str):
@@ -377,6 +539,8 @@ def process_and_execute_notebooks(
             Required. Should run notebooks in parallel using a thread pool as opposed to in sequence.
         timeout (str):
             Required. Timeout string according to https://cloud.google.com/build/docs/build-config-file-schema#timeout.
+        concurrent_notebooks (int): Max number of notebooks per minute to run in parallel.
+        aiplatform_whl: alternate whl version of Vertex AI SDK to install
     """
 
     # Calculate deadline
@@ -393,7 +557,9 @@ def process_and_execute_notebooks(
             print(
                 "Running notebooks in parallel, so no logs will be displayed. Please wait..."
             )
-            with concurrent.futures.ThreadPoolExecutor(max_workers=100) as executor:
+
+
+            with concurrent.futures.ThreadPoolExecutor(max_workers=concurrent_notebooks) as executor:
                 print(f"Max workers: {executor._max_workers}")
 
                 notebook_execution_results = list(
@@ -471,7 +637,7 @@ def process_and_execute_notebooks(
             print("=" * 100)
 
             build_id = results_sorted[0].build_id
-            logs_bucket_name = (results_sorted[0].logs_bucket).removeprefix("gs://")
+            logs_bucket_name = (results_sorted[0].logs_bucket).replace("gs://", "")
             log_file_name = f"log-{build_id}.txt"
 
             log_contents = util.download_blob_into_memory(
@@ -489,6 +655,10 @@ def process_and_execute_notebooks(
             else:
                 print(log_contents)
 
+        _save_results(results_sorted, 
+                      artifacts_bucket, 
+                      results_file)
+
         print("\n=== END RESULTS===\n")
 
         total_notebook_duration = functools.reduce(

.cloud-build/notebook-execution-test-cloudbuild.yaml

@@ -36,7 +36,7 @@ steps:
     - -c
     - |
       . workspace/env/bin/activate &&
-      python3 .cloud-build/execute_changed_notebooks_cli.py --test_paths_file "${_TEST_PATHS_FILE}" --base_branch "${_FORCED_BASE_BRANCH}" --container_uri ${_PYTHON_IMAGE} --staging_bucket ${_GCS_STAGING_BUCKET}  --artifacts_bucket ${_GCS_STAGING_BUCKET}/executed_notebooks/PR_${_PR_NUMBER}/BUILD_${BUILD_ID} --variable_project_id ${PROJECT_ID} --variable_region ${_GCP_REGION} --variable_service_account ${_GCP_SERVICE_ACCOUNT} --variable_vpc_network "${_GPC_VPC_NETWORK_NAME}" `if [ ! -z "${_PRIVATE_POOL_NAME}" ]; then echo "--private_pool_id ${_PRIVATE_POOL_NAME}"; fi`
+      python3 .cloud-build/execute_changed_notebooks_cli.py --test_paths_file "${_TEST_PATHS_FILE}" --base_branch "${_FORCED_BASE_BRANCH}" --container_uri ${_PYTHON_IMAGE} --staging_bucket ${_GCS_STAGING_BUCKET}  --artifacts_bucket ${_GCS_STAGING_BUCKET}/executed_notebooks/PR_${_PR_NUMBER}/BUILD_${BUILD_ID} --variable_project_id ${PROJECT_ID} --variable_region ${_GCP_REGION} --variable_service_account ${_GCP_SERVICE_ACCOUNT} --variable_vpc_network "${_GCP_VPC_NETWORK_NAME}" `if [ ! -z "${_PRIVATE_POOL_NAME}" ]; then echo "--private_pool_id ${_PRIVATE_POOL_NAME}"; fi` --build_id ${BUILD_ID} --test_percent=${_TEST_PERCENT} --concurrent_notebooks=${_CONCURRENT_NOTEBOOKS} --run_first_file=${_RUN_FIRST_FILE}
     env:
     - 'IS_TESTING=1'
 timeout: 86400s

.cloud-build/requirements.txt

@@ -3,11 +3,15 @@ numpy
 jupyter
 nbconvert
 papermill
-pandas
 matplotlib
 tabulate
 google-cloud-aiplatform
 google-cloud-storage
 google-cloud-build
+google-cloud-storage
 ratemate
 GitPython
+tqdm
+fsspec
+pandas
+

.cloud-build/run_first.csv

@@ -0,0 +1,8 @@
+notebooks/official/pipelines/google_cloud_pipeline_components_automl_text.ipynb
+notebooks/official/generative_ai/rlhf_tune_llm.ipynb
+notebooks/official/generative_ai/tune_peft.ipynb
+notebooks/official/prediction/llm_streaming_prediction.ipynb
+notebooks/official/migration/sdk-automl-text-classification-batch-prediction.ipynb
+notebooks/official/vizier/get_started_vertex_vizier.ipynb
+notebooks/official/workbench/sentiment_analysis/Sentiment_Analysis.ipynb
+notebooks/official/model_monitoring/get_started_with_model_monitoring_automl.ipynb

.cloud-build/tests/flaky.py

@@ -0,0 +1,46 @@
+# grep PASSED tests.txt | cut -c 10-100 >passed.txt
+
+import os
+
+repo_dir = '/home/jupyter/vertex-ai-samples/'
+repo_dir_len = len(repo_dir)
+official_dir = repo_dir + 'notebooks/official'
+
+entries = os.scandir(official_dir)
+folders = []
+for entry in entries:
+    if entry.is_dir():
+        folders.append(entry.path)
+
+# Passing
+with open('passed.txt', 'r') as pass_file:
+    notebook_names = pass_file.readlines()
+        
+notebooks = []
+for folder in folders:
+    entries = os.scandir(folder)
+    for entry in entries:
+        for notebook in notebook_names:
+            if entry.name == notebook.rstrip():
+                notebooks.append(entry.path[repo_dir_len:])
+                
+with open('passing_tests.txt', 'w') as f:
+    for notebook in notebooks:
+        f.write(notebook + '\n')
+        
+        
+# Failing
+with open('failed.txt', 'r') as fail_file:
+    notebook_names = fail_file.readlines()
+        
+notebooks = []
+for folder in folders:
+    entries = os.scandir(folder)
+    for entry in entries:
+        for notebook in notebook_names:
+            if entry.name == notebook.rstrip():
+                notebooks.append(entry.path[repo_dir_len:])
+                
+with open('failing_tests.txt', 'w') as f:
+    for notebook in notebooks:
+        f.write(notebook + '\n')

.cloud-build/tests/select_notebooks_test.py

@@ -0,0 +1,33 @@
+import sys
+
+from execute_changed_notebooks_helper import (load_results, select_notebook)
+
+
+def test_load_results():
+    bucket: str = "cloud-build-notebooks-presubmit"
+    bucket_file: str = "build_results"
+
+    accum = load_results(bucket, bucket_file)
+
+    print(accum)
+
+    assert len(accum) > 0
+
+def test_select_notebook():
+    bucket: str = "cloud-build-notebooks-presubmit"
+    bucket_file: str = "build_results"
+
+    accum = load_results(bucket, bucket_file)
+
+    n_select = 0
+    n_notselect = 0
+    for notebook in accum:
+        if select_notebook(notebook, accum, 50):
+            n_select += 1
+        else:
+            n_notselect += 1
+
+    print(f"SELECTED {n_select}, NOT SELECTED {n_notselect}")
+
+    assert n_select > 0
+    assert n_notselect > 0

.cloud-build/utils/NotebookProcessors.py

@@ -35,7 +35,7 @@ class RemoveNoExecuteCells(Preprocessor):
 
 
 class UpdateVariablesPreprocessor(Preprocessor):
-    def __init__(self, replacement_map: Dict):
+    def __init__(self, replacement_map: Dict[str, str]):
         self._replacement_map = replacement_map
 
     @staticmethod
@@ -98,3 +98,28 @@ class UniqueStringsPreprocessor(Preprocessor):
             executable_cells.append(cell)
         notebook.cells = executable_cells
         return notebook, resources
+
+class VertexAIInstallProprocessor(Preprocessor):
+    def __init__(self, vertex_ai_wheel):
+        self.vertex_ai_wheel = vertex_ai_wheel
+
+    @staticmethod
+    def update_vertex_ai_install(content: str):
+        if "google-cloud-aiplatform" not in content:
+            return content
+        return (
+            f"gsutil cp {self.vertex_ai_wheel} google-cloud-aiplatform.whl\n" + 
+            content.replace("google-cloud-aiplatform\n", "google-cloud-aiplatform.whl\n")
+            .replace("google-cloud-aiplatform ", "google-cloud-aiplatform.whl ")
+        )
+
+    def preprocess(self, notebook, resources=None):
+        executable_cells = []
+        for cell in notebook.cells:
+            if cell.cell_type == "code":
+                cell.source = self.update_vertex_ai_install(
+                    content=cell.source,
+                )
+
+            executable_cells.append(cell)
+        notebook.cells = executable_cells

.cloud-build/utils/build_results_viewer.py

@@ -0,0 +1,73 @@
+'''
+Viewer for the weekly regression testing of the official notebooks
+
+Cloud Storage location: gs://cloud-build-notebooks-presubmit/build_results/
+'''
+import argparse
+import json
+from  util import download_file
+import csv
+import datetime
+from google.cloud import storage
+
+BUILD_BUCKET = "cloud-build-notebooks-presubmit"
+BUILD_FOLDER = "build_results"
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--file', dest='file',
+                    default=None, type=str, help='build results filei (local or GCS)')
+args = parser.parse_args()
+
+investigate = {}
+with open('investigate.csv', 'r') as csvfile:
+    reader = csv.reader(csvfile)
+    for row in reader:
+        investigate[row[0][:-6]] = row[1]
+
+if not args.file:
+    client = storage.Client()
+    blobs = client.list_blobs(BUILD_BUCKET, prefix=BUILD_FOLDER)
+    newest_time = datetime.datetime(2000, 1, 1)
+    for blob in blobs:
+        # individual PR
+        if blob.size < 2000:
+            continue
+        time_created = blob.time_created.replace(tzinfo=None)
+        if time_created > newest_time:
+            newest_time = time_created
+            args.file = f"gs://{BUILD_BUCKET}/{blob.name}"
+
+if args.file.startswith("gs://"):
+    path = args.file[5:]
+    bucket = path.split('/')[0]
+    file   = path[len(bucket)+1:]
+    download_file(bucket, file, "build.json")
+    args.file = "build.json"
+
+with open(args.file, 'r') as f:
+    results = json.load(f)
+
+for item in results.items():
+    notebook = item[0][len("/notebooks/official/")-1:-6]
+    if item[1]['passed']:
+        passed = "PASS"
+    else:
+        if notebook in investigate:
+            passed = "INVG"
+        else:
+            passed = "FAIL"
+      
+    error = item[1]['error_type']
+  
+    if passed == "FAIL":
+        if error == '':
+            error = "undetermined"
+        if 'log_url' in item[1]:
+            log_url = item[1]['log_url']
+        else:
+            log_url = ''
+    else:
+        log_url = ''
+
+    print(f"{notebook:75} {passed} {error:10} {log_url}")

.cloud-build/utils/investigate.csv

@@ -0,0 +1,19 @@
+notebook,status
+prediction/llm_streaming_prediction.ipynb,wait_for_fix
+custom/get_started_with_vertex_endpoint_and_shared_vm.ipynb,issue 2527
+feature_store/online_feature_serving_and_fetching_bigquery_data_with_feature_store.ipynb,wait_for_reaper
+feature_store/online_feature_serving_and_vector_retrieval_bigquery_data_with_feature_store.ipynb,wait_for_reaper
+pipelines/google_cloud_pipeline_components_TPU_model_train_upload_deploy.ipynb,wait_for_fix
+explainable_ai/sdk_custom_image_classification_batch_explain.ipynb,issue 2528
+explainable_ai/sdk_custom_image_classification_online_explain.ipynb,issue 2528
+explainable_ai/sdk_custom_tabular_regression_online_explain.ipynb,issue 2528
+explainable_ai/sdk_automl_tabular_binary_classification_batch_explain.ipynb,issue 2528
+explainable_ai/xai_image_classification_feature_attributions.ipynb,issue 2528
+matching_engine,sdk_matching_engine_create_stack_overflow_embeddings.ipynb,issue 2530
+automl/automl_forecasting_bqml_arima_plus_comparison.ipynb,flaky
+model_evaluation/custom_tabular_regression_model_evaluation.ipynb,regr
+experiments/get_started_with_vertex_experiments.ipynb,regr
+experiments/comparing_local_trained_models.ipynb,regr
+generative_ai/tune_peft.ipynb,internal
+pipelines/custom_model_training_and_batch_prediction.ipynb,regr
+feature_store/online_feature_serving_and_fetching_bigquery_data_with_feature_store_optimized.ipynb,wait_for_reaper

.cloud-build/web-index/do_not_index.txt

@@ -0,0 +1,11 @@
+sdk2_remote_tabnet_training.ipynb
+remote_hyperparameter_tuning.ipynb
+remote_prediction.ipynb
+remote_training_bigframes_pytorch.ipynb
+remote_training_bigframes_sklearn.ipynb
+remote_training_bigframes_tensorflow.ipynb
+remote_training_lightning.ipynb
+remote_training_pytorch.ipynb
+remote_training_sklearn.ipynb
+remote_training_tensorflow_with_autologging.ipynb
+

.cloud-build/web-index/notebook-index-cloudbuild.yaml

@@ -0,0 +1,23 @@
+steps:
+  # Fetch full repo for diff purposes
+  - name: gcr.io/cloud-builders/git
+    args: [fetch, --unshallow, --quiet]
+  # Create a virtual environment
+  - name: ${_PYTHON_IMAGE}
+    entrypoint: /bin/sh
+    args:
+    - -c
+    - python3 -m venv workspace/env
+  # Install Python dependencies and run testing script
+  - name: ${_PYTHON_IMAGE}
+    entrypoint: /bin/sh
+    args:
+    - -c
+    - |
+      . workspace/env/bin/activate &&
+      python3 notebooks/notebook_template_review.py --web --title --steps --desc --linkback --notebook-dir=notebooks/official --skip-file=${_DO_NOT_INDEX_FILE} >web.html
+artifacts:
+  objects:
+          location: gs://${_GCS_ARTIFACTS_BUCKET}/webdoc
+          paths: ['web.html']
+timeout: 86400s

.github/workflows/ci.yaml

@@ -7,11 +7,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: '3.x'      
     - name: Fetch pull request branch
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         fetch-depth: 0      
     - name: Fetch base main branch

.github/workflows/linter/Dockerfile

@@ -4,7 +4,7 @@
 # 2. To lint specific notebooks:
 # docker run -v ${PWD}:/setup/app gcr.io/python-docs-samples-tests/notebook_linter:latest notebooks/1.ipynb notebooks/2.ipynb
 
-FROM python:3.10
+FROM python:3.12
 
 WORKDIR setup
 

.github/workflows/linter/requirements.txt

@@ -2,9 +2,9 @@ git+https://github.com/tensorflow/docs
 ipython
 jupyter
 nbconvert
-black==22.10.0
-pyupgrade==2.38.4
-isort==5.10.1
-flake8==4.0.1
-nbqa==1.5.3
+black==24.4.2
+pyupgrade==3.15.2
+isort==5.13.2
+flake8==7.0.0
+nbqa==1.8.5
 

.github/workflows/linter/run_linter.sh

@@ -84,7 +84,7 @@ if [ ${#notebooks[@]} -gt 0 ]; then
                 # python3 -m nbqa black "$notebook" --check
                 # BLACK_RTN=$?
                 echo "Running pyupgrade..."
-                python3 -m nbqa pyupgrade "$notebook"
+                python3 -m nbqa pyupgrade --exit-zero-even-if-changed "$notebook"
                 PYUPGRADE_RTN=$?
                 echo "Running isort..."
                 python3 -m nbqa isort "$notebook" --check
@@ -97,7 +97,7 @@ if [ ${#notebooks[@]} -gt 0 ]; then
                 python3 -m nbqa black "$notebook"
                 BLACK_RTN=$?
                 echo "Running pyupgrade..."
-                python3 -m nbqa pyupgrade "$notebook"
+                python3 -m nbqa pyupgrade --exit-zero-even-if-changed "$notebook"
                 PYUPGRADE_RTN=$?
                 echo "Running isort..."
                 python3 -m nbqa isort "$notebook"

CONTRIBUTING.md

@@ -44,12 +44,10 @@ Finally, run this code block to check for errors. Each step will attempt to
 automatically fix any issues. If the fixes can't be performed automatically,
 then you will need to manually address them before submitting your PR.
 
+Note: For official, only submit one notebook per PR.
+
 ```shell
-nbqa black "$notebook"
-nbqa pyupgrade "$notebook"
-nbqa isort "$notebook"
-nbqa flake8 "$notebook" --extend-ignore=W391,E501,F821,E402,F404,W503,E203,E722,W293,W291
-python3 -m tensorflow_docs.tools.nbfmt --remove_outputs "$notebook"
+docker run -v ${PWD}:/setup/app gcr.io/cloud-devrel-public-resources/notebook_linter:latest your_notebook
 ```
 
 ## Code Reviews

community-content/CODEOWNERS

@@ -9,3 +9,22 @@
 /Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines @Ark-kun
 /pipeline_components @Ark-kun
 /pipeline_components/image_ml_model_training @lakeyk
+/prediction_featurestore_integration @googleapis/vertex-prediction-team
+/vertex_model_garden/model_oss/util @weigary
+/vertex_model_garden/model_oss/diffusers @weigary
+/vertex_model_garden/model_oss/keras @dstnluong-google
+/vertex_model_garden/model_oss/transformers @dstnluong-google
+/vertex_model_garden/model_oss/pic2word @jismailyan-google
+/vertex_model_garden/model_oss/open_clip @lydhr
+/vertex_model_garden/model_oss/movinet @KCFindstr
+/vertex_model_garden/model_oss/data_converter @KCFindstr
+/vertex_model_garden/model_oss/peft @weigary
+/vertex_model_garden/model_oss/lm-evaluation-harness @kathyyu-google
+/vertex_model_garden/model_oss/tfvision @dstnluong-google
+/vertex_model_garden/model_oss/fvlm @minwoo33park
+/vertex_model_garden/model_oss/imagebind @kathyyu-google
+/vertex_model_garden/model_oss/llava @py4
+/vertex_model_garden/model_oss/vllm @kathyyu-google
+/vertex_model_garden/benchmarking_reports @lavraicse
+/vertex_model_garden/model_oss/autogluon @lavraicse
+

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_model_using_XGBoost_and_import_to_Vertex_AI/pipeline.py

@@ -9,7 +9,7 @@ binarize_column_using_Pandas_on_CSV_data_op = components.load_component_from_url
 split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
 train_XGBoost_model_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Train/component.yaml")
 xgboost_predict_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Predict/component.yaml")
-upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/component.yaml")
+upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/d5c9918850a6cc70004c4269dae066cfe2e664eb/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/component.yaml")
 deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
 
 # %% Pipeline definition

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_model_using_all_frameworks_and_import_to_Vertex_AI/pipeline.py

@@ -23,7 +23,7 @@ upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op = components.load_comp
 # XGBoost
 train_XGBoost_model_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Train/component.yaml")
 xgboost_predict_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Predict/component.yaml")
-upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/component.yaml")
+upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/d5c9918850a6cc70004c4269dae066cfe2e664eb/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/component.yaml")
 
 # Scikit-learn
 #train_linear_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/1f5cf6e06409b704064b2086c0a705e4e6b4fcde/community-content/pipeline_components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml")

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_regression_model_using_XGBoost_and_import_to_Vertex_AI/pipeline.py

@@ -8,7 +8,7 @@ fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_
 split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
 train_XGBoost_model_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Train/component.yaml")
 xgboost_predict_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Predict/component.yaml")
-upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/component.yaml")
+upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/d5c9918850a6cc70004c4269dae066cfe2e664eb/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/component.yaml")
 deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
 
 # %% Pipeline definition

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_regression_model_using_all_frameworks_and_import_to_Vertex_AI/pipeline.py

@@ -22,7 +22,7 @@ upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op = components.load_comp
 # XGBoost
 train_XGBoost_model_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Train/component.yaml")
 xgboost_predict_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Predict/component.yaml")
-upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/component.yaml")
+upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/d5c9918850a6cc70004c4269dae066cfe2e664eb/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/component.yaml")
 
 # Scikit-learn
 train_linear_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/1f5cf6e06409b704064b2086c0a705e4e6b4fcde/community-content/pipeline_components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml")

community-content/cpr-examples/timm_serving/requirements.txt

@@ -1,5 +1,5 @@
 absl-py==1.1.0
-fastapi==0.75.2
+fastapi==0.109.1
 uvicorn==0.18.2
 timm==0.5.4
 smart_open==6.0.0

community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/component.yaml

@@ -64,8 +64,8 @@ implementation:
           labels["component-source"] = "github-com-ark-kun-pipeline-components"
 
           # The serving container decides the model type based on the model file extension.
-          # So we need to rename the mode file (e.g. /tmp/inputs/model/data) to *.pkl
-          _, renamed_model_path = tempfile.mkstemp(suffix=".pkl")
+          # So we need to rename the mode file (e.g. /tmp/inputs/model/data) to *.bst
+          _, renamed_model_path = tempfile.mkstemp(suffix=".bst")
           shutil.copyfile(src=model_path, dst=renamed_model_path)
 
           model = aiplatform.Model.upload_xgboost_model_file(

community-content/pipeline_components/image_ml_model_training/load_image_classification_model/component.yaml

@@ -87,7 +87,7 @@ outputs:
 - {name: image_size_path, type: HeightWidth}
 implementation:
   container:
-    image: us-docker.pkg.dev/vertex-ai/ready-to-go-image-classification/image-components:v0.1
+    image: us-docker.pkg.dev/vertex-ai/ready-to-go-image-classification/image-components:v0.2
     # command is a list of strings (command-line arguments).
     # The YAML language has two syntaxes for lists and you can use either of them.
     # Here we use the "flow syntax" - comma-separated strings inside square brackets.
@@ -109,4 +109,4 @@ implementation:
       {inputValue: l2_regularization_penalty},
       --image-size-path,
       {outputPath: image_size_path},
-    ]
+    ]

community-content/pipeline_components/image_ml_model_training/preprocess_image_data/component.yaml

@@ -34,7 +34,7 @@ outputs:
     path for the validation data,'}
 implementation:
   container:
-    image: us-docker.pkg.dev/vertex-ai/ready-to-go-image-classification/image-components:v0.1
+    image: us-docker.pkg.dev/vertex-ai/ready-to-go-image-classification/image-components:v0.2
     # command is a list of strings (command-line arguments).
     # The YAML language has two syntaxes for lists and you can use either of them.
     # Here we use the "flow syntax" - comma-separated strings inside square brackets.

community-content/pipeline_components/image_ml_model_training/train_image_classification_model/component.yaml

@@ -55,7 +55,7 @@ outputs:
     for the saved model,'}
 implementation:
   container:
-    image: us-docker.pkg.dev/vertex-ai/ready-to-go-image-classification/image-components:v0.1
+    image: us-docker.pkg.dev/vertex-ai/ready-to-go-image-classification/image-components:v0.2
     # command is a list of strings (command-line arguments).
     # The YAML language has two syntaxes for lists and you can use either of them.
     # Here we use the "flow syntax" - comma-separated strings inside square brackets.

community-content/pipeline_components/image_ml_model_training/transcode_tfrecord_image_dataset_from_csv/component.yaml

@@ -20,7 +20,7 @@ outputs:
     path for the TFRecord image data}
 implementation:
   container:
-    image: us-docker.pkg.dev/vertex-ai/ready-to-go-image-classification/image-components:v0.1
+    image: us-docker.pkg.dev/vertex-ai/ready-to-go-image-classification/image-components:v0.2
     # command is a list of strings (command-line arguments).
     # The YAML language has two syntaxes for lists and you can use either of them.
     # Here we use the "flow syntax" - comma-separated strings inside square brackets.

community-content/pipeline_components/image_ml_model_training/transcode_tfrecord_image_dataset_from_jsonl/component.yaml

@@ -22,7 +22,7 @@ outputs:
     path for the TFRecord image data}
 implementation:
   container:
-    image: us-docker.pkg.dev/vertex-ai/ready-to-go-image-classification/image-components:v0.1
+    image: us-docker.pkg.dev/vertex-ai/ready-to-go-image-classification/image-components:v0.2
     # command is a list of strings (command-line arguments).
     # The YAML language has two syntaxes for lists and you can use either of them.
     # Here we use the "flow syntax" - comma-separated strings inside square brackets.

community-content/pytorch_image_classification_distributed_data_parallel_training_with_vertex_sdk/trainer/requirements.txt

@@ -1,3 +1,3 @@
-torch==1.8.1
+torch==1.13.1
 torchvision==0.9.1
 tensorboard==2.5.0

community-content/pytorch_image_classification_single_gpu_with_vertex_sdk_and_torchserve/trainer/requirements.txt

@@ -1,3 +1,3 @@
-torch==1.8.1
+torch==1.13.1
 torchvision==0.9.1
 tensorboard==2.5.0

community-content/pytorch_pre_built_images_deployment/pytorch-text-classification-vertex-ai-deploy.ipynb

@@ -31,17 +31,7 @@
       "source": [
         "# Deploying a PyTorch Text Classification Model on [Vertex AI](https://cloud.google.com/vertex-ai)\n",
         "\n",
-        "**This is an Experimental release**, covered by the Pre-GA Offerings Terms of your Google Cloud Platform [Terms of Service](https://cloud.google.com/terms).\n",
-        "\n",
-        "Experiments are focused on validating a prototype and are not guaranteed to be released. They are not intended for production use or covered by any SLA, support obligation, or deprecation policy and might be subject to backward-incompatible changes.\n",
-        "\n",
-        "**Kindly drop us a note before you run any scale tests.**\n",
-        "\n",
-        "**Do not hesitate to contact vertexai-prediction-preview-feedback@google.com if you have any questions or run into any issues.**\n",
-        "\n",
-        "The usage of the product is free during the Experimental release period: you will still incur charges for other GCP products usage, such as storage.\n",
-        "\n",
-        "The projects need to be added to the allowlist in order to deploy PyTorch models using Vertex AI Prediction pre-built PyTorch images. If you are interested in the feature, please send an email to vertexai-prediction-preview-feedback@google.com to provide your project numbers OR project ids."
+        "**Kindly reach out to Vertex AI before you run any scale tests or you have any questions.**\n"
       ]
     },
     {

community-content/tf_agents_bandits_movie_recommendation_with_kfp_and_vertex_sdk/mlops_pipeline_tf_agents_bandits_movie_recommendation/src/logger/requirements.txt

@@ -1,4 +1,4 @@
 google-cloud-bigquery==2.20.0
 tensorflow==2.7.2
-pillow==9.0.1
+pillow==10.3.0
 tf-agents==0.8.0

community-content/tf_agents_bandits_movie_recommendation_with_kfp_and_vertex_sdk/mlops_pipeline_tf_agents_bandits_movie_recommendation/src/prediction_container/requirements.txt

@@ -1,4 +1,4 @@
 google-cloud-pubsub==2.5.0
-pillow==9.0.1
+pillow==10.3.0
 tf-agents==0.8.0
 tensorflow==2.7.2

community-content/tf_agents_bandits_movie_recommendation_with_kfp_and_vertex_sdk/mlops_pipeline_tf_agents_bandits_movie_recommendation/src/simulator/requirements.txt

@@ -1,5 +1,5 @@
 dataclasses==0.6
 google-cloud-aiplatform==1.8.1
 tensorflow==2.7.2
-pillow==9.0.1
+pillow==10.3.0
 tf-agents==0.8.0

community-content/vertex_model_garden/.pylintrc

@@ -0,0 +1,4 @@
+[MASTER]
+
+generated-members=get_concrete_function,cv2.*
+ignored-modules=tensorflow,google.cloud

community-content/vertex_model_garden/benchmarking_reports/jax_vit_benchmarking_report.md

@@ -0,0 +1,253 @@
+# ViT PyTorch vs JAX training benchmarks on Vertex AI Training Platform
+
+Lav Rai, Software Engineer, Google Cloud
+
+Xiang Xu, Software Engineer, Google Cloud
+
+Andreas Steiner, Software Engineer, Google DeepMind
+
+Tao Wang, Software Engineer, Google DeepMind
+
+Alexander Kolesnikov, Research Engineer, Google DeepMind
+
+## Introduction
+
+Many repositories now offer both PyTorch and JAX versions of a model. For
+example, [Hugging Face offers many models such as GPT2, BERT][1]
+etc. Other examples are [OpenLLaMa][2] and [ViT][3]
+models which were first developed in JAX and then their corresponding PyTorch
+versions were made available. **Given both the PyTorch and JAX options for a
+model, it may not be obvious as to which option to choose**. To make such a
+decision, it is important for one to know about the training cost, effectiveness
+and efficiency for each choice.
+
+Apart from the framework choice, the other choice that one faces on Vertex AI
+training platform is the type and count of the accelerators. Although the
+[Vertex AI pricing table][4] lists the price per hour for each
+machine, **one may not know beforehand about the training speed of JAX and
+PyTorch frameworks for different types and count of the accelerators**.
+
+If one has access to some training benchmark numbers for the same model
+under (a) PyTorch and JAX frameworks and (b) for different types and count of
+the accelerators, then it will be easier for them to make a cost effective
+decision. Such a benchmark will also aid the developers in identifying strength
+and weakness of different choices and then figure out recipes to remove those
+weaknesses if possible.
+
+This blog uses the ViT [classification models][5] of varying sizes
+to benchmark the training performance of PyTorch and JAX versions on the Vertex
+AI Platform under different machine configurations. The goal is to:
+
+- Benchmark OSS ViT training for both PyTorch and JAX frameworks.
+- Benchmark OSS ViT L16, H14, g14, and G14 models.
+- Benchmark OSS ViT PyTorch training with A100 GPUs.
+- Benchmark OSS ViT JAX training with A100 GPUs and TPU V3 accelerators.
+
+## Benchmarking setup
+
+This section lays out the benchmarking set up for the [PyTorch][6] and [JAX][7]
+frameworks and provides a reasoning for choosing those settings.
+
+### PyTorch GPU
+
+#### Machine configuration
+
+We run training jobs on [Vertex AI Custom Training][8] using 1
+single node with 8 A100-40GB GPUs.
+
+- Machine type: [a2-highgpu-8g][9]
+- Machine count: 1
+- Accelerator type: [NVIDIA_TESLA_A100 (40GB)][10]
+- Accelerator count: 8
+
+#### Modeling
+
+We benchmark 4 variants of ViT model in different sizes:
+
+- [ViT-L16, 300M params][11]
+- [ViT-H14, 630M params][12]
+- [ViT-g14, 1B params][13]
+- [ViT-G14, 1.8B params][14]
+
+We use the Huggingface [transformers library][15] for ViT L16 and
+H14 variants, and the [TIMM library][16] for ViT g14 and G14
+variants.
+
+#### Dataset
+
+We run training against the [cifar10][17] dataset with 50K training
+images and 10K test images. To factor out network communication overhead for
+data loading, we copy the whole dataset to the local disk then load data from
+the local disk during training.
+
+#### Training parameters
+
+- Trainer
+  - We use [PyTorch Lightning][18] as the trainer for the
+    boilerplate data loading and train loop coding.
+- Precision
+  - Float16
+- Input resolution
+  - 224 x 224
+- Strategy
+  - We use [DDP][19] for models which can be entirely loaded to one
+    GPU, use [Deepspeed-ZeRO][20] otherwise:
+    - ViT-L16: DDP
+    - ViT-H14: DDP
+    - ViT-g14: DDP
+    - ViT-G14: Deepspeed-ZeRO stage-3
+- Batch size
+  - We use the max batch size as power of 2 without CUDA OOM for each model:
+    - ViT-L16: 64 per GPU
+    - ViT-H14: 16 per GPU
+    - ViT-g14: 16 per GPU
+    - ViT-G14: 32 per GPU
+- Compilation
+  - We apply [torch.compile][21] to model whenever it's applicable:
+    - ViT-L16: torch.compile
+    - ViT-H14: torch.compile
+    - ViT-g14: torch.compile
+    - ViT-G14: N/A
+
+### JAX TPU and GPU
+
+#### Machine configuration
+
+All the TPU and GPU training jobs are run on [Vertex AI Custom
+Training][8]. The following machine configurations were used for the
+TPU and GPU experiments:
+
+**Note**: TPU V3 POD requires multi-host supporting training code. For example,
+a 32 core POD runs on 4 hosts with each host using 8 cores.
+
+**Note**: 8 A100 are similar to TPU V3 32 cores in terms of [Vertex AI
+pricing][4].
+
+**Note**: [Each TPU v3 chip has 2 cores which can use 32 GB high-bandwidth
+memory][22] (16 GB per core) so total memory for 32 cores is 16x32 =
+512 GB. Therefore for the same price, TPUs offer more memory than 8 A100-40GB
+GPUs.
+
+#### Modeling
+
+We decided to use an OSS code repository for model implementation. Using an OSS
+repository helps anyone to independently verify the benchmarking results and
+also relate to the results well. For JAX, we selected the
+[Big Vision][23] code repository.
+
+Same as the PyTorch modeling, we benchmark 4 variants of ViT model in different
+sizes:
+
+- [ViT-L16, 300M params][24]
+- [ViT-H14, 630M params][24]
+- [ViT-g14, 1B params][24]
+- [ViT-G14, 1.8B params][24]
+
+**Note**: The [Big Vision code repo][23] has not made the
+checkpoints publicly available for the models larger than the ViT-L16. Therefore
+for the rest of the three variants, the experiments only used random
+initialization for benchmarking the training speed.
+
+#### Dataset
+
+We use training against the [cifar10 TensorFlow dataset][25] with
+50K training images and 10K test images. This dataset is the same as the one
+used for PyTorch experiments except that it is loaded as a TensorFlow dataset.
+Similar to the PyTorch experiments, we copy the whole dataset to the docker
+image to factor out network communication overhead for data loading.
+
+#### Training parameters
+
+- Precision
+  - "bfloat16" setting was used.
+- Input resolution
+  - 224 x 224 after resize (to 448x448) and random crop (to 224x224) before
+    training.
+    - This resolution for training was the same as the PyTorch settings.
+- Strategy
+  - Used DDP for all models except ViT-G14. ViT-G14 used the FSDP strategy.
+- Batch size
+  - We use the max batch size as power of 2 without OOM for each model. The
+    [Benchmarking results][26] section shows the final
+    batch size for each experiment.
+  - Once a maximum batch-size for TPU V3 8 cores was determined, we just scaled
+    it linearly for 32 cores.
+  - Once a maximum batch-size for 1 A100 GPU was determined, we just scaled it
+    linearly for 8 A100 GPUs.
+- Compilation
+  - [jax.jit() compilation][27] is used in JAX codes for efficient
+    execution in XLA.
+- GPU related flags
+  - The following flags are set in the dockerfile for the GPU runs.
+    - Note: _xla_gpu_enable_pipelined_collectives_ is set to false for the
+      ViT-G14 FSDP run.
+
+### Evaluation metric
+
+For both the PyTorch and JAX experiments, the following evaluation metrics are
+collected:
+
+- Throughput: Images-per-second observed for training.
+- Cost: The training-cost-per-epoch (USD).
+
+**Note**: The above metrics are not biased against any framework or machine
+configurations. In addition, these metrics will help one decide the most
+efficient training configurations on Vertex AI.
+
+## Benchmarking results
+
+The lowest cost experiment for each model is marked in **bold** in the last
+column.
+
+![vit_benchmarking_table](images/vit_benchmarking_table.png)
+
+The following bar charts summarize the performance visually:
+
+![vit_training_time](images/vit_training_time.png)
+
+![vit_training_cost](images/vit_training_cost.png)
+
+The following section provides observations and conclusions for these results.
+
+## Observation and Conclusions
+
+- Training with JAX TPU V3 POD with 32 cores costs 33% less than the PyTorch GPU
+  8 A100-40GBs runs.
+- Training with JAX GPU 8 A100-40GBs costs 23% less than the PyTorch GPU 8
+  A100-40GBs runs.
+- JAX TPU V3 POD with 32 cores was 4x faster and slightly more cost-effective
+  than the JAX TPU V3 8 core run for the ViT-large model. This indicates that it
+  might be better to use more cores. The JAX TPU V3 speed scales very well with
+  the number of cores.
+- Cloud TPU VM training speed numbers were the same as the Vertex AI for
+  TPU V3 8 cores. The dataset was copied to the docker in both the cases.
+- The training-cost-per-epoch increases with the model size irrespective of the
+  framework.
+
+[1]: https://github.com/huggingface/transformers/blob/main/examples/research_projects/jax-projects/README.md#quickstart-flax-and-jax-in-transformers
+[2]: https://github.com/openlm-research/open_llama
+[3]: https://github.com/google-research/vision_transformer
+[4]: https://cloud.google.com/vertex-ai/pricing#custom-trained_models
+[5]: https://arxiv.org/abs/2010.11929
+[6]: #pytorch-gpu
+[7]: #jax-tpu-and-gpu
+[8]: https://cloud.google.com/vertex-ai/docs/training/overview
+[9]: https://cloud.google.com/vertex-ai/docs/training/configure-compute#machine-types
+[10]: https://cloud.google.com/vertex-ai/docs/training/configure-compute#specifying_gpus
+[11]: https://huggingface.co/google/vit-large-patch16-224-in21k
+[12]: https://huggingface.co/google/vit-huge-patch14-224-in21k
+[13]: https://github.com/huggingface/pytorch-image-models/blob/v0.9.2/timm/models/vision_transformer.py#L1308
+[14]: https://github.com/huggingface/pytorch-image-models/blob/v0.9.2/timm/models/vision_transformer.py#L1312
+[15]: https://huggingface.co/docs/transformers/main/model_doc/vit#transformers.ViTModel
+[16]: https://github.com/huggingface/pytorch-image-models
+[17]: https://huggingface.co/datasets/cifar10
+[18]: https://lightning.ai/docs/pytorch/stable/
+[19]: https://pytorch.org/docs/stable/notes/ddp.html
+[20]: https://www.deepspeed.ai/tutorials/zero/
+[21]: https://pytorch.org/tutorials/intermediate/torch_compile_tutorial.html
+[22]: https://cloud.google.com/tpu/docs/system-architecture-tpu-vm#tpu_v3
+[23]: https://github.com/google-research/big_vision
+[24]: https://screenshot.googleplex.com/BximJgxsgvBVu38
+[25]: https://www.tensorflow.org/datasets/catalog/cifar10
+[26]: #benchmarking-results
+[27]: https://jax.readthedocs.io/en/latest/jax-101/02-jitting.html

community-content/vertex_model_garden/benchmarking_reports/pytorch_openllama_7b_finetune_benchmark_report.md

@@ -0,0 +1,227 @@
+# Benchmark report on fine tuning the OpenLLaMA 7B model on Google Cloud Vertex Model Garden
+
+Gary Wei, Software Engineer, Google Cloud
+Dustin Luong, Software Engineer, Google Cloud
+Changyu Zhu, Software Engineer, Google Cloud
+Genquan Duan, Software Engineer, Google Cloud
+
+## Introduction
+
+Fine-tuning of LLMs can be non-trivial to find an optimal configuration of
+machine types, training parameters, and other hyperparameters that achieves a
+good balance between cost efficiency and model performance. To facilitate users
+in conducting tuning experiments, this report benchmarks OpenLLaMA 7B
+fine-tuning on Google Cloud Vertex Model Garden, demonstrating both efficiency
+and effectiveness. The observations are general and can be applied to other LLM
+models.
+
+We benchmarked fine tuning algorithms [LoRA](https://arxiv.org/abs/2106.09685)
+and [QLoRA](https://arxiv.org/abs/2305.14314) supported by
+[huggingface PEFT libraries](https://github.com/huggingface/peft). LoRA, short
+for Low-Rank Adaptation of Large Language Models, is an improved fine tuning
+method where instead of fine tuning all the weights that constitute the weight
+matrix of the pre-trained large language model, two smaller matrices that
+approximate this larger matrix are fine-tuned. QLoRA is an even more
+memory-efficient version of LoRA, where the pretrained model is loaded to GPU
+memory as quantized 4-bit weights, while preserving similar effectiveness to
+LoRA. We also provide simple scripts and parameter settings to reproduce the
+results reported in this report.
+
+In general, there are many factors that affect the performance of fine-tuning
+experiments, such as hardware settings, parameters, cost, and accuracy. It is
+impractical to obtain benchmarks for all possible combinations of these factors.
+Instead, we focus on tuning a subset of related parameters and evaluating their
+impact on a set of chosen metrics. The evaluation metrics are GPU memory usage,
+percentage of parameters tuned, tuning speed, cost, and accuracy. The tuning
+parameters are batch size, lora rank, maximum sequence length, and maximum
+training steps.
+
+## Key takeaways
+
+-   **Use QLoRA to minimize the peak GPU requirements**: The QLoRA can
+    significantly reduce the peak GPU memory usage by ~75% compared to LoRA. For
+    OpenLLaMA7b, the peak memory is ~28G for LoRA and ~7G for QLoRA.
+-   **Use LoRA to maximize the tuning speed and minimize the tuning cost**: LoRA
+    is ~66% faster than QLoRA in fine tuning speed. LoRA/QLoRA tuning cost is
+    low generally, while LoRA is even ~40% cheaper than QLoRA with the same
+    parameters. Suggest to use QLoRA for limited GPU memories, and LoRA for
+    limited training budgets. For OpenLLaMA7b, the tuning speed for LoRA/QLoRA
+    ~5 samples / 3 samples per second, and the tuning cost for LoRA/QLoRA in 500
+    steps is ~$1/$1.7 on `a2-highgpu-1g` with 1 A100 40G GPU. The tuning cost
+    for QLoRA in 500 steps is $6.75 on n1-standard-8 with 1 V100 GPU, while LoRA
+    could not run because of OOM.
+-   **Use QLoRA to tune models with large sequence lengths**. For OpenLLaMA7b,
+    the max sequence length for QLoRA can be 2048 when consuming 16.3G GPU,
+    while the max sequence length for LoRA is 512 when consuming 28.2G GPU, and
+    encounter OOM when max sequence length is 1024.
+-   **Both LoRA and QLoRA give similar accuracy improvement after fine tuning.**
+    For OpenLLaMA7b, both LoRA/QLoRA can improve the average accuracy by ~4%
+    evaluating on 3 typical tasks (ARC challenge, HellaSwag and TruthfulQA),
+    after training 1875 steps on dataset
+    [timdettmers/openassistant-guanaco](https://huggingface.co/datasets/timdettmers/openassistant-guanaco).
+-   **Use a big batch size if GPU memory is not a constraint**. For OpenLLaMA7b
+    with other default parameters, we suggest using a batch size as 24 for
+    QLoRA, but 2 for LoRA when tuning with 1 A100 40G. We also suggest using a
+    batch size as 8 for QLoRA when tuning with 1 V100. Tuning with LoRA and
+    batch size as 1 got OOM and we don't recommend tuning LoRA with 1 V100.
+
+## Benchmark Details
+
+### Experiment Setup
+
+The benchmark dataset is
+[timdettmers/openassistant-guanaco](https://huggingface.co/datasets/timdettmers/openassistant-guanaco).
+The training dataset is directly downloaded from hugging face to the VM, before
+every experiment.
+
+The default tuning parameters during benchmark are:
+
+-   Host VM: a2-highgpu-1g
+-   Accelerator type: 1 A100 40G
+-   batch size: 2
+-   lora_rank: 16
+-   max_seq_length: 512
+-   precision_mode: float16
+-   max_train_steps: 500
+
+For simplicity, we set the precision mode to `float16` when tuning LoRA models,
+and set the precision to `4bit` for QLoRA.
+
+Sample script to start fine tuning dockers in a VM on GCP.
+
+```shell
+IMAGE_TAG=us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-peft-train:latest
+docker run  --runtime=nvidia  -e NVIDIA_VISIBLE_DEVICES=0 \
+--rm --name "test_gpu"  -it --pull=always ${IMAGE_TAG} \
+--task=instruct-lora \
+--pretrained_model_id=openlm-research/open_llama_7b \
+--dataset_name="timdettmers/openassistant-guanaco"  \
+--instruct_column_in_dataset="text" \
+--precision_mode="float16" \
+--output_dir=<OUTPUT DIR>  \
+--lora_rank=2 \
+--max_sequence_length=512 \
+--learning_rate=2e-4 \
+--max_steps=50
+```
+
+### GPU Memory
+
+In this benchmark, we investigated the impact of batch size, lora rank, and
+maximum sequence length on GPU memory, and then made recommendations on the
+maximum batch size for different GPUs.
+
+#### Peak GPU memory by batch size (GB)
+
+<img src="images/openllama_7b_fine_tune_benchmark_report/openllama-7b-peak-gpu-vs-batch-size.png" width="600">
+
+-   The QLoRA can significantly reduce the peak GPU memory usage by ~75%
+    compared to LoRA. The peak GPU memory is ~28G for LoRA and ~7G for QLoRA
+    when batch size is 2.
+-   QLoRA can support much larger batch sizes than LoRA
+    -   We can use a batch size as 32 for QLoRA, but only 2 for LoRA on 1 A100
+        40G.
+    -   We can use a batch size of 8 for QLoRA on 1 V100 GPU. LoRA will fail
+        with OOM even with a batch size of 1.
+
+#### Peak GPU memory by LoRA rank (GB)
+
+<img src="images/openllama_7b_fine_tune_benchmark_report/openllama-7b-peak-gpu-vs-lora-rank.png" width="600">
+
+-   Peak GPU memories are quite similar for different LoRA ranks for both
+    LoRA/QLoRA.
+-   The peak GPU memory increasing percentages are very small generally when
+    LoRA rank increases.
+    -   The peak GPU memory increases from 28G with LoRA rank 4 to 29.09G with
+        LoRA rank 64, and the increasing percentage is only ~3.9%.
+
+#### Peak GPU memory by max sequence length for LoRA/QLoRA (GB)
+
+<img src="images/openllama_7b_fine_tune_benchmark_report/openllama-7b-peak-gpu-vs-max-seq-length.png" width="600">
+
+-   The peak GPU increases quickly when max sequence length increases for both
+    LoRA/QLoRA, and the increasing rate of LoRA is much faster than QLoRA.
+    -   For LoRA tuning, the GPU memory increased from 20.5G (max sequence
+        length=256) to 28.2G (max sequence length=512), an increase of ~37%.
+    -   For QLoRA tuning, the GPU memory increased from 6.94G (max sequence
+        length=256) to 7.57G (max sequence length=512), an increase of ~9%.
+-   The max sequence length for QLoRA can be 2048 when consuming 16.3G GPU,
+    while the max sequence length for LoRA is 512 when consuming 28.2G GPU, and
+    encounter OOM when max sequence length is 1024.
+
+### Fine Tuning Parameters
+
+This section shows the number/percentage of trainable parameters, and the sizes
+of the fine tuned models. LoRA and QLoRA differ only in how they represent the
+precision of their parameters. The total number of parameters and the number of
+trainable parameters are the same for both methods.
+
+| LoRA Rank | Finetuned parameters | Total parameters | Trainable Parameter Percentage | Fine tuned model size (MB) |
+| --------- | -------------------- | ---------------- | ------------------------------ | -------------------------- |
+| 8         | 2.00E+07             | 6.76E+09         | 0.3%                           | 76.4                       |
+| 16        | 4.00E+07             | 6.78E+09         | 0.6%                           | 152.65                     |
+| 32        | 8.00E+07             | 6.82E+09         | 1.2%                           | 305.15                     |
+| 64        | 1.60E+08             | 6.90E+09         | 2.3%                           | 610.15                     |
+
+
+LoRA/QLoRA tunes quite a small fraction (only 0.3% with LoRA rank=8) of all
+parameters, and the tuned models are very small (only 76.4MB with LoRA rank=8).
+
+### Fine Tuning Speed And Costs
+
+The fine-tuning speed and cost are affected by various factors, such as the
+GPUs, LoRA ranks, and max sequence lengths.
+
+-   LoRA is ~66% faster than QLoRA in fine tuning speed. The tuning speed for
+    LoRA/QLoRA ~5 samples / 3 samples per second on 1 A100 40G GPU
+-   Higher LoRA ranks, slower tuning speed for both LoRA/QLoRA.
+    -   LoRA tuning speed reduces from ~5 samples per second with LoRA rank as 8
+        to ~4 samples per second with LoRA rank as 64, slowed down by 20%.
+    -   QLoRA tuning speed reduces from ~3 samples per second with LoRA rank as
+        8 to ~2.5 samples per second with LoRA rank as 64, slowed down by 17%.
+
+<img src="images/openllama_7b_fine_tune_benchmark_report/openllama-7b-tune-speed-vs-lora-rank.png" width="600">
+
+-   Longer sequence lengths, slower tuning speed.
+    -   LoRA tuning speed reduces from ~5.56 samples per second with max
+        sequence length as 256 to ~4.84 samples per second with max sequence
+        length as 512 slowed down by 13%.
+    -   LoRA tuning speed reduces from ~2.95 samples per second with max
+        sequence length as 256 to ~2.88 samples per second with max sequence
+        length as 512 slowed down by ~2.4%.
+
+<img src="images/openllama_7b_fine_tune_benchmark_report/openllama-7b-tune-speed-lora-qlora.png" width="600">
+
+-   LoRA/QLoRA tuning cost is low generally, while LoRA is even ~40% cheaper
+    than QLoRA with the same parameters.
+    -   The LoRA/QLoRA fine tuning cost for 500 steps is ~$1/$1.7 on 1 A100 40G.
+    -   The tuning cost for QLoRA in 500 steps is $6.75 on n1-standard-8 with 1
+        V100 GPU, while LoRA could not run because of OOM.
+
+<img src="images/openllama_7b_fine_tune_benchmark_report/openllama-7b-tune-cost-lora-qlora.png" width="600">
+
+### Accuracy
+
+We fine tuned Open Llama 7B model with
+[timdettmers/openassistant-guanaco](https://huggingface.co/datasets/timdettmers/openassistant-guanaco),
+and report accuracy similar to the
+[HuggingFace leaderboard](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard)
+using
+[Eleuther AI Language Model Evaluation Harness](https://github.com/EleutherAI/lm-evaluation-harness).
+[HuggingFace leaderboard](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard)
+mainly compares models on ARC, HellaSwag, MMLU, and TruthfulQA. The authors did
+not publish OpenLLaMA 7B on MMLU
+([link](https://huggingface.co/openlm-research/open_llama_7b)). Therefore, we
+only benchmark accuracies on ARC, HellaSwag, and TruthfulQA.
+
+|                                                              | Mean | ARC  | HellaSwag | TruthfulQA | Tuning Parameters                                            |
+| ------------------------------------------------------------ | ---- | ---- | --------- | ---------- | ------------------------------------------------------------ |
+| OpenLLaMA7B ([Original Report](https://huggingface.co/openlm-research/open_llama_7b)) | 0.49 | 0.41 | 0.73      | 0.34       | n/a                                                          |
+| OpenLLaMA7B ([Re-run with lm-evaluation-harness](https://github.com/EleutherAI/lm-evaluation-harness)) | 0.51 | 0.47 | 0.72      | 0.35       | n/a                                                          |
+| OpenLLaMA7B+LoRA                                             | 0.56 | 0.48 | 0.74      | 0.45       | LoRA Rank=16; Max Sequence Length=512;Learning Rate=1e-4; Train steps=1875 |
+| OpenLLaMA7B+QLoRA                                            | 0.53 | 0.45 | 0.73      | 0.42       | LoRA Rank=16; Max Sequence Length=512; Learning Rate=1e-4; Train steps=1875 |
+
+-   The base OpenLLaMA7B model gets better performance (2%) when using the
+    [Eleuther AI Language Model Evaluation Harness](https://github.com/EleutherAI/lm-evaluation-harness).
+-   LoRA/QLoRA can improve the performance by ~2-4% when trained for 1875 steps
+    with learning rate 1e-4.

community-content/vertex_model_garden/benchmarking_reports/pytorch_openllama_hyperparameter_tuning_benchmark_report.md

@@ -0,0 +1,188 @@
+# Benchmark report on hyperparameter tuning the OpenLLaMA models on Google Cloud Vertex Model Garden
+
+Changyu Zhu, Software Engineer, Google Cloud
+
+Dustin Luong, Software Engineer, Google Cloud
+
+Gary Wei, Software Engineer, Google Cloud
+
+Genquan Duan, Software Engineer, Google Cloud
+
+## Introduction
+
+Fine-tuning of LLMs can be non-trivial to find an optimal configuration of
+machine types, training parameters, and other hyperparameters that achieves a
+good balance between cost efficiency and model performance. To facilitate users
+in conducting tuning experiments, this report benchmarks fine-tuning OpenLLaMA
+models with [Vertex AI Hyperparameter Tuning Service](https://cloud.google.com/vertex-ai/docs/training/hyperparameter-tuning-overview), demonstrating both efficiency
+and effectiveness. Similar hyperparameter tuning techniques can apply to other models as well.
+
+## Key takeaways
+
+- **The hyperparameter tuning service finds good parameters**: The best model found by the hyperparameter tuning service has an average improvement of around 4% in accuracy in *ARC*, *HellaSwag*, and *TruthfulQA* datasets, while only tuning the learning rate.
+
+- **Hyperparameter tuning works with QLoRA on limited resources**: 4bit QLoRA is sufficient for hyperparameter tuning to find a set of good parameters. In this way, all OpenLLaMA models can run on 1 single `NVIDIA_L4` GPU. It is also possible to train for more steps on the good parameters discovered by hyperparameter tuning, avoiding the waste of computing resources on fine-tuning with suboptimal hyperparameters.
+
+- **Hyperparameter tuning is cost-effective**: While `NVIDIA_L4` is slower than `NVIDIA_TESLA_V100`, it costs less and avoids the overhead of multi-GPU training since it has more GPU memory. Finding a good 3B/7B/13B OpenLLaMA model costs $28.5671, $47.8016, and $87.9208, respectively.
+
+## Benchmarking setup
+
+This section describes the experiment setup of the hyperparameter tuning experiments. The default tuning parameters are:
+
+### Machine configuration
+
+- Machine type: g2-standard-8
+- Machine count: 1
+- Accelerator type: NVIDIA_L4
+- Accelerator count: 1
+
+### Modeling
+
+We benchmark all 3 OpenLLaMA models:
+
+- [open_llama_3b](https://huggingface.co/openlm-research/open_llama_3b)
+- [open_llama_7b](https://huggingface.co/openlm-research/open_llama_7b)
+- [open_llama_13b](https://huggingface.co/openlm-research/open_llama_13b)
+
+We use the Huggingface [PEFT](https://github.com/huggingface/peft) library for fine-tuning.
+
+### Training dataset
+
+We use the dataset [timdettmers/openassistant-guanaco](https://huggingface.co/datasets/timdettmers/openassistant-guanaco) loaded directly via Huggingface.
+
+### Training parameters
+
+The set of training parameters used during benchmarking:
+
+- Batch size: 4
+- Precision mode: 4bit QLoRA
+- LoRA rank: 32
+- LoRA alpha: 64
+- Max sequence length: 512
+- Max train steps: 1000
+
+### Evaluation dataset
+
+We use the [lm-evaluation-harness](https://github.com/EleutherAI/lm-evaluation-harness) library injected into the training loop for evaluation. The hyperparameter tuning job will pick the model according to the evaluation metrics.
+
+- Eval task: [ARC Challenge](https://huggingface.co/datasets/ai2_arc)
+- Eval metric: acc_norm
+- Max eval examples: 10000
+
+### Standalone evaluation dataset
+
+After finding the best model with Vertex hyperparameter tuning service, we run standalone evaluations with the model on the following datasets:
+
+- [ARC Challenge](https://huggingface.co/datasets/ai2_arc)
+- [HellaSwag](https://huggingface.co/datasets/Rowan/hellaswag)
+- [TruthfulQA](https://huggingface.co/datasets/EleutherAI/truthful_qa_mc)
+
+### Hyperparameter tuning
+
+We only tune the learning rate hyperparameter. It is considered a floating point value in the continuous range [1e-5, 1e-4]. We run 8 trials in total, with a parallelism of 1 or 2.
+
+### Code example
+
+The following code example launches an example hyperparameter tuning job of OpenLLaMA 7B model.
+
+```py
+from google.cloud import aiplatform
+from google.cloud.aiplatform import hyperparameter_tuning as hpt
+
+
+TRAIN_DOCKER_URI = 'us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-peft-train:20231130_0936_RC00'
+output_dir = "gs://path/to/output/dir"
+base_model_id = "openlm-research/open_llama_7b"
+dataset_name = "timdettmers/openassistant-guanaco"
+hpt_precision_mode = "4bit"
+machine_type = "g2-standard-8"
+accelerator_type = "NVIDIA_L4"
+accelerator_count = 1
+eval_task = "arc_challenge"
+eval_metric_name = "acc_norm"
+max_steps = 1000
+eval_limit = 10000
+
+flags = {
+    "learning_rate": 1e-5,
+    "precision_mode": hpt_precision_mode,
+    "task": "instruct-lora",
+    "pretrained_model_id": base_model_id,
+    "output_dir": output_dir,
+    "warmup_steps": 10,
+    "max_steps": max_steps,
+    "lora_rank": 32,
+    "lora_alpha": 64,
+    "lora_dropout": 0.05,
+    "dataset_name": dataset_name,
+    "eval_steps": max_steps + 1,  # Only evaluates at the end.
+    "eval_tasks": eval_task,
+    "eval_limit": eval_limit,
+    "eval_metric_name": eval_metric_name,
+}
+worker_pool_specs = [
+    {
+        "machine_spec": {
+            "machine_type": machine_type,
+            "accelerator_type": accelerator_type,
+            "accelerator_count": accelerator_count,
+        },
+        "replica_count": 1,
+        "container_spec": {
+            "image_uri": TRAIN_DOCKER_URI,
+            "args": ["--{}={}".format(k, v) for k, v in flags.items()],
+        },
+    }
+]
+metric_spec = {"model_performance": "maximize"}
+parameter_spec = {
+    "learning_rate": hpt.DoubleParameterSpec(
+        min=1e-5, max=1e-4, scale="linear"
+    ),
+}
+
+train_job = aiplatform.CustomJob(
+    display_name=job_name,
+    worker_pool_specs=worker_pool_specs,
+    staging_bucket=STAGING_BUCKET,
+)
+
+train_hpt_job = aiplatform.HyperparameterTuningJob(
+    display_name=f"{job_name}_hpt",
+    custom_job=train_job,
+    metric_spec=metric_spec,
+    parameter_spec=parameter_spec,
+    max_trial_count=8,
+    parallel_trial_count=2,
+)
+
+train_hpt_job.run()
+```
+
+## Benchmark results
+
+### Fine-tuning cost
+
+The fine-tuning cost is calculated from `us-central1` pricing and may be subject to changes.
+
+| Model         | Train time | Trials | Parallel Trials | Hourly cost | Cost     | Eval acc_norm (ARC-Challenge) |
+|---------------|------------|--------|-----------------|-------------|----------|-------------------------------|
+| OpenLLaMA 3B  | 16 hrs     | 8      | 2               | $1.7072     | $28.5671 | 39.9%                         |
+| OpenLLaMA 7B  | 28 hrs     | 8      | 2               | $1.7072     | $47.8016 | 45.8%                         |
+| OpenLLaMA 13B | 103 hrs    | 8      | 1               | $0.8536     | $87.9208 | 47.6%                         |
+
+### Fine-tuning performance
+
+Here are the evaluation results of the best model found by hyperparameter tuning, compared with the baseline model. The column `Eval acc_norm` is calculated during training, which is always lower than that during standalone evaluation, because the model is loaded and evaluated at a lower precision (4bit during training / float16 during standalone evaluation).
+
+| Model         | Eval acc_norm (ARC-Challenge) | ARC    | hellaswag | Truthfulqa_mc | ∆ARC   | ∆Hellaswag | ∆Truthfulqa_mc | ∆Average |
+|---------------|-------------------------------|--------|-----------|---------------|--------|------------|----------------|----------|
+| OpenLLaMA 3B  | 39.9%                         | 41.47% | 69.97%    | 38.31%        | +1.62% | +7.32%     | +3.34%         | +4.09%   |
+| OpenLLaMA 7B  | 45.8%                         | 49.83% | 75.53%    | 41.53%        | +2.82% | +3.55%     | +6.68%         | +4.35%   |
+| OpenLLaMA 13B | 47.6%                         | 52.20% | 78.90%    | 44.27%        | +1.01% | +3.67%     | +6.19%         | +3.62%   |
+
+## Related documents
+
+1. [Benchmark report on fine tuning the OpenLLaMA 7B model on Google Cloud Vertex Model Garden
+](
+https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/community-content/vertex_model_garden/benchmarking_reports/pytorch_openllama_7b_finetune_benchmark_report.md)

community-content/vertex_model_garden/benchmarking_reports/stable_diffusion_v1-5_benchmarking_report.md

@@ -0,0 +1,218 @@
+# Benchmark Stable Diffusion v1-5 Fine Tuning and Serving With Google Cloud Vertex Model Garden
+
+Dustin Luong, Software Engineer, Google Cloud
+Gary Wei, Software Engineer, Google Cloud
+Changyu Zhu, Software Engineer, Google Cloud
+Genquan Duan, Software Engineer, Google Cloud
+
+## Introduction
+[The public notebook][1] shows the full examples of fine tuning and serving of Stable diffusion v1-5. [The github repo][2] contains examples of building training and serving dockers for Google Cloud Vertex Model Garden. This report benchmarks Stable diffusion v1-5 fine tuning and serving in Google Cloud Vertex AI, showing both efficiencies and effectiveness.
+
+### Benchmark Highlights
+- Fine tuning
+  - Stable diffusion v1-5 with LoRA and Gradient checkpointing only requires ~10G GPU memory. Larger batch sizes, or larger resolutions require more GPU memories, but not does not change much for different LoRA ranks.
+  - The fine tuning speed is fast in ~11 minutes for 1k steps, and costs less than $1 in 1 A100. The fine tuning speed increases with batch sizes, decreases with resolution, but is not affected much by LoRA ranks.
+  - LoRA tunes a few percent (only 0.1% with LoRA rank=8) of all parameters, and the tuned models are very small (only 3.1MB with LoRA rank=8).
+  - Dreambooth+LoRA and Dreambooth can achieve similar performances, but Dreambooth LoRA can require much less GPU.
+  - Increasing batch size, reducing training steps, and increasing learning rate can result in models with the same performance for less cost.
+- Inference
+  - The optimized serving docker pytorch-peft-serve can speed up inference by 2x than current pytorch-diffuser-serve, and support both base models and fine tuned lora models.
+  - The optimized serving docker pytorch-peft-serve can generate 4 512*512 images in 4.1 seconds on 1 V100 and 1.7 seconds on 1 A100.
+
+Benchmark details are below.
+
+## Fine Tuning Benchmarks
+
+### Experiment Setup
+We mainly compare two tuning algorithms:
+- parameter efficient finetuning based on [dreambooth][3] and [LoRA][4] (shorten as Dreambooth+LoRA below)
+- full parameter fine tuning based on [dreambooth][3] (shorten as Dreambooth below)
+
+And then  report benchmark results on GPU memories, tuning parameters, tuning speeds, costs and accuracy, using the public oxford flowers dataset: [train][5] and [test][6], where the column blip_caption as texts, and column image as images. We also benchmark subject and prompt fidelity using the [dataset][7] from the Dreambooth paper.
+
+The default tuning parameters during benchmark are:
+- Hardware: 1 A100 40G
+- batch size: 4
+- lora_rank: 8
+- resolution: 512
+- max_train_steps: 10
+- use_lora: False
+- gradient_checkpointing: False
+
+```
+# Examples to start finetuning dockers.
+MODEL_NAME="runwayml/stable-diffusion-v1-5"
+OUTPUT_DIR=<OUTPUT_DIR>
+INSTANCE_DATA_DIR=<INSTANCE_DATA_DIR>
+INSTANCE_PROMPT=<INSTANCE_PROMPT>
+IMAGE="us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-peft-train"
+docker run \
+  --runtime=nvidia -e NVIDIA_VISIBLE_DEVICES=0 \
+  --rm --name "test_gpu" \
+  -it ${IMAGE} \
+  --task=text-to-image-dreambooth-lora-peft \
+  --pretrained_model_name_or_path=$MODEL_NAME \
+  --resolution=512 \
+  --instance_data_dir=$INSTANCE_DATA_DIR \
+  --instance_prompt=$INSTANCE_PROMPT \
+  --train_batch_size=4 \
+  --max_train_steps=10 \
+  --output_dir=${OUTPUT_DIR} \
+  --use_lora \
+  --lora_r=8 \
+  --gradient_checkpointing
+```
+
+### GPU Memories
+Many various factors will impact GPU memory usages. In this benchmark, we mainly benchmark with different finetuning algorithms, batch sizes, lora rank, resolution, and then recommended max batch size on different GPUs.
+
+
+![sd_v1-5_peak_gpu_algorithm](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_peak_gpu_algorithm.png)
+
+![sd_v1-5_peak_gpu_batch_size](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_peak_gpu_batch_size.png)
+
+![sd_v1-5_peak_gpu_lora_rank](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_peak_gpu_lora_rank.png)
+
+![sd_v1-5_peak_gpu_resolution](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_peak_gpu_resolution.png)
+
+- LoRA tuning reduced about 47% peak RAM and 42% peak VRAM for GPU memory, compared to full parameter fine tuning.
+- Gradient checkpointing decreases about 1% peak RAM and 31% peak VRAM for GPU memory further, compared without gradient checkpointing.
+- The GPU memory does not change much for different LoRA ranks.
+- Larger batch sizes require more GPU memories.
+- Larger resolutions require more GPU memories.
+- Dreambooth+LoRA+Gradient_Checkpointing can support max batch size as 32, or max resolution as 2048, but Dreambooth can only support max batch size as 8, or max resolution as 1024.
+
+### Fine Tuning Parameters
+This section shows the percentage of trainable parameters, and tuned model sizes.
+
+- LoRA tunes quite a few percent (only 0.1% with LoRA rank=8) of all parameters, and the tuned models are very small (only 3.1MB with LoRA rank=8).
+
+| LoRA Rank | Trainable parameters | Total parameters | Trainable Parameter Percentage | Fine tuned model size (MB) |
+|---|---|---|---|---|
+| 4 | 398592 | 859919556 | 0.05% | 1.57 |
+|8 | 797184 | 860318148 | 0.09% | 3.09 |
+| 16 | 1594368| 861115332| 0.19%| 6.13|
+| 32| 3188736| 862709700| 0.37%| 12.21|
+### Fine Tuning Speed And Costs
+Fine tuning speeds and costs are affected by many different factors, such as batch size, tuning parameters, image resolutions, GPUs, and datasets. In order to make the report easy to understand, we set the following values in this section:
+- Hardware: 1 A100 40G
+- use_lora: True
+- gradient_checkpointing: True
+
+![sd_v1-5_training_speed_batch_size](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_training_speed_batch_size.png)
+
+![sd_v1-5_training_speed_lora_rank](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_training_speed_lora_rank.png)
+
+![sd_v1-5_training_speed_resolution](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_training_speed_resolution.png)
+
+![sd_v1-5_training_cost_max_steps](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_training_cost_max_steps.png)
+
+- The fine tuning speed increases with batch sizes, decreases with resolution, but is not affected much by LoRA ranks.
+- The fine tuning speed is about 11 minutes for 1k steps, and costs less than $1 in 1 A100.
+
+### Fine Tuning Quality
+In this benchmark, we mainly benchmark Dreambooth and Dreambooth+LoRA to compare fine tuning quality. We compare [subject fidelity scored (DINO)][8], how well the subject is represented in the generated images, and [prompt fidelity scores (CoCa)][9], how well the generated images match the given prompt, for a single subject, a [dog][10] from the dataset released with the original Dreambooth paper. In practice, we recommend saving checkpoints periodically and inspecting validation prompts visually. We fine tuned the unet without fine tuning the text encoder and used the following hyperparameters:
+
+Dreambooth
+- Learning rate: 5e-6
+- Batch size: 1
+
+Dreambooth+LoRA
+- Learning rate: 1e-4
+- Batch size: 1
+
+![sd_v1-5_finetuning_quality_subject_fidelity](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_finetuning_quality_subject_fidelity.png)
+
+![sd_v1-5_finetuning_quality_prompt_fidelity](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_finetuning_quality_prompt_fidelity.png)
+
+- Fine tuning with Dreambooth or Dreambooth+LoRA can result in models with comparable performance. The base model produced images of the class rather than the instance.
+- Dreambooth+LoRA is able to achieve the same subject fidelity score as Dreambooth if trained for more epochs.
+- Increasing the number of training steps results in better subject fidelity but at the cost of prompt fidelity.
+
+### Suggested Max Batch Sizes By Resolutions
+We benchmarked and suggested max batch sizes by resolutions on 1 A100 and 1 V100 as below. This is with LoRA and gradient checkpointing enabled.
+
+![sd_v1-5_batch_size_by_resolution](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_batch_size_by_resolution.png)
+
+### Fine Tuning Cost Optimization
+Increasing batch size allows for more images to be considered at each training step for fine tuning. This allows models to be trained in fewer training steps. In this benchmark, we aim to show how batch size can be increased to reduce training costs while still preserving subject and prompt fidelity.
+
+Since the training dataset consists of 5 images, we train with a batch size of 5 and reduce the number of training steps from 400 to 80. Doing so results in a model that has not learned the subject since we’ve decreased the number of training steps. Conceptually, the model is taking a more precise step at each iteration, but it is taking fewer steps. To compensate for this, we increased the learning rate from 5e-6 and observed the best results at 1e-5 for full parameter finetuning.
+
+![sd_v1-5_subject_fidelity_batch_size_5_learning_rate](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_subject_fidelity_batch_size_5_learning_rate.png)
+
+![sd_v1-5_prompt_fidelity_batch_size_5_learning_rate](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_prompt_fidelity_batch_size_5_learning_rate.png)
+
+Comparing cost of training the “best” model for batch size 1 vs. batch size 5
+
+![sd_v1-5_cost_batch_size](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_cost_batch_size.png)
+
+
+| Train method| Training parameters| Sample image| CoCa (prompt fidelity)| DINO (subject fidelity) | Cost of training on A100 |
+|---|---|---|---|---|---|
+| dreambooth| dreambooth, num_train_steps=400, batch_size=1, lr=5e-6| ![dog1](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_dog1.png) | 0.12215| 0.76531| $0.26 |
+| dreambooth | dreambooth, num_train_steps=80, batch_size=5,lr=1e-5| ![dog2](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_dog2.png)| 0.12644| 0.74697 | $0.15 |
+| dreambooth-lora| num_train_steps=500, batch_size=1, lr=1e-4, gc|![dog3](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_dog3.png)| 0.12856| 0.78148 | $0.26|
+| dreambooth-lora | num_train_steps=50, batch_size=5, lr=1e-3, gc | ![dog4](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_dog4.png) | 0.12566 | 0.75479 | $0.09 |
+
+
+
+A followup question is that since finetuning can be run on a single GPU, should finetuning be run on 1 V100 or A100?
+
+Setup:
+- num_train_steps=800 / batch_size
+- Resolution=512
+
+![sd_v1-5_cost_training_method_batch_size](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_cost_training_method_batch_size.png)
+- Although V100 has a lower $/hr cost than an A100, the same training setup takes longer. Even given the longer training time, the cost on V100 is still lower.
+- Dreambooth+LoRA enables training with larger batch sizes, however, larger batch sizes will not necessarily mean faster training time.
+- It is possible to fine tune with 1 V100 on 512 resolution with Dreambooth+LoRA.
+- Dreambooth fine tuning must be run on 1 A100 at 512 resolution.
+
+## Inference Benchmarks
+We provide two serving dockers in vertex model garden for stable diffusion:
+- pytorch-diffuser-serve:
+  - us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-diffusers-serve
+  - This serving docker only serves base stable diffusion models and does not contain any optimizations yet.
+- pytorch-peft-serve:
+  - us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-peft-serve
+  - This serving docker can serve base stable diffusion models, and base stable diffusion models with fine tuned lora models, and contains optimization for serving.
+
+We run the two serving dockers on T4/V100/A100 to generate 4 512*512 images, and compare the inference speed without network considerations as:
+
+![sd_v1-5_inference_speed_gpu](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_inference_speed_gpu.png)
+The speed up of optimized pytorch-peft-serve is about 2x than current pytorch-diffuser-serve.
+
+### Serving cost comparison
+
+Pytorch-diffuser-serve (without any optimizations)
+
+| GPU type|  Time required to generate 4 512x512 images | Machine unit price ($ / hour) | Cost per image ($) |
+|---|---|---|---|
+| T4 | 28.6 | 0.4025| 0.00080 |
+| V100 | 8.8 | 2.852| 0.00174|
+| A100 | 4.2 | 4.2245 | 0.00123 |
+
+Pytorch-peft-serve (with optimizations)
+
+| GPU type | Time required to generate 4 512x512 images | Machine unit price ($ / hour) | Cost per image ($) |
+|--- |---|---|---|
+| T4 | 12.6 | 0.4025 | 0.00035 |
+| V100 | 4.1 | 2.852 | 0.00081 |
+|  A100 | 1.7 | 4.2245 | 0.00050 |
+
+- The optimized pytorch-peft-serve has approximately half the price per image, compared with the un-optimized pytorch-diffuser-serve.
+- Serving the model with a T4 is most cost effective, however, serving with an A100 still has the best throughput and fastest predictions.
+
+
+
+[1]: https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion.ipynb
+[2]: https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/community-content/vertex_model_garden/model_oss
+[3]: https://arxiv.org/abs/2208.12242
+[4]: https://arxiv.org/abs/2106.09685
+[5]: https://huggingface.co/datasets/Multimodal-Fatima/OxfordFlowers_train
+[6]: https://huggingface.co/datasets/Multimodal-Fatima/OxfordFlowers_test_facebook_opt_6.7b_Attributes_ns_6149
+[7]: https://github.com/google/dreambooth
+[8]: https://arxiv.org/abs/2104.14294
+[9]: https://arxiv.org/abs/2205.01917
+[10]: https://github.com/google/dreambooth/tree/main/dataset/dog6

community-content/vertex_model_garden/model_oss/autogluon/dockerfile/autogluon_serve.Dockerfile

@@ -0,0 +1,50 @@
+# Dockerfile for serving dockers with AutoGluon.
+#
+# To build:
+# docker build -f model_oss/autogluon/dockerfile/serve.Dockerfile . -t ${YOUR_IMAGE_TAG}
+#
+# To push to gcr:
+# docker tag ${YOUR_IMAGE_TAG} gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
+# docker push gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
+
+FROM pytorch/pytorch:2.1.2-cuda11.8-cudnn8-runtime
+
+USER root
+
+# AutoGluon might require libgomp for some dependencies.
+RUN apt-get update && apt-get -y upgrade && apt-get install -y --no-install-recommends \
+        curl \
+        wget \
+        vim \
+        libgomp1
+
+# Install AutoGluon and other dependencies.
+RUN pip install --upgrade pip
+RUN pip install autogluon==1.0.0
+RUN pip install flask==3.0.0
+
+# Dependencies needed to work with GCS.
+RUN pip install absl-py==2.0.0
+RUN pip install google-cloud-storage==2.7.0
+
+# Copy scripts into the container.
+COPY model_oss/autogluon /autogluon
+COPY model_oss/util /autogluon/util
+WORKDIR /autogluon
+
+# Copy license.
+RUN wget https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/LICENSE
+RUN wget https://github.com/pallets/flask/blob/main/LICENSE.rst
+
+# Expose the port the app runs on.
+EXPOSE 8501
+
+# Set the working directory to a specific path for consistency.
+WORKDIR /autogluon
+
+# Change to a non-root user for security purposes.
+RUN useradd -m autogluonuser
+USER autogluonuser
+
+# Run Flask application.
+CMD ["python", "serve.py"]

community-content/vertex_model_garden/model_oss/autogluon/dockerfile/autogluon_train.Dockerfile

@@ -0,0 +1,36 @@
+# Dockerfile for training dockers with Autogluon.
+#
+# To build:
+# docker build -f model_oss/autogluon/dockerfile/train.Dockerfile . -t ${YOUR_IMAGE_TAG}
+#
+# To push to gcr:
+# docker tag ${YOUR_IMAGE_TAG} gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
+# docker push gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
+
+FROM pytorch/pytorch:2.1.2-cuda11.8-cudnn8-runtime
+
+# Install tools.
+ENV DEBIAN_FRONTEND=noninteractive
+ENV PIP_ROOT_USER_ACTION=ignore
+
+RUN apt-get update && apt-get -y upgrade && apt-get install -y --no-install-recommends \
+apt-utils \
+curl \
+wget \
+git \
+jq \
+gnupg \
+build-essential \
+tesseract-ocr \
+vim
+
+# Install libraries.
+RUN pip install autogluon==1.0.0
+
+COPY model_oss/autogluon /autogluon
+WORKDIR /autogluon
+
+# Copy license.
+RUN wget https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/LICENSE
+
+ENTRYPOINT ["python", "train.py"]

community-content/vertex_model_garden/model_oss/autogluon/serve.py

@@ -0,0 +1,87 @@
+r"""AutoGluon serving binary.
+
+This module sets up a Flask web server for serving predictions from a
+trained AutoGluon model. The server exposes two endpoints:
+
+1.  `/ping`: A health check endpoint that returns "pong" to
+    indicate that the server is running.
+2.  `/predict`: An endpoint that accepts POST requests with JSON content.
+    Each request should contain one or more instances for which the
+    predictions are desired. The endpoint returns the predictions and
+    associated probabilities in a JSON response.
+
+The server expects an environment variable `model_path` that points to
+the directory where the AutoGluon model artifacts are
+stored. If `model_path` is not provided, it defaults to '/autogluon/models'.
+"""
+
+import json
+import logging
+import os
+
+from autogluon.tabular import TabularPredictor
+import flask
+import pandas as pd
+
+from util import constants
+from util import fileutils
+
+_SUCCESS_STATUS = 200
+_ERROR_STATUS = 500
+_PORT = 8501
+
+app = flask.Flask(__name__)
+# Check the environment variables.
+model_dir = os.getenv('model_path', '/autogluon/models')
+logging.info('Model directory passed by the user is: %s', model_dir)
+# If the model is on GCS then copy it to a local folder first.
+if model_dir.startswith(constants.GCS_URI_PREFIX):
+  gcs_path = model_dir[len(constants.GCS_URI_PREFIX) :]
+  local_model_dir = os.path.join(constants.LOCAL_MODEL_DIR, gcs_path)
+  logging.info('Download %s to %s', model_dir, local_model_dir)
+  fileutils.download_gcs_dir_to_local(model_dir, local_model_dir)
+  model_dir = local_model_dir
+  logging.info('Local model directory is: %s', model_dir)
+
+
+# Load the predictor at startup.
+predictor = TabularPredictor.load(model_dir)
+
+
+@app.route('/ping', methods=['GET'])
+def ping() -> flask.Response:
+  """Health check route."""
+  return flask.Response('pong', status=_SUCCESS_STATUS)
+
+
+@app.route('/predict', methods=['POST'])
+def predict() -> flask.Response:
+  """Prediction route."""
+  try:
+    # Extract JSON content from the POST request.
+    data = flask.request.get_json(force=True)
+    instances = data.get('instances', [])
+
+    # Convert instances to DataFrame.
+    df_to_predict = pd.DataFrame(instances)
+
+    # Perform prediction.
+    predictions = predictor.predict(df_to_predict).tolist()
+    response = {'predictions': predictions}
+
+    return flask.Response(
+        json.dumps(response),
+        status=_SUCCESS_STATUS,
+        mimetype='application/json',
+    )
+
+  except Exception as e:  # pylint: disable=broad-exception-caught
+    return flask.Response(
+        json.dumps({'error': str(e)}),
+        status=_ERROR_STATUS,
+        mimetype='application/json',
+    )
+
+
+if __name__ == '__main__':
+  app.run(host='0.0.0.0', port=_PORT)

community-content/vertex_model_garden/model_oss/autogluon/train.py

@@ -0,0 +1,144 @@
+"""AutoGluon training binary. """
+
+import argparse
+import json
+from typing import Any
+
+from autogluon.tabular import TabularPredictor
+import pandas as pd
+
+
+class BaseConfig:
+
+  def to_dict(self) -> dict[str, Any]:
+    return {
+        key: value for key, value in self.__dict__.items() if value is not None
+    }
+
+
+class DataConfig(BaseConfig):
+
+  def __init__(self, train_data_path: Any) -> None:
+    self.train_data_path = train_data_path
+
+
+class ProblemConfig(BaseConfig):
+
+  def __init__(self, label: Any, problem_type: Any) -> None:
+    self.label = label
+    self.problem_type = problem_type
+
+
+class EvaluationConfig(BaseConfig):
+
+  def __init__(self, eval_metric: Any) -> None:
+    self.eval_metric = eval_metric
+
+
+class TrainingConfig(BaseConfig):
+  """Config for training."""
+
+  def __init__(
+      self,
+      time_limit: Any,
+      presets: Any,
+      hyperparameters: Any,
+      model_save_path: str,
+  ) -> None:
+    self.time_limit = time_limit
+    self.hyperparameters = hyperparameters
+    self.presets = presets
+    self.model_save_path = model_save_path
+
+
+def parse_args() -> (
+    tuple[DataConfig, ProblemConfig, EvaluationConfig, TrainingConfig]
+):
+  """Parse command line arguments."""
+  parser = argparse.ArgumentParser(description="AutoGluon Tabular Predictor")
+  # Add arguments for each config class
+  parser.add_argument(
+      "--train_data_path",
+      type=str,
+      required=True,
+      help="Path to the input data CSV file.",
+  )
+  parser.add_argument(
+      "--label", type=str, required=True, help="Target variable column name."
+  )
+  parser.add_argument(
+      "--problem_type",
+      type=str,
+      choices=["binary", "multiclass", "regression", "quantile"],
+      default=None,
+      help="Problem type.",
+  )
+  parser.add_argument(
+      "--eval_metric", type=str, default=None, help="Evaluation metric to use."
+  )
+  # Add arguments for TrainingConfig if needed
+  parser.add_argument(
+      "--time_limit",
+      type=int,
+      default=None,
+      help="Time limit in seconds for training.",
+  )
+  parser.add_argument(
+      "--presets",
+      type=str,
+      default="medium_quality",
+      help="Presets used for training ",
+  )
+  parser.add_argument(
+      "--hyperparameters",
+      type=json.loads,
+      default=None,
+      help="Hyperparameter dictionary in JSON format.",
+  )
+  parser.add_argument(
+      "--model_save_path",
+      type=str,
+      default=None,
+      help="Path to save the trained model.",
+  )
+  args = parser.parse_args()
+
+  data_config = DataConfig(train_data_path=args.train_data_path)
+  problem_config = ProblemConfig(
+      label=args.label, problem_type=args.problem_type
+  )
+  eval_config = EvaluationConfig(eval_metric=args.eval_metric)
+  training_config = TrainingConfig(
+      time_limit=args.time_limit,
+      presets=args.presets,
+      hyperparameters=args.hyperparameters,
+      model_save_path=args.model_save_path,
+  )
+
+  return data_config, problem_config, eval_config, training_config
+
+
+def main() -> None:
+  data_config, problem_config, eval_config, training_config = parse_args()
+
+  # Load the training data.
+  data = pd.read_csv(data_config.train_data_path)
+
+  # Create a TabularPredictor.
+  predictor = TabularPredictor(
+      label=problem_config.label,
+      eval_metric=eval_config.eval_metric,
+      path=training_config.model_save_path,
+  )
+
+  # Fit the model
+  predictor.fit(
+      data,
+      presets=training_config.presets,
+      time_limit=training_config.time_limit,
+      hyperparameters=training_config.hyperparameters,
+  )
+
+
+if __name__ == "__main__":
+  main()

community-content/vertex_model_garden/model_oss/cloudnerf/configs/360.gin

@@ -0,0 +1,25 @@
+# The provided content is a configuration file for the ZipNeRF
+# PyTorch implementation.
+
+# Sets the name of the experiment to 'test'.
+Config.exp_name = 'test'
+# Specifies the dataset loader, in this case, 'llff' for light field.
+Config.dataset_loader = 'llff'
+# Defines the near and far clipping planes for the camera view.
+Config.near = 0.2
+Config.far = 1e6
+# Image downsampling.
+Config.factor = 4
+
+# For the model configurations.
+Model.raydist_fn = 'power_transformation'
+Model.opaque_background = True
+
+# Disables the computation of density normals and RGB values, and sets
+# the grid level dimension to 1 for PropMLP.
+PropMLP.disable_density_normals = True
+PropMLP.disable_rgb = True
+PropMLP.grid_level_dim = 1
+
+# Disable density normals for NerfMLP
+NerfMLP.disable_density_normals = True

community-content/vertex_model_garden/model_oss/cloudnerf/configs/360_glo.gin

@@ -0,0 +1,21 @@
+# The provided content is a configuration file for Generative
+# Latent Optimization (GLO) vectors in the Pytorch implemnetation of ZipNeRF.
+
+# Specifies the dataset loader, in this case, 'llff' for light field.
+Config.dataset_loader = 'llff'
+# Defines the near and far clipping planes for the camera view.
+Config.near = 0.2
+Config.far = 1e6
+# Image downsampling.
+Config.factor = 4
+
+# For the model configurations.
+Model.raydist_fn = 'power_transformation'
+Model.num_glo_features = 128
+Model.opaque_background = True
+
+PropMLP.disable_density_normals = True
+PropMLP.disable_rgb = True
+PropMLP.grid_level_dim = 1
+
+NerfMLP.disable_density_normals = True

community-content/vertex_model_garden/model_oss/cloudnerf/configs/accelerate_config.yaml

@@ -0,0 +1,18 @@
+# The provided content is a configuration file running ZipNeRF
+# training on 8 gpu machine.
+compute_environment: LOCAL_MACHINE
+debug: false
+distributed_type: MULTI_GPU
+downcast_bf16: 'no'
+gpu_ids: all
+machine_rank: 0
+main_training_function: main
+mixed_precision: fp16
+num_machines: 1
+num_processes: 8
+rdzv_backend: static
+same_network: true
+tpu_env: []
+tpu_use_cluster: false
+tpu_use_sudo: false
+use_cpu: false

community-content/vertex_model_garden/model_oss/cloudnerf/dockerfile/pytorch_cloudnerf_base.Dockerfile

@@ -0,0 +1,120 @@
+# Dockerfile for ZipNeRF base image.
+#
+# To build:
+# docker build -f model_oss/cloudnerf/dockerfile/pytorch_cloudnerf_base.Dockerfile . -t ${YOUR_IMAGE_TAG}
+#
+# To push to gcr:
+# docker tag ${YOUR_IMAGE_TAG} gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
+# docker push gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
+
+FROM pytorch/pytorch:2.1.0-cuda11.8-cudnn8-devel
+
+USER root
+
+ARG COLMAP_GIT_COMMIT=main
+ARG CUDA_ARCHITECTURES=60;70;75;80;86
+
+# Prevent stop building ubuntu at time zone selection.
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update -y --allow-releaseinfo-change && apt-get -y upgrade && apt-get install -y --no-install-recommends \
+        curl \
+        g++ \
+        wget \
+        vim \
+        bash \
+        cmake \
+        imagemagick \
+        ninja-build \
+        build-essential \
+        libboost-program-options-dev \
+        libboost-filesystem-dev \
+        libboost-graph-dev \
+        libboost-system-dev \
+        libeigen3-dev \
+        libflann-dev \
+        libfreeimage-dev \
+        libmetis-dev \
+        libgoogle-glog-dev \
+        libgtest-dev \
+        libsqlite3-dev \
+        libglew-dev \
+        qtbase5-dev \
+        libqt5opengl5-dev \
+        libcgal-dev \
+        libceres-dev \
+        git \
+        git-lfs \
+        python3-cffi \
+        python3-cryptography \
+        libffi-dev \
+        python-dev
+
+# Copy license.
+RUN wget https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/LICENSE
+
+# Install google cloud CLI.
+RUN wget -q https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-430.0.0-linux-x86.tar.gz
+RUN tar xzf google-cloud-cli-430.0.0-linux-x86.tar.gz
+RUN ./google-cloud-sdk/install.sh -q
+# Make sure gsutil will use the default service account.
+RUN echo '[GoogleCompute]\nservice_account = default' > /etc/boto.cfg
+
+# Install deps and install gsutil.
+RUN pip install gsutil==5.27
+
+# When building colmap in colab, the link error "undefined reference.
+# to '_glapi_tls_Current'" happens. A solution is to install "libglvnd"
+# as described in this page https://github.com/colmap/colmap/issues/1271.
+RUN git clone --depth 1 --branch v1.7.0 https://github.com/NVIDIA/libglvnd && \
+    apt-get install -y libxext-dev libx11-dev x11proto-gl-dev && \
+    cd libglvnd/  && \
+    apt-get install -y autoconf automake libtool && \
+    apt-get install -y libffi-dev && \
+    ./autogen.sh && \
+    ./configure && \
+    make  -j4 && \
+    make install
+
+RUN apt remove nvidia-cuda-toolkit -y \
+    nvidia-cuda-toolkit \
+    nvidia-cuda-toolkit-gcc
+
+# Install libraries.
+ENV PIP_ROOT_USER_ACTION=ignore
+RUN python3 -m pip install --upgrade pip
+
+ENV CUDA_HOME=/usr/local/cuda
+
+RUN git clone --branch main https://github.com/SuLvXiangXin/zipnerf-pytorch.git
+# Set current directory to the downloaded 'zipnerf-pytorch' repository.
+WORKDIR ./zipnerf-pytorch
+# Using git reset command to pin it down to a specific version.
+RUN git reset --hard 4de3d21ebb9e15412d36951b56e2d713fddd812b
+COPY model_oss/cloudnerf/requirements.txt requirements.txt
+RUN pip install -r requirements.txt
+
+# Install gridencoder extensions and nvdiffrast (for textured mesh).
+RUN cd .. && \
+    TORCH_CUDA_ARCH_LIST="6.0 7.0 7.5 8.0 8.6+PTX" CXX=g++ pip install ./zipnerf-pytorch/gridencoder
+
+# Install cuda version of torch_scatter.
+RUN pip install torch-scatter==2.1.2 -f https://data.pyg.org/whl/torch-2.0.1+cu118.html
+RUN pip install google-cloud-aiplatform==1.25.0
+RUN pip install google-cloud-storage==2.9.0
+
+# Build and install COLMAP.
+RUN git clone  --depth 1 --branch 3.8 https://github.com/colmap/colmap.git
+RUN cd colmap && \
+    git fetch https://github.com/colmap/colmap.git ${COLMAP_GIT_COMMIT} && \
+    mkdir build && \
+    cd build && \
+    cmake .. -GNinja -DCMAKE_CUDA_ARCHITECTURES=${CUDA_ARCHITECTURES} && \
+    ninja && \
+    ninja install && \
+    cd .. && rm -rf colmap
+
+RUN git clone  --depth 1 --branch v1.0.2 https://github.com/dranjan/python-plyfile.git
+
+RUN sed -i "20 i\sys.path.append('/workspace/zipnerf-pytorch/internal/pycolmap')" /workspace/zipnerf-pytorch/internal/datasets.py
+RUN sed -i "21 i\sys.path.append('/workspace/zipnerf-pytorch/internal/pycolmap/pycolmap')" /workspace/zipnerf-pytorch/internal/datasets.py

community-content/vertex_model_garden/model_oss/cloudnerf/dockerfile/pytorch_cloudnerf_calibrate.Dockerfile

@@ -0,0 +1,16 @@
+# Dockerfile for ZipNeRF COLMAP image calibration.
+#
+# To build:
+# docker build -f model_oss/cloudnerf/dockerfile/cloudnerf_pytorch_calibrate.Dockerfile . -t ${YOUR_IMAGE_TAG}
+#
+# To push to gcr:
+# docker tag ${YOUR_IMAGE_TAG} gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
+# docker push gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
+
+FROM us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-cloudnerf-base:20231206_0923_RC00
+
+COPY model_oss/cloudnerf/local_colmap_and_resize.sh /workspace/zipnerf-pytorch/scripts/local_colmap_and_resize.sh
+
+WORKDIR /workspace/zipnerf-pytorch/
+
+ENTRYPOINT ["bash","scripts/local_colmap_and_resize.sh"]

community-content/vertex_model_garden/model_oss/cloudnerf/dockerfile/pytorch_cloudnerf_render.Dockerfile

@@ -0,0 +1,22 @@
+# Dockerfile for ZipNeRF rendering.
+#
+# To build:
+# docker build -f model_oss/cloudnerf/dockerfile/pytorch_cloudnerf_render.Dockerfile . -t ${YOUR_IMAGE_TAG}
+#
+# To push to gcr:
+# docker tag ${YOUR_IMAGE_TAG} gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
+# docker push gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
+
+FROM us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-cloudnerf-base:20231206_0923_RC00
+
+COPY model_oss/cloudnerf/render.sh /workspace/zipnerf-pytorch/scripts/render.sh
+COPY model_oss/cloudnerf/configs/360.gin /workspace/zipnerf-pytorch/configs/360.gin
+COPY model_oss/cloudnerf/configs/360_glo.gin /workspace/zipnerf-pytorch/configs/360_glo.gin
+COPY model_oss/cloudnerf/configs/accelerate_config.yaml /root/.cache/huggingface/accelerate/default_config.yaml
+RUN sed -i '324s/.*/            keyframe_names = fp.read().splitlines()/' /workspace/zipnerf-pytorch/internal/camera_utils.py
+
+ENV PYTHONPATH "${PYTHONPATH}:/workspace/zipnerf-pytorch/util"
+
+WORKDIR /workspace/zipnerf-pytorch/
+
+ENTRYPOINT ["bash", "scripts/render.sh"]

community-content/vertex_model_garden/model_oss/cloudnerf/dockerfile/pytorch_cloudnerf_train.Dockerfile

@@ -0,0 +1,21 @@
+# Dockerfile for ZipNeRF training.
+#
+# To build:
+# docker build -f model_oss/cloudnerf/dockerfile/pytorch_cloudnerf_train.Dockerfile . -t ${YOUR_IMAGE_TAG}
+#
+# To push to gcr:
+# docker tag ${YOUR_IMAGE_TAG} gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
+# docker push gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
+
+FROM us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-cloudnerf-base:20231206_0923_RC00
+
+COPY model_oss/cloudnerf/train.sh /workspace/zipnerf-pytorch/scripts/train.sh
+COPY model_oss/cloudnerf/configs/360.gin /workspace/zipnerf-pytorch/configs/360.gin
+COPY model_oss/cloudnerf/configs/360_glo.gin /workspace/zipnerf-pytorch/configs/360_glo.gin
+COPY model_oss/cloudnerf/configs/accelerate_config.yaml /root/.cache/huggingface/accelerate/default_config.yaml
+
+ENV PYTHONPATH "${PYTHONPATH}:/workspace/zipnerf-pytorch/util"
+
+WORKDIR /workspace/zipnerf-pytorch/
+
+ENTRYPOINT ["bash", "scripts/train.sh"]

community-content/vertex_model_garden/model_oss/cloudnerf/local_colmap_and_resize.sh

@@ -0,0 +1,144 @@
+#!/bin/bash
+# This script runs colmap for scale invariant feature (SIFT) extraction and
+# matching to map camera extrinsics and intrinsics values for ZipNeRF,
+# given a folder of images and videos
+# from a GCS bucket. It uses ffmepg to extract an image from a video at
+# 1fps. The folder can contain images or videos. If both images and videos
+# are present, the extracted frames from the videos is added to the images
+# to create the final combined image dataset.
+# vv-docker:google3-begin(internal)
+# TODO(b/314042136): Specify cloudnerf colmap fps.
+# vv-docker:google3-end
+
+# Initialize variables.
+use_gpu=1  # Default to 1 (assuming the docker is run on a machine with GPU)
+gcs_dataset_path=""
+gcs_experiment_path=""
+camera=""
+
+# This loop processes command-line arguments for configuring the container.
+# It supports arguments for GPU usage, dataset and experiment paths,
+# and camera type.
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    -use_gpu)
+      use_gpu="$2"
+      if ! [[ $use_gpu =~ ^[0-9]+$ ]]; then
+        echo "Error: -use_gpu must be an integer."
+        exit 1
+      fi
+      shift # past argument
+      shift # past value
+      ;;
+    -gcs_dataset_path)
+      gcs_dataset_path="$2"
+      shift # past argument
+      shift # past value
+      ;;
+    -gcs_experiment_path)
+      gcs_experiment_path="$2"
+      shift # past argument
+      shift # past value
+      ;;
+    -camera)
+      camera="$2"
+      if [[ $camera != "OPENCV" && $camera != "OPENCV_FISHEYE" ]]; then
+        echo "Error: -camera must be either 'OPENCV' or 'OPENCV_FISHEYE'."
+        exit 1
+      fi
+      shift # past argument
+      shift # past value
+      ;;
+    *)    # unknown option
+      echo "Unknown option: $1" >&2
+      exit 1
+      ;;
+  esac
+done
+
+local_folder="dataset_content"
+images_folder="dataset_images"
+images_subfolder="images"
+output_folder="$images_folder/$images_subfolder"
+
+# Create the local folder if it doesn't exist
+mkdir -p "$local_folder"
+mkdir -p "$output_folder"
+
+# Download the content from the GCS URI
+gsutil -m cp -r "$gcs_dataset_path"/* "$local_folder/"
+
+# Process files in the local folder
+for file in "$local_folder"/*; do
+  if [[ -f "$file" ]]; then
+    # Check if the file is an image (e.g., jpg, png, etc.)
+    if file --mime-type "$file" | grep -q "image"; then
+      # Copy the image to the "images" subfolder within the "dataset_images" folder
+      cp "$file" "$output_folder/$(basename "$file")"
+    elif file --mime-type "$file" | grep -q "video"; then
+      # Use FFmpeg to extract an image every 30 frames from the video
+      ffmpeg -i "$file" -vf "select='not(mod(n,30))'" "$output_folder/$(basename "$file" ."${file##*.}")_%03d.jpg"
+    else
+      echo "Skipping unsupported file: $file"
+    fi
+  fi
+done
+
+# Run COLMAP Feature extraction
+colmap feature_extractor \
+    --database_path "$local_folder"/database.db \
+    --image_path "$output_folder" \
+    --ImageReader.single_camera 1 \
+    --ImageReader.camera_model "$camera" \
+    --SiftExtraction.use_gpu "$use_gpu"
+
+# Run COLMAP Feature matching
+colmap exhaustive_matcher \
+    --database_path "$local_folder"/database.db \
+    --SiftMatching.use_gpu "$use_gpu"
+
+# Bundle adjustment. The default Mapper tolerance is unnecessarily large,
+# decreasing it speeds up bundle adjustment steps.
+mkdir -p "$local_folder"/sparse
+colmap mapper \
+    --database_path "$local_folder"/database.db \
+    --image_path "$output_folder" \
+    --output_path "$local_folder"/sparse \
+    --Mapper.ba_global_function_tolerance=0.000001
+
+# Downsample images at 1/2, 1/4, 1/8 scales. Save feature matching to
+# sqlite database.
+# All input and output images:
+#   $gcs_dataset_path
+#   $gcs_experiment_path/data/images
+# Downsampled output images:
+#   $gcs_experiment_path/data/images_2/
+#   $gcs_experiment_path/data/images_4/
+#   $gcs_experiment_path/data/images_8/
+# COLMAP sparse reconstruction files: project.ini, images.bin,
+# cameras.bin, points3D.bin
+#   $gcs_experiment_path/data/sparse/0/
+cp -r "$output_folder" "$images_folder"/images_2
+pushd "$images_folder"/images_2
+ls | xargs -P 8 -I {} mogrify -resize 50% {}
+popd
+gsutil -m cp -r "$images_folder"/images_2/*  "$gcs_experiment_path"/data/images_2
+
+cp -r "$output_folder" "$images_folder"/images_4
+pushd "$images_folder"/images_4
+ls | xargs -P 8 -I {} mogrify -resize 25% {}
+popd
+gsutil -m cp -r "$images_folder"/images_4/*  "$gcs_experiment_path"/data/images_4
+
+cp -r "$output_folder" "$images_folder"/images_8
+pushd "$images_folder"/images_8
+ls | xargs -P 8 -I {} mogrify -resize 12.5% {}
+popd
+gsutil -m cp "$images_folder"/images_8/*  "$gcs_experiment_path"/data/images_8
+
+# Copy images and sparse reconstruction files to gcs experiment folder.
+gsutil -m cp "$images_folder"/images/*  "$gcs_experiment_path"/data/images
+gsutil -m cp -r "$local_folder"/sparse  "$gcs_experiment_path"/data
+gsutil -m cp "$local_folder"/database.db  "$gcs_experiment_path"/data
+
+echo "Processing complete."

community-content/vertex_model_garden/model_oss/cloudnerf/render.sh

@@ -0,0 +1,117 @@
+#!/bin/bash
+# This script runs rendering for ZipNeRF given an experiment folder
+# from a GCS bucket with colmap dataset.
+
+# Initialize associative array for arguments.
+declare -A args
+
+# vv-docker:google3-begin(internal)
+# TODO(b/311468174): Pass gin config file from gcs bucket.
+# vv-docker:google3-end
+# Function to parse named arguments.
+parse_args() {
+  while [[ $# -gt 0 ]]; do
+    key="$1"
+    case $key in
+      -gcs_experiment_path|-gin_config_file|-gcs_keyframes_file)
+        args[$key]="$2"
+        shift # past argument
+        shift # past value
+        ;;
+      -training_job_name)
+        training_job_name="$2"
+        shift # past argument
+        shift # past value
+        ;;
+      -rendering_job_name)
+        rendering_job_name="$2"
+        shift # past argument
+        shift # past value
+        ;;
+      -render_path_frames|-factor|-render_video_fps)
+        args[$key]="$2"
+        if ! [[ ${args[$key]} =~ ^[0-9]+$ ]]; then
+          echo "Error: $key must be an integer."
+          exit 1
+        fi
+        shift # past argument
+        shift # past value
+        ;;
+      *)
+        echo "Unknown option: $1" >&2
+        exit 1
+        ;;
+    esac
+  done
+}
+
+# Function to create a directory if it doesn't exist.
+create_dir_if_not_exists() {
+  local dir_path=$1
+  if [[ ! -d "$dir_path" ]]; then
+    echo "Creating folder: $dir_path"
+    mkdir "$dir_path"
+  else
+    echo "Folder $dir_path already exists."
+  fi
+}
+
+# Function to launch rendering.
+launch_rendering() {
+  local keyframes_file=$1
+  local render_bindings=(
+    "--gin_configs=${args[-gin_config_file]}"
+    "--gin_bindings=Config.data_dir='${DATASET_PATH}'"
+    "--gin_bindings=Config.exp_name='${EXPERIMENT}'"
+    "--gin_bindings=Config.render_path=True"
+    "--gin_bindings=Config.render_path_frames=${args[-render_path_frames]}"
+    "--gin_bindings=Config.render_video_fps=${args[-render_video_fps]}"
+    "--gin_bindings=Config.factor=${args[-factor]}"
+  )
+
+  if [[ -n $keyframes_file ]]; then
+    render_bindings+=("--gin_bindings=Config.render_spline_keyframes='${keyframes_file}'")
+  fi
+
+  accelerate launch render.py "${render_bindings[@]}"
+}
+
+# Parse arguments.
+parse_args "$@"
+
+# Extract folder names and paths.
+scene_folder_name=$(basename "${args[-gcs_experiment_path]}")
+local_dataset_path="local_dataset"
+local_experiment_path="exp"
+exp_folder_name=$(basename "${args[-gcs_experiment_path]}")
+DATASET_PATH="$local_experiment_path/$exp_folder_name/data"
+CHECKPOINTS_PATH="$local_experiment_path/$exp_folder_name/checkpoints"
+OUTPUT_RENDER_PATH="$local_experiment_path/$scene_folder_name/render"
+EXPERIMENT=$exp_folder_name
+
+# Create necessary directories.
+create_dir_if_not_exists "$local_dataset_path"
+create_dir_if_not_exists "$local_experiment_path"
+create_dir_if_not_exists "$local_experiment_path/$exp_folder_name"
+create_dir_if_not_exists "$CHECKPOINTS_PATH"
+
+# Create the file log_render.txt in the exp folder.
+touch "$local_experiment_path/$exp_folder_name/log_render.txt"
+
+# Copy experiment from GCS bucket to local
+gsutil -m cp -r "${args[-gcs_experiment_path]}/data" "$local_experiment_path/$exp_folder_name" || exit 1
+gsutil -m cp -r "${args[-gcs_experiment_path]}/checkpoints/${training_job_name}/*" "$CHECKPOINTS_PATH" || exit 1
+
+# Check and copy keyframes file.
+if [[ -n ${args[-gcs_keyframes_file]} ]]; then
+  keyframes_file_basename=$(basename "${args[-gcs_keyframes_file]}")
+  local_keyframes_file="$local_dataset_path/$keyframes_file_basename"
+  gsutil cp "${args[-gcs_keyframes_file]}" "$local_keyframes_file" || exit 1
+  echo "Local keyframe file: $local_keyframes_file"
+  launch_rendering "$local_keyframes_file"
+else
+  launch_rendering ""
+fi
+
+# Copy rendered data back to GCS.
+gsutil -m cp -r "$OUTPUT_RENDER_PATH" "${args[-gcs_experiment_path]}/render/${rendering_job_name}"

community-content/vertex_model_garden/model_oss/cloudnerf/requirements.txt

@@ -0,0 +1,24 @@
+--find-links https://download.pytorch.org/whl/torch_stable.html
+
+torch==2.0.1+cu118
+numpy==1.26.1
+absl_py==2.0.0
+accelerate==0.24.0
+gin_config==0.5.0
+imageio==2.31.6
+imageio-ffmpeg==0.4.9
+matplotlib==3.8.0
+mediapy==1.1.9
+ninja==1.11.1.1
+opencv_contrib_python==4.8.1.78
+opencv_python==4.8.1.78
+Pillow==10.3.0
+rawpy==0.18.1
+scipy==1.11.3
+scikit-image==0.22.0
+scikit-learn==1.3.2
+tensorboard==2.15.0
+tensorboardX==2.6.2.2
+tqdm==4.66.3
+trimesh==4.0.1
+xatlas==0.0.8

community-content/vertex_model_garden/model_oss/cloudnerf/train.sh

@@ -0,0 +1,94 @@
+#!/bin/bash
+
+# Initialize variables.
+training_job_name=""
+gcs_experiment_path=""
+gin_config_file="configs/360.gin"
+factor=4
+max_training_steps=25000
+
+# Parse named arguments.
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    -training_job_name)
+      training_job_name="$2"
+      shift # past argument
+      shift # past value
+      ;;
+    -gcs_experiment_path)
+      gcs_experiment_path="$2"
+      shift # past argument
+      shift # past value
+      ;;
+    -gin_config_file)
+      gin_config_file="$2"
+      shift # past argument
+      shift # past value
+      ;;
+    -factor)
+      factor="$2"
+      if ! [[ $factor =~ ^[0-9]+$ ]]; then
+        echo "Error: -factor must be an integer."
+        exit 1
+      fi
+      shift # past argument
+      shift # past value
+      ;;
+    -max_training_steps)
+      max_training_steps="$2"
+      if ! [[ $max_training_steps =~ ^[0-9]+$ ]]; then
+        echo "Error: -max_training_steps must be an integer."
+        exit 1
+      fi
+      shift # past argument
+      shift # past value
+      ;;
+    *)    # unknown option
+      echo "Unknown option: $1" >&2
+      exit 1
+      ;;
+  esac
+done
+
+# Function to create a directory if it doesn't exist.
+create_dir_if_not_exists() {
+  local dir_path=$1
+  if [[ ! -d "$dir_path" ]]; then
+    echo "Creating folder: $dir_path"
+    mkdir "$dir_path"
+  else
+    echo "Folder $dir_path already exists."
+  fi
+}
+
+# Extract folder names and paths.
+scene_folder_name=$(basename "${gcs_experiment_path}")
+local_dataset_path="local_dataset"
+local_experiment_path="exp"
+DATASET_PATH="$local_experiment_path/$scene_folder_name/data"
+EXPERIMENT=$scene_folder_name
+
+# Create necessary directories.
+create_dir_if_not_exists "$local_dataset_path"
+create_dir_if_not_exists "$local_experiment_path"
+create_dir_if_not_exists "$local_experiment_path/$scene_folder_name"
+
+# Copy experiment from GCS bucket to local.
+gsutil -m cp -r "${gcs_experiment_path}/data" "$local_experiment_path/$scene_folder_name" || exit 1
+
+echo "GCS Experiment: $gcs_experiment_path"
+echo "Gin Config File: $gin_config_file"
+echo "Factor: $factor"
+echo "Scene: $scene_folder_name"
+echo "Local Dataset: $DATASET_PATH"
+echo "Local Experiment: $EXPERIMENT"
+
+accelerate launch train.py --gin_configs="$gin_config_file" \
+  --gin_bindings="Config.data_dir = '${DATASET_PATH}'" \
+  --gin_bindings="Config.exp_name = '${EXPERIMENT}'" \
+  --gin_bindings="Config.factor = ${factor}" \
+  --gin_bindings="Config.max_steps = ${max_training_steps}"
+
+gsutil -m rm -r "${gcs_experiment_path}/checkpoints/${training_job_name}"
+gsutil -m cp -r "$local_experiment_path/$scene_folder_name/config.gin" "${gcs_experiment_path}/${training_job_name}_config.gin"
+gsutil -m cp -r "$local_experiment_path/$scene_folder_name/checkpoints/*/*" "${gcs_experiment_path}/checkpoints/${training_job_name}"

community-content/vertex_model_garden/model_oss/data_converter/common_lib.py

@@ -0,0 +1,623 @@
+"""Library with functions to use for data conversion."""
+
+import json
+import os
+import random
+from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple, Union
+import uuid
+
+from absl import logging
+import apache_beam as beam
+import cv2
+import numpy as np
+import pandas as pd
+import PIL
+from PIL import Image
+import tensorflow as tf
+import yaml
+
+from util import constants
+from util import fileutils
+from apache_beam.options import pipeline_options
+
+REFORMATTED_CSV_SUFFIX = '-reformatted.csv'
+
+LABEL_MAP_NAME = 'label_map.yaml'
+
+_SPLIT_RATIO_ERROR_THRESHOLD = 1e-5
+# Internal constant. Only for distinguishing rows without ML use.
+ML_USE_UNASSIGNED = 'unassigned'
+ALL_ML_USES = (
+    constants.ML_USE_TRAINING,
+    constants.ML_USE_VALIDATION,
+    constants.ML_USE_TEST,
+    ML_USE_UNASSIGNED,
+)
+COLUMN_NAME_ML_USE = 'ml_use'
+COLUMN_NAME_GCS_FILE_PATH = 'gcs_file_path'
+COLUMN_NAME_LABEL = 'label'
+COLUMN_NAME_START_SEC = 'start_sec'
+COLUMN_NAME_END_SEC = 'end_sec'
+# Output filenames
+TRAIN_TFRECORD_NAME = 'train.tfrecord'
+VALIDATION_TFRECORD_NAME = 'val.tfrecord'
+TEST_TFRECORD_NAME = 'test.tfrecord'
+# Jsonl keys
+JSON_GCS_URI_KEY = 'imageGcsUri'
+JSON_RESOURCE_LABEL_KEY = 'dataItemResourceLabels'
+JSON_ML_USE_KEY = 'aiplatform.googleapis.com/ml_use'
+# I/O parameters
+READ_CHUNK_SIZE = 1024 * 1024 * 1024  # 1GB
+
+
+class WriteToTFRecord(beam.DoFn):
+  """DoFn to write TF examples to sharded TF record files."""
+
+  def __init__(
+      self,
+      output_prefix: str,
+      num_shards: int,
+      convert_fn: Callable[[Dict[str, Any]], tf.train.Example],
+  ):
+    self.output_prefix = output_prefix
+    self.num_shards = num_shards
+    self.writer: list[tf.io.TFRecordWriter] = []
+    self.sharded_files: list[str] = []
+    self.convert_fn = convert_fn
+    self.success_counter = beam.metrics.Metrics.counter(
+        self.__class__.__name__, 'Success'
+    )
+    self.failure_counter = beam.metrics.Metrics.counter(
+        self.__class__.__name__, 'Failure'
+    )
+
+  def start_bundle(self):
+    logging.info('Start writing TF Record to %s.', self.output_prefix)
+    unique_str = uuid.uuid4().hex
+    for i in range(self.num_shards):
+      uri = f'{self.output_prefix}-{i}-{unique_str}'
+      self.sharded_files.append(uri)
+      self.writer.append(tf.io.TFRecordWriter(uri))
+
+  def process(self, data: Dict[str, Any]) -> Iterable[Tuple[int, str]]:
+    try:
+      example = self.convert_fn(data)
+      data = example.SerializeToString()
+      idx = hash(data) % self.num_shards
+      self.writer[idx].write(data)
+      self.success_counter.inc()
+      yield (idx, self.sharded_files[idx])
+    # pylint: disable-next=broad-exception-caught
+    except Exception as err:
+      logging.error('Failed to process %s', data)
+      logging.exception(err)
+      self.failure_counter.inc()
+
+  def finish_bundle(self):
+    logging.info('Finish writing TF Record to %s.', self.output_prefix)
+    for writer in self.writer:
+      writer.close()
+    self.writer = []
+
+
+def convert_to_feature(
+    value: Union[List[Union[int, float, bytes]], int, float, bytes],
+    value_type: Optional[str] = None,
+) -> tf.train.Feature:
+  """Converts the given python object to a tf.train.Feature.
+
+  This is copied from tensorflow_models/official/vision/data/tfrecord_lib.py.
+
+  Args:
+    value: int, float, bytes or a list of them.
+    value_type: optional, if specified, forces the feature to be of the given
+      type. Otherwise, type is inferred automatically. Can be one of ['bytes',
+      'int64', 'float', 'bytes_list', 'int64_list', 'float_list']
+
+  Returns:
+    feature: A tf.train.Feature object.
+  """
+
+  if value_type is None:
+    element = value[0] if isinstance(value, list) else value
+
+    if isinstance(element, bytes):
+      value_type = 'bytes'
+
+    elif isinstance(element, (int, np.integer)):
+      value_type = 'int64'
+
+    elif isinstance(element, (float, np.floating)):
+      value_type = 'float'
+
+    else:
+      raise ValueError(
+          'Cannot convert type {} to feature'.format(type(element))
+      )
+
+    if isinstance(value, list):
+      value_type = value_type + '_list'
+
+  if value_type == 'int64':
+    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))
+
+  elif value_type == 'int64_list':
+    value = np.asarray(value).astype(np.int64).reshape(-1)
+    return tf.train.Feature(int64_list=tf.train.Int64List(value=value))
+
+  elif value_type == 'float':
+    return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))
+
+  elif value_type == 'float_list':
+    value = np.asarray(value).astype(np.float32).reshape(-1)
+    return tf.train.Feature(float_list=tf.train.FloatList(value=value))
+
+  elif value_type == 'bytes':
+    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
+
+  elif value_type == 'bytes_list':
+    return tf.train.Feature(bytes_list=tf.train.BytesList(value=value))
+
+  else:
+    raise ValueError('Unknown value_type parameter - {}'.format(value_type))
+
+
+def convert_to_string_feature(
+    value: str, encoding: str = 'utf-8'
+) -> tf.train.Feature:
+  """Returns a bytes_list from an encoded string."""
+  return convert_to_feature(value.encode(encoding))
+
+
+def convert_to_list_string_feature(
+    lst: list[str], encoding: str = 'utf-8'
+) -> tf.train.Feature:
+  """Returns a bytes_list from a list of encoded strings."""
+  return convert_to_feature([value.encode(encoding) for value in lst])
+
+
+def create_ml_use_array_with_split(
+    total_size: int,
+    split_ratio: Sequence[float],
+) -> list[str]:
+  """Create randomized list of 'training', 'validation', 'test'.
+
+  The list of will be of length total_size with ratios according to train_size,
+  validation_size, and test_size.
+
+  Args:
+    total_size: Length of sequence to return
+    split_ratio: Proportions to split into 'training', 'validation', and 'test'
+
+  Returns:
+    List containing 'training', 'validation', and 'test'
+  """
+  train_size, validation_size, _ = split_ratio
+  num_train = round(train_size * total_size)
+  num_validation = round(validation_size * total_size)
+  num_test = total_size - num_train - num_validation
+  ml_use_row = (
+      [constants.ML_USE_TRAINING] * num_train
+      + [constants.ML_USE_VALIDATION] * num_validation
+      + [constants.ML_USE_TEST] * num_test
+  )
+  random.shuffle(ml_use_row)
+  return ml_use_row
+
+
+def format_ml_use_column(df: pd.DataFrame):
+  df[COLUMN_NAME_ML_USE].replace(
+      # We need to support non-standard ML uses other than documented ones,
+      # since they are used by some existing datasets.
+      [r'(?i)^train(ing)?$', r'(?i)^test$', r'(?i)^validat(ion|e)$'],
+      [
+          constants.ML_USE_TRAINING,
+          constants.ML_USE_TEST,
+          constants.ML_USE_VALIDATION,
+      ],
+      inplace=True,
+      regex=True,
+  )
+
+
+def insert_missing_ml_use(df: pd.DataFrame) -> None:
+  """For every row that does not have ml_use as the first column, insert a column containing 'unassigned' to the front.
+
+  Args:
+    df: The DataFrame to process. The first column should be 'ml_use'.
+  """
+  df[COLUMN_NAME_ML_USE].fillna(ML_USE_UNASSIGNED, inplace=True)
+  rows_to_fill = ~df[COLUMN_NAME_ML_USE].isin(ALL_ML_USES)
+  df.loc[rows_to_fill] = df[rows_to_fill].shift(
+      axis=1, fill_value=ML_USE_UNASSIGNED
+  )
+
+
+def replace_unassigned_ml_use(
+    ml_uses: List[str],
+    split_ratio: Sequence[float],
+):
+  """Replace `unassigned` in ml_uses with `training`, `validation`, and `test` with ratios according to split_ratio.
+
+  Args:
+    ml_uses: List of ml_use string values.
+    split_ratio: Proportions to split into `training`, `validation`, and `test`.
+  """
+  unassigned_indices = [
+      i for i, ml_use in enumerate(ml_uses) if ml_use == ML_USE_UNASSIGNED
+  ]
+  ml_use_arr = create_ml_use_array_with_split(
+      len(unassigned_indices), split_ratio
+  )
+  for unassigned_index, ml_use in zip(unassigned_indices, ml_use_arr):
+    ml_uses[unassigned_index] = ml_use
+
+
+def merge_seq_into_dicts(
+    key: str, values: Sequence[Any], dicts: Sequence[Dict[Any, Any]]
+):
+  """Merges a list of values into a list of dicts, inserted with the given key.
+
+  Args:
+    key: Key to insert or overwrite in the dictionary.
+    values: A list of values to insert.
+    dicts: A list of dictionaries. Each value will be inserted into the
+      corresponding dictionary. The original value will be overwritten if the
+      key already existed.
+
+  Raises:
+    ValueError: The values and dicts have different lengths.
+  """
+  if len(values) != len(dicts):
+    raise ValueError(
+        f'Length of values and dicts must match, got {len(values)} and'
+        f' {len(dicts)}'
+    )
+  for val, d in zip(values, dicts):
+    d[key] = val
+
+
+def drop_invalid_rows(df: pd.DataFrame) -> int:
+  """Drops DataFrame rows missing the gcs_file_path column or the label column.
+
+  Args:
+    df: The DataFrame to process in place.
+
+  Returns:
+    The number of rows dropped.
+  """
+  original_rows = df.shape[0]
+  df.dropna(subset=[COLUMN_NAME_GCS_FILE_PATH, COLUMN_NAME_LABEL], inplace=True)
+  dropped_num = original_rows - df.shape[0]
+  if dropped_num > 0:
+    df.reset_index(drop=True, inplace=True)
+  return dropped_num
+
+
+def check_split_ratio(split_ratio: Sequence[float]):
+  """Checks if the give split ratio is valid.
+
+  Args:
+    split_ratio: Proportions to split into 'training', 'validation', and 'test'
+
+  Raises:
+    ValueError: Must have valid entries, correct length, and sum to 1.
+  """
+  if len(split_ratio) != 3:
+    raise ValueError('split_ratio must contain exactly 3 values.')
+  if abs(sum(split_ratio) - 1) > _SPLIT_RATIO_ERROR_THRESHOLD:
+    raise ValueError('split_ratio must sum to 1.')
+  if not all([0 <= val <= 1 for val in split_ratio]):
+    raise ValueError('Entries of split_ratio must be in the range [0, 1].')
+
+
+def check_num_shard(num_shard: Sequence[int]):
+  """Checks if the number of shards is valid.
+
+  Args:
+    num_shard: The number of shards for each tfrecord.
+
+  Raises:
+    ValueError: Must have valid entries and correct length.
+  """
+  if len(num_shard) != 3:
+    raise ValueError('num_shard must contain exactly 3 values.')
+  if not all([val >= 1 for val in num_shard]):
+    raise ValueError('Shards must be at least 1.')
+
+
+def create_label_map_yaml(meta_data_path: str, output_dir: str) -> None:
+  """Generate label_map.yaml from meta_data.yaml.
+
+  Args:
+    meta_data_path: Path to a meta_data.yaml file.
+    output_dir: Directory to output label_map.yaml.
+  """
+  tf.io.gfile.copy(
+      meta_data_path, os.path.join(output_dir, LABEL_MAP_NAME), overwrite=True
+  )
+
+
+def reformat_bbox(
+    bbox: Sequence[int], img_width: int, img_height: int
+) -> Tuple[float, float, float, float]:
+  """Converts XYWH unnormalized bounding box with to a normalized XYXY bounding box.
+
+  Args:
+    bbox: Relative bounding box with unnormalized coordinates as [x, y, width,
+      height].
+    img_width: Image's pixel width.
+    img_height: Image's pixel height.
+
+  Returns:
+    Absolute bounding box with normalized coordinates as
+      [xmin, ymin, xmax, ymax].
+  """
+  x, y, width, height = bbox
+  xmin = x / img_width
+  ymin = y / img_height
+  xmax = (x + width) / img_width
+  ymax = (y + height) / img_height
+  return xmin, ymin, xmax, ymax
+
+
+def encode_image(
+    filepath: str,
+    output_shape: Optional[Sequence[int]] = None,
+    image_format: str = 'png',
+) -> Tuple[bytes, Sequence[int]]:
+  """Encodes an image at the given path.
+
+  Args:
+    filepath: Path to the image.
+    output_shape: The output shape of the image, (height, width).
+    image_format: The format of the output image.
+
+  Returns:
+    The encoded image data in bytes and the shape of the image, (height, width).
+
+  Raises:
+    IOError: The image file is corrupt.
+  """
+  filepath = fileutils.force_gcs_fuse_path(filepath)
+  with open(filepath, 'rb') as f:
+    # If an output_shape is specified, resize the image and set data to the new
+    # bytes.
+    try:
+      img = Image.open(f)
+    except PIL.UnidentifiedImageError as e:
+      raise IOError(f'Failed to open {filepath}') from e
+
+    try:
+      if output_shape is not None:
+        rgb_img = img.resize((output_shape[1], output_shape[0])).convert('RGB')
+      else:
+        rgb_img = img.convert('RGB')
+      rgb_img = np.array(rgb_img)
+
+      _, data = cv2.imencode(f'.{image_format}', rgb_img)
+      data = data.tobytes()
+      return data, rgb_img.shape
+    except cv2.error as e:
+      raise IOError(f'Failed to encode {filepath}') from e
+    finally:
+      img.close()
+
+
+def encode_video(
+    filepath: str,
+    start_sec: float,
+    end_sec: float,
+    output_fps: int = 5,
+    output_shape: Optional[Sequence[int]] = None,
+    image_format: str = 'jpg',
+) -> Sequence[bytes]:
+  """Encodes a video clip at the given path with start and end timestamps.
+
+  Args:
+    filepath: Path to the video.
+    start_sec: Start timestamp of the video clip in seconds.
+    end_sec: End timestamp of the video clip in seconds.
+    output_fps: The output frame rate per second.
+    output_shape: The output shape of each frame, (height, width).
+    image_format: The format of the encoded frames.
+
+  Returns:
+    A list of the encoded frames data in bytes.
+
+  Raises:
+    IOError if the video file is corrupt.
+  """
+  filepath = fileutils.force_gcs_fuse_path(filepath)
+  video = None
+
+  try:
+    video = cv2.VideoCapture(filepath)
+    frames = []
+    frame_interval = 1 / output_fps
+    total_frames = video.get(cv2.CAP_PROP_FRAME_COUNT)
+    original_fps = video.get(cv2.CAP_PROP_FPS)
+    if not original_fps:
+      # 0 or None indicates the video is invalid
+      raise IOError(f'Failed to load {filepath}')
+    video_length = total_frames / original_fps
+    start_sec = max(start_sec, 0)
+    end_sec = min(end_sec, video_length)
+    for t in np.arange(start_sec, end_sec, frame_interval):
+      frame_idx = min(total_frames - 1, round(t * original_fps))
+      video.set(cv2.CAP_PROP_POS_FRAMES, frame_idx)
+      ret, frame = video.read()
+      if not ret:
+        raise IOError(f'Failed to load {filepath} at frame {frame_idx}')
+      if output_shape is not None:
+        frame = cv2.resize(frame, (output_shape[1], output_shape[0]))
+      _, data = cv2.imencode(f'.{image_format}', frame)
+      frames.append(data.tobytes())
+  except cv2.error as e:
+    raise IOError(f'Failed to load {filepath}') from e
+  finally:
+    if video:
+      video.release()
+  return frames
+
+
+def create_label_map(
+    labels: Sequence[str],
+) -> Tuple[Sequence[int], Dict[int, str]]:
+  """Creates a label map from a sequence of label strings.
+
+  Args:
+    labels: The sequence of labels to create label map from. Must not contain
+      invalid values, which means data without labels should be filtered first.
+
+  Returns:
+    The integer labels and the mapping from integers to the original strings.
+  """
+  inverse_label_map: Dict[str, int] = dict()
+  num_labels = 0
+  for label in labels:
+    if label not in inverse_label_map:
+      num_labels += 1
+      inverse_label_map[label] = num_labels
+  int_labels = [inverse_label_map[label] for label in labels]
+  label_map = {value: key for key, value in inverse_label_map.items()}
+  return int_labels, label_map
+
+
+def write_label_map(output_file: str, label_map: Dict[int, str]) -> None:
+  """Writes a label map to the output file, which can be a GCS uri."""
+  with tf.io.gfile.GFile(output_file, 'w') as f:
+    yaml.dump({'label_map': label_map}, f)
+
+
+def detectron_json_to_image_rows(input_json: str) -> list[Dict[str, Any]]:
+  """Converts a Detectron JSON file to a list of image rows.
+
+  Args:
+    input_json: A path to a Detectron JSON or JSONL file.
+
+  Returns:
+    A list of dictionaries, where each dictionary contains Detectron format
+    entry.
+
+  Raises:
+    ValueError: If the input JSON is invalid.
+  """
+
+  image_rows = []
+  with tf.io.gfile.GFile(input_json, 'r') as f:
+    for line in f:
+      json_data = json.loads(line)
+      if isinstance(json_data, dict):
+        image_rows.append(json_data)
+      elif isinstance(json_data, list):
+        image_rows.extend(json_data)
+      else:
+        raise ValueError(
+            'The input JSON is invalid. Dict or list is expected, but got '
+            f'{type(json_data)}.'
+        )
+  return image_rows
+
+
+def coco_json_to_image_rows(
+    input_json: str,
+) -> List[Dict[str, Any]]:
+  """Converts a COCO JSON file to a list of image rows.
+
+  Args:
+    input_json: A path to a COCO JSON or JSONL file.
+
+  Returns:
+    A list of dictionaries, where each dictionary contains COCO format entry.
+
+  Raises:
+    ValueError: If the input JSON is invalid.
+  """
+
+  with tf.io.gfile.GFile(input_json, 'r') as f:
+    coco_json = json.load(f)
+  if 'annotations' not in coco_json:
+    raise ValueError('"annotations" is not in the dataset.')
+  if 'images' not in coco_json:
+    raise ValueError('"images" is not in the dataset.')
+
+  images = coco_json['images']
+  return images
+
+
+def partition_by_ml_use(element: Dict[str, Any], num_partitions: int) -> int:
+  """Beam partition function to split data by ml_use."""
+  del num_partitions
+  try:
+    partition = ALL_ML_USES.index(element[COLUMN_NAME_ML_USE])
+  except Exception as e:
+    raise ValueError(f'Invalid ML use: {element[COLUMN_NAME_ML_USE]}') from e
+  return partition
+
+
+def run_beam_pipeline(pipeline: Any) -> None:
+  """Runs a beam pipeline. Works in both internal and docker environment."""
+  options = pipeline_options.PipelineOptions([
+      '--runner=FlinkRunner',
+      '--faster_copy',
+      '--max_parallelism', '8',
+  ])
+  p = beam.Pipeline(options=options)
+  pipeline(p)
+  result = p.run()
+  result.wait_until_finish()
+  for counter in result.metrics().query()['counters']:
+    logging.info('%s counter: %s.', counter.key.metric.name, counter)
+  logging.info('Completing beam pipeline.')
+
+
+def beam_convert_tfexamples(
+    root: beam.Pipeline,
+    data_list: Sequence[Dict[str, Any]],
+    convert_fn: Callable[[Dict[str, Any]], tf.train.Example],
+    output_dir: str,
+    num_shards: Sequence[int],
+) -> None:
+  """Constructs beam pipelines to convert train, val, test TF Examples."""
+  names = [TRAIN_TFRECORD_NAME, VALIDATION_TFRECORD_NAME, TEST_TFRECORD_NAME]
+  split_data = (
+      root
+      | 'Create PCollection' >> beam.Create(data_list)
+      | 'Data split' >> beam.Partition(partition_by_ml_use, 3)
+  )
+  for i in range(3):
+    ml_use: str = ALL_ML_USES[i]
+    num_shard = num_shards[i]
+    output_prefix = os.path.join(output_dir, names[i])
+    _ = (
+        split_data[i]
+        | f'Convert {ml_use} TF Examples'
+        >> beam.ParDo(WriteToTFRecord(output_prefix, num_shard, convert_fn))
+        | f'Group {ml_use} TF Record files' >> beam.GroupBy(lambda x: x[0])
+        | f'Merge {ml_use} TF Record files'
+        >> beam.Map(merge_tfrecords_func(output_prefix, num_shard))
+    )
+
+
+def merge_tfrecords_func(output_prefix: str, num_shard: int) -> ...:
+  """Returns a function to merge sharded worker output into expected shards."""
+  output_prefix = fileutils.force_gcs_fuse_path(output_prefix)
+
+  def merge_tfrecords(worker_output: Tuple[int, Sequence[Tuple[int, str]]]):
+    idx = worker_output[0]
+    files: Sequence[str] = np.unique([x[1] for x in worker_output[1]])
+    output_file = f'{output_prefix}-{idx:05d}-of-{num_shard:05d}'
+    with open(output_file, 'wb') as f:
+      for file in files:
+        logging.info('Merging %s.', file)
+        file = fileutils.force_gcs_fuse_path(file)
+        with open(file, 'rb') as fin:
+          while True:
+            data = fin.read(READ_CHUNK_SIZE)
+            if not data:
+              break
+            f.write(data)
+        os.remove(file)
+
+  return merge_tfrecords

community-content/vertex_model_garden/model_oss/data_converter/convert_coco_label_map_iod.py

@@ -0,0 +1,111 @@
+r"""Converts COCO labels as yamls for model garden playground (IOD).
+"""
+
+import os
+import urllib.request
+
+from absl import app
+from absl import flags
+import tensorflow as tf
+import yaml
+
+from object_detection.utils import label_map_util
+
+_CONVERT_LABEL_TYPE_COCO_80 = 'coco_80'
+_CONVERT_LABEL_TYPE_COCO_91 = 'coco_91'
+
+_CONVERT_LABEL_TYPE = flags.DEFINE_enum(
+    'convert_label_type',
+    None,
+    [
+        _CONVERT_LABEL_TYPE_COCO_80,
+        _CONVERT_LABEL_TYPE_COCO_91,
+    ],
+    'Different types of label type conversion.',
+    required=True,
+)
+
+_TEMPORARY_PATH = flags.DEFINE_string(
+    'temporary_path',
+    None,
+    'The tempory path.',
+    required=True,
+)
+
+_OUTPUT_YAML_FILEPATH = flags.DEFINE_string(
+    'output_yaml_filepath',
+    None,
+    'The output yaml filepath.',
+    required=True,
+)
+
+
+def convert_coco_label_map_91(
+    output_yaml_filepath: str,
+) -> None:
+  """Converts coco label map 91."""
+  input_proto_filepath = 'https://raw.githubusercontent.com/tensorflow/models/master/research/object_detection/data/mscoco_label_map.pbtxt'
+  local_input_proto_filepath = os.path.join(
+      _TEMPORARY_PATH.value, 'mscoco_label_map.pbtxt'
+  )
+  with open(local_input_proto_filepath, 'w') as writer:
+    contents = (
+        urllib.request.urlopen(input_proto_filepath).read().decode('utf-8')
+    )
+    writer.write(contents)
+
+  label_map = label_map_util.load_labelmap(local_input_proto_filepath)
+  label_map_dict = label_map_util.get_label_map_dict(
+      label_map, use_display_name=True
+  )
+  swapped_label_map_dict = {v: k for k, v in label_map_dict.items()}
+  print(swapped_label_map_dict)
+
+  # Saves new label maps as yamls.
+  with tf.io.gfile.GFile(output_yaml_filepath, 'w') as writer:
+    writer.write(yaml.dump(swapped_label_map_dict))
+
+
+def convert_coco_label_map_80(
+    output_yaml_filepath: str,
+) -> None:
+  """Converts coco label map 80."""
+  # Loads label maps from texts.
+  input_text_filepath = 'https://gist.githubusercontent.com/AruniRC/7b3dadd004da04c80198557db5da4bda/raw/2f10965ace1e36c4a9dca76ead19b744f5eb7e88/ms_coco_classnames.txt'
+  local_input_text_filepath = os.path.join(
+      _TEMPORARY_PATH.value, 'ms_coco_classnames.txt'
+  )
+  with open(local_input_text_filepath, 'w') as writer:
+    contents = (
+        urllib.request.urlopen(input_text_filepath).read().decode('utf-8')
+    )
+    writer.write(contents)
+  with open(local_input_text_filepath, 'r') as file:
+    content = file.read()
+    label_map = yaml.safe_load(content)
+
+  # Removes background in label maps.
+  new_label_map = {}
+  for k, v in label_map.items():
+    if k == 0:
+      continue
+    new_label_map[k - 1] = v
+  print(new_label_map)
+  # Saves new label maps as yamls.
+  with tf.io.gfile.GFile(output_yaml_filepath, 'w') as writer:
+    writer.write(yaml.dump(new_label_map))
+
+
+def main(_) -> None:
+  if _CONVERT_LABEL_TYPE.value == _CONVERT_LABEL_TYPE_COCO_80:
+    convert_coco_label_map_80(_OUTPUT_YAML_FILEPATH.value)
+  elif _CONVERT_LABEL_TYPE.value == _CONVERT_LABEL_TYPE_COCO_91:
+    convert_coco_label_map_91(
+        _OUTPUT_YAML_FILEPATH.value,
+    )
+  else:
+    print('Not supported convert label type: ', _CONVERT_LABEL_TYPE.value)
+
+
+if __name__ == '__main__':
+  app.run(main)

community-content/vertex_model_garden/model_oss/data_converter/convert_imagenet_label_map.py

@@ -0,0 +1,86 @@
+r"""Converts ImageNet label texts as yamls for model garden playground.
+
+# ImageNet1K will have label maps with background.
+"""
+
+import urllib.request
+from absl import app
+from absl import flags
+import tensorflow as tf
+import yaml
+
+
+_INPUT_TEXT_FILEPATH = flags.DEFINE_string(
+    'input_text_filepath',
+    None,
+    'The input text filepath.',
+    required=True,
+)
+
+_ADD_BACKGROUND_LABEL = flags.DEFINE_boolean(
+    'add_background_label',
+    None,
+    'Whether or not add background labels.',
+    required=True,
+)
+
+_ADD_IDS = flags.DEFINE_boolean(
+    'add_ids',
+    None,
+    'Whether or not add ids.',
+    required=True,
+)
+
+_OUTPUT_YAML_FILEPATH = flags.DEFINE_string(
+    'output_yaml_filepath',
+    None,
+    'The output yaml filepath.',
+    required=True,
+)
+
+
+def convert_imagenet_label_map_from_text_to_yaml(
+    input_text_filepath: str,
+    add_background_label: bool,
+    add_ids: bool,
+    output_yaml_filepath: str,
+) -> None:
+  """Converts imagenet label map from text to yamls."""
+  label_map = {}
+
+  # Shifts all keys by 1, and add 0 as 'background'.
+  if add_background_label:
+    label_map = yaml.safe_load(
+        urllib.request.urlopen(input_text_filepath).read()
+    )
+    new_label_map = {}
+    for key, value in label_map.items():
+      new_label_map[key + 1] = value
+    new_label_map[0] = 'background'
+    label_map = new_label_map
+
+  # Adds maps from id to each line.
+  if add_ids:
+    lines = urllib.request.urlopen(input_text_filepath).readlines()
+    current_id = 0
+    for line in lines:
+      label_map[current_id] = line.decode('ascii').strip()
+      print(label_map[current_id])
+      current_id += 1
+
+  # Saves new label maps as yamls.
+  with tf.io.gfile.GFile(output_yaml_filepath, 'w') as writer:
+    writer.write(yaml.dump(label_map))
+
+
+def main(_) -> None:
+  convert_imagenet_label_map_from_text_to_yaml(
+      _INPUT_TEXT_FILEPATH.value,
+      _ADD_BACKGROUND_LABEL.value,
+      _ADD_IDS.value,
+      _OUTPUT_YAML_FILEPATH.value,
+  )
+
+
+if __name__ == '__main__':
+  app.run(main)

community-content/vertex_model_garden/model_oss/data_converter/data_converter_icn_lib.py

@@ -0,0 +1,199 @@
+"""Converts ICN CSV/JSONL files to TFRecord with apache beam."""
+
+import json
+from os import path
+from typing import Any, Dict, Sequence, Union, cast
+
+from absl import logging
+import apache_beam as beam
+import pandas as pd
+import tensorflow as tf
+
+from data_converter import common_lib
+
+
+_COLUMN_NAMES = [
+    common_lib.COLUMN_NAME_ML_USE,
+    common_lib.COLUMN_NAME_GCS_FILE_PATH,
+    common_lib.COLUMN_NAME_LABEL,
+]
+_JSON_GCS_URI_KEY = 'imageGcsUri'
+_JSON_CLASS_ANNOTATION_KEY = 'classificationAnnotation'
+_JSON_RESOURCE_LABEL_KEY = 'dataItemResourceLabels'
+_JSON_CLASS_NAME_KEY = 'displayName'
+_JSON_ML_USE_KEY = 'aiplatform.googleapis.com/ml_use'
+
+
+def build_tf_example(element: Dict[str, Union[str, int]]) -> tf.train.Example:
+  """Builds a TF Example from an image uri and label.
+
+  Args:
+    element: A dict with the keys gcs_file_path and label.
+
+  Returns:
+    The created TF Example.
+  """
+  image_uri = cast(str, element[common_lib.COLUMN_NAME_GCS_FILE_PATH])
+  label = cast(int, element[common_lib.COLUMN_NAME_LABEL])
+  image_bytes, shape = common_lib.encode_image(image_uri, image_format='jpeg')
+  features = tf.train.Features(
+      feature={
+          'image/encoded': common_lib.convert_to_feature(image_bytes),
+          'image/format': common_lib.convert_to_string_feature('jpeg'),
+          'image/height': common_lib.convert_to_feature(shape[0]),
+          'image/width': common_lib.convert_to_feature(shape[1]),
+          'image/class/label': common_lib.convert_to_feature(label),
+      },
+  )
+  return tf.train.Example(features=features)
+
+
+def _run_convert_pipeline(
+    output_dir: str, df: pd.DataFrame, num_shards: Sequence[int]
+) -> None:
+  """Starts a Beam pipeline to write DataFrame as TF Records.
+
+  Args:
+    output_dir: TF Records output directory.
+    df: DataFrame to convert from.
+    num_shards: Number of shards for train/validation/test TFRecord files.
+  """
+  images_list = df.to_dict('records')
+
+  def pipeline(root: beam.Pipeline):
+    common_lib.beam_convert_tfexamples(
+        root,
+        images_list,
+        build_tf_example,
+        output_dir,
+        num_shards,
+    )
+
+  common_lib.run_beam_pipeline(pipeline)
+
+
+def _convert_df_to_tfrecord(
+    df: pd.DataFrame,
+    output_dir: str,
+    split_ratio: Sequence[float],
+    num_shard: Sequence[int],
+) -> None:
+  """Converts a DataFrame into three separate tfrecords for training, validation, and testing into output_dir.
+
+  Args:
+    df: DataFrame to convert.
+    output_dir: The directory to save TFRecords and label_map.yaml.
+    split_ratio: List specifying the training, validation, and testing splits
+      for unassigned TFRecords.
+    num_shard: Number of shards for train/validation/test TFRecord files.
+  """
+  # Replaces ml_use with common_lib string constants for consistency.
+  common_lib.format_ml_use_column(df)
+  common_lib.insert_missing_ml_use(df)
+
+  # Ignores invalid rows.
+  dropped_row_num = common_lib.drop_invalid_rows(df)
+  if dropped_row_num > 0:
+    logging.warning('Ignored %d invalid rows.', dropped_row_num)
+
+  common_lib.replace_unassigned_ml_use(
+      df[common_lib.COLUMN_NAME_ML_USE], split_ratio
+  )
+
+  # Converts labels to integers as required by training.
+  new_labels, label_map = common_lib.create_label_map(
+      df[common_lib.COLUMN_NAME_LABEL]
+  )
+  df[common_lib.COLUMN_NAME_LABEL] = new_labels
+  label_map_path = path.join(output_dir, common_lib.LABEL_MAP_NAME)
+  logging.info('Writing label map to %s.', label_map_path)
+  common_lib.write_label_map(label_map_path, label_map)
+
+  _run_convert_pipeline(output_dir, df, num_shard)
+
+
+def convert_csv_to_tfrecord(
+    input_csv: str,
+    output_dir: str,
+    split_ratio: Sequence[float] = (0.8, 0.1, 0.1),
+    num_shard: Sequence[int] = (10, 10, 10),
+) -> None:
+  """Parses input_csv file into three separate tfrecords for training, validation, and testing into output_dir.
+
+  The csv format is shown in
+    https://cloud.google.com/vertex-ai/docs/image-data/classification/prepare-data#csv.
+
+  If an ml_use column is not provided, one will be created.
+
+  label_map.yaml containing the label map will be placed in output_dir.
+
+  Args:
+    input_csv: Name of the csv file.
+    output_dir: The directory to save TFRecords and label_map.yaml.
+    split_ratio: List specifying the training, validation, and testing splits
+      for unassigned TFRecords.
+    num_shard: Number of shards for train/validation/test TFRecord files.
+  """
+  with tf.io.gfile.GFile(input_csv, 'r') as f:
+    df: pd.DataFrame = pd.read_csv(
+        f, header=None, names=_COLUMN_NAMES, on_bad_lines='warn'
+    )
+
+  _convert_df_to_tfrecord(df, output_dir, split_ratio, num_shard)
+
+
+def convert_jsonl_to_tfrecord(
+    input_jsonl: str,
+    output_dir: str,
+    split_ratio: Sequence[float] = (0.8, 0.1, 0.1),
+    num_shard: Sequence[int] = (10, 10, 10),
+) -> None:
+  """Parses input_jsonl file into three separate tfrecords for training, validation, and testing into output_dir.
+
+  The JSONL format is shown in
+    https://cloud.google.com/vertex-ai/docs/image-data/classification/prepare-data#json-lines.
+
+  If an ml_use column is not provided, one will be created.
+
+  label_map.yaml containing the label map will be placed in output_dir.
+
+  Args:
+    input_jsonl: Name of the JSONL file.
+    output_dir: The directory to save TFRecords and label_map.yaml.
+    split_ratio: List specifying the training, validation, and testing splits
+      for unassigned TFRecords.
+    num_shard: Number of shards for train/validation/test TFRecord files.
+  """
+  df_rows = []
+  with tf.io.gfile.GFile(input_jsonl, 'r') as f:
+    lines = f.read().rstrip().splitlines()
+
+  for i, line in enumerate(lines, 1):
+    try:
+      item: Dict[str, Any] = json.loads(line)
+
+      gcs_uri = item.get(_JSON_GCS_URI_KEY)
+      label = item.get(_JSON_CLASS_ANNOTATION_KEY, {}).get(_JSON_CLASS_NAME_KEY)
+      if not gcs_uri or not label:
+        logging.warning('Invalid JSON at line %d, skipped.', i)
+        continue
+
+      ml_use = item.get(_JSON_RESOURCE_LABEL_KEY, {}).get(
+          _JSON_ML_USE_KEY, common_lib.ML_USE_UNASSIGNED
+      )
+    except (json.JSONDecodeError, AttributeError):
+      logging.warning('Invalid JSON at line %d, skipped.', i)
+      continue
+
+    df_rows.append([ml_use, gcs_uri, label])
+
+  df = pd.DataFrame(
+      data=df_rows,
+      columns=[
+          common_lib.COLUMN_NAME_ML_USE,
+          common_lib.COLUMN_NAME_GCS_FILE_PATH,
+          common_lib.COLUMN_NAME_LABEL,
+      ],
+  )
+
+  _convert_df_to_tfrecord(df, output_dir, split_ratio, num_shard)

community-content/vertex_model_garden/model_oss/data_converter/data_converter_iod_lib.py

@@ -0,0 +1,430 @@
+"""Converts IOD dataset files to TFRecord with apache beam."""
+
+import collections
+import json
+from os import path
+from typing import Any, Dict, Sequence
+
+from absl import logging
+import apache_beam as beam
+import pandas as pd
+import tensorflow as tf
+
+from data_converter import common_lib
+from util import constants
+
+COLUMN_NAME_LABEL_INT = 'label_int'
+_COLUMN_NAME_XMIN = 'X_MIN'
+_COLUMN_NAME_YMIN = 'Y_MIN'
+_COLUMN_NAME_XMAX = 'X_MAX'
+_COLUMN_NAME_YMAX = 'Y_MAX'
+COLUMN_NAMES = [
+    common_lib.COLUMN_NAME_ML_USE,
+    common_lib.COLUMN_NAME_GCS_FILE_PATH,
+    common_lib.COLUMN_NAME_LABEL,
+    _COLUMN_NAME_XMIN,
+    _COLUMN_NAME_YMIN,
+    'XMAX_NOT_USED',
+    'YMIN_NOT_USED',
+    _COLUMN_NAME_XMAX,
+    _COLUMN_NAME_YMAX,
+    'XMIN_NOT_USED',
+    'YMAX_NOT_USED',
+]
+_BOUNDING_BOX_COLUMNS = [
+    _COLUMN_NAME_XMIN,
+    _COLUMN_NAME_YMIN,
+    _COLUMN_NAME_XMAX,
+    _COLUMN_NAME_YMAX,
+]
+_JSON_BBOX_ANNOTATIONS_KEY = 'boundingBoxAnnotations'
+_JSON_DISPLAY_NAME_KEY = 'displayName'
+_JSON_X_MIN_KEY = 'xMin'
+_JSON_X_MAX_KEY = 'xMax'
+_JSON_Y_MIN_KEY = 'yMin'
+_JSON_Y_MAX_KEY = 'yMax'
+
+
+def build_tf_example(image_row: Dict[str, Any]) -> tf.train.Example:
+  """Builds a TF Example from an image row.
+
+  Args:
+    image_row: A dictionary containing information about the image, such as its
+      GCS uri, labels, and bounding box coordinates.
+
+  Returns:
+    A tf.train.Example containing the encoded image and optionally a
+      bounding box and label.
+  """
+  image_uri = image_row[common_lib.COLUMN_NAME_GCS_FILE_PATH]
+  image_bytes, shape = common_lib.encode_image(image_uri, image_format='jpeg')
+  feature = {
+      'image/encoded': common_lib.convert_to_feature(image_bytes),
+      'image/format': common_lib.convert_to_string_feature('jpeg'),
+      'image/height': common_lib.convert_to_feature(shape[0]),
+      'image/width': common_lib.convert_to_feature(shape[1]),
+      'image/source_id': common_lib.convert_to_string_feature(image_uri),
+      'image/object/bbox/xmin': common_lib.convert_to_feature(
+          image_row[_COLUMN_NAME_XMIN]
+      ),
+      'image/object/bbox/ymin': common_lib.convert_to_feature(
+          image_row[_COLUMN_NAME_YMIN]
+      ),
+      'image/object/bbox/xmax': common_lib.convert_to_feature(
+          image_row[_COLUMN_NAME_XMAX]
+      ),
+      'image/object/bbox/ymax': common_lib.convert_to_feature(
+          image_row[_COLUMN_NAME_YMAX]
+      ),
+      'image/object/class/text': common_lib.convert_to_list_string_feature(
+          image_row[common_lib.COLUMN_NAME_LABEL]
+      ),
+      'image/object/class/label': common_lib.convert_to_feature(
+          image_row[COLUMN_NAME_LABEL_INT]
+      ),
+  }
+  return tf.train.Example(features=tf.train.Features(feature=feature))
+
+
+def _run_convert_pipeline(
+    output_dir: str,
+    image_rows: Sequence[Dict[str, Any]],
+    num_shards: Sequence[int],
+) -> None:
+  """Starts a Beam pipeline to write DataFrame as TF Records.
+
+  Args:
+    output_dir: TF Records output directory.
+    image_rows: Contains all necessary information to create a TF Example.
+    num_shards: Number of shards for train/validation/test TFRecord files.
+  """
+
+  def pipeline(root: beam.Pipeline):
+    common_lib.beam_convert_tfexamples(
+        root,
+        image_rows,
+        build_tf_example,
+        output_dir,
+        num_shards,
+    )
+
+  common_lib.run_beam_pipeline(pipeline)
+
+
+def _convert_df_to_tfrecord(
+    df: pd.DataFrame,
+    output_dir: str,
+    split_ratio: Sequence[float],
+    num_shard: Sequence[int],
+) -> None:
+  """Converts a DataFrame into three separate tfrecords for training, validation, and testing into output_dir.
+
+  Args:
+    df: DataFrame to convert.
+    output_dir: The directory to save TFRecords and label_map.yaml.
+    split_ratio: List specifying the training, validation, and testing splits
+      for unassigned TFRecords.
+    num_shard: Number of shards for train/validation/test TFRecord files.
+  """
+  # Replaces ml_use with common_lib string constants for consistency.
+  common_lib.format_ml_use_column(df)
+  common_lib.insert_missing_ml_use(df)
+
+  # Specify bounding box columns to be numeric.
+  df[_BOUNDING_BOX_COLUMNS] = df[_BOUNDING_BOX_COLUMNS].apply(pd.to_numeric)
+
+  # Ignores invalid rows.
+  dropped_row_num = common_lib.drop_invalid_rows(df)
+  dropped_row_num += drop_rows_without_bbox(df)
+  if dropped_row_num > 0:
+    logging.warning('Ignored %d invalid rows.', dropped_row_num)
+
+  # Converts labels to integers as required by training.
+  int_labels, label_map = common_lib.create_label_map(
+      df[common_lib.COLUMN_NAME_LABEL]
+  )
+  df[COLUMN_NAME_LABEL_INT] = int_labels
+  label_map_path = path.join(output_dir, common_lib.LABEL_MAP_NAME)
+  logging.info('Writing label map to %s.', label_map_path)
+  common_lib.write_label_map(label_map_path, label_map)
+
+  image_rows = _condense_bounding_boxes(df.to_dict(orient='records'))
+  ml_uses = [row[common_lib.COLUMN_NAME_ML_USE] for row in image_rows]
+  common_lib.replace_unassigned_ml_use(ml_uses, split_ratio)
+  common_lib.merge_seq_into_dicts(
+      common_lib.COLUMN_NAME_ML_USE, ml_uses, image_rows
+  )
+
+  _run_convert_pipeline(output_dir, image_rows, num_shard)
+
+
+def _condense_bounding_boxes(
+    image_rows: Sequence[Dict[str, Any]]
+) -> Sequence[Dict[str, Any]]:
+  """Gather all the bounding boxes in an image and put them in the same dictionary.
+
+  Args:
+    image_rows: List of dictionaries, each containing information about the
+      image, such as its GCS uri, labels, and bounding box coordinates.
+
+  Returns:
+    List of dictionaries such that each contains all the bounding boxes for a
+    given gcs_file_path.
+
+  Raises:
+    RuntimeError: This is raised when the input data contains images that have
+      annotations in different ml_use classes.
+  """
+  output = {}
+  for image_row in image_rows:
+    ml_use = image_row[common_lib.COLUMN_NAME_ML_USE]
+    gcs_file_path = image_row[common_lib.COLUMN_NAME_GCS_FILE_PATH]
+    label = image_row[common_lib.COLUMN_NAME_LABEL]
+    xmin = image_row[_COLUMN_NAME_XMIN]
+    ymin = image_row[_COLUMN_NAME_YMIN]
+    xmax = image_row[_COLUMN_NAME_XMAX]
+    ymax = image_row[_COLUMN_NAME_YMAX]
+    label_int = image_row[COLUMN_NAME_LABEL_INT]
+    if gcs_file_path in output:
+      d = output[gcs_file_path]
+      if ml_use != common_lib.ML_USE_UNASSIGNED:
+        if d[common_lib.COLUMN_NAME_ML_USE] == common_lib.ML_USE_UNASSIGNED:
+          d[common_lib.COLUMN_NAME_ML_USE] = ml_use
+        elif ml_use != d[common_lib.COLUMN_NAME_ML_USE]:
+          raise RuntimeError(
+              f'Image {gcs_file_path} can only be placed in one of'
+              f' training/validation/test. It is currently in {ml_use} and'
+              f' {d[common_lib.COLUMN_NAME_ML_USE]}.'
+          )
+      d[common_lib.COLUMN_NAME_LABEL].append(label)
+      d[_COLUMN_NAME_XMIN].append(xmin)
+      d[_COLUMN_NAME_YMIN].append(ymin)
+      d[_COLUMN_NAME_XMAX].append(xmax)
+      d[_COLUMN_NAME_YMAX].append(ymax)
+      d[COLUMN_NAME_LABEL_INT].append(label_int)
+    else:
+      output[gcs_file_path] = {
+          common_lib.COLUMN_NAME_ML_USE: ml_use,
+          common_lib.COLUMN_NAME_GCS_FILE_PATH: gcs_file_path,
+          common_lib.COLUMN_NAME_LABEL: [label],
+          _COLUMN_NAME_XMIN: [xmin],
+          _COLUMN_NAME_YMIN: [ymin],
+          _COLUMN_NAME_XMAX: [xmax],
+          _COLUMN_NAME_YMAX: [ymax],
+          COLUMN_NAME_LABEL_INT: [label_int],
+      }
+  return list(output.values())
+
+
+def convert_csv_to_tfrecord(
+    input_csv: str,
+    output_dir: str,
+    split_ratio: Sequence[float] = (0.8, 0.1, 0.1),
+    num_shard: Sequence[int] = (10, 10, 10),
+) -> None:
+  """Parses input_csv file into three separate tfrecords for training, validation, and testing into output_dir.
+
+  The csv format is shown in
+    https://cloud.google.com/vertex-ai/docs/image-data/object-detection/prepare-data#csv.
+
+  If an ml_use column is not provided, one will be created.
+
+  label_map.yaml containing the label map will be placed in output_dir.
+
+  Args:
+    input_csv: Name of the csv file.
+    output_dir: The directory to save TFRecords and label_map.yaml.
+    split_ratio: List specifying the train, validation, and test splits for
+      unassigned TFRecords.
+    num_shard: Number of shards for train/validation/test TFRecord files.
+  """
+  with tf.io.gfile.GFile(input_csv, 'r') as f:
+    df: pd.DataFrame = pd.read_csv(
+        f, header=None, names=COLUMN_NAMES, on_bad_lines='warn'
+    )
+
+  _convert_df_to_tfrecord(df, output_dir, split_ratio, num_shard)
+
+
+def drop_rows_without_bbox(df: pd.DataFrame) -> int:
+  """Drops DataFrame rows without bounding_boxes.
+
+  Args:
+    df: The DataFrame to process in place.
+
+  Returns:
+    The number of rows dropped.
+  """
+  invalid_rows = df.index[~(df[_BOUNDING_BOX_COLUMNS].notnull().all(axis=1))]
+  dropped_num = len(invalid_rows)
+  if dropped_num > 0:
+    invalid_df = df.loc[invalid_rows].to_dict(orient='records')
+    for entry in invalid_df:
+      logging.warning('Skipping entry due to missing bounding box: %s.', entry)
+    df.drop(invalid_rows, inplace=True)
+    df.reset_index(drop=True, inplace=True)
+  return dropped_num
+
+
+def convert_coco_json_categories_to_label_map(
+    categories: Sequence[Dict[str, Any]]
+) -> Dict[int, str]:
+  return {category['id']: category['name'] for category in categories}
+
+
+def convert_coco_json_to_tfrecord(
+    input_coco_json: str,
+    output_dir: str,
+    split_ratio: Sequence[float] = (0.8, 0.1, 0.1),
+    num_shard: Sequence[int] = (10, 10, 10),
+) -> None:
+  """Parses input_csv file into three separate tfrecords for training, validation, and testing into output_dir.
+
+  The COCO json format is shown here: https://cocodataset.org/#format-data.
+
+  label_map.yaml containing the label map will be placed in output_dir.
+
+  Args:
+    input_coco_json: Name of coco json file.
+    output_dir: The directory to save TFRecords and label_map.yaml.
+    split_ratio: List specifying the train, validation, and test splits for
+      dataset.
+    num_shard: Number of shards for train/validation/test TFRecord files.
+  """
+  with tf.io.gfile.GFile(input_coco_json, 'r') as f:
+    coco_json = json.load(f)
+  # Writes label map from coco json categories.
+  label_map = convert_coco_json_categories_to_label_map(
+      coco_json[constants.COCO_JSON_CATEGORIES]
+  )
+  label_map_path = path.join(output_dir, common_lib.LABEL_MAP_NAME)
+  logging.info('Writes label map to %s.', label_map_path)
+  common_lib.write_label_map(label_map_path, label_map)
+
+  img_to_anns = collections.defaultdict(list)
+  imgs = {}
+  if constants.COCO_JSON_ANNOTATIONS in coco_json:
+    for ann in coco_json[constants.COCO_JSON_ANNOTATIONS]:
+      img_to_anns[ann[constants.COCO_JSON_ANNOTATION_IMAGE_ID]].append(ann)
+
+  if constants.COCO_JSON_IMAGES in coco_json:
+    for img in coco_json[constants.COCO_JSON_IMAGES]:
+      imgs[img[constants.COCO_JSON_IMAGE_ID]] = img
+
+  df_rows = []
+
+  for image_id, annotations in img_to_anns.items():
+    img = imgs[image_id]
+    for ann in annotations:
+      xmin, ymin, xmax, ymax = common_lib.reformat_bbox(
+          ann[constants.COCO_ANNOTATION_BBOX],
+          img[constants.COCO_JSON_IMAGE_WIDTH],
+          img[constants.COCO_JSON_IMAGE_HEIGHT],
+      )
+      df_rows.append([
+          common_lib.ML_USE_UNASSIGNED,
+          img[constants.COCO_JSON_IMAGE_COCO_URL],
+          label_map[ann[constants.COCO_JSON_ANNOTATION_CATEGORY_ID]],
+          xmin,
+          ymin,
+          xmax,
+          ymin,
+          xmax,
+          ymax,
+          xmin,
+          ymax,
+          ann[constants.COCO_JSON_ANNOTATION_CATEGORY_ID],
+      ])
+  df = pd.DataFrame(
+      data=df_rows,
+      columns=COLUMN_NAMES + [COLUMN_NAME_LABEL_INT],
+  )
+
+  # Replaces ml_use with common_lib string constants for consistency.
+  common_lib.format_ml_use_column(df)
+  common_lib.insert_missing_ml_use(df)
+
+  # Species bounding box columns to be numeric.
+  df[_BOUNDING_BOX_COLUMNS] = df[_BOUNDING_BOX_COLUMNS].apply(pd.to_numeric)
+
+  # Ignores invalid rows.
+  dropped_row_num = common_lib.drop_invalid_rows(df)
+  dropped_row_num += drop_rows_without_bbox(df)
+  if dropped_row_num > 0:
+    logging.warning('Ignored %d invalid rows.', dropped_row_num)
+
+  image_rows = _condense_bounding_boxes(df.to_dict(orient='records'))
+  ml_uses = [row[common_lib.COLUMN_NAME_ML_USE] for row in image_rows]
+  common_lib.replace_unassigned_ml_use(ml_uses, split_ratio)
+  common_lib.merge_seq_into_dicts(
+      common_lib.COLUMN_NAME_ML_USE, ml_uses, image_rows
+  )
+
+  _run_convert_pipeline(output_dir, image_rows, num_shard)
+
+
+def convert_jsonl_to_tfrecord(
+    input_jsonl: str,
+    output_dir: str,
+    split_ratio: Sequence[float] = (0.8, 0.1, 0.1),
+    num_shard: Sequence[int] = (10, 10, 10),
+) -> None:
+  """Parses input_jsonl file into three separate tfrecords for training, validation, and testing into output_dir.
+
+  The JSONL format is shown in
+    https://cloud.google.com/vertex-ai/docs/image-data/object-detection/prepare-data#json-lines.
+
+  If an ml_use column is not provided, one will be created.
+
+  label_map.yaml containing the label map will be placed in output_dir.
+
+  Args:
+    input_jsonl: Name of the JSONL file.
+    output_dir: The directory to save TFRecords and label_map.yaml.
+    split_ratio: List specifying the training, validation, and testing splits
+      for unassigned TFRecords.
+    num_shard: Number of shards for train/validation/test TFRecord files.
+  """
+  df_rows = []
+  with tf.io.gfile.GFile(input_jsonl, 'r') as f:
+    lines = f.read().rstrip().splitlines()
+
+  for i, line in enumerate(lines, start=1):
+    try:
+      item: Dict[str, Any] = json.loads(line)
+    except (json.JSONDecodeError, AttributeError):
+      logging.warning('Invalid JSON at line %d skipped.', i)
+      continue
+
+    gcs_uri = item.get(common_lib.JSON_GCS_URI_KEY)
+    if not gcs_uri:
+      logging.warning(
+          'Invalid JSON at line %d skipped. Missing gcs_uri_key.', i
+      )
+      continue
+    ml_use = item.get(common_lib.JSON_RESOURCE_LABEL_KEY, {}).get(
+        common_lib.JSON_ML_USE_KEY, common_lib.ML_USE_UNASSIGNED
+    )
+
+    for bbox in item.get(_JSON_BBOX_ANNOTATIONS_KEY, []):
+      label = bbox.get(_JSON_DISPLAY_NAME_KEY)
+      xmin = bbox.get(_JSON_X_MIN_KEY)
+      ymin = bbox.get(_JSON_Y_MIN_KEY)
+      xmax = bbox.get(_JSON_X_MAX_KEY)
+      ymax = bbox.get(_JSON_Y_MAX_KEY)
+
+      df_rows.append([ml_use, gcs_uri, label, xmin, ymin, xmax, ymax])
+
+  df = pd.DataFrame(
+      data=df_rows,
+      columns=[
+          common_lib.COLUMN_NAME_ML_USE,
+          common_lib.COLUMN_NAME_GCS_FILE_PATH,
+          common_lib.COLUMN_NAME_LABEL,
+          _COLUMN_NAME_XMIN,
+          _COLUMN_NAME_YMIN,
+          _COLUMN_NAME_XMAX,
+          _COLUMN_NAME_YMAX,
+      ],
+  )
+  _convert_df_to_tfrecord(df, output_dir, split_ratio, num_shard)

community-content/vertex_model_garden/model_oss/data_converter/data_converter_isg_lib.py

@@ -0,0 +1,328 @@
+"""Python script to convert different file formats for ISG to tfrecords."""
+
+import hashlib
+import os
+from typing import Any, Dict, Iterator, List, Optional, Tuple, Union
+
+from absl import logging
+import apache_beam as beam
+from apache_beam.io import tfrecordio
+import cv2
+import numpy as np
+from pycocotools import coco
+import tensorflow as tf
+import yaml
+
+from data_converter import common_lib
+from util import constants
+from util import fileutils
+
+_IMAGE_FORMAT = 'PNG'
+
+
+def build_tf_example(
+    image_info: dict[str, Union[str, int]],
+    segmentation_image: List[List[int]],
+    output_shape: Optional[Tuple[int, int]] = None,
+) -> tf.train.Example:
+  """Encodes an image and its segmentation mask into a tf.train.Example.
+
+  Args:
+    image_info: A dictionary containing information about the image, such as its
+      file name, height, and width.
+    segmentation_image: 2D image in list of lists having category ids.
+    output_shape: The desired output shape of the image. If None, the original
+      image shape will be used.
+
+  Returns:
+    A tf.train.Example containing the encoded image and segmentation mask.
+
+  Raises:
+    IOError: If image cannot be found in the path.
+  """
+  file_name = image_info[constants.COCO_JSON_FILE_NAME]
+  height = int(image_info[constants.COCO_JSON_IMAGE_HEIGHT])
+  width = int(image_info[constants.COCO_JSON_IMAGE_WIDTH])
+
+  segmentation_image = np.expand_dims(
+      np.asarray(segmentation_image, dtype=np.int32), axis=-1
+  )
+  _, encoded_seg = cv2.imencode(f'.{_IMAGE_FORMAT.lower()}', segmentation_image)
+  encoded_seg = encoded_seg.tobytes()
+
+  encoded_img, _ = common_lib.encode_image(
+      image_info[constants.COCO_JSON_IMAGE_COCO_URL],
+      output_shape=output_shape,
+      image_format=_IMAGE_FORMAT.lower(),
+  )
+
+  key = hashlib.sha256(encoded_img).hexdigest()
+
+  return tf.train.Example(
+      features=tf.train.Features(
+          feature={
+              'image/height': common_lib.convert_to_feature(height),
+              'image/width': common_lib.convert_to_feature(width),
+              'image/filename': common_lib.convert_to_string_feature(file_name),
+              'image/sha256': common_lib.convert_to_string_feature(key),
+              'image/encoded': common_lib.convert_to_feature(encoded_img),
+              'image/format': common_lib.convert_to_string_feature(
+                  _IMAGE_FORMAT
+              ),
+              'image/segmentation/class/encoded': common_lib.convert_to_feature(
+                  encoded_seg
+              ),
+              'image/segmentation/class/format': (
+                  common_lib.convert_to_string_feature(_IMAGE_FORMAT)
+              ),
+              'image/segmentation/class/height': common_lib.convert_to_feature(
+                  height
+              ),
+              'image/segmentation/class/width': common_lib.convert_to_feature(
+                  width
+              ),
+          }
+      )
+  )
+
+
+class AcquireTFExampleDoFn(beam.DoFn):
+  """Beam DoFn to build TF Examples from a single row of image_info data."""
+
+  # These tags will be used to tag the outputs of this DoFn.
+  output_tag_train = constants.ML_USE_TRAINING
+  output_tag_validation = constants.ML_USE_VALIDATION
+  output_tag_test = constants.ML_USE_TEST
+
+  valid_ml_use_set = set(
+      [output_tag_train, output_tag_validation, output_tag_test]
+  )
+
+  def __init__(self, output_shape: Optional[Tuple[int, int]] = None):
+    self.acquired_examples_counter = beam.metrics.Metrics.counter(
+        self.__class__.__name__, 'Success'
+    )
+    self.failure_counter = beam.metrics.Metrics.counter(
+        self.__class__.__name__, 'Failure'
+    )
+    self.output_shape = output_shape
+
+  def process(
+      self,
+      row: Tuple[str, Dict[str, Union[str, int]], List[List[int]]],
+  ) -> Iterator[tf.train.Example]:
+    ml_use, image_info, annotation_info = row
+    if ml_use not in self.valid_ml_use_set:
+      logging.warning('ml_use invalid: %s', ml_use)
+      self.failure_counter.inc()
+      return
+
+    try:
+      tf_example = build_tf_example(
+          image_info, annotation_info, self.output_shape
+      )
+    except IOError as e:
+      logging.warning('Failed to build TF Example: %s', e)
+      self.failure_counter.inc()
+    else:
+      self.acquired_examples_counter.inc()
+      yield beam.pvalue.TaggedOutput(ml_use, tf_example)
+
+
+def _define_data_conversion_pipeline(
+    root: beam.Pipeline,
+    ml_use_rows: List[str],
+    image_rows: List[Dict[str, Union[str, int]]],
+    segmentation_rows: List[List[List[int]]],
+    output_dir: str,
+    output_shape: Optional[Tuple[int, int]],
+    num_shard_list: List[int],
+):
+  """Define a data conversion pipeline.
+
+  Args:
+    root: A Beam pipeline.
+    ml_use_rows: List containing the ml_use.
+    image_rows: List of dictionaries containing information about the image,
+      such as its file name, height, and width.
+    segmentation_rows: List of 2D images of integers representing segmentation
+      masks.
+    output_dir: Directory where the output TFRecords will be written.
+    output_shape: Desired output shape of the image. If None, the original image
+      shape will be used.
+    num_shard_list: Number of shards to write to each output TFRecord.
+
+  Returns:
+    A Beam pipeline.
+  """
+  train, validation, test = (
+      root
+      | 'Load ml use and image rows to beam'
+      >> beam.Create(zip(ml_use_rows, image_rows, segmentation_rows))
+      | 'Build TF Examples'
+      >> beam.ParDo(AcquireTFExampleDoFn(output_shape)).with_outputs(
+          AcquireTFExampleDoFn.output_tag_train,
+          AcquireTFExampleDoFn.output_tag_validation,
+          AcquireTFExampleDoFn.output_tag_test,
+      )
+  )
+
+  # Save each split to TFRecord.
+  _ = train | 'Save train split to TFRecord' >> tfrecordio.WriteToTFRecord(
+      os.path.join(output_dir, common_lib.TRAIN_TFRECORD_NAME),
+      coder=beam.coders.ProtoCoder(tf.train.Example),
+      num_shards=num_shard_list[0],
+  )
+  _ = (
+      validation
+      | 'Save validation split to TFRecord'
+      >> tfrecordio.WriteToTFRecord(
+          os.path.join(output_dir, common_lib.VALIDATION_TFRECORD_NAME),
+          coder=beam.coders.ProtoCoder(tf.train.Example),
+          num_shards=num_shard_list[1],
+      )
+  )
+  _ = test | 'Save test split to TFRecord' >> tfrecordio.WriteToTFRecord(
+      os.path.join(output_dir, common_lib.TEST_TFRECORD_NAME),
+      coder=beam.coders.ProtoCoder(tf.train.Example),
+      num_shards=num_shard_list[2],
+  )
+
+
+def _image_info_to_segmentation_image(
+    img: Dict[str, Any],
+    coco_dataset: coco.COCO,
+    label_id_by_category_id: Dict[int, int],
+) -> List[List[int]]:
+  """Convert image information to a segmentation image.
+
+  Args:
+      img: The image information.
+      coco_dataset: The COCO dataset.
+      label_id_by_category_id: The mapping from label id used for training to
+        category_id defined in dataset.
+
+  Returns:
+      The segmentation image.
+
+  Raises:
+    ValueError: If the mask size does not match the image or if a pixel has
+      multiple labels.
+  """
+  seg_img = np.zeros(
+      shape=(
+          img[constants.COCO_JSON_IMAGE_HEIGHT],
+          img[constants.COCO_JSON_IMAGE_WIDTH],
+      ),
+      dtype=np.int32,
+  )
+  for ann in coco_dataset.imgToAnns[img[constants.COCO_JSON_IMAGE_ID]]:
+    new_category_id = ann[constants.COCO_JSON_ANNOTATION_CATEGORY_ID]
+    binary_mask = coco_dataset.annToMask(ann)
+    if seg_img.shape != binary_mask.shape:
+      raise ValueError(
+          'Binary mask does not have the same shape as image. image_id:'
+          f' {img["id"]}'
+      )
+    boolean_mask = binary_mask == 1
+    if (seg_img[boolean_mask] != 0).any():
+      raise ValueError(
+          'Error: Some pixels have more than one label in image_id:'
+          f' {img["id"]}.'
+      )
+    seg_img[boolean_mask] = label_id_by_category_id[new_category_id]
+
+  return seg_img.tolist()
+
+
+def get_input_rows(
+    coco_dataset: coco.COCO,
+    split_ratio: List[float],
+    label_id_by_category_id: Dict[int, int],
+) -> Tuple[List[str], List[Dict[str, Union[str, int]]], List[List[List[int]]]]:
+  """Get input rows for training and validation.
+
+  Args:
+      coco_dataset: The COCO dataset.
+      split_ratio: The split ratio for training and validation.
+      label_id_by_category_id: The mapping from label id used for training to
+        category_id defined in dataset.
+
+  Returns:
+      - A list of ml_use strings.
+      - A list of image informations.
+      - A list of segmentation images for the corresponding images.
+  """
+  image_rows = coco_dataset.dataset[constants.COCO_JSON_IMAGES]
+
+  segmentation_rows = [
+      _image_info_to_segmentation_image(
+          img, coco_dataset, label_id_by_category_id
+      )
+      for img in image_rows
+  ]
+
+  ml_use_rows = common_lib.create_ml_use_array_with_split(
+      len(image_rows), split_ratio
+  )
+  return ml_use_rows, image_rows, segmentation_rows
+
+
+def beam_build_tfrecord_from_coco_json(
+    input_json: str,
+    output_dir: str,
+    split_ratio: List[float],
+    num_shard_list: List[int],
+    output_shape: Optional[Tuple[int, int]] = None,
+) -> None:
+  """Builds TFRecord files from COCO dataset.
+
+  The output file names are `_TRAIN_TFRECORD_NAME`, `_VALIDATION_TFRECORD_NAME`,
+  and `_TEST_TFRECORD_NAME`.
+
+  Args:
+    input_json: Path to a COCO JSON or JSONL file.
+    output_dir: Directory to output the TFRecord files.
+    split_ratio: List of how to split entries to train, validation, and test
+      TFRecords.
+    num_shard_list: List of the number of shards for each TFRecord file.
+    output_shape: The desired output shape of the image. If None, the original
+      image shape will be used.
+  """
+  # `coco` cannot access gcs uri. Use gcsfuse, it is faster.
+  input_json = fileutils.force_gcs_fuse_path(input_json)
+  coco_dataset = coco.COCO(input_json)
+
+  label_map = {}
+  label_id_by_category_id = {}
+  for idx, category in enumerate(
+      coco_dataset.dataset[constants.COCO_JSON_CATEGORIES], start=1
+  ):
+    label_map[idx] = category[constants.COCO_JSON_CATEGORY_NAME]
+    label_id_by_category_id[category[constants.COCO_JSON_CATEGORY_ID]] = idx
+  label_map_path = os.path.join(output_dir, common_lib.LABEL_MAP_NAME)
+  logging.info('Writing label map to %s.', label_map_path)
+  common_lib.write_label_map(label_map_path, label_map)
+
+  with tf.io.gfile.GFile(
+      os.path.join(output_dir, 'label_id_by_category_id.yaml'), 'w'
+  ) as f:
+    yaml.dump(label_id_by_category_id, f)
+
+  ml_use_rows, image_rows, segmentation_rows = get_input_rows(
+      coco_dataset, split_ratio, label_id_by_category_id
+  )
+
+  def pipeline(root):
+    _define_data_conversion_pipeline(
+        root,
+        ml_use_rows,
+        image_rows,
+        segmentation_rows,
+        output_dir,
+        output_shape,
+        num_shard_list,
+    )
+
+  logging.info('Beginning beam pipeline to acquire tfrecords.')
+  common_lib.run_beam_pipeline(pipeline)

community-content/vertex_model_garden/model_oss/data_converter/data_converter_main.py

@@ -0,0 +1,166 @@
+r"""Python script to convert user input data to training docker format.
+
+
+Note: the training format is designed to be tfrecord as in the design doc.
+If there are training efficiency issues for pytorch algorithms, we will also
+support pytorch formats as well.
+"""
+
+from absl import app
+from absl import flags
+from absl import logging
+from data_converter import common_lib
+from data_converter import data_converter_icn_lib
+from data_converter import data_converter_iod_lib
+from data_converter import data_converter_isg_lib
+from data_converter import data_converter_vcn_lib
+from util import constants
+
+
+_INPUT_FILE_PATH = flags.DEFINE_string(
+    'input_file_path',
+    None,
+    'Input file path.',
+    required=True,
+)
+_INPUT_FILE_TYPE = flags.DEFINE_enum(
+    'input_file_type',
+    None,
+    [
+        constants.INPUT_FILE_TYPE_CSV,
+        constants.INPUT_FILE_TYPE_JSONL,
+        constants.INPUT_FILE_TYPE_COCO_JSON,
+    ],
+    'Input file type.',
+    required=True,
+)
+_OBJECTIVE = flags.DEFINE_enum(
+    'objective',
+    None,
+    [
+        constants.OBJECTIVE_IMAGE_CLASSIFICATION,
+        constants.OBJECTIVE_IMAGE_OBJECT_DETECTION,
+        constants.OBJECTIVE_IMAGE_SEGMENTATION,
+        constants.OBJECTIVE_VIDEO_CLASSIFICATION,
+    ],
+    'The objective of this training job.',
+    required=True,
+)
+_OUTPUT_DIR = flags.DEFINE_string(
+    'output_dir',
+    None,
+    'The output directory for converted data and label map files.',
+    required=True,
+)
+_SPLIT_RATIO = flags.DEFINE_list(
+    'split_ratio',
+    '0.8,0.1,0.1',
+    'Proportion of data to split into train/validation/test.',
+)
+_NUM_SHARD = flags.DEFINE_list(
+    'num_shard', '10,10,10', 'The number of shards for train/validation/test.'
+)
+_OUTPUT_FPS = flags.DEFINE_integer(
+    'output_fps', 5, 'For videos only. The output frames rate per second.'
+)
+
+
+def main(_) -> None:
+  logging.info(
+      (
+          'Start data converter on: %s (type: %s) with split: %s for %s'
+          ' (shard=%s), and output to %s.'
+      ),
+      _INPUT_FILE_PATH.value,
+      _INPUT_FILE_TYPE.value,
+      _SPLIT_RATIO.value,
+      _OBJECTIVE.value,
+      _NUM_SHARD.value,
+      _OUTPUT_DIR.value,
+  )
+  split_ratio = list(map(float, _SPLIT_RATIO.value))
+  num_shard = list(map(int, _NUM_SHARD.value))
+  common_lib.check_split_ratio(split_ratio)
+  common_lib.check_num_shard(num_shard)
+  if (
+      _OBJECTIVE.value == constants.OBJECTIVE_IMAGE_OBJECT_DETECTION
+      and _INPUT_FILE_TYPE.value == constants.INPUT_FILE_TYPE_CSV
+  ):
+    data_converter_iod_lib.convert_csv_to_tfrecord(
+        _INPUT_FILE_PATH.value,
+        _OUTPUT_DIR.value,
+        split_ratio,
+        num_shard,
+    )
+  elif (
+      _OBJECTIVE.value == constants.OBJECTIVE_IMAGE_OBJECT_DETECTION
+      and _INPUT_FILE_TYPE.value == constants.INPUT_FILE_TYPE_JSONL
+  ):
+    data_converter_iod_lib.convert_jsonl_to_tfrecord(
+        _INPUT_FILE_PATH.value, _OUTPUT_DIR.value, split_ratio, num_shard
+    )
+  elif (
+      _OBJECTIVE.value == constants.OBJECTIVE_IMAGE_OBJECT_DETECTION
+      and _INPUT_FILE_TYPE.value == constants.INPUT_FILE_TYPE_COCO_JSON
+  ):
+    data_converter_iod_lib.convert_coco_json_to_tfrecord(
+        _INPUT_FILE_PATH.value,
+        _OUTPUT_DIR.value,
+        split_ratio,
+        num_shard,
+    )
+  elif _OBJECTIVE.value == constants.OBJECTIVE_IMAGE_SEGMENTATION:
+    data_converter_isg_lib.beam_build_tfrecord_from_coco_json(
+        _INPUT_FILE_PATH.value,
+        _OUTPUT_DIR.value,
+        split_ratio,
+        num_shard,
+    )
+  elif (
+      _OBJECTIVE.value == constants.OBJECTIVE_IMAGE_CLASSIFICATION
+      and _INPUT_FILE_TYPE.value == constants.INPUT_FILE_TYPE_CSV
+  ):
+    data_converter_icn_lib.convert_csv_to_tfrecord(
+        _INPUT_FILE_PATH.value,
+        _OUTPUT_DIR.value,
+        split_ratio,
+        num_shard,
+    )
+  elif (
+      _OBJECTIVE.value == constants.OBJECTIVE_IMAGE_CLASSIFICATION
+      and _INPUT_FILE_TYPE.value == constants.INPUT_FILE_TYPE_JSONL
+  ):
+    data_converter_icn_lib.convert_jsonl_to_tfrecord(
+        _INPUT_FILE_PATH.value, _OUTPUT_DIR.value, split_ratio, num_shard
+    )
+  elif (
+      _OBJECTIVE.value == constants.OBJECTIVE_VIDEO_CLASSIFICATION
+      and _INPUT_FILE_TYPE.value == constants.INPUT_FILE_TYPE_CSV
+  ):
+    data_converter_vcn_lib.convert_csv_to_tfrecord(
+        _INPUT_FILE_PATH.value,
+        _OUTPUT_DIR.value,
+        _OUTPUT_FPS.value,
+        split_ratio,
+        num_shard,
+    )
+  elif (
+      _OBJECTIVE.value == constants.OBJECTIVE_VIDEO_CLASSIFICATION
+      and _INPUT_FILE_TYPE.value == constants.INPUT_FILE_TYPE_JSONL
+  ):
+    data_converter_vcn_lib.convert_jsonl_to_tfrecord(
+        _INPUT_FILE_PATH.value,
+        _OUTPUT_DIR.value,
+        _OUTPUT_FPS.value,
+        split_ratio,
+        num_shard,
+    )
+  else:
+    raise NotImplementedError(
+        f'File format {_INPUT_FILE_TYPE.value} is not supported for'
+        f' {_OBJECTIVE.value}.'
+    )
+
+
+if __name__ == '__main__':
+  app.run(main)

community-content/vertex_model_garden/model_oss/data_converter/data_converter_vcn_lib.py

@@ -0,0 +1,289 @@
+"""Converts VCN CSV/JSONL files to TFRecord with apache beam."""
+
+import json
+from os import path
+from typing import Any, Dict, Iterator, Sequence, Union, cast
+
+from absl import logging
+import apache_beam as beam
+from apache_beam.io import tfrecordio
+import numpy as np
+import pandas as pd
+import tensorflow as tf
+
+from data_converter import common_lib
+from util import constants
+
+
+_COLUMN_NAMES = [
+    common_lib.COLUMN_NAME_ML_USE,
+    common_lib.COLUMN_NAME_GCS_FILE_PATH,
+    common_lib.COLUMN_NAME_LABEL,
+    common_lib.COLUMN_NAME_START_SEC,
+    common_lib.COLUMN_NAME_END_SEC,
+]
+_JSON_GCS_URI_KEY = 'videoGcsUri'
+_JSON_CLASS_ANNOTATION_KEY = 'timeSegmentAnnotations'
+_JSON_CLASS_NAME_KEY = 'displayName'
+_JSON_START_TIME_KEY = 'startTime'
+_JSON_END_TIME_KEY = 'endTime'
+_JSON_RESOURCE_LABEL_KEY = 'dataItemResourceLabels'
+_JSON_ML_USE_KEY = 'aiplatform.googleapis.com/ml_use'
+
+
+def build_tf_example(
+    video_uri: str,
+    label: int,
+    start_sec: float,
+    end_sec: float,
+    output_fps: int,
+) -> tf.train.SequenceExample:
+  """Builds a TF Example from a video clip.
+
+  Args:
+    video_uri: GCS URI to the video file.
+    label: Class label as an integer.
+    start_sec: Start timestamp of the video clip in seconds.
+    end_sec: End timestamp of the video clip in seconds.
+    output_fps: The output frame rate per second.
+
+  Returns:
+    The created TF Example.
+  """
+  frame_bytes = common_lib.encode_video(
+      video_uri, start_sec, end_sec, output_fps, image_format='jpg'
+  )
+  seq_example = tf.train.SequenceExample()
+  seq_example.context.feature['clip/label/index'].int64_list.value[:] = [label]
+  for frame in frame_bytes:
+    seq_example.feature_lists.feature_list.get_or_create(
+        'image/encoded'
+    ).feature.add().bytes_list.value[:] = [frame]
+
+  return seq_example
+
+
+class AcquireTFExampleDoFn(beam.DoFn):
+  """Beam DoFn to build TF Examples from a DataFrame row dict for VCN."""
+
+  def __init__(self, output_fps: int):
+    self._success_counter = beam.metrics.Metrics.counter(
+        self.__class__.__name__, 'Success'
+    )
+    self._failure_counter = beam.metrics.Metrics.counter(
+        self.__class__.__name__, 'Failure'
+    )
+    self._output_fps = output_fps
+
+  def process(
+      self, element: Dict[str, Union[float, int, str]]
+  ) -> Iterator[tf.train.SequenceExample]:
+    ml_use: str = cast(str, element[common_lib.COLUMN_NAME_ML_USE])
+    video_uri: str = cast(str, element[common_lib.COLUMN_NAME_GCS_FILE_PATH])
+
+    try:
+      label: int = int(element[common_lib.COLUMN_NAME_LABEL])
+      start_sec: float = float(element[common_lib.COLUMN_NAME_START_SEC])
+      end_sec: float = float(element[common_lib.COLUMN_NAME_END_SEC])
+
+      tf_example = build_tf_example(
+          video_uri,
+          label,
+          start_sec,
+          end_sec,
+          self._output_fps,
+      )
+      self._success_counter.inc()
+      yield beam.pvalue.TaggedOutput(ml_use, tf_example)
+    except (ValueError, IOError) as err:
+      logging.error('Failed to process %s', video_uri)
+      logging.exception(err)
+      self._failure_counter.inc()
+
+
+def _run_convert_pipeline(
+    output_dir: str,
+    df: pd.DataFrame,
+    num_shards: Sequence[int],
+    output_fps: int,
+) -> None:
+  """Starts a Beam pipeline to write DataFrame as TF Records.
+
+  Args:
+    output_dir: TF Records output directory.
+    df: DataFrame to convert from.
+    num_shards: Number of shards for train/validation/test TFRecord files.
+    output_fps: The output frame rate per second.
+  """
+  clip_list = df.to_dict('records')
+
+  def pipeline(root):
+    train, val, test = (
+        root
+        | 'Create PCollection' >> beam.Create(clip_list)
+        | 'Convert to TF Example'
+        >> beam.ParDo(AcquireTFExampleDoFn(output_fps)).with_outputs(
+            constants.ML_USE_TRAINING,
+            constants.ML_USE_VALIDATION,
+            constants.ML_USE_TEST,
+        )
+    )
+    _ = train | 'Save train TF Record' >> tfrecordio.WriteToTFRecord(
+        path.join(output_dir, common_lib.TRAIN_TFRECORD_NAME),
+        coder=beam.coders.ProtoCoder(tf.train.Example),
+        num_shards=num_shards[0],
+    )
+    _ = val | 'Save val TF Record' >> tfrecordio.WriteToTFRecord(
+        path.join(output_dir, common_lib.VALIDATION_TFRECORD_NAME),
+        coder=beam.coders.ProtoCoder(tf.train.Example),
+        num_shards=num_shards[1],
+    )
+    _ = test | 'Save test TF Record' >> tfrecordio.WriteToTFRecord(
+        path.join(output_dir, common_lib.TEST_TFRECORD_NAME),
+        coder=beam.coders.ProtoCoder(tf.train.Example),
+        num_shards=num_shards[2],
+    )
+
+  common_lib.run_beam_pipeline(pipeline)
+
+
+def _convert_df_to_tfrecord(
+    df: pd.DataFrame,
+    output_dir: str,
+    split_ratio: Sequence[float],
+    num_shard: Sequence[int],
+    output_fps: int,
+) -> None:
+  """Converts a DataFrame into three separate tfrecords for training, validation, and testing into output_dir.
+
+  Args:
+    df: DataFrame to convert.
+    output_dir: The directory to save TFRecords and label_map.yaml.
+    split_ratio: List specifying the training, validation, and testing splits
+      for unassigned TFRecords.
+    num_shard: Number of shards for train/validation/test TFRecord files.
+    output_fps: The output frame rate per second.
+  """
+  # Replaces ml_use with common_lib string constants for consistency.
+  common_lib.format_ml_use_column(df)
+  common_lib.insert_missing_ml_use(df)
+
+  # Ignores invalid rows.
+  dropped_row_num = common_lib.drop_invalid_rows(df)
+  if dropped_row_num > 0:
+    logging.warning('Ignored %d invalid rows.', dropped_row_num)
+
+  common_lib.replace_unassigned_ml_use(
+      df[common_lib.COLUMN_NAME_ML_USE], split_ratio
+  )
+
+  # Converts labels to integers as required by training.
+  new_labels, label_map = common_lib.create_label_map(
+      df[common_lib.COLUMN_NAME_LABEL]
+  )
+  df[common_lib.COLUMN_NAME_LABEL] = new_labels
+  label_map_path = path.join(output_dir, common_lib.LABEL_MAP_NAME)
+  logging.info('Writing label map to %s.', label_map_path)
+  common_lib.write_label_map(label_map_path, label_map)
+
+  # Missing start / end times are treated as 0, inf, respectively.
+  df[common_lib.COLUMN_NAME_START_SEC].fillna(0, inplace=True)
+  df[common_lib.COLUMN_NAME_END_SEC].fillna(np.inf, inplace=True)
+
+  _run_convert_pipeline(output_dir, df, num_shard, output_fps)
+
+
+def convert_csv_to_tfrecord(
+    input_csv: str,
+    output_dir: str,
+    output_fps: int,
+    split_ratio: Sequence[float] = (0.8, 0.1, 0.1),
+    num_shard: Sequence[int] = (10, 10, 10),
+) -> None:
+  """Parses input_csv file into three separate tfrecords for training, validation, and testing into output_dir.
+
+  The csv format is shown in
+    https://cloud.google.com/vertex-ai/docs/video-data/classification/prepare-data#csv
+
+  If an ml_use column is not provided, one will be created.
+
+  label_map.yaml containing the label map will be placed in output_dir.
+
+  Args:
+    input_csv: Name of the csv file.
+    output_dir: The directory to save TFRecords and label_map.yaml.
+    output_fps: The output frame rate per second.
+    split_ratio: List specifying the training, validation, and testing splits
+      for unassigned TFRecords.
+    num_shard: Number of shards for train/validation/test TFRecord files.
+  """
+  with tf.io.gfile.GFile(input_csv, 'r') as f:
+    df: pd.DataFrame = pd.read_csv(
+        f, header=None, names=_COLUMN_NAMES, on_bad_lines='warn'
+    )
+
+  _convert_df_to_tfrecord(df, output_dir, split_ratio, num_shard, output_fps)
+
+
+def convert_jsonl_to_tfrecord(
+    input_jsonl: str,
+    output_dir: str,
+    output_fps: int,
+    split_ratio: Sequence[float] = (0.8, 0.1, 0.1),
+    num_shard: Sequence[int] = (10, 10, 10),
+) -> None:
+  """Parses input_jsonl file into three separate tfrecords for training, validation, and testing into output_dir.
+
+  The JSONL format is shown in
+    https://cloud.google.com/vertex-ai/docs/video-data/classification/prepare-data#jsonl.
+
+  If an ml_use column is not provided, one will be created.
+
+  label_map.yaml containing the label map will be placed in output_dir.
+
+  Args:
+    input_jsonl: Name of the JSONL file.
+    output_dir: The directory to save TFRecords and label_map.yaml.
+    output_fps: The output frame rate per second.
+    split_ratio: List specifying the training, validation, and testing splits
+      for unassigned TFRecords.
+    num_shard: Number of shards for train/validation/test TFRecord files.
+  """
+  df_rows = []
+  with tf.io.gfile.GFile(input_jsonl, 'r') as f:
+    lines = f.read().rstrip().splitlines()
+
+  for i, line in enumerate(lines, 1):
+    try:
+      item: Dict[str, Any] = json.loads(line)
+
+      gcs_uri = item.get(_JSON_GCS_URI_KEY)
+      if not gcs_uri:
+        logging.warning('Invalid JSON at line %d, skipped.', i)
+        continue
+
+      annotations = item.get(_JSON_CLASS_ANNOTATION_KEY, [])
+      ml_use = item.get(_JSON_RESOURCE_LABEL_KEY, {}).get(
+          _JSON_ML_USE_KEY, common_lib.ML_USE_UNASSIGNED
+      )
+
+      for j, annotation in enumerate(annotations):
+        label = annotation.get(_JSON_CLASS_NAME_KEY)
+        if not label:
+          logging.warning('Invalid annotation #%d at line %d, skipped.', j, i)
+          continue
+        # The example in external documentation uses strings like "1.0s", so we
+        # need to remove the "s" suffix.
+        start_time = annotation.get(_JSON_START_TIME_KEY, '0').removesuffix('s')
+        end_time = annotation.get(_JSON_END_TIME_KEY, 'inf').removesuffix('s')
+        df_rows.append([ml_use, gcs_uri, label, start_time, end_time])
+    except (json.JSONDecodeError, AttributeError):
+      logging.warning('Invalid JSON at line %d, skipped.', i)
+      continue
+
+  df = pd.DataFrame(
+      data=df_rows,
+      columns=_COLUMN_NAMES,
+  )
+
+  _convert_df_to_tfrecord(df, output_dir, split_ratio, num_shard, output_fps)

community-content/vertex_model_garden/model_oss/data_converter/dockerfile/base.Dockerfile

@@ -0,0 +1,50 @@
+FROM python:3.9
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Install basic libs.
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        cmake \
+        curl \
+        wget \
+        sudo \
+        gnupg \
+        python3-opencv \
+        lsb-release \
+        ca-certificates \
+        build-essential \
+        git \
+        vim \
+        screen \
+        libportaudio2 \
+        libusb-1.0-0-dev \
+        openjdk-17-jre
+
+# Add gcsfuse distribution URL as a package source and import its public key.
+RUN echo "deb https://packages.cloud.google.com/apt gcsfuse-`lsb_release -c -s` main" | sudo tee /etc/apt/sources.list.d/gcsfuse.list
+RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
+
+# Install gcsfuse.
+RUN apt-get update && apt-get install -y --no-install-recommends gcsfuse
+
+# Install google cloud SDK.
+RUN wget -q https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-359.0.0-linux-x86_64.tar.gz
+RUN tar xzf google-cloud-sdk-359.0.0-linux-x86_64.tar.gz
+RUN ./google-cloud-sdk/install.sh -q
+# Make sure gsutil will use the default service account.
+RUN echo '[GoogleCompute]\nservice_account = default' > /etc/boto.cfg
+
+
+# Install required libs.
+RUN pip install --upgrade pip
+RUN pip install pyyaml==5.4.1
+RUN pip install pycocotools==2.0.6
+RUN pip install opencv-python-headless==4.7.0.72
+RUN pip install numpy==1.24.2
+RUN pip install pandas==1.5.3
+RUN pip install Pillow==9.4.0
+RUN pip install apache-beam[gcp]==2.45.0
+RUN pip install object-detection==0.0.3
+RUN pip install google-cloud-storage==1.42.3
+RUN pip install gcsfs==2021.10.1
+RUN pip install pylint==2.17.2

community-content/vertex_model_garden/model_oss/data_converter/dockerfile/run.Dockerfile

@@ -0,0 +1,23 @@
+FROM gcr.io/automl-migration-test/automl-vision-data-converter-base:latest
+
+# Copy license.
+RUN wget https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/LICENSE
+
+COPY model_oss/data_converter /automl_vision/data_converter
+COPY model_oss/util /automl_vision/util
+
+WORKDIR /automl_vision
+ENV PYTHONPATH "${PYTHONPATH}:/automl_vision"
+
+# Run pylint to validate code.
+COPY .pylintrc /automl_vision/.pylintrc
+RUN find . -type f -name "*.py" | xargs pylint --rcfile=./.pylintrc --errors-only
+
+ENTRYPOINT ["python3","data_converter/data_converter_main.py"]
+
+CMD ["--input_file_path=YOUR_INPUT_FILE",\
+"--input_file_type=csv",\
+"--objective=iod",\
+"--output_dir=YOUR_OUTPUT_DIR",\
+"--num_shard=10,10,10",\
+"--split_ratio=0.8,0.1,0.1"]

community-content/vertex_model_garden/model_oss/detectron2/dockerfile/serving.Dockerfile

@@ -0,0 +1,90 @@
+# Dockerfile for Detectron2 serving.
+#
+# To build:
+# docker build -f model_oss/detectron2/dockerfile/serving.Dockerfile . -t ${YOUR_IMAGE_TAG}
+#
+# To push to gcr:
+# docker tag ${YOUR_IMAGE_TAG} gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
+# docker push gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
+FROM pytorch/torchserve:0.7.0-cpu
+
+USER root
+
+# Install tools.
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        curl \
+        wget \
+        vim
+
+# run and update some basic packages software packages, including security libs
+RUN apt-get update &&  apt-get install -y \
+    software-properties-common && \
+    add-apt-repository -y ppa:ubuntu-toolchain-r/test && \
+    apt-get update && apt-get install -y \
+    gcc-9 g++-9 apt-transport-https ca-certificates gnupg curl
+
+# Install gcloud tools for gsutil as well as debugging
+RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | \
+    tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \
+    curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | \
+    apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - && \
+    apt-get update -y && apt-get install google-cloud-sdk -y
+
+USER model-server
+
+# install detectron2 dependencies
+RUN python3 -m pip install --upgrade pip
+RUN python3 -m pip install --user numpy==1.24.2
+RUN python3 -m pip install --user opencv-python==4.7.0.72
+RUN python3 -m pip install --user 'git+https://github.com/facebookresearch/detectron2.git@v0.6'
+
+# Install GCS storage library.
+RUN pip install google-cloud-storage==2.6.0
+
+# For mask encoding.
+RUN pip install --upgrade pycocotools==2.0.6
+
+ARG MODEL_NAME=detectron2_serving
+ENV MODEL_NAME="${MODEL_NAME}"
+
+# health and prediction listener ports
+ARG AIP_HTTP_PORT=7080
+ENV AIP_HTTP_PORT="${AIP_HTTP_PORT}"
+
+ARG MODEL_MGMT_PORT=7081
+
+# expose health and prediction listener ports from the image
+EXPOSE "${AIP_HTTP_PORT}"
+EXPOSE "${MODEL_MGMT_PORT}"
+EXPOSE 8080 8081 8082 7070 7071
+
+# create torchserve configuration file
+USER root
+RUN echo "service_envelope=json\n" \
+    "inference_address=http://0.0.0.0:${AIP_HTTP_PORT}\n"  \
+    "management_address=http://0.0.0.0:${MODEL_MGMT_PORT}" >> /home/model-server/config.properties
+USER model-server
+
+# Copy license.
+RUN wget https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/LICENSE
+
+# Copy model artifacts.
+COPY ./model_oss/detectron2/handler.py /home/model-server/handler.py
+WORKDIR /home/model-server/
+
+# Create model archive file packaging model artifacts and dependencies.
+# Note(lavrai): The model `.pth` file and `cfg.yaml` file will be set by the
+# customer as an environment variable and will be later loaded by the
+# `handler.py` file.
+RUN torch-model-archiver \
+  --model-name="${MODEL_NAME}" \
+  --version=1.0 \
+  --handler=/home/model-server/handler.py \
+  --export-path=/home/model-server/model-store \
+  -f
+
+# run Torchserve HTTP serve to respond to prediction requests
+CMD ["ls", "-ltr", "/home/model-server/model-store/", ";", \
+    "torchserve", "--start", "--ts-config=/home/model-server/config.properties", \
+    "--models", "${MODEL_NAME}=${MODEL_NAME}.mar", \
+    "--model-store", "/home/model-server/model-store"]

community-content/vertex_model_garden/model_oss/detectron2/dockerfile/train.Dockerfile

@@ -0,0 +1,100 @@
+# Dockerfile for Detectron2 training.
+#
+# To build:
+# docker build -f model_oss/detectron2/dockerfile/train.Dockerfile . -t ${YOUR_IMAGE_TAG}
+#
+# To push to gcr:
+# docker tag ${YOUR_IMAGE_TAG} gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
+# docker push gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
+
+FROM nvidia/cuda:11.1.1-cudnn8-devel-ubuntu18.04
+# Using an older system (18.04) to avoid opencv incompatibility (issue#3524).
+
+ENV DEBIAN_FRONTEND noninteractive
+RUN apt-get update && apt-get install -y \
+    python3.7 python3.7-dev python3.7-distutils \
+    python3-opencv ca-certificates git wget sudo ninja-build \
+    curl wget vim
+
+# Make python3 available for python3.7.
+RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.6 1
+RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 2
+RUN update-alternatives --config python3
+# Make python available for python3.7.
+RUN ln -sv /usr/bin/python3.7 /usr/bin/python
+
+# Create a non-root user.
+ARG USER_ID=1000
+RUN useradd -m --no-log-init --system  --uid ${USER_ID} appuser -g sudo
+RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+USER appuser
+WORKDIR /home/appuser
+
+ENV PATH="/home/appuser/.local/bin:${PATH}"
+RUN wget https://bootstrap.pypa.io/pip/get-pip.py && \
+    python3.7 get-pip.py --user && \
+    rm get-pip.py
+
+# Important! Otherwise, it uses existing numpy from host-modules
+# which throws error.
+RUN pip install --user numpy==1.20.3
+
+# Install dependencies:
+# See https://pytorch.org/ for other options if you use
+# a different version of CUDA.
+RUN pip install --user tensorboard==2.11.0
+# cmake from apt-get is too old.
+RUN pip install --user cmake==3.25.2
+RUN pip install --user torch==1.10.0+cu111 torchvision==0.11.0+cu111 -f https://download.pytorch.org/whl/torch_stable.html
+RUN pip install --user setuptools==59.5.0
+RUN pip install --user opencv-python==4.7.0.72
+RUN pip install --user cloudml-hypertune==0.1.0.dev6
+RUN pip install --user fvcore==0.1.5.post20221221
+# Install detectron2.
+RUN git clone -b v0.6 https://github.com/facebookresearch/detectron2 detectron2_repo
+# Set FORCE_CUDA because during `docker build` cuda is not accessible.
+ENV FORCE_CUDA="1"
+# This will by default build detectron2 for all common cuda
+# architectures and take a lot more time,
+# because inside `docker build`, there is no way to tell
+# which architecture will be used.
+ARG TORCH_CUDA_ARCH_LIST="Kepler;Kepler+Tesla;Maxwell;Maxwell+Tegra;Pascal;Volta;Turing"
+ENV TORCH_CUDA_ARCH_LIST="${TORCH_CUDA_ARCH_LIST}"
+RUN pip install --user -e detectron2_repo
+
+# Set a fixed model cache directory.
+ENV FVCORE_CACHE="/tmp"
+
+# Copy license.
+RUN wget https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/LICENSE
+
+# Copy model-garden detectron2 files to '/home/appuser/trainer' folder.
+ADD ./model_oss/detectron2 /home/appuser/trainer
+
+################ Copy plain_train_net.py to task.py and
+# then modify it using sed commands. ###################
+# Src: https://github.com/facebookresearch/detectron2/blob/v0.6/tools/plain_train_net.py
+RUN sudo cp /home/appuser/detectron2_repo/tools/plain_train_net.py /home/appuser/trainer/task.py
+# Make additional changes to task.py.
+# Note(lavrai): Start adding SED commands from end of file towards the top
+# so that the line numbers do not keep changing for the source file.
+# For entry-point:
+RUN sudo sed -i "214 d" /home/appuser/trainer/task.py
+RUN sudo sed -i "213 a\    default_arg_parser = default_argument_parser()" /home/appuser/trainer/task.py
+RUN sudo sed -i "214 a\    extended_parser = trainer_utils.extend_parser_arguments(default_arg_parser)" /home/appuser/trainer/task.py
+RUN sudo sed -i "215 a\    args = extended_parser.parse_args()" /home/appuser/trainer/task.py
+# For main() function:
+RUN sudo sed -i "192 a\    trainer_utils.register_dataset(args)" /home/appuser/trainer/task.py
+# For setup() function:
+RUN sudo sed -i "184 a\    cfg.SOLVER.BASE_LR = args.lr" /home/appuser/trainer/task.py
+RUN sudo sed -i "185 a\    cfg.OUTPUT_DIR = args.output_dir" /home/appuser/trainer/task.py
+RUN sudo sed -i "186 a\    cfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url(config_file_copy)" /home/appuser/trainer/task.py
+RUN sudo sed -i "182 a\    config_file_copy = args.config_file" /home/appuser/trainer/task.py
+RUN sudo sed -i "183 a\    args.config_file = model_zoo.get_config_file(args.config_file)" /home/appuser/trainer/task.py
+# For new import:
+RUN sudo sed -i "27 a\from detectron2 import model_zoo" /home/appuser/trainer/task.py
+RUN sudo sed -i "21 a\import trainer_utils" /home/appuser/trainer/task.py
+
+ENV PYTHONPATH /home/appuser/trainer
+
+ENTRYPOINT ["python", "-m", "trainer.task"]