debug: set explicit timeout on operation.result()

.cloud-build/cleanup/cleanup.py

@@ -1,30 +1,10 @@
 from typing import List
 from ratemate import RateLimit
-import argparse
-
-parser = argparse.ArgumentParser()
-parser.add_argument("--dry_run",
-                    type=bool,
-                    default=False)
-args = parser.parse_args()
-
-
 from resource_cleanup_manager import (
     DatasetResourceCleanupManager,
     ModelResourceCleanupManager,
     EndpointResourceCleanupManager,
     ResourceCleanupManager,
-    MatchingEngineIndexEndpointResourceCleanupManager,
-    MatchingEngineIndexResourceCleanupManager,
-    FeatureStoreLegacyCleanupManager,
-    FeatureStoreCleanupManager,
-    PipelineJobCleanupManager,
-    TrainingJobCleanupManager,
-    HyperparameterTuningCleanupManager,
-    BatchPredictionJobCleanupManager,
-    ExperimentCleanupManager,
-    BucketCleanupManager,
-    ArtifactRegistryCleanupManager
 )
 
 rate_limit = RateLimit(max_count=25, per=60, greedy=False)
@@ -36,14 +16,12 @@ def run_cleanup_managers(managers: List[ResourceCleanupManager], is_dry_run: boo
 
         print(f"Fetching {type_name}'s...")
         resources = manager.list()
-        try:
-            print(f"Found {len(resources)} {type_name}'s")
-        except Exception as e:
-            print(f"{type_name} {e}")
+        print(f"Found {len(resources)} {type_name}'s")
         for resource in resources:
             try:
                 if not manager.is_deletable(resource):
                     continue
+
                 if is_dry_run:
                     resource_name = manager.resource_name(resource)
                     print(f"Will delete '{type_name}': {resource_name}")
@@ -56,25 +34,16 @@ def run_cleanup_managers(managers: List[ResourceCleanupManager], is_dry_run: boo
         print("")
 
 
-if args.dry_run:
+is_dry_run = False
+
+if is_dry_run:
     print("Starting cleanup in dry run mode...")
 
 # List of all cleanup managers
-managers: List[ResourceCleanupManager] = [
+managers = [
     DatasetResourceCleanupManager(),
     EndpointResourceCleanupManager(),
     ModelResourceCleanupManager(),  # ModelResourceCleanupManager must follow EndpointResourceCleanupManager due to deployed models blocking model deletion.
-    MatchingEngineIndexEndpointResourceCleanupManager(),
-    MatchingEngineIndexResourceCleanupManager(),
-    FeatureStoreLegacyCleanupManager(),
-    FeatureStoreCleanupManager(),
-    PipelineJobCleanupManager(),
-    TrainingJobCleanupManager(),
-    HyperparameterTuningCleanupManager(),
-    BatchPredictionJobCleanupManager(),
-    ExperimentCleanupManager(), # Experiment missing _resource_noun
-    BucketCleanupManager(),
-    ArtifactRegistryCleanupManager()
 ]
 
-run_cleanup_managers(managers=managers, is_dry_run=args.dry_run)
+run_cleanup_managers(managers=managers, is_dry_run=is_dry_run)

.cloud-build/cleanup/resource_cleanup_manager.py

@@ -1,26 +1,10 @@
-'''
-READ FIRST BEFORE MAKING CHANGES
--  Create a convention for resources created from vertex-ai-samples GH. We already have one IIRC
-- Only delete those objects as part of our clean-up script.
-- Don't run any tests on python-docs-samples-tests project, especially ones that affect resources created outside of our purview
-- Add --dry-run option to the clean-up script. This option will just output the list of resources the script will delete instead of actually deleting the resources.
-- Have a larger conversation in DEE before touching any resources that were not created as part of vertex-ai-samples
-'''
-import os
 import abc
 from typing import Any, Type
 
 from google.cloud import aiplatform
 from google.cloud.aiplatform import base
-from google.cloud.aiplatform_v1beta1 import (FeatureOnlineStoreAdminServiceClient,
-                                             FeatureOnlineStore)
-from google.cloud import storage
 from proto.datetime_helpers import DatetimeWithNanoseconds
 
-PROJECT_ID = "python-docs-samples-tests"
-REGION = "us-central1"
-API_ENDPOINT = f"{REGION}-aiplatform.googleapis.com"
-
 # If a resource was updated within this number of seconds, do not delete.
 RESOURCE_UPDATE_BUFFER_IN_SECONDS = 60 * 60 * 8
 
@@ -85,7 +69,7 @@ class VertexAIResourceCleanupManager(ResourceCleanupManager):
     def delete(self, resource):
         resource.delete()
 
-    def get_seconds_since_modification(self, resource: Any) -> float:
+    def get_seconds_since_modification(self, resource: Any) -> bool:
         update_time = resource.update_time
         current_time = DatetimeWithNanoseconds.now(tz=update_time.tzinfo)
         return (current_time - update_time).total_seconds()
@@ -113,193 +97,16 @@ class EndpointResourceCleanupManager(VertexAIResourceCleanupManager):
     vertex_ai_resource = aiplatform.Endpoint
 
     def delete(self, resource):
+        # TODO: Remove this once https://github.com/googleapis/python-aiplatform/issues/1441 is fixed
+        resource._sync_gca_resource()
         for deployed_model_id in [
             models.id for models in resource._gca_resource.deployed_models
         ]:
             resource._undeploy(deployed_model_id=deployed_model_id)
+
         resource.delete(force=True)
 
 
 class ModelResourceCleanupManager(VertexAIResourceCleanupManager):
     vertex_ai_resource = aiplatform.Model
 
-
-class MatchingEngineIndexResourceCleanupManager(VertexAIResourceCleanupManager):
-    vertex_ai_resource = aiplatform.MatchingEngineIndex
-
-
-class MatchingEngineIndexEndpointResourceCleanupManager(VertexAIResourceCleanupManager):
-    vertex_ai_resource = aiplatform.MatchingEngineIndexEndpoint
-
-    def delete(self, resource):
-        resource.undeploy_all()
-        resource.delete(force=True)
-
-class FeatureStoreLegacyCleanupManager(VertexAIResourceCleanupManager):
-    # TODO: only deleting legacy
-    #    not deleting ingestions jobs
-    #       ingest_from_xxx methods do not return a job ID, there is no list command, aka no python way to delete
-    #    not deleting batch serving jobs
-    #       batch_serve_to_xxx methods do not return a job ID, there is no list command, aka no python way to delete
-    vertex_ai_resource = aiplatform.Featurestore
-
-    def resource_name(self, resource: Any) -> str:
-        return resource.name
-
-    def delete(self, resource):
-        resource.delete(force=True)
-
-
-class FeatureStoreCleanupManager(VertexAIResourceCleanupManager):
-    # for FS 2.0
-    # TODO: use _v1beta1, and gapic clients
-    #    delete features, feature groups, feature views, feature online stores
-    vertex_ai_resource = FeatureOnlineStore
-
-    admin_client = FeatureOnlineStoreAdminServiceClient(
-        client_options={"api_endpoint": API_ENDPOINT}
-    )
-
-    def resource_name(self, resource: Any) -> str:
-        return resource.name
-
-    def type_name(self) -> str:
-        return "FeatureOnlineStore"
-
-    def list(self) -> Any:
-        try:
-            return self.admin_client.list_feature_online_stores(parent=f"projects/{PROJECT_ID}/locations/{REGION}")
-        except Exception as e:
-            print(e)
-            return []
-
-    def delete(self, resource):
-        try:
-            self.admin_client.delete_feature_online_store(name=resource.name, force=True)
-        except Exception as e:
-            print(e)
-
-
-class PipelineJobCleanupManager(VertexAIResourceCleanupManager):
-    vertex_ai_resource = aiplatform.PipelineJob
-
-class TrainingJobCleanupManager(VertexAIResourceCleanupManager):
-    vertex_ai_resource = aiplatform.training_jobs._CustomTrainingJob
-
-    job_types = [
-            aiplatform.AutoMLImageTrainingJob,
-            aiplatform.AutoMLTextTrainingJob,
-            aiplatform.AutoMLTabularTrainingJob,
-            aiplatform.AutoMLVideoTrainingJob,
-            aiplatform.AutoMLForecastingTrainingJob,
-            aiplatform.CustomJob,
-            aiplatform.CustomTrainingJob,
-            aiplatform.CustomContainerTrainingJob,
-            aiplatform.CustomPythonPackageTrainingJob
-    ]
-
-    def list(self) -> Any:
-        return [
-            job
-            for job_type in self.job_types
-            for job in job_type.list()
-        ]
-
-class HyperparameterTuningCleanupManager(VertexAIResourceCleanupManager):
-    vertex_ai_resource = aiplatform.HyperparameterTuningJob
-
-
-class BatchPredictionJobCleanupManager(VertexAIResourceCleanupManager):
-    vertex_ai_resource = aiplatform.BatchPredictionJob
-
-class ExperimentCleanupManager(VertexAIResourceCleanupManager):
-    vertex_ai_resource = aiplatform.Experiment
-
-    @property
-    def type_name(self) -> str:
-        return "Experiment"
-
-    def resource_name(self, resource: Any) -> str:
-        return resource.name
-
-    def get_seconds_since_modification(self, resource: Any) -> float:
-        update_time = resource._metadata_context.update_time
-        current_time = DatetimeWithNanoseconds.now()
-        return float(current_time.timestamp() - update_time.timestamp())
-
-class BucketCleanupManager(ResourceCleanupManager):
-    vertex_ai_resource = storage.bucket.Bucket
-
-    def list(self) -> Any:
-        storage_client = storage.Client()
-        return list(storage_client.list_buckets())
-
-    def delete(self, resource):
-        try:
-            resource.delete(force=True)
-        except Exception as e:
-            print(e)
-
-    @property
-    def type_name(self) -> str:
-        return "Bucket"
-
-    def get_seconds_since_modification(self, resource: Any) -> float:
-        # Bucket has no last_update property, only time created
-        created_time = resource.time_created
-        current_time = DatetimeWithNanoseconds.now()
-        return float(current_time.timestamp() - created_time.timestamp())
-
-    def resource_name(self, resource: Any) -> str:
-        return resource.name
-
-    def is_deletable(self, resource: Any) -> bool:
-        time_difference = self.get_seconds_since_modification(resource)
-
-        if not self.resource_name(resource).startswith('your-bucket-name'):
-            print(f"Skipping '{resource}' not a Vertex AI notebook bucket")
-            return False
-
-        # Check that it wasn't created too recently, to prevent race conditions
-        if time_difference <= RESOURCE_UPDATE_BUFFER_IN_SECONDS:
-            print(
-                f"Skipping '{resource}' due to update_time being '{time_difference}', which is less than '{RESOURCE_UPDATE_BUFFER_IN_SECONDS}'."
-            )
-            return False
-        return True
-
-class ArtifactRegistryCleanupManager(ResourceCleanupManager):
-    vertex_ai_resource = "Artifact Registry"
-
-    def list(self) -> Any:
-        import subprocess
-
-        result = subprocess.run(["gcloud artifacts repositories list --location=us-central1"], 
-                                shell=True, capture_output=True, text=True)
-
-        ret = []
-        lines = result.stdout.split('\n')[2:]
-        for line in lines:
-            repo = line.split(' ')[0]
-            if repo.startswith("my-docker-repo"):
-                ret.append(repo)
-
-        return ret
-
-    def delete(self, resource):
-        os.system(f"! gcloud artifacts repositories delete {resource} --location=us-central1")
-
-    @property
-    def type_name(self) -> str:
-        return "ArtifactRepository"
-
-    def resource_name(self, resource: Any) -> str:
-        return resource
-
-    # delete repository regardless of age
-    def get_seconds_since_modification(self, resource: Any) -> float:
-        return RESOURCE_UPDATE_BUFFER_IN_SECONDS + 1
-    
-    def is_deleteable(self, resource: Any) -> bool:
-        return True
-

.cloud-build/execute_changed_notebooks_cli.py

@@ -17,8 +17,6 @@
 
 import argparse
 import pathlib
-import os
-import csv
 
 import execute_changed_notebooks_helper
 
@@ -38,22 +36,9 @@ parser = argparse.ArgumentParser(description="Run changed notebooks.")
 parser.add_argument(
     "--test_paths_file",
     type=pathlib.Path,
-    help="The path to the file that has newline-delimited folders of notebooks that should be tested.",
+    help="The path to the file that has newline-limited folders of notebooks that should be tested.",
     required=True,
 )
-parser.add_argument(
-    "--test_percent",
-    type=int,
-    help="The percent of notebooks to be tested (between 1 and 100).",
-    required=False,
-    default=100,
-)
-parser.add_argument(
-    "--build_id",
-    type=str,
-    help="The build id (which may be a Cloud Build job specific or user explicit.",
-    required=True
-)
 parser.add_argument(
     "--base_branch",
     help="The base git branch to diff against to find changed files.",
@@ -122,98 +107,24 @@ parser.add_argument(
     default=True,
     help="Should run notebooks in parallel.",
 )
-parser.add_argument(
-    "--concurrent_notebooks",
-    type=int,
-    help="Maximum number of parallel notebook executions per minute",
-    default=10,
-    required=False,
-)
-parser.add_argument(
-    "--run_first_file",
-    type=pathlib.Path,
-    help="The path to the file that has newline-delimited of notebooks to run in the first batch",
-    default=None,
-    required=False,
-)
-parser.add_argument(
-    "--aiplatform_whl",
-    type=str,
-    help="The GCS path to a whl version google-cloud-aiplatform",
-    default=None,
-    required=False,
-)
-parser.add_argument(
-    "--dry_run",
-    type=str2bool,
-    default=False,
-    help="Dry run for testing - no execution",
-)
 
 args = parser.parse_args()
 
-changed_notebooks = execute_changed_notebooks_helper.get_changed_notebooks(
+notebooks = execute_changed_notebooks_helper.get_changed_notebooks(
     test_paths_file=args.test_paths_file,
     base_branch=args.base_branch,
 )
 
-
-results_bucket = f"{args.artifacts_bucket}"
-# artifacts_bucket may get set by trigger to a full gs:// folder path
-if results_bucket.startswith("gs://"):
-    results_bucket = results_bucket[5:]
-results_bucket = results_bucket.split('/')[0]
-results_file = f"build_results/{args.build_id}.json"
-
-if args.test_percent == 100:
-    notebooks = changed_notebooks
-    accumulative_results = {}
-else:
-    accumulative_results = execute_changed_notebooks_helper.load_results(results_bucket, results_file)
-
-    notebooks = [changed_notebook for changed_notebook in changed_notebooks if execute_changed_notebooks_helper.select_notebook(changed_notebook, accumulative_results, args.test_percent)]
-    # cap the number of notebooks to the specified percentage
-    max_notebooks = int((len(changed_notebooks) * (args.test_percent/100)))
-    if (len(notebooks) > max_notebooks):
-        notebooks = notebooks[:max_notebooks]
-
-run_first = []
-if args.run_first_file:
-    if not os.path.isfile(args.run_first_file):
-        print("Error: file does not exist", args.run_first_file)
-    else:
-        with open(args.run_first_file, 'r') as csvfile:
-            reader = csv.reader(csvfile)
-            for row in reader:
-                notebook = row[0]
-                run_first.append(notebook)
-
-    for notebook in run_first:
-        if notebook in notebooks:
-            # remove from existing list
-            notebooks.remove(notebook)
-            # add back to the front of the list
-            notebooks.insert(0, notebook)
-            print(f"Run first: {notebook}")
-
-if args.dry_run:
-    print("Dry run ...\n")
-    for notebook in notebooks:
-        print(f"Would execute: {notebook}")
-else:
-    execute_changed_notebooks_helper.process_and_execute_notebooks(
-        notebooks=notebooks,
-        container_uri=args.container_uri,
-        staging_bucket=args.staging_bucket,
-        artifacts_bucket=args.artifacts_bucket,
-        results_file=results_file,
-        should_parallelize=args.should_parallelize,
-        timeout=args.timeout,
-        variable_project_id=args.variable_project_id,
-        variable_region=args.variable_region,
-        variable_service_account=args.variable_service_account,
-        variable_vpc_network=args.variable_vpc_network,
-        private_pool_id=args.private_pool_id,
-        concurrent_notebooks=args.concurrent_notebooks,
-        aiplatform_whl=args.aiplatform_whl
+execute_changed_notebooks_helper.process_and_execute_notebooks(
+    notebooks=notebooks,
+    container_uri=args.container_uri,
+    staging_bucket=args.staging_bucket,
+    artifacts_bucket=args.artifacts_bucket,
+    should_parallelize=args.should_parallelize,
+    timeout=args.timeout,
+    variable_project_id=args.variable_project_id,
+    variable_region=args.variable_region,
+    variable_service_account=args.variable_service_account,
+    variable_vpc_network=args.variable_vpc_network,
+    private_pool_id=args.private_pool_id,
 )

.cloud-build/execute_changed_notebooks_helper.py

@@ -21,34 +21,25 @@ import json
 import git
 import operator
 import os
-import io
-import json
 import pathlib
 import re
 import subprocess
-import random
-from google.cloud import storage
 import utils
-from typing import List, Optional, Dict, Any
+from typing import List, Optional
 from utils import util
 
 import execute_notebook_helper
 import execute_notebook_remote
 import nbformat
 from google.cloud.devtools.cloudbuild_v1.types import BuildOperationMetadata
+from ratemate import RateLimit
 from tabulate import tabulate
 from utils import NotebookProcessors, util
 
 # A buffer so that workers finish before the orchestrating job
 WORKER_TIMEOUT_BUFFER_IN_SECONDS: int = 60 * 60
-
 PYTHON_VERSION = "3.9"  # Set default python version
 
-# rolling time window for accumulating build results for selecting notebooks
-MAX_RESULTS_AGE_SECONDS: int = (60 * 60) * 24 * 60  # 60 days
-# maximum time since last run to force a run on the current build
-MAX_AGE_BEFORE_FORCE_RUN: int = (60 * 60) * 24 * 30
-
 
 def format_timedelta(delta: datetime.timedelta) -> str:
     """Formats a timedelta duration to [N days] %H:%M:%S format"""
@@ -74,9 +65,7 @@ def format_timedelta(delta: datetime.timedelta) -> str:
 @dataclasses.dataclass
 class NotebookExecutionResult:
     name: str
-    path: str
     duration: datetime.timedelta
-    start_time: datetime.datetime
     is_pass: bool
     log_url: str
     output_uri: str
@@ -92,97 +81,6 @@ class NotebookExecutionResult:
             return None
 
 
-def load_results(results_bucket: str,
-                 results_file: str) -> Dict[str, Any]:
-    '''
-    Load accumulated notebook test results
-    '''
-
-    print("Loading existing accumulative results ...")
-    accumulative_results = {}
-    try:
-        client = storage.Client()
-        bucket = client.bucket(results_bucket)
-
-        build_results_dir = os.path.dirname(results_file)
-        blobs = client.list_blobs(results_bucket, prefix=build_results_dir)
-        for blob in blobs:
-            time_created = blob.time_created.replace(tzinfo=None)
-            if (datetime.datetime.now().replace(tzinfo=None) - time_created).total_seconds() > MAX_RESULTS_AGE_SECONDS:
-                continue
-
-            content = util.download_blob_into_memory(results_bucket, blob.name, download_as_text=True)
-
-            try:
-                build_results = json.loads(content)
-            except:
-                continue  # skip corrupted build results files
-            for notebook in build_results:
-                if notebook in accumulative_results:
-                    accumulative_results[notebook]['passed'] += build_results[notebook]['passed']
-                    accumulative_results[notebook]['failed'] += build_results[notebook]['failed']
-                    if accumulative_results[notebook]['last_time_ran'] < time_created:
-                        accumulative_results[notebook]['last_time_ran'] = time_created
-                else:
-                    accumulative_results[notebook] = build_results[notebook]
-                    accumulative_results[notebook]['failed_on_latest_run'] = build_results[notebook]['failed']
-                    accumulative_results[notebook]['last_time_ran'] = time_created
-
-        print(accumulative_results)
-    except Exception as e:
-        print(e)
-
-    # If there are no accumulative results, an empty dict is returned
-    return accumulative_results
-
-def select_notebook(changed_notebook: str,
-                    accumulative_results: Dict[str, Any],
-                    test_percent: int) -> bool:
-    '''
-    Algorithm to randomly select a notebook, but weight the propbability of selected based on past failures
-    '''
-
-    if changed_notebook in accumulative_results:
-        pass_count = accumulative_results[changed_notebook]['passed']
-        fail_count = accumulative_results[changed_notebook]['failed']
-        failed_on_latest_run = accumulative_results[changed_notebook]['failed_on_latest_run']
-        last_time_ran = accumulative_results[changed_notebook]['last_time_ran']
-    else:
-        pass_count = 1
-        fail_count = 0
-        failed_on_latest_run = 0
-        last_time_ran = datetime.datetime.now().replace(tzinfo=None)
-
-    # If notebook has not been ran in a long time, force running it
-    if (datetime.datetime.now().replace(tzinfo=None) - last_time_ran).total_seconds() > MAX_AGE_BEFORE_FORCE_RUN:
-        should_test_do_to_age = True
-    else:
-        should_test_do_to_age  = False
-
-
-    # if failed on the last time it was ran, select the notebook
-    if failed_on_latest_run:
-        inferred_failure_rate = 1
-    # otherwise, calculate the frequency of failure
-    else:
-        inferred_failure_rate = fail_count / (pass_count + fail_count)
-
-    # If failure rate is high, the chance of testing should be higher
-    should_test_due_to_failure = random.uniform(0, 1) <= inferred_failure_rate
-
-    #if accumulative_resultsi[changed_notebook]['latest_date_ran']
-
-    # Additionally, only test a percentage of these
-    should_test_due_to_random_subset = random.uniform(0, 1) <= (test_percent / 100)
-
-    if should_test_due_to_failure or should_test_due_to_random_subset or should_test_do_to_age:
-        print(f"Selected: {changed_notebook}, {should_test_due_to_failure}, {should_test_due_to_random_subset}")
-        return True
-    else:
-        print(f"Not Selected: {changed_notebook}, pass {pass_count}, fail {fail_count}")
-        return False
-
-
 def _process_notebook(
     notebook_path: str,
     variable_project_id: str,
@@ -238,7 +136,7 @@ def _get_notebook_python_version(notebook_path: str) -> str:
 
             # Look for the python version specification pattern
             re_match = re.search(
-                "python version = (\d+\.\d+)", markdown, flags=re.IGNORECASE
+                "python version = (\d\.\d)", markdown, flags=re.IGNORECASE
             )
             if re_match:
                 # get the version number
@@ -258,6 +156,7 @@ def _create_tag(filepath: str) -> str:
     return tag
 
 
+rate_limit = RateLimit(max_count=50, per=60, greedy=True)
 
 
 def process_and_execute_notebook(
@@ -271,8 +170,9 @@ def process_and_execute_notebook(
     private_pool_id: Optional[str],
     deadline: datetime.datetime,
     notebook: str,
-    should_get_tail_logs: bool = True,
+    should_get_tail_logs: bool = False,
 ) -> NotebookExecutionResult:
+    rate_limit.wait()  # wait before creating the task
 
     print(f"Running notebook: {notebook}")
 
@@ -291,9 +191,7 @@ def process_and_execute_notebook(
 
     result = NotebookExecutionResult(
         name=tag,
-        path=notebook,
         duration=datetime.timedelta(seconds=0),
-        start_time=datetime.datetime.now(),
         is_pass=False,
         output_uri=notebook_output_uri,
         log_url="",
@@ -303,6 +201,7 @@ def process_and_execute_notebook(
     )
 
     # TODO: Handle cases where multiple notebooks have the same name
+    time_start = datetime.datetime.now()
     operation = None
     try:
         # Get the python version for running the notebook if specified
@@ -346,14 +245,15 @@ def process_and_execute_notebook(
         result.logs_bucket = operation_metadata.build.logs_bucket
 
         # Block and wait for the result
-        operation_result = operation.result(timeout=timeout_in_seconds)
+        operation_result = operation.result(timeout=84600)
 
-        result.duration = datetime.datetime.now() - result.start_time
+        result.duration = datetime.datetime.now() - time_start
         result.is_pass = True
         print(f"{notebook} PASSED in {format_timedelta(result.duration)}.")
-
     except Exception as error:
         result.error_message = str(error)
+        import traceback
+        traceback.print_exc()
 
         if operation and should_get_tail_logs:
             # Extract the logs
@@ -370,7 +270,7 @@ def process_and_execute_notebook(
             except Exception as error:
                 result.error_message = str(error)
 
-        result.duration = datetime.datetime.now() - result.start_time
+        result.duration = datetime.datetime.now() - time_start
         result.is_pass = False
 
         print(
@@ -438,68 +338,12 @@ def get_changed_notebooks(
 
     return notebooks
 
-def _save_results(results: List[NotebookExecutionResult],
-                  artifacts_bucket: str,
-                  results_file: str):
-
-    artifacts_bucket = artifacts_bucket.replace("gs://", "").split('/')[0]
-
-    print("Updating build results ...")
-    build_results = {}
-    for result in results:
-        if result.is_pass:
-            pass_count = 1
-            fail_count = 0
-        else:
-            pass_count = 0
-            fail_count = 1
-        if result.error_message is None:
-            error_type = ''
-        elif '500 Internal' in result.error_message or 'INTERNAL' in result.error_message or 'internal error' in result.error_message:
-            error_type = 'INTERNAL'
-        elif 'context deadline exceeded' in result.error_message or 'TIMEOUT' in result.error_message:
-            error_type = 'TIMEOUT'
-        elif 'Quota' in result.error_message or 'quotas are exceeded' in result.error_message:
-            error_type = 'QUOTA'
-        elif 'ServiceUnavailable' in result.error_message:
-            error_type = 'SERVICEUNAVAILABLE'
-        elif 'ModuleNotFoundError' in result.error_message:
-            error_type = 'IMPORT'
-        elif result.is_pass:
-            error_type = ''
-        else:
-            error_type = 'undetermined'
-
-        if error_type != '':
-            log_url = result.log_url
-        else:
-            log_url = ''
-
-        build_results[result.path] = {
-                'duration': result.duration.total_seconds(),
-                'start_time': str(result.start_time),
-                'passed': pass_count,
-                'failed': fail_count,
-                'error_type': error_type,
-                'log_url': log_url
-        }
-        print(f"adding {result.path}")
-
-    print(f"Saving accumulative results to {results_file}, nentries {len(build_results)}")
-    content = json.dumps(build_results)
-
-    client = storage.Client()
-    bucket = client.get_bucket(artifacts_bucket)
-    bucket.blob(str(results_file)).upload_from_string(content, 'text/json')
-
-
 
 def process_and_execute_notebooks(
     notebooks: List[str],
     container_uri: str,
     staging_bucket: str,
     artifacts_bucket: str,
-    results_file: str,
     should_parallelize: bool,
     timeout: int,
     variable_project_id: str,
@@ -507,8 +351,6 @@ def process_and_execute_notebooks(
     variable_service_account: str,
     variable_vpc_network: Optional[str] = None,
     private_pool_id: Optional[str] = None,
-    concurrent_notebooks: Optional[int] = 10,
-    aiplatform_whl: Optional[str] = None,
 ):
     """
     Run the notebooks that exist under the folders defined in the test_paths_file.
@@ -529,8 +371,6 @@ def process_and_execute_notebooks(
             Required. The GCS staging bucket to write source code to.
         artifacts_bucket (str):
             Required. The GCS staging bucket to write executed notebooks to.
-        results_file (str):
-            Required: The path to the artifacts bucket to save results 
         variable_project_id (str):
             Required. The value for PROJECT_ID to inject into notebooks.
         variable_region (str):
@@ -539,8 +379,6 @@ def process_and_execute_notebooks(
             Required. Should run notebooks in parallel using a thread pool as opposed to in sequence.
         timeout (str):
             Required. Timeout string according to https://cloud.google.com/build/docs/build-config-file-schema#timeout.
-        concurrent_notebooks (int): Max number of notebooks per minute to run in parallel.
-        aiplatform_whl: alternate whl version of Vertex AI SDK to install
     """
 
     # Calculate deadline
@@ -557,9 +395,7 @@ def process_and_execute_notebooks(
             print(
                 "Running notebooks in parallel, so no logs will be displayed. Please wait..."
             )
-
-
-            with concurrent.futures.ThreadPoolExecutor(max_workers=concurrent_notebooks) as executor:
+            with concurrent.futures.ThreadPoolExecutor(max_workers=100) as executor:
                 print(f"Max workers: {executor._max_workers}")
 
                 notebook_execution_results = list(
@@ -637,7 +473,7 @@ def process_and_execute_notebooks(
             print("=" * 100)
 
             build_id = results_sorted[0].build_id
-            logs_bucket_name = (results_sorted[0].logs_bucket).replace("gs://", "")
+            logs_bucket_name = (results_sorted[0].logs_bucket).removeprefix("gs://")
             log_file_name = f"log-{build_id}.txt"
 
             log_contents = util.download_blob_into_memory(
@@ -655,10 +491,6 @@ def process_and_execute_notebooks(
             else:
                 print(log_contents)
 
-        _save_results(results_sorted, 
-                      artifacts_bucket, 
-                      results_file)
-
         print("\n=== END RESULTS===\n")
 
         total_notebook_duration = functools.reduce(

.cloud-build/execute_notebook_helper.py

@@ -66,6 +66,7 @@ def execute_notebook(
 
     # Execute notebook
     try:
+        print("DEBUG HERE\n")
         # Execute notebook
         pm.execute_notebook(
             input_path=notebook_source,

.cloud-build/execute_notebook_remote.py

@@ -45,6 +45,9 @@ def execute_notebook_remote(
     """Create and execute a single notebook on Google Cloud Build"""
     # Load build steps from YAML
 
+    print(f"DEBUG TIMEOUT {timeout_in_seconds}\n")
+    
+
     cloudbuild_config = yaml.load(open(CLOUD_BUILD_FILEPATH), Loader=FullLoader)
 
     substitutions = {
@@ -95,7 +98,14 @@ def execute_notebook_remote(
     if tag:
         build.tags = [tag]
 
-    operation = client.create_build(project_id=project_id, build=build)
+    try:
+        print("DEBUG: START\n")
+        operation = client.create_build(project_id=project_id, build=build)
+    except Exception as e:
+        import traceback
+        traceback.print_exc()
+    print("DEBUG: FINISH\n")
+    print(operation)
     # Print the in-progress operation
     # print("IN PROGRESS:")
     # print(operation.metadata)

.cloud-build/notebook-execution-test-cloudbuild.yaml

@@ -36,7 +36,7 @@ steps:
     - -c
     - |
       . workspace/env/bin/activate &&
-      python3 .cloud-build/execute_changed_notebooks_cli.py --test_paths_file "${_TEST_PATHS_FILE}" --base_branch "${_FORCED_BASE_BRANCH}" --container_uri ${_PYTHON_IMAGE} --staging_bucket ${_GCS_STAGING_BUCKET}  --artifacts_bucket ${_GCS_STAGING_BUCKET}/executed_notebooks/PR_${_PR_NUMBER}/BUILD_${BUILD_ID} --variable_project_id ${PROJECT_ID} --variable_region ${_GCP_REGION} --variable_service_account ${_GCP_SERVICE_ACCOUNT} --variable_vpc_network "${_GCP_VPC_NETWORK_NAME}" `if [ ! -z "${_PRIVATE_POOL_NAME}" ]; then echo "--private_pool_id ${_PRIVATE_POOL_NAME}"; fi` --build_id ${BUILD_ID} --test_percent=${_TEST_PERCENT} --concurrent_notebooks=${_CONCURRENT_NOTEBOOKS} --run_first_file=${_RUN_FIRST_FILE}
+      python3 .cloud-build/execute_changed_notebooks_cli.py --test_paths_file "${_TEST_PATHS_FILE}" --base_branch "${_FORCED_BASE_BRANCH}" --container_uri ${_PYTHON_IMAGE} --staging_bucket ${_GCS_STAGING_BUCKET}  --artifacts_bucket ${_GCS_STAGING_BUCKET}/executed_notebooks/PR_${_PR_NUMBER}/BUILD_${BUILD_ID} --variable_project_id ${PROJECT_ID} --variable_region ${_GCP_REGION} --variable_service_account ${_GCP_SERVICE_ACCOUNT} --variable_vpc_network "${_GPC_VPC_NETWORK_NAME}" --timeout 86400 `if [ ! -z "${_PRIVATE_POOL_NAME}" ]; then echo "--private_pool_id ${_PRIVATE_POOL_NAME}"; fi`
     env:
     - 'IS_TESTING=1'
 timeout: 86400s

.cloud-build/requirements.txt

@@ -3,15 +3,12 @@ numpy
 jupyter
 nbconvert
 papermill
+pandas
 matplotlib
 tabulate
 google-cloud-aiplatform
 google-cloud-storage
-google-cloud-build
-google-cloud-storage
+google-cloud-build==3.9.3
+protobuf==4.21.9
 ratemate
 GitPython
-tqdm
-fsspec
-pandas
-

.cloud-build/run_first.csv

@@ -1,8 +0,0 @@
-notebooks/official/pipelines/google_cloud_pipeline_components_automl_text.ipynb
-notebooks/official/generative_ai/rlhf_tune_llm.ipynb
-notebooks/official/generative_ai/tune_peft.ipynb
-notebooks/official/prediction/llm_streaming_prediction.ipynb
-notebooks/official/migration/sdk-automl-text-classification-batch-prediction.ipynb
-notebooks/official/vizier/get_started_vertex_vizier.ipynb
-notebooks/official/workbench/sentiment_analysis/Sentiment_Analysis.ipynb
-notebooks/official/model_monitoring/get_started_with_model_monitoring_automl.ipynb

.cloud-build/tests/flaky.py

@@ -1,46 +0,0 @@
-# grep PASSED tests.txt | cut -c 10-100 >passed.txt
-
-import os
-
-repo_dir = '/home/jupyter/vertex-ai-samples/'
-repo_dir_len = len(repo_dir)
-official_dir = repo_dir + 'notebooks/official'
-
-entries = os.scandir(official_dir)
-folders = []
-for entry in entries:
-    if entry.is_dir():
-        folders.append(entry.path)
-
-# Passing
-with open('passed.txt', 'r') as pass_file:
-    notebook_names = pass_file.readlines()
-        
-notebooks = []
-for folder in folders:
-    entries = os.scandir(folder)
-    for entry in entries:
-        for notebook in notebook_names:
-            if entry.name == notebook.rstrip():
-                notebooks.append(entry.path[repo_dir_len:])
-                
-with open('passing_tests.txt', 'w') as f:
-    for notebook in notebooks:
-        f.write(notebook + '\n')
-        
-        
-# Failing
-with open('failed.txt', 'r') as fail_file:
-    notebook_names = fail_file.readlines()
-        
-notebooks = []
-for folder in folders:
-    entries = os.scandir(folder)
-    for entry in entries:
-        for notebook in notebook_names:
-            if entry.name == notebook.rstrip():
-                notebooks.append(entry.path[repo_dir_len:])
-                
-with open('failing_tests.txt', 'w') as f:
-    for notebook in notebooks:
-        f.write(notebook + '\n')

.cloud-build/tests/select_notebooks_test.py

@@ -1,33 +0,0 @@
-import sys
-
-from execute_changed_notebooks_helper import (load_results, select_notebook)
-
-
-def test_load_results():
-    bucket: str = "cloud-build-notebooks-presubmit"
-    bucket_file: str = "build_results"
-
-    accum = load_results(bucket, bucket_file)
-
-    print(accum)
-
-    assert len(accum) > 0
-
-def test_select_notebook():
-    bucket: str = "cloud-build-notebooks-presubmit"
-    bucket_file: str = "build_results"
-
-    accum = load_results(bucket, bucket_file)
-
-    n_select = 0
-    n_notselect = 0
-    for notebook in accum:
-        if select_notebook(notebook, accum, 50):
-            n_select += 1
-        else:
-            n_notselect += 1
-
-    print(f"SELECTED {n_select}, NOT SELECTED {n_notselect}")
-
-    assert n_select > 0
-    assert n_notselect > 0

.cloud-build/utils/NotebookProcessors.py

@@ -35,7 +35,7 @@ class RemoveNoExecuteCells(Preprocessor):
 
 
 class UpdateVariablesPreprocessor(Preprocessor):
-    def __init__(self, replacement_map: Dict[str, str]):
+    def __init__(self, replacement_map: Dict):
         self._replacement_map = replacement_map
 
     @staticmethod
@@ -98,28 +98,3 @@ class UniqueStringsPreprocessor(Preprocessor):
             executable_cells.append(cell)
         notebook.cells = executable_cells
         return notebook, resources
-
-class VertexAIInstallProprocessor(Preprocessor):
-    def __init__(self, vertex_ai_wheel):
-        self.vertex_ai_wheel = vertex_ai_wheel
-
-    @staticmethod
-    def update_vertex_ai_install(content: str):
-        if "google-cloud-aiplatform" not in content:
-            return content
-        return (
-            f"gsutil cp {self.vertex_ai_wheel} google-cloud-aiplatform.whl\n" + 
-            content.replace("google-cloud-aiplatform\n", "google-cloud-aiplatform.whl\n")
-            .replace("google-cloud-aiplatform ", "google-cloud-aiplatform.whl ")
-        )
-
-    def preprocess(self, notebook, resources=None):
-        executable_cells = []
-        for cell in notebook.cells:
-            if cell.cell_type == "code":
-                cell.source = self.update_vertex_ai_install(
-                    content=cell.source,
-                )
-
-            executable_cells.append(cell)
-        notebook.cells = executable_cells

.cloud-build/utils/build_results_viewer.py

@@ -1,73 +0,0 @@
-'''
-Viewer for the weekly regression testing of the official notebooks
-
-Cloud Storage location: gs://cloud-build-notebooks-presubmit/build_results/
-'''
-import argparse
-import json
-from  util import download_file
-import csv
-import datetime
-from google.cloud import storage
-
-BUILD_BUCKET = "cloud-build-notebooks-presubmit"
-BUILD_FOLDER = "build_results"
-
-
-parser = argparse.ArgumentParser()
-parser.add_argument('--file', dest='file',
-                    default=None, type=str, help='build results filei (local or GCS)')
-args = parser.parse_args()
-
-investigate = {}
-with open('investigate.csv', 'r') as csvfile:
-    reader = csv.reader(csvfile)
-    for row in reader:
-        investigate[row[0][:-6]] = row[1]
-
-if not args.file:
-    client = storage.Client()
-    blobs = client.list_blobs(BUILD_BUCKET, prefix=BUILD_FOLDER)
-    newest_time = datetime.datetime(2000, 1, 1)
-    for blob in blobs:
-        # individual PR
-        if blob.size < 2000:
-            continue
-        time_created = blob.time_created.replace(tzinfo=None)
-        if time_created > newest_time:
-            newest_time = time_created
-            args.file = f"gs://{BUILD_BUCKET}/{blob.name}"
-
-if args.file.startswith("gs://"):
-    path = args.file[5:]
-    bucket = path.split('/')[0]
-    file   = path[len(bucket)+1:]
-    download_file(bucket, file, "build.json")
-    args.file = "build.json"
-
-with open(args.file, 'r') as f:
-    results = json.load(f)
-
-for item in results.items():
-    notebook = item[0][len("/notebooks/official/")-1:-6]
-    if item[1]['passed']:
-        passed = "PASS"
-    else:
-        if notebook in investigate:
-            passed = "INVG"
-        else:
-            passed = "FAIL"
-      
-    error = item[1]['error_type']
-  
-    if passed == "FAIL":
-        if error == '':
-            error = "undetermined"
-        if 'log_url' in item[1]:
-            log_url = item[1]['log_url']
-        else:
-            log_url = ''
-    else:
-        log_url = ''
-
-    print(f"{notebook:75} {passed} {error:10} {log_url}")

.cloud-build/utils/investigate.csv

@@ -1,19 +0,0 @@
-notebook,status
-prediction/llm_streaming_prediction.ipynb,wait_for_fix
-custom/get_started_with_vertex_endpoint_and_shared_vm.ipynb,issue 2527
-feature_store/online_feature_serving_and_fetching_bigquery_data_with_feature_store.ipynb,wait_for_reaper
-feature_store/online_feature_serving_and_vector_retrieval_bigquery_data_with_feature_store.ipynb,wait_for_reaper
-pipelines/google_cloud_pipeline_components_TPU_model_train_upload_deploy.ipynb,wait_for_fix
-explainable_ai/sdk_custom_image_classification_batch_explain.ipynb,issue 2528
-explainable_ai/sdk_custom_image_classification_online_explain.ipynb,issue 2528
-explainable_ai/sdk_custom_tabular_regression_online_explain.ipynb,issue 2528
-explainable_ai/sdk_automl_tabular_binary_classification_batch_explain.ipynb,issue 2528
-explainable_ai/xai_image_classification_feature_attributions.ipynb,issue 2528
-matching_engine,sdk_matching_engine_create_stack_overflow_embeddings.ipynb,issue 2530
-automl/automl_forecasting_bqml_arima_plus_comparison.ipynb,flaky
-model_evaluation/custom_tabular_regression_model_evaluation.ipynb,regr
-experiments/get_started_with_vertex_experiments.ipynb,regr
-experiments/comparing_local_trained_models.ipynb,regr
-generative_ai/tune_peft.ipynb,internal
-pipelines/custom_model_training_and_batch_prediction.ipynb,regr
-feature_store/online_feature_serving_and_fetching_bigquery_data_with_feature_store_optimized.ipynb,wait_for_reaper

.cloud-build/web-index/do_not_index.txt

@@ -1,11 +0,0 @@
-sdk2_remote_tabnet_training.ipynb
-remote_hyperparameter_tuning.ipynb
-remote_prediction.ipynb
-remote_training_bigframes_pytorch.ipynb
-remote_training_bigframes_sklearn.ipynb
-remote_training_bigframes_tensorflow.ipynb
-remote_training_lightning.ipynb
-remote_training_pytorch.ipynb
-remote_training_sklearn.ipynb
-remote_training_tensorflow_with_autologging.ipynb
-

.cloud-build/web-index/notebook-index-cloudbuild.yaml

@@ -1,23 +0,0 @@
-steps:
-  # Fetch full repo for diff purposes
-  - name: gcr.io/cloud-builders/git
-    args: [fetch, --unshallow, --quiet]
-  # Create a virtual environment
-  - name: ${_PYTHON_IMAGE}
-    entrypoint: /bin/sh
-    args:
-    - -c
-    - python3 -m venv workspace/env
-  # Install Python dependencies and run testing script
-  - name: ${_PYTHON_IMAGE}
-    entrypoint: /bin/sh
-    args:
-    - -c
-    - |
-      . workspace/env/bin/activate &&
-      python3 notebooks/notebook_template_review.py --web --title --steps --desc --linkback --notebook-dir=notebooks/official --skip-file=${_DO_NOT_INDEX_FILE} >web.html
-artifacts:
-  objects:
-          location: gs://${_GCS_ARTIFACTS_BUCKET}/webdoc
-          paths: ['web.html']
-timeout: 86400s

.github/dependabot.yml

@@ -1,10 +0,0 @@
-version: 2
-updates:
-
-  # Ignore model garden dockerfiles:
-  - package-ecosystem: "npm"
-    directory: "/community-content/vertex_model_garden"
-    schedule:
-      interval: "monthly"
-    ignore:
-      - dependency-name: "*"

.github/workflows/ci.yaml

@@ -7,11 +7,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Set up Python
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@v4
       with:
         python-version: '3.x'      
     - name: Fetch pull request branch
-      uses: actions/checkout@v4
+      uses: actions/checkout@v3
       with:
         fetch-depth: 0      
     - name: Fetch base main branch

.github/workflows/linter/Dockerfile

@@ -4,7 +4,7 @@
 # 2. To lint specific notebooks:
 # docker run -v ${PWD}:/setup/app gcr.io/python-docs-samples-tests/notebook_linter:latest notebooks/1.ipynb notebooks/2.ipynb
 
-FROM python:3.13
+FROM python:3.10
 
 WORKDIR setup
 

.github/workflows/linter/requirements.txt

@@ -2,9 +2,9 @@ git+https://github.com/tensorflow/docs
 ipython
 jupyter
 nbconvert
-black==25.1.0
-pyupgrade==3.19.1
-isort==6.0.1
-flake8==7.1.1
-nbqa==1.9.1
+black==22.10.0
+pyupgrade==2.38.4
+isort==5.10.1
+flake8==4.0.1
+nbqa==1.5.3
 

.github/workflows/linter/run_linter.sh

@@ -58,7 +58,7 @@ done
 # Only check notebooks in test folders modified in this pull request.
 # Note: Use process substitution to persist the data in the array
 if [ ${#notebooks[@]} -eq 0 ]; then
-    echo "Checking for changed notebooks using git"
+    echo "Checking for changed notebooked using git"
     while read -r file || [ -n "$line" ]; do
         notebooks+=("$file")
     done < <(git diff --name-only main... | grep '\.ipynb$')
@@ -84,7 +84,7 @@ if [ ${#notebooks[@]} -gt 0 ]; then
                 # python3 -m nbqa black "$notebook" --check
                 # BLACK_RTN=$?
                 echo "Running pyupgrade..."
-                python3 -m nbqa pyupgrade --exit-zero-even-if-changed "$notebook"
+                python3 -m nbqa pyupgrade "$notebook"
                 PYUPGRADE_RTN=$?
                 echo "Running isort..."
                 python3 -m nbqa isort "$notebook" --check
@@ -97,7 +97,7 @@ if [ ${#notebooks[@]} -gt 0 ]; then
                 python3 -m nbqa black "$notebook"
                 BLACK_RTN=$?
                 echo "Running pyupgrade..."
-                python3 -m nbqa pyupgrade --exit-zero-even-if-changed "$notebook"
+                python3 -m nbqa pyupgrade "$notebook"
                 PYUPGRADE_RTN=$?
                 echo "Running isort..."
                 python3 -m nbqa isort "$notebook"

CONTRIBUTING.md

@@ -44,10 +44,12 @@ Finally, run this code block to check for errors. Each step will attempt to
 automatically fix any issues. If the fixes can't be performed automatically,
 then you will need to manually address them before submitting your PR.
 
-Note: For official, only submit one notebook per PR.
-
 ```shell
-docker run -v ${PWD}:/setup/app gcr.io/cloud-devrel-public-resources/notebook_linter:latest your_notebook
+nbqa black "$notebook"
+nbqa pyupgrade "$notebook"
+nbqa isort "$notebook"
+nbqa flake8 "$notebook" --extend-ignore=W391,E501,F821,E402,F404,W503,E203,E722,W293,W291
+python3 -m tensorflow_docs.tools.nbfmt --remove_outputs "$notebook"
 ```
 
 ## Code Reviews

README.md

@@ -1,176 +1,37 @@
-# ![Google Cloud](https://avatars.githubusercontent.com/u/2810941?s=60&v=4) Google Cloud Vertex AI Samples
+# Google Cloud Vertex AI Samples
 
-This repository contains notebooks, code samples, sample apps, and other resources that demonstrate how to use, develop and manage machine learning and generative AI workflows using Google Cloud Vertex AI.
+[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](LICENSE)
+
+Welcome to the Google Cloud [Vertex AI](https://cloud.google.com/vertex-ai/docs/) sample repository.
 
 ## Overview
 
-[Vertex AI](https://cloud.google.com/vertex-ai) is a fully-managed, unified AI development platform for building and using generative AI. This repository is designed to help you get started with Vertex AI. Whether you're new to Vertex AI or an experienced ML practitioner, you'll find valuable resources here.
-
-For more Vertex AI Generative AI notebook samples, please visit the Vertex AI [Generative AI](https://github.com/GoogleCloudPlatform/generative-ai) GitHub repository.
-
-## Explore, learn and contribute
-
-You can explore, learn, and contribute to this repository to unleash the full potential of machine learning on Vertex AI! 
-
-### Explore and learn
-
-Explore this repository, follow the links in the header section of each of the notebooks to -
-
-![Colab](https://cloud.google.com/ml-engine/images/colab-logo-32px.png)  Open and run the notebook in [Colab](https://colab.google/)\
-![Colab Enterprise](https://cloud.google.com/ml-engine/images/colab-enterprise-logo-32px.png)  Open and run the notebook in [Colab Enterprise](https://cloud.google.com/colab/docs/introduction)\
-![Workbench](https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32)  Open and run the notebook in [Vertex AI Workbench](https://cloud.google.com/vertex-ai/docs/workbench/introduction)\
-![Github](https://cloud.google.com/ml-engine/images/github-logo-32px.png)  View the notebook on Github
-
-### Contribute
-
-See the [Contributing Guide](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/CONTRIBUTING.md).
-
-## Get started
-
-To get started using Vertex AI, you must have a Google Cloud project. 
-
-- If you don't have a Google Cloud project, you can learn and build on GCP for free using [Free Trail](https://cloud.google.com/free).
-- Once you have a Google Cloud project, you can learn more about [setting up a project and a development environment](https://cloud.google.com/vertex-ai/docs/start/cloud-environment).
-
+The repository contains [notebooks](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/master/notebooks) and [community content](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/master/community-content) that demonstrate how to develop and manage ML workflows using Google Cloud Vertex AI.
 
 ## Repository structure
 
 ```bash
+├── community-content - Sample code and tutorials contributed by the community
 ├── notebooks
+│   ├── community - Notebooks contributed by the community
 │   ├── official - Notebooks demonstrating use of each Vertex AI service
 │   │   ├── automl
 │   │   ├── custom
 │   │   ├── ...
-│   ├── community - Notebooks contributed by the community
-│   │   ├── model_garden
-│   │   ├── ...
-├── community-content - Sample code and tutorials contributed by the community
-
 ```
-## Examples
 
-<!-- markdownlint-disable MD033 -->
-<table>
+## Contributing
 
-  <tr>
-    <th style="text-align: center;">Category</th>
-    <th style="text-align: center;">Product</th>
-    <th style="text-align: center;">Description</th>
-  </tr>
-  <tr>
-    <td>Model</td>
-    <td>
-      <a href="notebooks/community/model_garden"><code>Model Garden/</code></a>
-    </td>
-    <td>
-      Curated collection of first-party, open-source, and third-party models available on Vertex AI including Gemini, Gemma, Llama 3, Claude 3 and many more.
-    </td>
-  </tr>
-  <tr>
-    <td>Data</td>
-    <td>
-      <a href="notebooks/official/feature_store"><code>Feature Store/</code></a>
-    </td>
-    <td>
-      Set up and manage online serving using Vertex AI Feature Store.
-    </td>
-  </tr>
-  <tr>
-    <td></td>
-    <td>
-      <a href="notebooks/official/datasets"><code>datasets/</code></a>
-    </td>
-    <td>
-      Use BigQuery and Data Labeling service with Vertex AI.
-    </td>
-  </tr>
-  <tr>
-    <td>Model development</td>
-    <td>
-      <a href="notebooks/official/automl"><code>automl/</code></a>
-    </td>
-    <td>
-      Train and make predictions on AutoML models
-    </td>
-  </tr>
-  <tr>
-    <td></td>
-    <td>
-      <a href="notebooks/official/custom"><code>custom/</code></a>
-    </td>
-    <td>
-      Create, deploy and serve custom  models on Vertex AI
-    </td>
-  </tr>
-  <tr>
-    <td></td>
-    <td>
-      <a href="notebooks/official/ray_on_vertex_ai"><code>ray_on_vertex_ai/</code></a>
-    </td>
-    <td>
-      Use Colab Enterprise and Vertex AI SDK for Python to connect to the Ray Cluster.
-    </td>
-  </tr>
-  <tr>
-    <td>Deploy and use</td>
-    <td>
-      <a href="notebooks/official/prediction"><code>prediction/</code></a>
-    </td>
-    <td>
-      Build, train and deploy models using prebuilt containers for custom training and prediction.
-    </td>
-  </tr>
-  <tr>
-    <td></td>
-    <td>
-      <a href="notebooks/official/model_registry"><code>model_registry/</code></a>
-    </td>
-    <td>
-      Use Model Registry to create and register a model.
-    </td>
-  </tr>
-  <tr>
-    <td></td>
-    <td>
-      <a href="notebooks/official/explainable_ai"><code>Explainable AI/</code></a>
-    </td>
-    <td>
-      Use Vertex Explainable AI's feature-based and example-based explanations to explain how or why a model produced a specific prediction.
-    </td>
-  </tr>
-  <tr>
-    <td></td>
-    <td>
-      <a href="notebooks/official/ml_metadata"><code>ml_metadata/</code></a>
-    </td>
-    <td>
-      Record the metadata and artifacts and query that metadata to help analyze, debug, and audit the performance of your ML system.
-    </td>
-  </tr>
-  <tr>
-    <td>Tools</td>
-    <td>
-      <a href="notebooks/official/pipelines"><code>Pipelines/</code></a>
-    </td>
-    <td>
-      Use `Vertex AI Pipelines` and `Google Cloud Pipeline Components` to build, tune, or deploy a custom model.
-    </td>
-  </tr>
-</table>
-<!-- markdownlint-enable MD033 -->
+Contributions welcome! See the [Contributing Guide](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/CONTRIBUTING.md).
 
+## Getting help
 
-## Get help
-
-Please use the [Issues page](https://github.com/GoogleCloudPlatform/vertex-ai-samples/issues) to provide feedback or submit a bug report.
+Please use the [issues page](https://github.com/GoogleCloudPlatform/vertex-ai-samples/issues) to provide feedback or submit a bug report.
 
 ## Disclaimer
 
 This is not an officially supported Google product. The code in this repository is for demonstrative purposes only.
 
+## Feedback
 
-## References
-- [Vertex AI Jupyter Notebook tutorials](https://cloud.google.com/vertex-ai/docs/tutorials/jupyter-notebooks)
-- Vertex AI [Generative AI](https://github.com/GoogleCloudPlatform/generative-ai) GitHub repository
-- [Vertex AI documentaton](https://cloud.google.com/vertex-ai/docs)
-  
+Please feel free to fill out our [survey](https://bit.ly/vertex-ai-samples-survey) to give us feedback on the repo and its content.

community-content/CODEOWNERS

@@ -8,25 +8,3 @@
 /cpr-examples @samthrasher
 /Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines @Ark-kun
 /pipeline_components @Ark-kun
-/pipeline_components/image_ml_model_training @lakeyk
-/prediction_featurestore_integration @googleapis/vertex-prediction-team
-/vertex_model_garden/model_oss/notebook_util @minwoo33park
-/vertex_model_garden/model_oss/util @weigary
-/vertex_model_garden/model_oss/diffusers @weigary
-/vertex_model_garden/model_oss/keras @dstnluong-google
-/vertex_model_garden/model_oss/transformers @dstnluong-google
-/vertex_model_garden/model_oss/pic2word @jismailyan-google
-/vertex_model_garden/model_oss/open_clip @lydhr
-/vertex_model_garden/model_oss/movinet @KCFindstr
-/vertex_model_garden/model_oss/data_converter @KCFindstr
-/vertex_model_garden/model_oss/peft @weigary
-/vertex_model_garden/model_oss/peft/templates @rayandasoriya
-/vertex_model_garden/model_oss/lm-evaluation-harness @kathyyu-google
-/vertex_model_garden/model_oss/tfvision @dstnluong-google
-/vertex_model_garden/model_oss/fvlm @minwoo33park
-/vertex_model_garden/model_oss/imagebind @kathyyu-google
-/vertex_model_garden/model_oss/llava @py4
-/vertex_model_garden/model_oss/vllm @kathyyu-google
-/vertex_model_garden/benchmarking_reports @lavraicse
-/vertex_model_garden/model_oss/autogluon @lavraicse
-

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_logistic_regression_model_using_Scikit_learn_and_import_to_Vertex_AI/pipeline.py

@@ -6,8 +6,8 @@ download_from_gcs_op = components.load_component_from_url("https://raw.githubuse
 select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml")
 fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
 binarize_column_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Binarize_column/in_CSV_format/component.yaml")
-train_logistic_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/1f5cf6e06409b704064b2086c0a705e4e6b4fcde/community-content/pipeline_components/ML_frameworks/Scikit_learn/Train_logistic_regression_model/from_CSV/component.yaml")
-upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/component.yaml")
+train_logistic_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/ML_frameworks/Scikit_learn/Train_logistic_regression_model/from_CSV/component.yaml")
+upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/1f5cf6e06409b704064b2086c0a705e4e6b4fcde/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/component.yaml")
 deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
 
 # %% Pipeline definition

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_model_using_XGBoost_and_import_to_Vertex_AI/pipeline.py

@@ -9,7 +9,7 @@ binarize_column_using_Pandas_on_CSV_data_op = components.load_component_from_url
 split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
 train_XGBoost_model_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Train/component.yaml")
 xgboost_predict_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Predict/component.yaml")
-upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/d5c9918850a6cc70004c4269dae066cfe2e664eb/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/component.yaml")
+upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/component.yaml")
 deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
 
 # %% Pipeline definition

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_model_using_all_frameworks_and_import_to_Vertex_AI/pipeline.py

@@ -23,7 +23,7 @@ upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op = components.load_comp
 # XGBoost
 train_XGBoost_model_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Train/component.yaml")
 xgboost_predict_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Predict/component.yaml")
-upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/d5c9918850a6cc70004c4269dae066cfe2e664eb/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/component.yaml")
+upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/component.yaml")
 
 # Scikit-learn
 #train_linear_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/1f5cf6e06409b704064b2086c0a705e4e6b4fcde/community-content/pipeline_components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml")

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_regression_model_using_XGBoost_and_import_to_Vertex_AI/pipeline.py

@@ -8,7 +8,7 @@ fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_
 split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
 train_XGBoost_model_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Train/component.yaml")
 xgboost_predict_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Predict/component.yaml")
-upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/d5c9918850a6cc70004c4269dae066cfe2e664eb/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/component.yaml")
+upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/component.yaml")
 deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
 
 # %% Pipeline definition

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_regression_model_using_all_frameworks_and_import_to_Vertex_AI/pipeline.py

@@ -22,7 +22,7 @@ upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op = components.load_comp
 # XGBoost
 train_XGBoost_model_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Train/component.yaml")
 xgboost_predict_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Predict/component.yaml")
-upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/d5c9918850a6cc70004c4269dae066cfe2e664eb/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/component.yaml")
+upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/component.yaml")
 
 # Scikit-learn
 train_linear_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/1f5cf6e06409b704064b2086c0a705e4e6b4fcde/community-content/pipeline_components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml")

community-content/cpr-examples/timm_serving/requirements.txt

@@ -1,5 +1,5 @@
 absl-py==1.1.0
-fastapi==0.109.1
+fastapi==0.75.2
 uvicorn==0.18.2
 timm==0.5.4
 smart_open==6.0.0

community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/component.yaml

@@ -64,8 +64,8 @@ implementation:
           labels["component-source"] = "github-com-ark-kun-pipeline-components"
 
           # The serving container decides the model type based on the model file extension.
-          # So we need to rename the mode file (e.g. /tmp/inputs/model/data) to *.bst
-          _, renamed_model_path = tempfile.mkstemp(suffix=".bst")
+          # So we need to rename the mode file (e.g. /tmp/inputs/model/data) to *.pkl
+          _, renamed_model_path = tempfile.mkstemp(suffix=".pkl")
           shutil.copyfile(src=model_path, dst=renamed_model_path)
 
           model = aiplatform.Model.upload_xgboost_model_file(

community-content/pipeline_components/image_ml_model_training/load_image_classification_model/component.yaml

@@ -1,112 +0,0 @@
-name: Load image classification model from tfhub
-description: |
-  Loads specified model from TFHub, creates layer to receive additional (3 channel) imagery data.
-  Args:
-    class_names (Sequence[str]):
-        Sequence of strings of categories for classification corresponding to input data.
-    loaded_model_path (str):
-        Output path for the loaded model.
-    image_size_path (str):
-        Output path for the model expected image size.
-    model_name (Optional[str]):
-        Name of the pre-trained image classification model to load from TFHub.
-        Eligible model_name:
-        - efficientnetv2-s
-        - efficientnetv2-m
-        - efficientnetv2-l
-        - efficientnetv2-s-21k
-        - efficientnetv2-m-21k
-        - efficientnetv2-l-21k
-        - efficientnetv2-xl-21k
-        - efficientnetv2-b0-21k
-        - efficientnetv2-b1-21k
-        - efficientnetv2-b2-21k
-        - efficientnetv2-b3-21k
-        - efficientnetv2-s-21k-ft1k
-        - efficientnetv2-m-21k-ft1k
-        - efficientnetv2-l-21k-ft1k
-        - efficientnetv2-xl-21k-ft1k
-        - efficientnetv2-b0-21k-ft1k
-        - efficientnetv2-b1-21k-ft1k
-        - efficientnetv2-b2-21k-ft1k
-        - efficientnetv2-b3-21k-ft1k
-        - efficientnetv2-b0
-        - efficientnetv2-b1
-        - efficientnetv2-b2
-        - efficientnetv2-b3
-        - efficientnet_b0
-        - efficientnet_b1
-        - efficientnet_b2
-        - efficientnet_b3
-        - efficientnet_b4
-        - efficientnet_b5
-        - efficientnet_b6
-        - efficientnet_b7
-        - bit_s-r50x1
-        - inception_v3
-        - inception_resnet_v2
-        - resnet_v1_50
-        - resnet_v1_101
-        - resnet_v1_152
-        - resnet_v2_50
-        - resnet_v2_101
-        - resnet_v2_152
-        - nasnet_large
-        - nasnet_mobile
-        - pnasnet_large
-        - mobilenet_v2_100_224
-        - mobilenet_v2_130_224
-        - mobilenet_v2_140_224
-        - mobilenet_v3_small_100_224
-        - mobilenet_v3_small_075_224
-        - mobilenet_v3_large_100_224
-        - mobilenet_v3_large_075_224
-    dropout_rate (Optional[float]):
-        Fraction of input units to drop in the last layer. Value should be between 0.0 and 1.0.
-    trainable (Optional[bool]):
-        If true fine tuning will be performed on entire Hub model. If false only additional
-        layers will be trained.
-    l2_regularization_penalty (Optional[float]):
-        l2 regularization penalty.
-inputs:
-- {name: class_names, type: 'typing.List[str]', description: List of class names corresponding
-    to the input image data}
-- {name: model_name, type: String, description: Name of the TFHub model to load, default: efficientnetv2-xl-21k,
-  optional: true}
-- {name: dropout_rate, type: Float, description: Dropout rate, default: '0.2', optional: true}
-- name: trainable
-  type: Boolean
-  description: True if fine tuning should be performed
-  default: "True"
-  optional: true
-- {name: l2_regularization_penalty, type: Float, description: Regularization penalty,
-  default: '0.0001', optional: true}
-outputs:
-- {name: loaded_model_path, type: TensorflowSavedModel, description: Output path for
-    the loaded model}
-- {name: image_size_path, type: HeightWidth}
-implementation:
-  container:
-    image: us-docker.pkg.dev/vertex-ai/ready-to-go-image-classification/image-components:v0.2
-    # command is a list of strings (command-line arguments).
-    # The YAML language has two syntaxes for lists and you can use either of them.
-    # Here we use the "flow syntax" - comma-separated strings inside square brackets.
-    command: [
-      python3,
-      # Path of the program inside the container
-      /pipelines/component/src/loading_component.py,
-      --loaded-model-path,
-      {outputPath: loaded_model_path},
-      --class-names,
-      {inputValue: class_names},
-      --model-name,
-      {inputValue: model_name},
-      --dropout-rate,
-      {inputValue: dropout_rate},
-      --trainable,
-      {inputValue: trainable},
-      --l2-regularization-penalty,
-      {inputValue: l2_regularization_penalty},
-      --image-size-path,
-      {outputPath: image_size_path},
-    ]

community-content/pipeline_components/image_ml_model_training/pipeline.py

@@ -1,62 +0,0 @@
-# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
-from kfp import components
-from kfp.v2 import dsl
-
-# %% Loading components
-upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url('https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/component.yaml')
-deploy_model_to_endpoint_op = components.load_component_from_url('https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml')
-transcode_imagedataset_tfrecord_from_csv_op = components.load_component_from_url('https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/community-content/pipeline_components/image_ml_model_training/transcode_tfrecord_image_dataset_from_csv/component.yaml')
-load_image_classification_model_from_tfhub_op = components.load_component_from_url('https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/b5b65198a6c2ffe8c0fa2aa70127e3325752df68/community-content/pipeline_components/image_ml_model_training/load_image_classification_model/component.yaml')
-preprocess_image_data_op = components.load_component_from_url('https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/community-content/pipeline_components/image_ml_model_training/preprocess_image_data/component.yaml')
-train_tensorflow_image_classification_model_op = components.load_component_from_url('https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/community-content/pipeline_components/image_ml_model_training/train_image_classification_model/component.yaml')
-
-
-# %% Pipeline definition
-def image_classification_pipeline():
-    class_names = ['daisy', 'dandelion', 'roses', 'sunflowers', 'tulips']
-    csv_image_data_path = 'gs://cloud-samples-data/ai-platform/flowers/flowers.csv'
-    deploy_model = False
-
-    image_data = dsl.importer(
-        artifact_uri=csv_image_data_path, artifact_class=dsl.Dataset).output
-
-    image_tfrecord_data = transcode_imagedataset_tfrecord_from_csv_op(
-        csv_image_data_path=image_data,
-        class_names=class_names
-    ).outputs['tfrecord_image_data_path']
-
-    loaded_model_outputs = load_image_classification_model_from_tfhub_op(
-        class_names=class_names,
-    ).outputs
-
-    preprocessed_data = preprocess_image_data_op(
-        image_tfrecord_data,
-        height_width_path=loaded_model_outputs['image_size_path'],
-    ).outputs
-
-    trained_model = (train_tensorflow_image_classification_model_op(
-        preprocessed_training_data_path = preprocessed_data['preprocessed_training_data_path'],
-        preprocessed_validation_data_path = preprocessed_data['preprocessed_validation_data_path'],
-        model_path=loaded_model_outputs['loaded_model_path']).
-                   set_cpu_limit('96').
-                   set_memory_limit('128G').
-                   add_node_selector_constraint('cloud.google.com/gke-accelerator', 'NVIDIA_TESLA_A100').
-                   set_gpu_limit('8').
-                   outputs['trained_model_path'])
-
-    vertex_model_name = upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op(
-        model=trained_model,
-    ).outputs['model_name']
-
-    # Deploying the model might incur additional costs over time
-    if deploy_model:
-        vertex_endpoint_name = deploy_model_to_endpoint_op(
-            model_name=vertex_model_name,
-        ).outputs['endpoint_name']
-
-pipeline_func = image_classification_pipeline
-
-# %% Pipeline submission
-if __name__ == '__main__':
-    from google.cloud import aiplatform
-    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

community-content/pipeline_components/image_ml_model_training/preprocess_image_data/component.yaml

@@ -1,57 +0,0 @@
-name: Preprocess image data
-description: |
-  Preprocess the image data and split between train and validation.
-  Args:
-    input_data_path (str):
-        Input path for the TFRecord image data. Data will be formatted as 'label' (encoded image
-        label), and 'image_raw' (the binary string of the image data).
-    height_width_path (str):
-        Path to square height and width to resize images to. File should contain single float value.
-        Value is dependent on training model.
-    preprocessed_training_data_path (str):
-        Output path for the TFRecord training data. Data will be formatted as 'label' (encoded image
-        label), and 'image_raw' (the binary string of the image data).
-    preprocessed_validation_data_path (str):
-        Output path for the TFRecord validation data. Data will be formatted as 'label' (encoded
-        image label), and 'image_raw' (the binary string of the image data).
-    validation_split (Optional[float]):
-        Fraction of data that will make up validation dataset. Value should be between 0.0 and 1.0.
-    seed (Optional[int]):
-        The global random seed to ensure the system gets a unique random sequence
-        that is deterministic (https://www.tensorflow.org/api_docs/python/tf/random/set_seed).
-inputs:
-- {name: input_data_path, type: ImageDatasetTFRecord, description: 'Input path for
-    the TFRecord image data,'}
-- {name: height_width_path, type: HeightWidth, description: 'Path to square height and width to
-    resize images to,'}
-- {name: validation_split, type: Float, description: 'Fraction of data that will make
-    up validation dataset,', default: '0.2', optional: true}
-- {name: seed, type: Integer, description: Random seed, default: '0', optional: true}
-outputs:
-- {name: preprocessed_training_data_path, type: ImageDatasetTFRecord, description: 'Output
-    path for the training data,'}
-- {name: preprocessed_validation_data_path, type: ImageDatasetTFRecord, description: 'Output
-    path for the validation data,'}
-implementation:
-  container:
-    image: us-docker.pkg.dev/vertex-ai/ready-to-go-image-classification/image-components:v0.2
-    # command is a list of strings (command-line arguments).
-    # The YAML language has two syntaxes for lists and you can use either of them.
-    # Here we use the "flow syntax" - comma-separated strings inside square brackets.
-    command: [
-      python3,
-      # Path of the program inside the container
-      /pipelines/component/src/preprocessing_component.py,
-      --input-data-path,
-      {inputPath: input_data_path},
-      --height-width-path,
-      {inputPath: height_width_path},
-      --validation-split,
-      {inputValue: validation_split},
-      --seed,
-      {inputValue: seed},
-      --preprocessed-training-data-path,
-      {outputPath: preprocessed_training_data_path},
-      --preprocessed-validation-data-path,
-      {outputPath: preprocessed_validation_data_path},
-    ]

community-content/pipeline_components/image_ml_model_training/train_image_classification_model/component.yaml

@@ -1,90 +0,0 @@
-name: Train tensorflow image classification model
-description: |
-  Creates a trained image classification TensorFlow model.
-  Args:
-    preprocessed_training_data_path (str):
-        Input path to the TFRecord training data. Data will be formatted as 'label' (encoded image
-        label), and 'image_raw' (the binary string of the image data).
-    preprocessed_validation_data_path (str):
-        Input path to the TFRecord validation data. Data will be formatted as 'label' (encoded
-        image label), and 'image_raw' (the binary string of the image data).
-    model_path (str):
-        Input path to the loaded pre-trained model.
-    trained_model_path (str):
-        Output path to save the trained model to.
-    optimizer_name (Optional[str]):
-        Name of the tf.keras optimizer. Available optimizers are listed at
-        https://keras.io/api/optimizers/
-    optimizer_parameters (Optional[Dict[str, str]]):
-        Optimizer parameters.
-    loss_function_name (Optional[str]):
-        Name of the loss function.
-    loss_function_parameters (Optional[Dict[str, str]]):
-        Loss function parameters.
-    number_of_epochs (Optional[int]):
-        Number of training iterations over data.
-    metric_names (Optional[Sequence[str]]):
-        List of tf.keras.metrics to be evaluated by the model during training and testing. Available
-        metrics are listed at https://keras.io/api/metrics/.
-    seed Optional(int):
-        The global random seed to ensure the system gets a unique random sequence
-        that is deterministic (https://www.tensorflow.org/api_docs/python/tf/random/set_seed).
-inputs:
-- {name: preprocessed_training_data_path, type: ImageDatasetTFRecord, description: 'Input
-    path for the training data,'}
-- {name: preprocessed_validation_data_path, type: ImageDatasetTFRecord, description: 'Input
-    path for the validation data,'}
-- {name: model_path, type: TensorflowSavedModel, description: 'Input path for the
-    model,'}
-- {name: optimizer_name, type: String, description: 'Name of the optimizer,', default: SGD,
-  optional: true}
-- {name: optimizer_parameters, type: 'typing.Dict[str, str]', description: 'Optimizer
-    parameters,', default: '{}', optional: true}
-- {name: loss_function_name, type: String, description: 'Name of the loss function,',
-  default: CategoricalCrossentropy, optional: true}
-- {name: loss_function_parameters, type: 'typing.Dict[str, str]', description: 'Loss
-    function parameters,', default: '{}', optional: true}
-- {name: number_of_epochs, type: Integer, description: 'Number of epochs,', default: '10',
-  optional: true}
-- {name: metric_names, type: 'typing.List[str]', description: 'List of metrics to
-    use,', default: '["accuracy"]', optional: true}
-- {name: seed, type: Integer, description: 'Random seed,', default: '0', optional: true}
-- {name: batch_size, type: Integer, description: Batch size, default: '16', optional: true}
-outputs:
-- {name: trained_model_path, type: TensorflowSavedModel, description: 'Output path
-    for the saved model,'}
-implementation:
-  container:
-    image: us-docker.pkg.dev/vertex-ai/ready-to-go-image-classification/image-components:v0.2
-    # command is a list of strings (command-line arguments).
-    # The YAML language has two syntaxes for lists and you can use either of them.
-    # Here we use the "flow syntax" - comma-separated strings inside square brackets.
-    command: [
-      python3,
-      # Path of the program inside the container
-      /pipelines/component/src/training_component.py,
-      --preprocessed-training-data-path,
-      {inputPath: preprocessed_training_data_path},
-      --preprocessed-validation-data-path,
-      {inputPath: preprocessed_validation_data_path},
-      --model-path,
-      {inputPath: model_path},
-      --trained-model-path,
-      {outputPath: trained_model_path},
-      --optimizer-name,
-      {inputValue: optimizer_name},
-      --loss-function-name,
-      {inputValue: loss_function_name},
-      --number-of-epochs,
-      {inputValue: number_of_epochs},
-      --seed,
-      {inputValue: seed},
-      --batch-size,
-      {inputValue: batch_size},
-      --metric-names,
-      {inputValue: metric_names},
-      --optimizer-parameters,
-      {inputValue: optimizer_parameters},
-      --loss-function-parameters,
-      {inputValue: loss_function_parameters},
-    ]

community-content/pipeline_components/image_ml_model_training/transcode_tfrecord_image_dataset_from_csv/component.yaml

@@ -1,37 +0,0 @@
-name: Transcode imagedataset tfrecord from csv
-description: |
-  Transcodes CSV Data into TFRecord file of TFExamples.
-  Args:
-    csv_image_data_path (str):
-        Path to the CSV image data. Data must include 'image_filepath' (Path to image file) and
-        'image_label' (output for a prediction) fields.
-    class_names (Sequence[str]):
-        Sequence of strings of categories for classification corresponding to input data.
-    tfrecord_image_data_path (str):
-        Output path for the TFRecord image data. Data will be formatted as 'label' (encoded image
-        label), and 'image_raw' (the binary string of the image data).
-inputs:
-- {name: csv_image_data_path, type: ImageDatasetCSV, description: Input path for the
-    CSV image data}
-- {name: class_names, type: 'typing.List[str]', description: List of class names corresponding
-    to the input image data}
-outputs:
-- {name: tfrecord_image_data_path, type: ImageDatasetTFRecord, description: Output
-    path for the TFRecord image data}
-implementation:
-  container:
-    image: us-docker.pkg.dev/vertex-ai/ready-to-go-image-classification/image-components:v0.2
-    # command is a list of strings (command-line arguments).
-    # The YAML language has two syntaxes for lists and you can use either of them.
-    # Here we use the "flow syntax" - comma-separated strings inside square brackets.
-    command: [
-      python3,
-      # Path of the program inside the container
-      /pipelines/component/src/transcoding_csv_component.py,
-      --csv-image-data-path,
-      {inputPath: csv_image_data_path},
-      --tfrecord-image-data-path,
-      {outputPath: tfrecord_image_data_path},
-      --class-names,
-      {inputValue: class_names},
-    ]

community-content/pipeline_components/image_ml_model_training/transcode_tfrecord_image_dataset_from_jsonl/component.yaml

@@ -1,39 +0,0 @@
-name: Transcode imagedataset tfrecord from jsonlines
-description: |
-  Transcodes JSONL Data into TFRecord file of TFExamples.
-  Args:
-    jsonl_image_data_path (str):
-        Input path for the JSONL image data
-        Path to the JSONL image data. Each line corresponds to a JSON input describing an image.
-        Schema follows AutoML image classification JSONL format
-        https://cloud.google.com/vertex-ai/docs/image-data/classification/prepare-data#json-lines.
-    class_names (Sequence[str]):
-        Sequence of strings of categories for classification corresponding to input data.
-    tfrecord_image_data_path (str):
-        Output path for the TFRecord image data. Data will be formatted as 'label' (encoded image
-        label), and 'image_raw' (the binary string of the image data).
-inputs:
-- {name: jsonl_image_data_path, type: ImageDatasetJsonLines, description: Input path
-    for the JSONL image data}
-- {name: class_names, type: 'typing.List[str]', description: List of class names corresponding
-    to the input image data}
-outputs:
-- {name: tfrecord_image_data_path, type: ImageDatasetTFRecord, description: Output
-    path for the TFRecord image data}
-implementation:
-  container:
-    image: us-docker.pkg.dev/vertex-ai/ready-to-go-image-classification/image-components:v0.2
-    # command is a list of strings (command-line arguments).
-    # The YAML language has two syntaxes for lists and you can use either of them.
-    # Here we use the "flow syntax" - comma-separated strings inside square brackets.
-    command: [
-      python3,
-      # Path of the program inside the container
-      /pipelines/component/src/transcoding_jsonl_component.py,
-      --jsonl-image-data-path,
-      {inputPath: jsonl_image_data_path},
-      --tfrecord-image-data-path,
-      {outputPath: tfrecord_image_data_path},
-      --class-names,
-      {inputValue: class_names},
-    ]

community-content/pytorch_efficient_training/README.md

@@ -15,19 +15,15 @@ pip install -r requirements.txt
 * resnet_dp.py - Train ResNet-50 on single node multiple GPUs with `DataParallel` strategy.
 * resnet_ddp.py - Train ResNet-50 on single node multiple GPUs with `DistributedDataParallel` strategy.
 * resnet_ddp_wds.py - Train ResNet-50 on single node multiple GPUs with `DistributedDataParallel` strategy and `Webdataset`.
-* resnet_fsdp.py - Train ResNet-50 on single node multiple GPUs with `FullyShardedDataParallel` strategy.
-* resnet_fsdp_wds.py - Train ResNet-50 on single node multiple GPUs with `FullyShardedDataParallel` strategy and `Webdataset`.
 * shard_imagenet.py - Shard ImagNet individual files into `tar` files.
 
 ## Benchmark
 
 When run the benchmark on Nvidia T4 GPUs using ImageNet validation dataset, you can get the result like:
-Strategy               | Seconds/Epoch - Local Data | Seconds/Epoch - Cloud Data
----------------------- | -------------------------- | --------------------------
-On 1 GPU               | 489                        | 804 (2x slower)
-On 4 GPUs (DP)         | 157                        | 738 (5x slower)
-On 4 GPUs (DDP)        | 134                        | 432 (3x slower)
-On 4 GPUs (DDP + WDS)  | 131                        | 133 (same performance)
-On 4 GPUs (FSDP)       | 139                        | 353 (3x slower)
-On 4 GPUs (FSDP + WDS) | 138                        | 135 (same performance)
+Strategy              | Seconds/Epoch - Local Data | Seconds/Epoch - Cloud Data
+--------------------- | -------------------------- | --------------------------
+On 1 GPU              | 489                        | 804 (2x slower)
+On 4 GPUs (DP)        | 157                        | 738 (5x slower)
+On 4 GPUs (DDP)       | 134                        | 432 (3x slower)
+On 4 GPUs (DDP + WDS) | 131                        | 133 (same performance)
 

community-content/pytorch_efficient_training/resnet_fsdp.py

@@ -1,242 +0,0 @@
-# Copyright 2022 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the \"License\");
-# you may not use this file except in compliance with the License.\n",
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an \"AS IS\" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Train resnet on multiple GPUs with FSDP."""
-
-import argparse
-import functools
-import os
-import time
-
-from PIL import Image
-import torch
-from torch import nn
-import torch.distributed as dist
-from torch.distributed.fsdp import FullyShardedDataParallel as FSDP
-from torch.distributed.fsdp.wrap import size_based_auto_wrap_policy
-import torch.multiprocessing as mp
-import torchmetrics
-import torchvision
-from torchvision.models import resnet50
-
-
-class ImageFolder(torchvision.datasets.ImageFolder):
-  """Class for loading imagenet."""
-
-  def __init__(self, image_list_file, transform=None, target_transform=None):
-    self.samples = self._make_dataset(image_list_file)
-    self.loader = self._loader
-
-    self.imgs = self.samples
-    self.targets = [s[1] for s in self.samples]
-
-    self.transform = transform
-    self.target_transform = target_transform
-
-  def _make_dataset(self, image_list_file):
-    items = []
-    with open(image_list_file, 'r') as f:
-      for line in f:
-        item = line.strip().split(' ')
-        items.append((item[0], int(item[1])))
-    return items
-
-  def _loader(self, image_path):
-    with open(image_path, 'rb') as f:
-      img = Image.open(f)
-      img = img.convert('RGB')
-      return img
-
-
-def train(model, device, dataloader, optimizer):
-  model.train()
-  for image, target in dataloader:
-    image = image.to(device, non_blocking=True)
-    target = target.to(device, non_blocking=True)
-    pred = model(image)
-    # pred.shape (N, C), target.shape (N)
-    loss = nn.functional.cross_entropy(pred, target)
-    optimizer.zero_grad()
-    loss.backward()
-    optimizer.step()
-  return loss
-
-
-def evaluate(model, device, dataloader, metric):
-  model.eval()
-  with torch.no_grad():
-    for image, target in dataloader:
-      image = image.to(device, non_blocking=True)
-      target = target.to(device, non_blocking=True)
-      pred = model(image)
-      metric.update(pred, target)
-  accuracy = metric.compute()
-  metric.reset()
-  return accuracy
-
-
-def worker(gpu, args):
-  """Run training and evaluation."""
-  # Init process group.
-  print(f'Initiating process {gpu}')
-  dist.init_process_group(
-      backend='nccl',
-      init_method='env://',
-      world_size=args.gpus,
-      rank=gpu)
-
-  # Create train dataloader.
-  train_dataset = ImageFolder(
-      image_list_file=args.train_data_path,
-      transform=torchvision.transforms.Compose([
-          torchvision.transforms.RandomResizedCrop(224),
-          torchvision.transforms.RandomHorizontalFlip(),
-          torchvision.transforms.ToTensor(),
-          torchvision.transforms.Normalize(
-              mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
-      ]))
-  train_sampler = torch.utils.data.distributed.DistributedSampler(
-      train_dataset, num_replicas=args.gpus, rank=gpu)
-  train_dataloader = torch.utils.data.DataLoader(
-      dataset=train_dataset,
-      batch_size=args.train_batch_size,
-      shuffle=False,
-      num_workers=args.dataloader_num_workers,
-      pin_memory=True,
-      sampler=train_sampler)
-  if gpu == 0:
-    print(f'Train dataloader | samples: {len(train_dataloader.dataset)}, '
-          f'num workers: {train_dataloader.num_workers}, '
-          f'global batch size: {args.train_batch_size * args.gpus}, '
-          f'batches/epoch: {len(train_dataloader)}')
-
-  # Create eval dataloader.
-  eval_dataset = ImageFolder(
-      image_list_file=args.eval_data_path,
-      transform=torchvision.transforms.Compose([
-          torchvision.transforms.Resize(256),
-          torchvision.transforms.CenterCrop(224),
-          torchvision.transforms.ToTensor(),
-          torchvision.transforms.Normalize(
-              mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
-      ]))
-  eval_sampler = torch.utils.data.distributed.DistributedSampler(
-      eval_dataset, num_replicas=args.gpus, rank=gpu)
-  eval_dataloader = torch.utils.data.DataLoader(
-      dataset=eval_dataset,
-      batch_size=args.eval_batch_size,
-      shuffle=False,
-      num_workers=args.dataloader_num_workers,
-      pin_memory=True,
-      drop_last=True,
-      sampler=eval_sampler)
-  if gpu == 0:
-    print(f'Eval dataloader | samples: {len(eval_dataloader.dataset)}, '
-          f'num workers: {eval_dataloader.num_workers}, '
-          f'batch size: {args.eval_batch_size}, '
-          f'batches/epoch: {len(eval_dataloader)}')
-
-  # Wrap policy.
-  my_auto_wrap_policy = functools.partial(
-      size_based_auto_wrap_policy, min_num_params=100)
-  torch.cuda.set_device(gpu)
-
-  # Create model.
-  model = resnet50(weights=None)
-  model.to(args.device)
-  model = FSDP(model, auto_wrap_policy=my_auto_wrap_policy)
-
-  # Optimizer.
-  optimizer = torch.optim.SGD(model.parameters(), 0.1)
-
-  # Main loop.
-  metric = torchmetrics.classification.Accuracy(top_k=1).to(args.device)
-  for epoch in range(1, args.epochs + 1):
-    if gpu == 0:
-      print(f'Running epoch {epoch}')
-    train_sampler.set_epoch(epoch)
-
-    start = time.time()
-    train(model, args.device, train_dataloader, optimizer)
-    end = time.time()
-    if gpu == 0:
-      print(f'Training finished in {(end - start):>0.3f} seconds')
-
-    start = time.time()
-    evaluate(model, args.device, eval_dataloader, metric)
-    end = time.time()
-    if gpu == 0:
-      print(f'Evaluation finished in {(end - start):>0.3f} seconds')
-
-  if gpu == 0:
-    print('Done')
-  dist.destroy_process_group()
-
-
-def create_args():
-  """Create main args."""
-  parser = argparse.ArgumentParser(
-      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-  parser.add_argument(
-      '--gpus',
-      default=4,
-      type=int,
-      help='number of gpus to use')
-  parser.add_argument(
-      '--epochs',
-      default=2,
-      type=int,
-      help='number of total epochs to run')
-  parser.add_argument(
-      '--dataloader_num_workers',
-      default=2,
-      type=int,
-      help='number of workders for dataloader')
-  parser.add_argument(
-      '--train_data_path',
-      default='',
-      type=str,
-      help='path to training data')
-  parser.add_argument(
-      '--train_batch_size',
-      default=32,
-      type=int,
-      help='batch size for training per gpu')
-  parser.add_argument(
-      '--eval_data_path',
-      default='',
-      type=str,
-      help='path to evaluation data')
-  parser.add_argument(
-      '--eval_batch_size',
-      default=32,
-      type=int,
-      help='batch size for evaluation per gpu')
-  args = parser.parse_args()
-  return args
-
-
-def main():
-  args = create_args()
-
-  os.environ['MASTER_ADDR'] = 'localhost'
-  os.environ['MASTER_PORT'] = '8888'
-
-  args.device = 'cuda' if torch.cuda.is_available() else 'cpu'
-  print(f'Launch job on {args.gpus} GPUs with FSDP')
-  mp.spawn(worker, nprocs=args.gpus, args=(args,))
-
-
-if __name__ == '__main__':
-  main()

community-content/pytorch_efficient_training/resnet_fsdp_wds.py

@@ -1,240 +0,0 @@
-"""Train resnet on multiple GPUs with DDP."""
-
-import argparse
-import functools
-import itertools
-import math
-import os
-import time
-
-import torch
-from torch import nn
-import torch.distributed as dist
-from torch.distributed.fsdp import FullyShardedDataParallel as FSDP
-from torch.distributed.fsdp.wrap import size_based_auto_wrap_policy
-import torch.multiprocessing as mp
-import torchmetrics
-from torchvision.models import resnet50
-from torchvision.transforms import transforms
-import webdataset as wds
-
-
-def wds_split(src, rank, world_size):
-  """Shards split function for webdataset."""
-  # The context of caller of this function is within multiple processes
-  # (by DDP world_size) and multiple workers (by dataloader_num_workers).
-  # So we totally have (world_size * num_workers) workers for processing data.
-  # NOTE: Raw data should be sharded to enough shards to make sure one process
-  # can handle at least one shard, otherwise the process may hang.
-  worker_id = 0
-  num_workers = 1
-  worker_info = torch.utils.data.get_worker_info()
-  if worker_info:
-    worker_id = worker_info.id
-    num_workers = worker_info.num_workers
-  for s in itertools.islice(src, rank * num_workers + worker_id, None,
-                            world_size * num_workers):
-    yield s
-
-
-def identity(x):
-  return x
-
-
-def create_wds_dataloader(rank, args, mode):
-  """Create webdataset dataset and dataloader."""
-  if mode == 'train':
-    transform = transforms.Compose([
-        transforms.RandomResizedCrop(224),
-        transforms.RandomHorizontalFlip(),
-        transforms.ToTensor(),
-        transforms.Normalize(
-            mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
-    ])
-    data_path = args.train_data_path
-    data_size = args.train_data_size
-    batch_size_local = args.train_batch_size
-    batch_size_global = args.train_batch_size * args.gpus
-    # Since webdataset disallows partial batch, we pad the last batch for train.
-    batches = int(math.ceil(data_size / batch_size_global))
-  else:
-    transform = transforms.Compose([
-        transforms.Resize(256),
-        transforms.CenterCrop(224),
-        transforms.ToTensor(),
-        transforms.Normalize(
-            mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
-    ])
-    data_path = args.eval_data_path
-    data_size = args.eval_data_size
-    batch_size_local = args.eval_batch_size
-    batch_size_global = args.eval_batch_size * args.gpus
-    # Since webdataset disallows partial batch, we drop the last batch for eval.
-    batches = int(data_size / batch_size_global)
-
-  dataset = wds.DataPipeline(
-      wds.SimpleShardList(data_path),
-      functools.partial(wds_split, rank=rank, world_size=args.gpus),
-      wds.tarfile_to_samples(),
-      wds.decode('pil'),
-      wds.to_tuple('jpg;png;jpeg cls'),
-      wds.map_tuple(transform, identity),
-      wds.batched(batch_size_local, partial=False),
-  )
-  num_workers = args.dataloader_num_workers
-  dataloader = wds.WebLoader(
-      dataset=dataset,
-      batch_size=None,
-      shuffle=False,
-      num_workers=num_workers,
-      persistent_workers=True if num_workers > 0 else False,
-      pin_memory=True).repeat(nbatches=batches)
-  print(f'{mode} dataloader | samples: {data_size}, '
-        f'num_workers: {num_workers}, '
-        f'local batch size: {batch_size_local}, '
-        f'global batch size: {batch_size_global}, '
-        f'batches: {batches}')
-  return dataloader
-
-
-def train(model, device, dataloader, optimizer):
-  model.train()
-  for image, target in dataloader:
-    image = image.to(device, non_blocking=True)
-    target = target.to(device, non_blocking=True)
-    pred = model(image)
-    # pred.shape (N, C), target.shape (N)
-    loss = nn.functional.cross_entropy(pred, target)
-    optimizer.zero_grad()
-    loss.backward()
-    optimizer.step()
-  return loss
-
-
-def evaluate(model, device, dataloader, metric):
-  model.eval()
-  with torch.no_grad():
-    for image, target in dataloader:
-      image = image.to(device, non_blocking=True)
-      target = target.to(device, non_blocking=True)
-      pred = model(image)
-      metric.update(pred, target)
-  accuracy = metric.compute()
-  metric.reset()
-  return accuracy
-
-
-def worker(gpu, args):
-  """Run training and evaluation."""
-  # Init process group.
-  print(f'Initiating process {gpu}')
-  dist.init_process_group(
-      backend='nccl',
-      init_method='env://',
-      world_size=args.gpus,
-      rank=gpu)
-
-  # Create dataloader.
-  train_dataloader = create_wds_dataloader(gpu, args, 'train')
-  eval_dataloader = create_wds_dataloader(gpu, args, 'eval')
-
-  # Wrap policy.
-  my_auto_wrap_policy = functools.partial(
-      size_based_auto_wrap_policy, min_num_params=100)
-  torch.cuda.set_device(gpu)
-
-  # Create model.
-  model = resnet50(weights=None)
-  model.to(args.device)
-  model = FSDP(model, auto_wrap_policy=my_auto_wrap_policy)
-
-  # Optimizer.
-  optimizer = torch.optim.SGD(model.parameters(), 0.1)
-
-  # Main loop.
-  metric = torchmetrics.classification.Accuracy(top_k=1).to(args.device)
-  for epoch in range(1, args.epochs + 1):
-    if gpu == 0:
-      print(f'Running epoch {epoch}')
-
-    start = time.time()
-    train(model, args.device, train_dataloader, optimizer)
-    end = time.time()
-    if gpu == 0:
-      print(f'Training finished in {(end - start):>0.3f} seconds')
-
-    start = time.time()
-    evaluate(model, args.device, eval_dataloader, metric)
-    end = time.time()
-    if gpu == 0:
-      print(f'Evaluation finished in {(end - start):>0.3f} seconds')
-
-  if gpu == 0:
-    print('Done')
-
-
-def create_args():
-  """Create main args."""
-  parser = argparse.ArgumentParser(
-      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-  parser.add_argument(
-      '--gpus',
-      default=4,
-      type=int,
-      help='number of gpus to use')
-  parser.add_argument(
-      '--epochs',
-      default=2,
-      type=int,
-      help='number of total epochs to run')
-  parser.add_argument(
-      '--dataloader_num_workers',
-      default=2,
-      type=int,
-      help='number of workders for dataloader')
-  parser.add_argument(
-      '--train_data_path',
-      default='',
-      type=str,
-      help='path to training data')
-  parser.add_argument(
-      '--train_batch_size',
-      default=32,
-      type=int,
-      help='batch size for training per gpu')
-  parser.add_argument(
-      '--train_data_size',
-      default=50000,
-      type=int,
-      help='data size for training')
-  parser.add_argument(
-      '--eval_data_path',
-      default='',
-      type=str,
-      help='path to evaluation data')
-  parser.add_argument(
-      '--eval_batch_size',
-      default=32,
-      type=int,
-      help='batch size for evaluation per gpu')
-  parser.add_argument(
-      '--eval_data_size',
-      default=50000,
-      type=int,
-      help='data size for evaluation')
-  args = parser.parse_args()
-  return args
-
-
-def main():
-  args = create_args()
-  os.environ['MASTER_ADDR'] = 'localhost'
-  os.environ['MASTER_PORT'] = '8888'
-
-  args.device = 'cuda' if torch.cuda.is_available() else 'cpu'
-  print(f'Launch job on {args.gpus} GPUs with FSDP')
-  mp.spawn(worker, nprocs=args.gpus, args=(args,))
-
-
-if __name__ == '__main__':
-  main()

community-content/pytorch_image_classification_distributed_data_parallel_training_with_vertex_sdk/trainer/Dockerfile

@@ -1,40 +1,16 @@
-# Stage 1: Build Environment
-FROM pytorch/pytorch:1.8.1-cuda11.1-cudnn8-runtime AS builder
-
-# Install necessary tools and dependencies
-RUN apt-get update && \
-    apt-get install -y curl gnupg && \
-    echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \
-    curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - && \
-    apt-get update -y && \
-    apt-get install -y google-cloud-sdk
-
-# Copy application code
-COPY . /trainer
-
-# Set working directory
-WORKDIR /trainer
-
-# Install Python dependencies
-RUN pip install --no-cache-dir -r requirements.txt
-
-# Stage 2: Runtime Environment
 FROM pytorch/pytorch:1.8.1-cuda11.1-cudnn8-runtime
 
-# Install Google Cloud SDK
 RUN apt-get update && \
     apt-get install -y curl gnupg && \
     echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \
-    curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - && \
+    curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg  add - && \
     apt-get update -y && \
-    apt-get install -y google-cloud-sdk && \
-    apt-get clean && rm -rf /var/lib/apt/lists/*
+    apt-get install google-cloud-sdk -y
 
-# Copy from the builder stage
-COPY --from=builder /trainer /trainer
+COPY . /trainer
 
-# Set working directory
 WORKDIR /trainer
 
-# Set the entry point
-ENTRYPOINT ["python", "-m", "task"]
+RUN pip install -r requirements.txt
+
+ENTRYPOINT ["python", "-m", "task"]

community-content/pytorch_image_classification_distributed_data_parallel_training_with_vertex_sdk/trainer/requirements.txt

@@ -1,3 +1,3 @@
-torch==2.2.0
+torch==1.8.1
 torchvision==0.9.1
 tensorboard==2.5.0

community-content/pytorch_image_classification_single_gpu_with_vertex_sdk_and_torchserve/trainer/requirements.txt

@@ -1,3 +1,3 @@
-torch==2.2.0
+torch==1.8.1
 torchvision==0.9.1
 tensorboard==2.5.0

community-content/pytorch_pre_built_images_deployment/pytorch-text-classification-vertex-ai-deploy.ipynb

@@ -31,7 +31,17 @@
       "source": [
         "# Deploying a PyTorch Text Classification Model on [Vertex AI](https://cloud.google.com/vertex-ai)\n",
         "\n",
-        "**Kindly reach out to Vertex AI before you run any scale tests or you have any questions.**\n"
+        "**This is an Experimental release**, covered by the Pre-GA Offerings Terms of your Google Cloud Platform [Terms of Service](https://cloud.google.com/terms).\n",
+        "\n",
+        "Experiments are focused on validating a prototype and are not guaranteed to be released. They are not intended for production use or covered by any SLA, support obligation, or deprecation policy and might be subject to backward-incompatible changes.\n",
+        "\n",
+        "**Kindly drop us a note before you run any scale tests.**\n",
+        "\n",
+        "**Do not hesitate to contact vertexai-prediction-preview-feedback@google.com if you have any questions or run into any issues.**\n",
+        "\n",
+        "The usage of the product is free during the Experimental release period: you will still incur charges for other GCP products usage, such as storage.\n",
+        "\n",
+        "The projects need to be added to the allowlist in order to deploy PyTorch models using Vertex AI Prediction pre-built PyTorch images. If you are interested in the feature, please send an email to vertexai-prediction-preview-feedback@google.com to provide your project numbers OR project ids."
       ]
     },
     {

community-content/tf_agents_bandits_movie_recommendation_with_kfp_and_vertex_sdk/mlops_pipeline_tf_agents_bandits_movie_recommendation/src/logger/requirements.txt

@@ -1,4 +1,4 @@
 google-cloud-bigquery==2.20.0
-tensorflow==2.12.1
-pillow==10.3.0
+tensorflow==2.7.2
+pillow==9.0.1
 tf-agents==0.8.0

community-content/tf_agents_bandits_movie_recommendation_with_kfp_and_vertex_sdk/mlops_pipeline_tf_agents_bandits_movie_recommendation/src/prediction_container/requirements.txt

@@ -1,4 +1,4 @@
 google-cloud-pubsub==2.5.0
-pillow==10.3.0
+pillow==9.0.1
 tf-agents==0.8.0
-tensorflow==2.12.1
+tensorflow==2.7.2

community-content/tf_agents_bandits_movie_recommendation_with_kfp_and_vertex_sdk/mlops_pipeline_tf_agents_bandits_movie_recommendation/src/simulator/requirements.txt

@@ -1,5 +1,5 @@
 dataclasses==0.6
 google-cloud-aiplatform==1.8.1
-tensorflow==2.12.1
-pillow==10.3.0
+tensorflow==2.7.2
+pillow==9.0.1
 tf-agents==0.8.0

community-content/tf_keras_image_classification_distributed_multi_worker_with_vertex_sdk/trainer/requirements.txt

@@ -1 +1 @@
-tensorflow==2.12.1
+tensorflow==2.7.2

community-content/tf_keras_text_classification_distributed_single_worker_gpus_with_gcloud_local_run_and_vertex_sdk/trainer/requirements.txt

@@ -1 +1 @@
-tensorflow==2.12.1
+tensorflow==2.7.2

community-content/vertex_cpr_samples/README.md

@@ -1,15 +0,0 @@
-# Vertex AI custom prediction routines samples
-
-## Overview
-Vertex Custom Prediction Routines(CPR) simplify the process of building custom containers 
-and make local model testing easy. Here are the sameple codes for different libraries.
-
-
-### Objectives
-The objective is to provide various samples for Vertex Custom Prediction Routine(CPR).
-
-
-### Supporting libraries
-* torch
-* sklearn
-* xgboost

community-content/vertex_cpr_samples/sklearn/predictor_Classifier.py

@@ -1,33 +0,0 @@
-import numpy as np
-import os
-import pickle
-
-from google.cloud.aiplatform.constants import prediction
-from google.cloud.aiplatform.utils import prediction_utils
-from google.cloud.aiplatform.prediction.predictor import Predictor
-from sklearn.datasets import load_breast_cancer
-from sklearn.linear_model import RidgeClassifier
-
-class LinearRegressionPredictor(Predictor):
-
-    def __init__(self):
-        return
-
-    def load(self, artifacts_uri: str) -> None:
-        prediction_utils.download_model_artifacts(artifacts_uri)
-        if os.path.exists(prediction.MODEL_FILENAME_PKL):
-            self._model = pickle.load(open(prediction.MODEL_FILENAME_PKL, "rb"))
-        else:
-            self._model = RidgeClassifier()
-        X, y = load_breast_cancer(return_X_y=True)
-        self._model.fit(X, y)
-
-    def preprocess(self, prediction_input: dict) -> np.ndarray:
-        instances = prediction_input["instances"]
-        return np.asarray(instances)
-
-    def predict(self, instances: np.ndarray) -> np.ndarray:
-        return self._model.predict(instances)
-
-    def postprocess(self, prediction_results: np.ndarray) -> dict:
-        return {"predictions": prediction_results.tolist()}

community-content/vertex_cpr_samples/sklearn/predictor_LinearRegression.py

@@ -1,33 +0,0 @@
-import numpy as np
-import os
-import pickle
-
-from google.cloud.aiplatform.constants import prediction
-from google.cloud.aiplatform.utils import prediction_utils
-from google.cloud.aiplatform.prediction.predictor import Predictor
-from sklearn.datasets import make_blobs
-from sklearn.linear_model import LinearRegression 
-
-class LinearRegressionPredictor(Predictor):
-
-    def __init__(self):
-        return
-
-    def load(self, artifacts_uri: str) -> None:
-        prediction_utils.download_model_artifacts(artifacts_uri)
-        if os.path.exists(prediction.MODEL_FILENAME_PKL):
-            self._model = pickle.load(open(prediction.MODEL_FILENAME_PKL, "rb"))
-        else:
-            self._model = LogisticRegression()
-        X, y = make_blobs(n_samples=100, centers=2, n_features=2, random_state=1)
-        self._model.fit(X, y)
-
-    def preprocess(self, prediction_input: dict) -> np.ndarray:
-        instances = prediction_input["instances"]
-        return np.asarray(instances)
-
-    def predict(self, instances: np.ndarray) -> np.ndarray:
-        return self._model.predict_proba(instances)
-
-    def postprocess(self, prediction_results: np.ndarray) -> dict:
-        return {"predictions": prediction_results.tolist()}

community-content/vertex_cpr_samples/sklearn/predictor_SGDClassifier.py

@@ -1,33 +0,0 @@
-import numpy as np
-import os
-import pickle
-
-from google.cloud.aiplatform.constants import prediction
-from google.cloud.aiplatform.utils import prediction_utils
-from google.cloud.aiplatform.prediction.predictor import Predictor
-from sklearn.linear_model import SGDClassifier
-
-class SGDClassifierPredictor(Predictor):
-
-    def __init__(self):
-        return
-
-    def load(self, artifacts_uri: str) -> None:
-        prediction_utils.download_model_artifacts(artifacts_uri)
-        if os.path.exists(prediction.MODEL_FILENAME_PKL):
-            self._model = pickle.load(open(prediction.MODEL_FILENAME_PKL, "rb"))
-        else:
-            self._model = SGDClassifier(max_iter=5)
-        X = [[0., 0.], [1., 1.]]
-        y = [0, 1]
-        self._model.fit(X, y)
-
-    def preprocess(self, prediction_input: dict) -> np.ndarray:
-        instances = prediction_input["instances"]
-        return np.asarray(instances)
-
-    def predict(self, instances: np.ndarray) -> np.ndarray:
-        return self._model.predict(instances)
-
-    def postprocess(self, prediction_results: np.ndarray) -> dict:
-        return {"predictions": prediction_results.tolist()}

community-content/vertex_cpr_samples/torch/predictor_resnet.py

@@ -1,34 +0,0 @@
-import os
-import torch
-
-from google.cloud.aiplatform.utils import prediction_utils
-from google.cloud.aiplatform.prediction.predictor import Predictor
-from torchvision.models import detection, resnet50, ResNet50_Weights
-from typing import Dict, List
-
-class ResNetPredictor(Predictor):
-
-    def __init__(self):
-        return
-
-    def load(self, artifacts_uri: str) -> None:
-        prediction_utils.download_model_artifacts(artifacts_uri)
-        if os.path.exists("model.pth.tar"):
-            self.model = detection.fasterrcnn_resnet50_fpn(pretrained=True)
-            stat_dic = torch.load("model.pth.tar")
-            self.model.load_state_dict(stat_dic['state_dict'])
-        else:
-            weights = ResNet50_Weights.DEFAULT
-            self.model = resnet50(weights=weights)
-        self.model.eval()
-
-    def preprocess(self, prediction_input: dict) -> torch.Tensor:
-        instances = prediction_input["instances"]
-        return torch.Tensor(instances)
-
-    @torch.inference_mode()
-    def predict(self, instances: torch.Tensor) -> List[str]:
-        return self._model(instances)
-
-    def postprocess(self, prediction_results: List[str]) -> Dict:
-        return {"predictions": prediction_results}

community-content/vertex_cpr_samples/torch/predictor_transformers.py

@@ -1,73 +0,0 @@
-import ast
-import json
-import os
-import pickle
-import torch
-
-from google.cloud.aiplatform.utils import prediction_utils
-from google.cloud.aiplatform.prediction.predictor import Predictor
-from transformers import AutoModelForQuestionAnswering
-from typing import Dict, List
-
-class TorchTransformersPredictor(Predictor):
-
-    def __init__(self):
-        return
-
-    def load(self, artifacts_uri: str) -> None:
-        prediction_utils.download_model_artifacts(artifacts_uri)
-        
-        if os.path.isfile("setup_config.json"):
-            with open("setup_config.json") as setup_config_file:
-                self.setup_config = json.load(setup_config_file)
-
-        if os.path.exists("model.pt"):
-            self.model = AutoModelForQuestionAnswering.from_pretrained("model.pt")
-            self.model.eval()
-        else:
-            raise ValueError("One of the following model files must be provided: model.pt.")
-
-    def preprocess(self, prediction_input: dict) -> torch.Tensor:
-        max_length = self.setup_config["max_length"]
-        instances = prediction_input["instances"]
-        question_context = ast.literal_eval(instances)
-        question = question_context["question"]
-        context = question_context["context"]
-        inputs = self.tokenizer.encode_plus(
-            question,
-            context,
-            max_length=int(max_length),
-            pad_to_max_length=True,
-            add_special_tokens=True,
-            return_tensors="pt",
-        )
-        input_ids = inputs["input_ids"]
-        attention_mask = inputs["attention_mask"]
-        return torch.Tensor(input_ids, attention_mask)
-    
-    @torch.inference_mode()
-    def predict(self, instances: torch.Tensor) -> List[str]:
-        input_ids, attention_mask = instances
-        outputs = self._model(input_ids, attention_mask)
-        answer_start_scores = outputs.start_logits
-        answer_end_scores = outputs.end_logits
-        
-        num_rows, num_cols = answer_start_scores.shape
-        inferences = []
-        for i in range(num_rows):
-            answer_start_scores_one_seq = answer_start_scores[i].unsqueeze(0)
-            answer_start = torch.argmax(answer_start_scores_one_seq)
-            answer_end_scores_one_seq = answer_end_scores[i].unsqueeze(0)
-            answer_end = torch.argmax(answer_end_scores_one_seq) + 1
-            prediction = self.tokenizer.convert_tokens_to_string(
-                self.tokenizer.convert_ids_to_tokens(
-                    input_ids[i].tolist()[answer_start:answer_end]
-                )
-            )
-            inferences.append(prediction)
-        return inferences
-
-    def postprocess(self, prediction_results: List[str]) -> Dict:
-        return {"predictions": prediction_results}
-     
-

community-content/vertex_cpr_samples/xgboost/predictor_Classifier.py

@@ -1,37 +0,0 @@
-import os
-import numpy as np
-import pickle
-import xgboost as xgb
-
-from google.cloud.aiplatform.constants import prediction
-from google.cloud.aiplatform.utils import prediction_utils
-from google.cloud.aiplatform.prediction.predictor import Predictor
-from sklearn.datasets import make_blobs
-from xgboost import XGBClassifier
-
-
-class ClassifierPredictor(Predictor):
-
-    def __init__(self):
-        return
-
-    def load(self, artifacts_uri: str) -> None:
-        prediction_utils.download_model_artifacts(artifacts_uri)
-        if os.path.exists(prediction.MODEL_FILENAME_PKL):
-            booster = pickle.load(open(prediction.MODEL_FILENAME_PKL, "rb"))
-        else:
-            X, y = make_blobs(n_samples=100, centers=2, n_features=2, random_state=1)
-            model = XGBClassifier()
-            model.fit(X, y)
-            booster = model.get_booster()
-        self._booster = booster
-
-    def preprocess(self, prediction_input: dict) -> xgb.DMatrix:
-        instances = prediction_input["instances"]
-        return xgb.DMatrix(instances)
-
-    def predict(self, instances: xgb.DMatrix) -> np.ndarray:
-        return self._booster.predict(instances)
-
-    def postprocess(self, prediction_results: np.ndarray) -> dict:
-        return {"predictions": prediction_results.tolist()}

community-content/vertex_cpr_samples/xgboost/predictor_xgbranker.py

@@ -1,41 +0,0 @@
-import os
-import numpy as np
-import pandas as pd
-import pickle
-import xgboost as xgb
-
-from google.cloud.aiplatform.constants import prediction
-from google.cloud.aiplatform.utils import prediction_utils
-from google.cloud.aiplatform.prediction.predictor import Predictor
-
-class XGBRankerPredictor(Predictor):
-
-    def __init__(self):
-        return
-
-    def load(self, artifacts_uri: str) -> None:
-        prediction_utils.download_model_artifacts(artifacts_uri)
-        if os.path.exists(prediction.MODEL_FILENAME_PKL):
-            booster = pickle.load(open(prediction.MODEL_FILENAME_PKL, "rb"))
-            self._booster = booster
-        else:
-            N = 500
-            dates = pd.date_range(start='2023-01-01', end='2023-01-12', periods=N)
-            X = pd.DataFrame(np.random.randn(N, 5), columns=list('ABCDE'), index=dates)
-            y = pd.Series(np.random.randint(0, 10, size=N), index=dates, name='label')
-            group = X.groupby(dates + pd.offsets.MonthEnd(0)).size()
-            sample_weight = pd.Series(np.arange(len(group)), index=group.index)
-            model = xgb.XGBRanker(objective='rank:pairwise', max_depth=3, learning_rate=0.1, booster='gbtree', tree_method='hist', n_jobs=4, n_estimators=50, enable_categorical=False, random_state=42)
-            model.fit(X=X, y=y, group=group, sample_weight=sample_weight, verbose=True)
-            booster = model.get_booster()
-            self._booster = booster
-
-    def preprocess(self, prediction_input: dict) -> xgb.DMatrix:
-        instances = prediction_input["instances"]
-        return xgb.DMatrix(instances)
-
-    def predict(self, instances: xgb.DMatrix) -> np.ndarray:
-        return self._booster.predict(instances, output_margin=False, ntree_limit=0)
-
-    def postprocess(self, prediction_results: np.ndarray) -> dict:
-        return {"predictions": prediction_results.tolist()}

community-content/vertex_model_garden/.pylintrc

@@ -1,4 +0,0 @@
-[MASTER]
-
-generated-members=get_concrete_function,cv2.*
-ignored-modules=tensorflow,google.cloud

community-content/vertex_model_garden/benchmarking_reports/jax_vit_benchmarking_report.md

@@ -1,253 +0,0 @@
-# ViT PyTorch vs JAX training benchmarks on Vertex AI Training Platform
-
-Lav Rai, Software Engineer, Google Cloud
-
-Xiang Xu, Software Engineer, Google Cloud
-
-Andreas Steiner, Software Engineer, Google DeepMind
-
-Tao Wang, Software Engineer, Google DeepMind
-
-Alexander Kolesnikov, Research Engineer, Google DeepMind
-
-## Introduction
-
-Many repositories now offer both PyTorch and JAX versions of a model. For
-example, [Hugging Face offers many models such as GPT2, BERT][1]
-etc. Other examples are [OpenLLaMa][2] and [ViT][3]
-models which were first developed in JAX and then their corresponding PyTorch
-versions were made available. **Given both the PyTorch and JAX options for a
-model, it may not be obvious as to which option to choose**. To make such a
-decision, it is important for one to know about the training cost, effectiveness
-and efficiency for each choice.
-
-Apart from the framework choice, the other choice that one faces on Vertex AI
-training platform is the type and count of the accelerators. Although the
-[Vertex AI pricing table][4] lists the price per hour for each
-machine, **one may not know beforehand about the training speed of JAX and
-PyTorch frameworks for different types and count of the accelerators**.
-
-If one has access to some training benchmark numbers for the same model
-under (a) PyTorch and JAX frameworks and (b) for different types and count of
-the accelerators, then it will be easier for them to make a cost effective
-decision. Such a benchmark will also aid the developers in identifying strength
-and weakness of different choices and then figure out recipes to remove those
-weaknesses if possible.
-
-This blog uses the ViT [classification models][5] of varying sizes
-to benchmark the training performance of PyTorch and JAX versions on the Vertex
-AI Platform under different machine configurations. The goal is to:
-
-- Benchmark OSS ViT training for both PyTorch and JAX frameworks.
-- Benchmark OSS ViT L16, H14, g14, and G14 models.
-- Benchmark OSS ViT PyTorch training with A100 GPUs.
-- Benchmark OSS ViT JAX training with A100 GPUs and TPU V3 accelerators.
-
-## Benchmarking setup
-
-This section lays out the benchmarking set up for the [PyTorch][6] and [JAX][7]
-frameworks and provides a reasoning for choosing those settings.
-
-### PyTorch GPU
-
-#### Machine configuration
-
-We run training jobs on [Vertex AI Custom Training][8] using 1
-single node with 8 A100-40GB GPUs.
-
-- Machine type: [a2-highgpu-8g][9]
-- Machine count: 1
-- Accelerator type: [NVIDIA_TESLA_A100 (40GB)][10]
-- Accelerator count: 8
-
-#### Modeling
-
-We benchmark 4 variants of ViT model in different sizes:
-
-- [ViT-L16, 300M params][11]
-- [ViT-H14, 630M params][12]
-- [ViT-g14, 1B params][13]
-- [ViT-G14, 1.8B params][14]
-
-We use the Huggingface [transformers library][15] for ViT L16 and
-H14 variants, and the [TIMM library][16] for ViT g14 and G14
-variants.
-
-#### Dataset
-
-We run training against the [cifar10][17] dataset with 50K training
-images and 10K test images. To factor out network communication overhead for
-data loading, we copy the whole dataset to the local disk then load data from
-the local disk during training.
-
-#### Training parameters
-
-- Trainer
-  - We use [PyTorch Lightning][18] as the trainer for the
-    boilerplate data loading and train loop coding.
-- Precision
-  - Float16
-- Input resolution
-  - 224 x 224
-- Strategy
-  - We use [DDP][19] for models which can be entirely loaded to one
-    GPU, use [Deepspeed-ZeRO][20] otherwise:
-    - ViT-L16: DDP
-    - ViT-H14: DDP
-    - ViT-g14: DDP
-    - ViT-G14: Deepspeed-ZeRO stage-3
-- Batch size
-  - We use the max batch size as power of 2 without CUDA OOM for each model:
-    - ViT-L16: 64 per GPU
-    - ViT-H14: 16 per GPU
-    - ViT-g14: 16 per GPU
-    - ViT-G14: 32 per GPU
-- Compilation
-  - We apply [torch.compile][21] to model whenever it's applicable:
-    - ViT-L16: torch.compile
-    - ViT-H14: torch.compile
-    - ViT-g14: torch.compile
-    - ViT-G14: N/A
-
-### JAX TPU and GPU
-
-#### Machine configuration
-
-All the TPU and GPU training jobs are run on [Vertex AI Custom
-Training][8]. The following machine configurations were used for the
-TPU and GPU experiments:
-
-**Note**: TPU V3 POD requires multi-host supporting training code. For example,
-a 32 core POD runs on 4 hosts with each host using 8 cores.
-
-**Note**: 8 A100 are similar to TPU V3 32 cores in terms of [Vertex AI
-pricing][4].
-
-**Note**: [Each TPU v3 chip has 2 cores which can use 32 GB high-bandwidth
-memory][22] (16 GB per core) so total memory for 32 cores is 16x32 =
-512 GB. Therefore for the same price, TPUs offer more memory than 8 A100-40GB
-GPUs.
-
-#### Modeling
-
-We decided to use an OSS code repository for model implementation. Using an OSS
-repository helps anyone to independently verify the benchmarking results and
-also relate to the results well. For JAX, we selected the
-[Big Vision][23] code repository.
-
-Same as the PyTorch modeling, we benchmark 4 variants of ViT model in different
-sizes:
-
-- [ViT-L16, 300M params][24]
-- [ViT-H14, 630M params][24]
-- [ViT-g14, 1B params][24]
-- [ViT-G14, 1.8B params][24]
-
-**Note**: The [Big Vision code repo][23] has not made the
-checkpoints publicly available for the models larger than the ViT-L16. Therefore
-for the rest of the three variants, the experiments only used random
-initialization for benchmarking the training speed.
-
-#### Dataset
-
-We use training against the [cifar10 TensorFlow dataset][25] with
-50K training images and 10K test images. This dataset is the same as the one
-used for PyTorch experiments except that it is loaded as a TensorFlow dataset.
-Similar to the PyTorch experiments, we copy the whole dataset to the docker
-image to factor out network communication overhead for data loading.
-
-#### Training parameters
-
-- Precision
-  - "bfloat16" setting was used.
-- Input resolution
-  - 224 x 224 after resize (to 448x448) and random crop (to 224x224) before
-    training.
-    - This resolution for training was the same as the PyTorch settings.
-- Strategy
-  - Used DDP for all models except ViT-G14. ViT-G14 used the FSDP strategy.
-- Batch size
-  - We use the max batch size as power of 2 without OOM for each model. The
-    [Benchmarking results][26] section shows the final
-    batch size for each experiment.
-  - Once a maximum batch-size for TPU V3 8 cores was determined, we just scaled
-    it linearly for 32 cores.
-  - Once a maximum batch-size for 1 A100 GPU was determined, we just scaled it
-    linearly for 8 A100 GPUs.
-- Compilation
-  - [jax.jit() compilation][27] is used in JAX codes for efficient
-    execution in XLA.
-- GPU related flags
-  - The following flags are set in the dockerfile for the GPU runs.
-    - Note: _xla_gpu_enable_pipelined_collectives_ is set to false for the
-      ViT-G14 FSDP run.
-
-### Evaluation metric
-
-For both the PyTorch and JAX experiments, the following evaluation metrics are
-collected:
-
-- Throughput: Images-per-second observed for training.
-- Cost: The training-cost-per-epoch (USD).
-
-**Note**: The above metrics are not biased against any framework or machine
-configurations. In addition, these metrics will help one decide the most
-efficient training configurations on Vertex AI.
-
-## Benchmarking results
-
-The lowest cost experiment for each model is marked in **bold** in the last
-column.
-
-![vit_benchmarking_table](images/vit_benchmarking_table.png)
-
-The following bar charts summarize the performance visually:
-
-![vit_training_time](images/vit_training_time.png)
-
-![vit_training_cost](images/vit_training_cost.png)
-
-The following section provides observations and conclusions for these results.
-
-## Observation and Conclusions
-
-- Training with JAX TPU V3 POD with 32 cores costs 33% less than the PyTorch GPU
-  8 A100-40GBs runs.
-- Training with JAX GPU 8 A100-40GBs costs 23% less than the PyTorch GPU 8
-  A100-40GBs runs.
-- JAX TPU V3 POD with 32 cores was 4x faster and slightly more cost-effective
-  than the JAX TPU V3 8 core run for the ViT-large model. This indicates that it
-  might be better to use more cores. The JAX TPU V3 speed scales very well with
-  the number of cores.
-- Cloud TPU VM training speed numbers were the same as the Vertex AI for
-  TPU V3 8 cores. The dataset was copied to the docker in both the cases.
-- The training-cost-per-epoch increases with the model size irrespective of the
-  framework.
-
-[1]: https://github.com/huggingface/transformers/blob/main/examples/research_projects/jax-projects/README.md#quickstart-flax-and-jax-in-transformers
-[2]: https://github.com/openlm-research/open_llama
-[3]: https://github.com/google-research/vision_transformer
-[4]: https://cloud.google.com/vertex-ai/pricing#custom-trained_models
-[5]: https://arxiv.org/abs/2010.11929
-[6]: #pytorch-gpu
-[7]: #jax-tpu-and-gpu
-[8]: https://cloud.google.com/vertex-ai/docs/training/overview
-[9]: https://cloud.google.com/vertex-ai/docs/training/configure-compute#machine-types
-[10]: https://cloud.google.com/vertex-ai/docs/training/configure-compute#specifying_gpus
-[11]: https://huggingface.co/google/vit-large-patch16-224-in21k
-[12]: https://huggingface.co/google/vit-huge-patch14-224-in21k
-[13]: https://github.com/huggingface/pytorch-image-models/blob/v0.9.2/timm/models/vision_transformer.py#L1308
-[14]: https://github.com/huggingface/pytorch-image-models/blob/v0.9.2/timm/models/vision_transformer.py#L1312
-[15]: https://huggingface.co/docs/transformers/main/model_doc/vit#transformers.ViTModel
-[16]: https://github.com/huggingface/pytorch-image-models
-[17]: https://huggingface.co/datasets/cifar10
-[18]: https://lightning.ai/docs/pytorch/stable/
-[19]: https://pytorch.org/docs/stable/notes/ddp.html
-[20]: https://www.deepspeed.ai/tutorials/zero/
-[21]: https://pytorch.org/tutorials/intermediate/torch_compile_tutorial.html
-[22]: https://cloud.google.com/tpu/docs/system-architecture-tpu-vm#tpu_v3
-[23]: https://github.com/google-research/big_vision
-[24]: https://screenshot.googleplex.com/BximJgxsgvBVu38
-[25]: https://www.tensorflow.org/datasets/catalog/cifar10
-[26]: #benchmarking-results
-[27]: https://jax.readthedocs.io/en/latest/jax-101/02-jitting.html

community-content/vertex_model_garden/benchmarking_reports/pytorch_openllama_hyperparameter_tuning_benchmark_report.md

@@ -1,188 +0,0 @@
-# Benchmark report on hyperparameter tuning the OpenLLaMA models on Google Cloud Vertex Model Garden
-
-Changyu Zhu, Software Engineer, Google Cloud
-
-Dustin Luong, Software Engineer, Google Cloud
-
-Gary Wei, Software Engineer, Google Cloud
-
-Genquan Duan, Software Engineer, Google Cloud
-
-## Introduction
-
-Fine-tuning of LLMs can be non-trivial to find an optimal configuration of
-machine types, training parameters, and other hyperparameters that achieves a
-good balance between cost efficiency and model performance. To facilitate users
-in conducting tuning experiments, this report benchmarks fine-tuning OpenLLaMA
-models with [Vertex AI Hyperparameter Tuning Service](https://cloud.google.com/vertex-ai/docs/training/hyperparameter-tuning-overview), demonstrating both efficiency
-and effectiveness. Similar hyperparameter tuning techniques can apply to other models as well.
-
-## Key takeaways
-
-- **The hyperparameter tuning service finds good parameters**: The best model found by the hyperparameter tuning service has an average improvement of around 4% in accuracy in *ARC*, *HellaSwag*, and *TruthfulQA* datasets, while only tuning the learning rate.
-
-- **Hyperparameter tuning works with QLoRA on limited resources**: 4bit QLoRA is sufficient for hyperparameter tuning to find a set of good parameters. In this way, all OpenLLaMA models can run on 1 single `NVIDIA_L4` GPU. It is also possible to train for more steps on the good parameters discovered by hyperparameter tuning, avoiding the waste of computing resources on fine-tuning with suboptimal hyperparameters.
-
-- **Hyperparameter tuning is cost-effective**: While `NVIDIA_L4` is slower than `NVIDIA_TESLA_V100`, it costs less and avoids the overhead of multi-GPU training since it has more GPU memory. Finding a good 3B/7B/13B OpenLLaMA model costs $28.5671, $47.8016, and $87.9208, respectively.
-
-## Benchmarking setup
-
-This section describes the experiment setup of the hyperparameter tuning experiments. The default tuning parameters are:
-
-### Machine configuration
-
-- Machine type: g2-standard-8
-- Machine count: 1
-- Accelerator type: NVIDIA_L4
-- Accelerator count: 1
-
-### Modeling
-
-We benchmark all 3 OpenLLaMA models:
-
-- [open_llama_3b](https://huggingface.co/openlm-research/open_llama_3b)
-- [open_llama_7b](https://huggingface.co/openlm-research/open_llama_7b)
-- [open_llama_13b](https://huggingface.co/openlm-research/open_llama_13b)
-
-We use the Huggingface [PEFT](https://github.com/huggingface/peft) library for fine-tuning.
-
-### Training dataset
-
-We use the dataset [timdettmers/openassistant-guanaco](https://huggingface.co/datasets/timdettmers/openassistant-guanaco) loaded directly via Huggingface.
-
-### Training parameters
-
-The set of training parameters used during benchmarking:
-
-- Batch size: 4
-- Precision mode: 4bit QLoRA
-- LoRA rank: 32
-- LoRA alpha: 64
-- Max sequence length: 512
-- Max train steps: 1000
-
-### Evaluation dataset
-
-We use the [lm-evaluation-harness](https://github.com/EleutherAI/lm-evaluation-harness) library injected into the training loop for evaluation. The hyperparameter tuning job will pick the model according to the evaluation metrics.
-
-- Eval task: [ARC Challenge](https://huggingface.co/datasets/ai2_arc)
-- Eval metric: acc_norm
-- Max eval examples: 10000
-
-### Standalone evaluation dataset
-
-After finding the best model with Vertex hyperparameter tuning service, we run standalone evaluations with the model on the following datasets:
-
-- [ARC Challenge](https://huggingface.co/datasets/ai2_arc)
-- [HellaSwag](https://huggingface.co/datasets/Rowan/hellaswag)
-- [TruthfulQA](https://huggingface.co/datasets/EleutherAI/truthful_qa_mc)
-
-### Hyperparameter tuning
-
-We only tune the learning rate hyperparameter. It is considered a floating point value in the continuous range [1e-5, 1e-4]. We run 8 trials in total, with a parallelism of 1 or 2.
-
-### Code example
-
-The following code example launches an example hyperparameter tuning job of OpenLLaMA 7B model.
-
-```py
-from google.cloud import aiplatform
-from google.cloud.aiplatform import hyperparameter_tuning as hpt
-
-
-TRAIN_DOCKER_URI = 'us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-peft-train:20231130_0936_RC00'
-output_dir = "gs://path/to/output/dir"
-base_model_id = "openlm-research/open_llama_7b"
-dataset_name = "timdettmers/openassistant-guanaco"
-hpt_precision_mode = "4bit"
-machine_type = "g2-standard-8"
-accelerator_type = "NVIDIA_L4"
-accelerator_count = 1
-eval_task = "arc_challenge"
-eval_metric_name = "acc_norm"
-max_steps = 1000
-eval_limit = 10000
-
-flags = {
-    "learning_rate": 1e-5,
-    "precision_mode": hpt_precision_mode,
-    "task": "instruct-lora",
-    "pretrained_model_id": base_model_id,
-    "output_dir": output_dir,
-    "warmup_steps": 10,
-    "max_steps": max_steps,
-    "lora_rank": 32,
-    "lora_alpha": 64,
-    "lora_dropout": 0.05,
-    "dataset_name": dataset_name,
-    "eval_steps": max_steps + 1,  # Only evaluates at the end.
-    "eval_tasks": eval_task,
-    "eval_limit": eval_limit,
-    "eval_metric_name": eval_metric_name,
-}
-worker_pool_specs = [
-    {
-        "machine_spec": {
-            "machine_type": machine_type,
-            "accelerator_type": accelerator_type,
-            "accelerator_count": accelerator_count,
-        },
-        "replica_count": 1,
-        "container_spec": {
-            "image_uri": TRAIN_DOCKER_URI,
-            "args": ["--{}={}".format(k, v) for k, v in flags.items()],
-        },
-    }
-]
-metric_spec = {"model_performance": "maximize"}
-parameter_spec = {
-    "learning_rate": hpt.DoubleParameterSpec(
-        min=1e-5, max=1e-4, scale="linear"
-    ),
-}
-
-train_job = aiplatform.CustomJob(
-    display_name=job_name,
-    worker_pool_specs=worker_pool_specs,
-    staging_bucket=STAGING_BUCKET,
-)
-
-train_hpt_job = aiplatform.HyperparameterTuningJob(
-    display_name=f"{job_name}_hpt",
-    custom_job=train_job,
-    metric_spec=metric_spec,
-    parameter_spec=parameter_spec,
-    max_trial_count=8,
-    parallel_trial_count=2,
-)
-
-train_hpt_job.run()
-```
-
-## Benchmark results
-
-### Fine-tuning cost
-
-The fine-tuning cost is calculated from `us-central1` pricing and may be subject to changes.
-
-| Model         | Train time | Trials | Parallel Trials | Hourly cost | Cost     | Eval acc_norm (ARC-Challenge) |
-|---------------|------------|--------|-----------------|-------------|----------|-------------------------------|
-| OpenLLaMA 3B  | 16 hrs     | 8      | 2               | $1.7072     | $28.5671 | 39.9%                         |
-| OpenLLaMA 7B  | 28 hrs     | 8      | 2               | $1.7072     | $47.8016 | 45.8%                         |
-| OpenLLaMA 13B | 103 hrs    | 8      | 1               | $0.8536     | $87.9208 | 47.6%                         |
-
-### Fine-tuning performance
-
-Here are the evaluation results of the best model found by hyperparameter tuning, compared with the baseline model. The column `Eval acc_norm` is calculated during training, which is always lower than that during standalone evaluation, because the model is loaded and evaluated at a lower precision (4bit during training / float16 during standalone evaluation).
-
-| Model         | Eval acc_norm (ARC-Challenge) | ARC    | hellaswag | Truthfulqa_mc | ∆ARC   | ∆Hellaswag | ∆Truthfulqa_mc | ∆Average |
-|---------------|-------------------------------|--------|-----------|---------------|--------|------------|----------------|----------|
-| OpenLLaMA 3B  | 39.9%                         | 41.47% | 69.97%    | 38.31%        | +1.62% | +7.32%     | +3.34%         | +4.09%   |
-| OpenLLaMA 7B  | 45.8%                         | 49.83% | 75.53%    | 41.53%        | +2.82% | +3.55%     | +6.68%         | +4.35%   |
-| OpenLLaMA 13B | 47.6%                         | 52.20% | 78.90%    | 44.27%        | +1.01% | +3.67%     | +6.19%         | +3.62%   |
-
-## Related documents
-
-1. [Benchmark report on fine tuning the OpenLLaMA 7B model on Google Cloud Vertex Model Garden
-](
-https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/community-content/vertex_model_garden/benchmarking_reports/pytorch_openllama_7b_finetune_benchmark_report.md)

community-content/vertex_model_garden/benchmarking_reports/stable_diffusion_v1-5_benchmarking_report.md

@@ -1,218 +0,0 @@
-# Benchmark Stable Diffusion v1-5 Fine Tuning and Serving With Google Cloud Vertex Model Garden
-
-Dustin Luong, Software Engineer, Google Cloud
-Gary Wei, Software Engineer, Google Cloud
-Changyu Zhu, Software Engineer, Google Cloud
-Genquan Duan, Software Engineer, Google Cloud
-
-## Introduction
-[The public notebook][1] shows the full examples of fine tuning and serving of Stable diffusion v1-5. [The github repo][2] contains examples of building training and serving dockers for Google Cloud Vertex Model Garden. This report benchmarks Stable diffusion v1-5 fine tuning and serving in Google Cloud Vertex AI, showing both efficiencies and effectiveness.
-
-### Benchmark Highlights
-- Fine tuning
-  - Stable diffusion v1-5 with LoRA and Gradient checkpointing only requires ~10G GPU memory. Larger batch sizes, or larger resolutions require more GPU memories, but not does not change much for different LoRA ranks.
-  - The fine tuning speed is fast in ~11 minutes for 1k steps, and costs less than $1 in 1 A100. The fine tuning speed increases with batch sizes, decreases with resolution, but is not affected much by LoRA ranks.
-  - LoRA tunes a few percent (only 0.1% with LoRA rank=8) of all parameters, and the tuned models are very small (only 3.1MB with LoRA rank=8).
-  - Dreambooth+LoRA and Dreambooth can achieve similar performances, but Dreambooth LoRA can require much less GPU.
-  - Increasing batch size, reducing training steps, and increasing learning rate can result in models with the same performance for less cost.
-- Inference
-  - The optimized serving docker pytorch-peft-serve can speed up inference by 2x than current pytorch-diffuser-serve, and support both base models and fine tuned lora models.
-  - The optimized serving docker pytorch-peft-serve can generate 4 512*512 images in 4.1 seconds on 1 V100 and 1.7 seconds on 1 A100.
-
-Benchmark details are below.
-
-## Fine Tuning Benchmarks
-
-### Experiment Setup
-We mainly compare two tuning algorithms:
-- parameter efficient finetuning based on [dreambooth][3] and [LoRA][4] (shorten as Dreambooth+LoRA below)
-- full parameter fine tuning based on [dreambooth][3] (shorten as Dreambooth below)
-
-And then  report benchmark results on GPU memories, tuning parameters, tuning speeds, costs and accuracy, using the public oxford flowers dataset: [train][5] and [test][6], where the column blip_caption as texts, and column image as images. We also benchmark subject and prompt fidelity using the [dataset][7] from the Dreambooth paper.
-
-The default tuning parameters during benchmark are:
-- Hardware: 1 A100 40G
-- batch size: 4
-- lora_rank: 8
-- resolution: 512
-- max_train_steps: 10
-- use_lora: False
-- gradient_checkpointing: False
-
-```
-# Examples to start finetuning dockers.
-MODEL_NAME="runwayml/stable-diffusion-v1-5"
-OUTPUT_DIR=<OUTPUT_DIR>
-INSTANCE_DATA_DIR=<INSTANCE_DATA_DIR>
-INSTANCE_PROMPT=<INSTANCE_PROMPT>
-IMAGE="us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-peft-train"
-docker run \
-  --runtime=nvidia -e NVIDIA_VISIBLE_DEVICES=0 \
-  --rm --name "test_gpu" \
-  -it ${IMAGE} \
-  --task=text-to-image-dreambooth-lora-peft \
-  --pretrained_model_name_or_path=$MODEL_NAME \
-  --resolution=512 \
-  --instance_data_dir=$INSTANCE_DATA_DIR \
-  --instance_prompt=$INSTANCE_PROMPT \
-  --train_batch_size=4 \
-  --max_train_steps=10 \
-  --output_dir=${OUTPUT_DIR} \
-  --use_lora \
-  --lora_r=8 \
-  --gradient_checkpointing
-```
-
-### GPU Memories
-Many various factors will impact GPU memory usages. In this benchmark, we mainly benchmark with different finetuning algorithms, batch sizes, lora rank, resolution, and then recommended max batch size on different GPUs.
-
-
-![sd_v1-5_peak_gpu_algorithm](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_peak_gpu_algorithm.png)
-
-![sd_v1-5_peak_gpu_batch_size](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_peak_gpu_batch_size.png)
-
-![sd_v1-5_peak_gpu_lora_rank](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_peak_gpu_lora_rank.png)
-
-![sd_v1-5_peak_gpu_resolution](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_peak_gpu_resolution.png)
-
-- LoRA tuning reduced about 47% peak RAM and 42% peak VRAM for GPU memory, compared to full parameter fine tuning.
-- Gradient checkpointing decreases about 1% peak RAM and 31% peak VRAM for GPU memory further, compared without gradient checkpointing.
-- The GPU memory does not change much for different LoRA ranks.
-- Larger batch sizes require more GPU memories.
-- Larger resolutions require more GPU memories.
-- Dreambooth+LoRA+Gradient_Checkpointing can support max batch size as 32, or max resolution as 2048, but Dreambooth can only support max batch size as 8, or max resolution as 1024.
-
-### Fine Tuning Parameters
-This section shows the percentage of trainable parameters, and tuned model sizes.
-
-- LoRA tunes quite a few percent (only 0.1% with LoRA rank=8) of all parameters, and the tuned models are very small (only 3.1MB with LoRA rank=8).
-
-| LoRA Rank | Trainable parameters | Total parameters | Trainable Parameter Percentage | Fine tuned model size (MB) |
-|---|---|---|---|---|
-| 4 | 398592 | 859919556 | 0.05% | 1.57 |
-|8 | 797184 | 860318148 | 0.09% | 3.09 |
-| 16 | 1594368| 861115332| 0.19%| 6.13|
-| 32| 3188736| 862709700| 0.37%| 12.21|
-### Fine Tuning Speed And Costs
-Fine tuning speeds and costs are affected by many different factors, such as batch size, tuning parameters, image resolutions, GPUs, and datasets. In order to make the report easy to understand, we set the following values in this section:
-- Hardware: 1 A100 40G
-- use_lora: True
-- gradient_checkpointing: True
-
-![sd_v1-5_training_speed_batch_size](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_training_speed_batch_size.png)
-
-![sd_v1-5_training_speed_lora_rank](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_training_speed_lora_rank.png)
-
-![sd_v1-5_training_speed_resolution](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_training_speed_resolution.png)
-
-![sd_v1-5_training_cost_max_steps](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_training_cost_max_steps.png)
-
-- The fine tuning speed increases with batch sizes, decreases with resolution, but is not affected much by LoRA ranks.
-- The fine tuning speed is about 11 minutes for 1k steps, and costs less than $1 in 1 A100.
-
-### Fine Tuning Quality
-In this benchmark, we mainly benchmark Dreambooth and Dreambooth+LoRA to compare fine tuning quality. We compare [subject fidelity scored (DINO)][8], how well the subject is represented in the generated images, and [prompt fidelity scores (CoCa)][9], how well the generated images match the given prompt, for a single subject, a [dog][10] from the dataset released with the original Dreambooth paper. In practice, we recommend saving checkpoints periodically and inspecting validation prompts visually. We fine tuned the unet without fine tuning the text encoder and used the following hyperparameters:
-
-Dreambooth
-- Learning rate: 5e-6
-- Batch size: 1
-
-Dreambooth+LoRA
-- Learning rate: 1e-4
-- Batch size: 1
-
-![sd_v1-5_finetuning_quality_subject_fidelity](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_finetuning_quality_subject_fidelity.png)
-
-![sd_v1-5_finetuning_quality_prompt_fidelity](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_finetuning_quality_prompt_fidelity.png)
-
-- Fine tuning with Dreambooth or Dreambooth+LoRA can result in models with comparable performance. The base model produced images of the class rather than the instance.
-- Dreambooth+LoRA is able to achieve the same subject fidelity score as Dreambooth if trained for more epochs.
-- Increasing the number of training steps results in better subject fidelity but at the cost of prompt fidelity.
-
-### Suggested Max Batch Sizes By Resolutions
-We benchmarked and suggested max batch sizes by resolutions on 1 A100 and 1 V100 as below. This is with LoRA and gradient checkpointing enabled.
-
-![sd_v1-5_batch_size_by_resolution](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_batch_size_by_resolution.png)
-
-### Fine Tuning Cost Optimization
-Increasing batch size allows for more images to be considered at each training step for fine tuning. This allows models to be trained in fewer training steps. In this benchmark, we aim to show how batch size can be increased to reduce training costs while still preserving subject and prompt fidelity.
-
-Since the training dataset consists of 5 images, we train with a batch size of 5 and reduce the number of training steps from 400 to 80. Doing so results in a model that has not learned the subject since we’ve decreased the number of training steps. Conceptually, the model is taking a more precise step at each iteration, but it is taking fewer steps. To compensate for this, we increased the learning rate from 5e-6 and observed the best results at 1e-5 for full parameter finetuning.
-
-![sd_v1-5_subject_fidelity_batch_size_5_learning_rate](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_subject_fidelity_batch_size_5_learning_rate.png)
-
-![sd_v1-5_prompt_fidelity_batch_size_5_learning_rate](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_prompt_fidelity_batch_size_5_learning_rate.png)
-
-Comparing cost of training the “best” model for batch size 1 vs. batch size 5
-
-![sd_v1-5_cost_batch_size](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_cost_batch_size.png)
-
-
-| Train method| Training parameters| Sample image| CoCa (prompt fidelity)| DINO (subject fidelity) | Cost of training on A100 |
-|---|---|---|---|---|---|
-| dreambooth| dreambooth, num_train_steps=400, batch_size=1, lr=5e-6| ![dog1](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_dog1.png) | 0.12215| 0.76531| $0.26 |
-| dreambooth | dreambooth, num_train_steps=80, batch_size=5,lr=1e-5| ![dog2](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_dog2.png)| 0.12644| 0.74697 | $0.15 |
-| dreambooth-lora| num_train_steps=500, batch_size=1, lr=1e-4, gc|![dog3](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_dog3.png)| 0.12856| 0.78148 | $0.26|
-| dreambooth-lora | num_train_steps=50, batch_size=5, lr=1e-3, gc | ![dog4](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_dog4.png) | 0.12566 | 0.75479 | $0.09 |
-
-
-
-A followup question is that since finetuning can be run on a single GPU, should finetuning be run on 1 V100 or A100?
-
-Setup:
-- num_train_steps=800 / batch_size
-- Resolution=512
-
-![sd_v1-5_cost_training_method_batch_size](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_cost_training_method_batch_size.png)
-- Although V100 has a lower $/hr cost than an A100, the same training setup takes longer. Even given the longer training time, the cost on V100 is still lower.
-- Dreambooth+LoRA enables training with larger batch sizes, however, larger batch sizes will not necessarily mean faster training time.
-- It is possible to fine tune with 1 V100 on 512 resolution with Dreambooth+LoRA.
-- Dreambooth fine tuning must be run on 1 A100 at 512 resolution.
-
-## Inference Benchmarks
-We provide two serving dockers in vertex model garden for stable diffusion:
-- pytorch-diffuser-serve:
-  - us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-diffusers-serve
-  - This serving docker only serves base stable diffusion models and does not contain any optimizations yet.
-- pytorch-peft-serve:
-  - us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-peft-serve
-  - This serving docker can serve base stable diffusion models, and base stable diffusion models with fine tuned lora models, and contains optimization for serving.
-
-We run the two serving dockers on T4/V100/A100 to generate 4 512*512 images, and compare the inference speed without network considerations as:
-
-![sd_v1-5_inference_speed_gpu](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_inference_speed_gpu.png)
-The speed up of optimized pytorch-peft-serve is about 2x than current pytorch-diffuser-serve.
-
-### Serving cost comparison
-
-Pytorch-diffuser-serve (without any optimizations)
-
-| GPU type|  Time required to generate 4 512x512 images | Machine unit price ($ / hour) | Cost per image ($) |
-|---|---|---|---|
-| T4 | 28.6 | 0.4025| 0.00080 |
-| V100 | 8.8 | 2.852| 0.00174|
-| A100 | 4.2 | 4.2245 | 0.00123 |
-
-Pytorch-peft-serve (with optimizations)
-
-| GPU type | Time required to generate 4 512x512 images | Machine unit price ($ / hour) | Cost per image ($) |
-|--- |---|---|---|
-| T4 | 12.6 | 0.4025 | 0.00035 |
-| V100 | 4.1 | 2.852 | 0.00081 |
-|  A100 | 1.7 | 4.2245 | 0.00050 |
-
-- The optimized pytorch-peft-serve has approximately half the price per image, compared with the un-optimized pytorch-diffuser-serve.
-- Serving the model with a T4 is most cost effective, however, serving with an A100 still has the best throughput and fastest predictions.
-
-
-
-[1]: https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion.ipynb
-[2]: https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/community-content/vertex_model_garden/model_oss
-[3]: https://arxiv.org/abs/2208.12242
-[4]: https://arxiv.org/abs/2106.09685
-[5]: https://huggingface.co/datasets/Multimodal-Fatima/OxfordFlowers_train
-[6]: https://huggingface.co/datasets/Multimodal-Fatima/OxfordFlowers_test_facebook_opt_6.7b_Attributes_ns_6149
-[7]: https://github.com/google/dreambooth
-[8]: https://arxiv.org/abs/2104.14294
-[9]: https://arxiv.org/abs/2205.01917
-[10]: https://github.com/google/dreambooth/tree/main/dataset/dog6

community-content/vertex_model_garden/model_oss/autogluon/dockerfile/autogluon_serve.Dockerfile

@@ -1,50 +0,0 @@
-# Dockerfile for serving dockers with AutoGluon.
-#
-# To build:
-# docker build -f model_oss/autogluon/dockerfile/serve.Dockerfile . -t ${YOUR_IMAGE_TAG}
-#
-# To push to gcr:
-# docker tag ${YOUR_IMAGE_TAG} gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
-# docker push gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
-
-FROM pytorch/pytorch:2.1.2-cuda11.8-cudnn8-runtime
-
-USER root
-
-# AutoGluon might require libgomp for some dependencies.
-RUN apt-get update && apt-get -y upgrade && apt-get install -y --no-install-recommends \
-        curl \
-        wget \
-        vim \
-        libgomp1
-
-# Install AutoGluon and other dependencies.
-RUN pip install --upgrade pip
-RUN pip install autogluon==1.0.0
-RUN pip install flask==3.0.0
-
-# Dependencies needed to work with GCS.
-RUN pip install absl-py==2.0.0
-RUN pip install google-cloud-storage==2.7.0
-
-# Copy scripts into the container.
-COPY model_oss/autogluon /autogluon
-COPY model_oss/util /autogluon/util
-WORKDIR /autogluon
-
-# Copy license.
-RUN wget https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/LICENSE
-RUN wget https://github.com/pallets/flask/blob/main/LICENSE.rst
-
-# Expose the port the app runs on.
-EXPOSE 8501
-
-# Set the working directory to a specific path for consistency.
-WORKDIR /autogluon
-
-# Change to a non-root user for security purposes.
-RUN useradd -m autogluonuser
-USER autogluonuser
-
-# Run Flask application.
-CMD ["python", "serve.py"]

community-content/vertex_model_garden/model_oss/autogluon/dockerfile/autogluon_train.Dockerfile

@@ -1,36 +0,0 @@
-# Dockerfile for training dockers with Autogluon.
-#
-# To build:
-# docker build -f model_oss/autogluon/dockerfile/train.Dockerfile . -t ${YOUR_IMAGE_TAG}
-#
-# To push to gcr:
-# docker tag ${YOUR_IMAGE_TAG} gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
-# docker push gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
-
-FROM pytorch/pytorch:2.1.2-cuda11.8-cudnn8-runtime
-
-# Install tools.
-ENV DEBIAN_FRONTEND=noninteractive
-ENV PIP_ROOT_USER_ACTION=ignore
-
-RUN apt-get update && apt-get -y upgrade && apt-get install -y --no-install-recommends \
-apt-utils \
-curl \
-wget \
-git \
-jq \
-gnupg \
-build-essential \
-tesseract-ocr \
-vim
-
-# Install libraries.
-RUN pip install autogluon==1.0.0
-
-COPY model_oss/autogluon /autogluon
-WORKDIR /autogluon
-
-# Copy license.
-RUN wget https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/LICENSE
-
-ENTRYPOINT ["python", "train.py"]

community-content/vertex_model_garden/model_oss/autogluon/serve.py

@@ -1,87 +0,0 @@
-r"""AutoGluon serving binary.
-
-This module sets up a Flask web server for serving predictions from a
-trained AutoGluon model. The server exposes two endpoints:
-
-1.  `/ping`: A health check endpoint that returns "pong" to
-    indicate that the server is running.
-2.  `/predict`: An endpoint that accepts POST requests with JSON content.
-    Each request should contain one or more instances for which the
-    predictions are desired. The endpoint returns the predictions and
-    associated probabilities in a JSON response.
-
-The server expects an environment variable `model_path` that points to
-the directory where the AutoGluon model artifacts are
-stored. If `model_path` is not provided, it defaults to '/autogluon/models'.
-"""
-
-import json
-import logging
-import os
-
-from autogluon.tabular import TabularPredictor
-import flask
-import pandas as pd
-
-from util import constants
-from util import fileutils
-
-_SUCCESS_STATUS = 200
-_ERROR_STATUS = 500
-_PORT = 8501
-
-app = flask.Flask(__name__)
-# Check the environment variables.
-model_dir = os.getenv('model_path', '/autogluon/models')
-logging.info('Model directory passed by the user is: %s', model_dir)
-# If the model is on GCS then copy it to a local folder first.
-if model_dir.startswith(constants.GCS_URI_PREFIX):
-  gcs_path = model_dir[len(constants.GCS_URI_PREFIX) :]
-  local_model_dir = os.path.join(constants.LOCAL_MODEL_DIR, gcs_path)
-  logging.info('Download %s to %s', model_dir, local_model_dir)
-  fileutils.download_gcs_dir_to_local(model_dir, local_model_dir)
-  model_dir = local_model_dir
-  logging.info('Local model directory is: %s', model_dir)
-
-
-# Load the predictor at startup.
-predictor = TabularPredictor.load(model_dir)
-
-
-@app.route('/ping', methods=['GET'])
-def ping() -> flask.Response:
-  """Health check route."""
-  return flask.Response('pong', status=_SUCCESS_STATUS)
-
-
-@app.route('/predict', methods=['POST'])
-def predict() -> flask.Response:
-  """Prediction route."""
-  try:
-    # Extract JSON content from the POST request.
-    data = flask.request.get_json(force=True)
-    instances = data.get('instances', [])
-
-    # Convert instances to DataFrame.
-    df_to_predict = pd.DataFrame(instances)
-
-    # Perform prediction.
-    predictions = predictor.predict(df_to_predict).tolist()
-    response = {'predictions': predictions}
-
-    return flask.Response(
-        json.dumps(response),
-        status=_SUCCESS_STATUS,
-        mimetype='application/json',
-    )
-
-  except Exception as e:  # pylint: disable=broad-exception-caught
-    return flask.Response(
-        json.dumps({'error': str(e)}),
-        status=_ERROR_STATUS,
-        mimetype='application/json',
-    )
-
-
-if __name__ == '__main__':
-  app.run(host='0.0.0.0', port=_PORT)

community-content/vertex_model_garden/model_oss/autogluon/train.py

@@ -1,144 +0,0 @@
-"""AutoGluon training binary. """
-
-import argparse
-import json
-from typing import Any
-
-from autogluon.tabular import TabularPredictor
-import pandas as pd
-
-
-class BaseConfig:
-
-  def to_dict(self) -> dict[str, Any]:
-    return {
-        key: value for key, value in self.__dict__.items() if value is not None
-    }
-
-
-class DataConfig(BaseConfig):
-
-  def __init__(self, train_data_path: Any) -> None:
-    self.train_data_path = train_data_path
-
-
-class ProblemConfig(BaseConfig):
-
-  def __init__(self, label: Any, problem_type: Any) -> None:
-    self.label = label
-    self.problem_type = problem_type
-
-
-class EvaluationConfig(BaseConfig):
-
-  def __init__(self, eval_metric: Any) -> None:
-    self.eval_metric = eval_metric
-
-
-class TrainingConfig(BaseConfig):
-  """Config for training."""
-
-  def __init__(
-      self,
-      time_limit: Any,
-      presets: Any,
-      hyperparameters: Any,
-      model_save_path: str,
-  ) -> None:
-    self.time_limit = time_limit
-    self.hyperparameters = hyperparameters
-    self.presets = presets
-    self.model_save_path = model_save_path
-
-
-def parse_args() -> (
-    tuple[DataConfig, ProblemConfig, EvaluationConfig, TrainingConfig]
-):
-  """Parse command line arguments."""
-  parser = argparse.ArgumentParser(description="AutoGluon Tabular Predictor")
-  # Add arguments for each config class
-  parser.add_argument(
-      "--train_data_path",
-      type=str,
-      required=True,
-      help="Path to the input data CSV file.",
-  )
-  parser.add_argument(
-      "--label", type=str, required=True, help="Target variable column name."
-  )
-  parser.add_argument(
-      "--problem_type",
-      type=str,
-      choices=["binary", "multiclass", "regression", "quantile"],
-      default=None,
-      help="Problem type.",
-  )
-  parser.add_argument(
-      "--eval_metric", type=str, default=None, help="Evaluation metric to use."
-  )
-  # Add arguments for TrainingConfig if needed
-  parser.add_argument(
-      "--time_limit",
-      type=int,
-      default=None,
-      help="Time limit in seconds for training.",
-  )
-  parser.add_argument(
-      "--presets",
-      type=str,
-      default="medium_quality",
-      help="Presets used for training ",
-  )
-  parser.add_argument(
-      "--hyperparameters",
-      type=json.loads,
-      default=None,
-      help="Hyperparameter dictionary in JSON format.",
-  )
-  parser.add_argument(
-      "--model_save_path",
-      type=str,
-      default=None,
-      help="Path to save the trained model.",
-  )
-  args = parser.parse_args()
-
-  data_config = DataConfig(train_data_path=args.train_data_path)
-  problem_config = ProblemConfig(
-      label=args.label, problem_type=args.problem_type
-  )
-  eval_config = EvaluationConfig(eval_metric=args.eval_metric)
-  training_config = TrainingConfig(
-      time_limit=args.time_limit,
-      presets=args.presets,
-      hyperparameters=args.hyperparameters,
-      model_save_path=args.model_save_path,
-  )
-
-  return data_config, problem_config, eval_config, training_config
-
-
-def main() -> None:
-  data_config, problem_config, eval_config, training_config = parse_args()
-
-  # Load the training data.
-  data = pd.read_csv(data_config.train_data_path)
-
-  # Create a TabularPredictor.
-  predictor = TabularPredictor(
-      label=problem_config.label,
-      eval_metric=eval_config.eval_metric,
-      path=training_config.model_save_path,
-  )
-
-  # Fit the model
-  predictor.fit(
-      data,
-      presets=training_config.presets,
-      time_limit=training_config.time_limit,
-      hyperparameters=training_config.hyperparameters,
-  )
-
-
-if __name__ == "__main__":
-  main()

community-content/vertex_model_garden/model_oss/cloudnerf/configs/360.gin

@@ -1,25 +0,0 @@
-# The provided content is a configuration file for the ZipNeRF
-# PyTorch implementation.
-
-# Sets the name of the experiment to 'test'.
-Config.exp_name = 'test'
-# Specifies the dataset loader, in this case, 'llff' for light field.
-Config.dataset_loader = 'llff'
-# Defines the near and far clipping planes for the camera view.
-Config.near = 0.2
-Config.far = 1e6
-# Image downsampling.
-Config.factor = 4
-
-# For the model configurations.
-Model.raydist_fn = 'power_transformation'
-Model.opaque_background = True
-
-# Disables the computation of density normals and RGB values, and sets
-# the grid level dimension to 1 for PropMLP.
-PropMLP.disable_density_normals = True
-PropMLP.disable_rgb = True
-PropMLP.grid_level_dim = 1
-
-# Disable density normals for NerfMLP
-NerfMLP.disable_density_normals = True

community-content/vertex_model_garden/model_oss/cloudnerf/configs/360_glo.gin

@@ -1,21 +0,0 @@
-# The provided content is a configuration file for Generative
-# Latent Optimization (GLO) vectors in the Pytorch implemnetation of ZipNeRF.
-
-# Specifies the dataset loader, in this case, 'llff' for light field.
-Config.dataset_loader = 'llff'
-# Defines the near and far clipping planes for the camera view.
-Config.near = 0.2
-Config.far = 1e6
-# Image downsampling.
-Config.factor = 4
-
-# For the model configurations.
-Model.raydist_fn = 'power_transformation'
-Model.num_glo_features = 128
-Model.opaque_background = True
-
-PropMLP.disable_density_normals = True
-PropMLP.disable_rgb = True
-PropMLP.grid_level_dim = 1
-
-NerfMLP.disable_density_normals = True

community-content/vertex_model_garden/model_oss/cloudnerf/configs/accelerate_config.yaml

@@ -1,18 +0,0 @@
-# The provided content is a configuration file running ZipNeRF
-# training on 8 gpu machine.
-compute_environment: LOCAL_MACHINE
-debug: false
-distributed_type: MULTI_GPU
-downcast_bf16: 'no'
-gpu_ids: all
-machine_rank: 0
-main_training_function: main
-mixed_precision: fp16
-num_machines: 1
-num_processes: 8
-rdzv_backend: static
-same_network: true
-tpu_env: []
-tpu_use_cluster: false
-tpu_use_sudo: false
-use_cpu: false

community-content/vertex_model_garden/model_oss/cloudnerf/dockerfile/pytorch_cloudnerf_base.Dockerfile

@@ -1,120 +0,0 @@
-# Dockerfile for ZipNeRF base image.
-#
-# To build:
-# docker build -f model_oss/cloudnerf/dockerfile/pytorch_cloudnerf_base.Dockerfile . -t ${YOUR_IMAGE_TAG}
-#
-# To push to gcr:
-# docker tag ${YOUR_IMAGE_TAG} gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
-# docker push gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
-
-FROM pytorch/pytorch:2.1.0-cuda11.8-cudnn8-devel
-
-USER root
-
-ARG COLMAP_GIT_COMMIT=main
-ARG CUDA_ARCHITECTURES=60;70;75;80;86
-
-# Prevent stop building ubuntu at time zone selection.
-ENV DEBIAN_FRONTEND=noninteractive
-
-RUN apt-get update -y --allow-releaseinfo-change && apt-get -y upgrade && apt-get install -y --no-install-recommends \
-        curl \
-        g++ \
-        wget \
-        vim \
-        bash \
-        cmake \
-        imagemagick \
-        ninja-build \
-        build-essential \
-        libboost-program-options-dev \
-        libboost-filesystem-dev \
-        libboost-graph-dev \
-        libboost-system-dev \
-        libeigen3-dev \
-        libflann-dev \
-        libfreeimage-dev \
-        libmetis-dev \
-        libgoogle-glog-dev \
-        libgtest-dev \
-        libsqlite3-dev \
-        libglew-dev \
-        qtbase5-dev \
-        libqt5opengl5-dev \
-        libcgal-dev \
-        libceres-dev \
-        git \
-        git-lfs \
-        python3-cffi \
-        python3-cryptography \
-        libffi-dev \
-        python-dev
-
-# Copy license.
-RUN wget https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/LICENSE
-
-# Install google cloud CLI.
-RUN wget -q https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-430.0.0-linux-x86.tar.gz
-RUN tar xzf google-cloud-cli-430.0.0-linux-x86.tar.gz
-RUN ./google-cloud-sdk/install.sh -q
-# Make sure gsutil will use the default service account.
-RUN echo '[GoogleCompute]\nservice_account = default' > /etc/boto.cfg
-
-# Install deps and install gsutil.
-RUN pip install gsutil==5.27
-
-# When building colmap in colab, the link error "undefined reference.
-# to '_glapi_tls_Current'" happens. A solution is to install "libglvnd"
-# as described in this page https://github.com/colmap/colmap/issues/1271.
-RUN git clone --depth 1 --branch v1.7.0 https://github.com/NVIDIA/libglvnd && \
-    apt-get install -y libxext-dev libx11-dev x11proto-gl-dev && \
-    cd libglvnd/  && \
-    apt-get install -y autoconf automake libtool && \
-    apt-get install -y libffi-dev && \
-    ./autogen.sh && \
-    ./configure && \
-    make  -j4 && \
-    make install
-
-RUN apt remove nvidia-cuda-toolkit -y \
-    nvidia-cuda-toolkit \
-    nvidia-cuda-toolkit-gcc
-
-# Install libraries.
-ENV PIP_ROOT_USER_ACTION=ignore
-RUN python3 -m pip install --upgrade pip
-
-ENV CUDA_HOME=/usr/local/cuda
-
-RUN git clone --branch main https://github.com/SuLvXiangXin/zipnerf-pytorch.git
-# Set current directory to the downloaded 'zipnerf-pytorch' repository.
-WORKDIR ./zipnerf-pytorch
-# Using git reset command to pin it down to a specific version.
-RUN git reset --hard 4de3d21ebb9e15412d36951b56e2d713fddd812b
-COPY model_oss/cloudnerf/requirements.txt requirements.txt
-RUN pip install -r requirements.txt
-
-# Install gridencoder extensions and nvdiffrast (for textured mesh).
-RUN cd .. && \
-    TORCH_CUDA_ARCH_LIST="6.0 7.0 7.5 8.0 8.6+PTX" CXX=g++ pip install ./zipnerf-pytorch/gridencoder
-
-# Install cuda version of torch_scatter.
-RUN pip install torch-scatter==2.1.2 -f https://data.pyg.org/whl/torch-2.0.1+cu118.html
-RUN pip install google-cloud-aiplatform==1.25.0
-RUN pip install google-cloud-storage==2.9.0
-
-# Build and install COLMAP.
-RUN git clone  --depth 1 --branch 3.8 https://github.com/colmap/colmap.git
-RUN cd colmap && \
-    git fetch https://github.com/colmap/colmap.git ${COLMAP_GIT_COMMIT} && \
-    mkdir build && \
-    cd build && \
-    cmake .. -GNinja -DCMAKE_CUDA_ARCHITECTURES=${CUDA_ARCHITECTURES} && \
-    ninja && \
-    ninja install && \
-    cd .. && rm -rf colmap
-
-RUN git clone  --depth 1 --branch v1.0.2 https://github.com/dranjan/python-plyfile.git
-
-RUN sed -i "20 i\sys.path.append('/workspace/zipnerf-pytorch/internal/pycolmap')" /workspace/zipnerf-pytorch/internal/datasets.py
-RUN sed -i "21 i\sys.path.append('/workspace/zipnerf-pytorch/internal/pycolmap/pycolmap')" /workspace/zipnerf-pytorch/internal/datasets.py

community-content/vertex_model_garden/model_oss/cloudnerf/dockerfile/pytorch_cloudnerf_calibrate.Dockerfile

@@ -1,16 +0,0 @@
-# Dockerfile for ZipNeRF COLMAP image calibration.
-#
-# To build:
-# docker build -f model_oss/cloudnerf/dockerfile/cloudnerf_pytorch_calibrate.Dockerfile . -t ${YOUR_IMAGE_TAG}
-#
-# To push to gcr:
-# docker tag ${YOUR_IMAGE_TAG} gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
-# docker push gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
-
-FROM us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-cloudnerf-base:20231206_0923_RC00
-
-COPY model_oss/cloudnerf/local_colmap_and_resize.sh /workspace/zipnerf-pytorch/scripts/local_colmap_and_resize.sh
-
-WORKDIR /workspace/zipnerf-pytorch/
-
-ENTRYPOINT ["bash","scripts/local_colmap_and_resize.sh"]