Update ray_cluster_management.ipynb

From Krishna Movva: "Change 'Google Cloud ML' to 'Vertex AI' in the "Objective" subsection. I thought Cloud ML was an old name for the AI/ML services on Google Cloud."

.cloud-build/cleanup/cleanup.py

@@ -16,6 +16,7 @@ from resource_cleanup_manager import (
     ResourceCleanupManager,
     MatchingEngineIndexEndpointResourceCleanupManager,
     MatchingEngineIndexResourceCleanupManager,
+    FeatureStoreLegacyCleanupManager,
     FeatureStoreCleanupManager,
     PipelineJobCleanupManager,
     TrainingJobCleanupManager,
@@ -35,7 +36,10 @@ def run_cleanup_managers(managers: List[ResourceCleanupManager], is_dry_run: boo
 
         print(f"Fetching {type_name}'s...")
         resources = manager.list()
-        print(f"Found {len(resources)} {type_name}'s")
+        try:
+            print(f"Found {len(resources)} {type_name}'s")
+        except Exception as e:
+            print(f"{type_name} {e}")
         for resource in resources:
             try:
                 if not manager.is_deletable(resource):
@@ -62,6 +66,7 @@ managers: List[ResourceCleanupManager] = [
     ModelResourceCleanupManager(),  # ModelResourceCleanupManager must follow EndpointResourceCleanupManager due to deployed models blocking model deletion.
     MatchingEngineIndexEndpointResourceCleanupManager(),
     MatchingEngineIndexResourceCleanupManager(),
+    FeatureStoreLegacyCleanupManager(),
     FeatureStoreCleanupManager(),
     PipelineJobCleanupManager(),
     TrainingJobCleanupManager(),

.cloud-build/cleanup/resource_cleanup_manager.py

@@ -12,9 +12,15 @@ from typing import Any, Type
 
 from google.cloud import aiplatform
 from google.cloud.aiplatform import base
+from google.cloud.aiplatform_v1beta1 import (FeatureOnlineStoreAdminServiceClient,
+                                             FeatureOnlineStore)
 from google.cloud import storage
 from proto.datetime_helpers import DatetimeWithNanoseconds
 
+PROJECT_ID = "python-docs-samples-tests"
+REGION = "us-central1"
+API_ENDPOINT = f"{REGION}-aiplatform.googleapis.com"
+
 # If a resource was updated within this number of seconds, do not delete.
 RESOURCE_UPDATE_BUFFER_IN_SECONDS = 60 * 60 * 8
 
@@ -129,12 +135,51 @@ class MatchingEngineIndexEndpointResourceCleanupManager(VertexAIResourceCleanupM
         resource.undeploy_all()
         resource.delete(force=True)
 
-class FeatureStoreCleanupManager(VertexAIResourceCleanupManager):
+class FeatureStoreLegacyCleanupManager(VertexAIResourceCleanupManager):
+    # TODO: only deleting legacy
+    #    not deleting ingestions jobs
+    #       ingest_from_xxx methods do not return a job ID, there is no list command, aka no python way to delete
+    #    not deleting batch serving jobs
+    #       batch_serve_to_xxx methods do not return a job ID, there is no list command, aka no python way to delete
     vertex_ai_resource = aiplatform.Featurestore
 
     def resource_name(self, resource: Any) -> str:
         return resource.name
 
+    def delete(self, resource):
+        resource.delete(force=True)
+
+
+class FeatureStoreCleanupManager(VertexAIResourceCleanupManager):
+    # for FS 2.0
+    # TODO: use _v1beta1, and gapic clients
+    #    delete features, feature groups, feature views, feature online stores
+    vertex_ai_resource = FeatureOnlineStore
+
+    admin_client = FeatureOnlineStoreAdminServiceClient(
+        client_options={"api_endpoint": API_ENDPOINT}
+    )
+
+    def resource_name(self, resource: Any) -> str:
+        return resource.name
+
+    def type_name(self) -> str:
+        return "FeatureOnlineStore"
+
+    def list(self) -> Any:
+        try:
+            return self.admin_client.list_feature_online_stores(parent=f"projects/{PROJECT_ID}/locations/{REGION}")
+        except Exception as e:
+            print(e)
+            return []
+
+    def delete(self, resource):
+        try:
+            self.admin_client.delete_feature_online_store(name=resource.name, force=True)
+        except Exception as e:
+            print(e)
+
+
 class PipelineJobCleanupManager(VertexAIResourceCleanupManager):
     vertex_ai_resource = aiplatform.PipelineJob
 

.cloud-build/execute_changed_notebooks_cli.py

@@ -18,6 +18,7 @@
 import argparse
 import pathlib
 import os
+import csv
 
 import execute_changed_notebooks_helper
 
@@ -37,7 +38,7 @@ parser = argparse.ArgumentParser(description="Run changed notebooks.")
 parser.add_argument(
     "--test_paths_file",
     type=pathlib.Path,
-    help="The path to the file that has newline-limited folders of notebooks that should be tested.",
+    help="The path to the file that has newline-delimited folders of notebooks that should be tested.",
     required=True,
 )
 parser.add_argument(
@@ -128,6 +129,20 @@ parser.add_argument(
     default=10,
     required=False,
 )
+parser.add_argument(
+    "--run_first_file",
+    type=pathlib.Path,
+    help="The path to the file that has newline-delimited of notebooks to run in the first batch",
+    default=None,
+    required=False,
+)
+parser.add_argument(
+    "--aiplatform_whl",
+    type=str,
+    help="The GCS path to a whl version google-cloud-aiplatform",
+    default=None,
+    required=False,
+)
 parser.add_argument(
     "--dry_run",
     type=str2bool,
@@ -157,6 +172,29 @@ else:
     accumulative_results = execute_changed_notebooks_helper.load_results(results_bucket, results_file)
 
     notebooks = [changed_notebook for changed_notebook in changed_notebooks if execute_changed_notebooks_helper.select_notebook(changed_notebook, accumulative_results, args.test_percent)]
+    # cap the number of notebooks to the specified percentage
+    max_notebooks = int((len(changed_notebooks) * (args.test_percent/100)))
+    if (len(notebooks) > max_notebooks):
+        notebooks = notebooks[:max_notebooks]
+
+run_first = []
+if args.run_first_file:
+    if not os.path.isfile(args.run_first_file):
+        print("Error: file does not exist", args.run_first_file)
+    else:
+        with open(args.run_first_file, 'r') as csvfile:
+            reader = csv.reader(csvfile)
+            for row in reader:
+                notebook = row[0]
+                run_first.append(notebook)
+
+    for notebook in run_first:
+        if notebook in notebooks:
+            # remove from existing list
+            notebooks.remove(notebook)
+            # add back to the front of the list
+            notebooks.insert(0, notebook)
+            print(f"Run first: {notebook}")
 
 if args.dry_run:
     print("Dry run ...\n")
@@ -177,4 +215,5 @@ else:
         variable_vpc_network=args.variable_vpc_network,
         private_pool_id=args.private_pool_id,
         concurrent_notebooks=args.concurrent_notebooks,
+        aiplatform_whl=args.aiplatform_whl
 )

.cloud-build/execute_changed_notebooks_helper.py

@@ -41,10 +41,13 @@ from utils import NotebookProcessors, util
 
 # A buffer so that workers finish before the orchestrating job
 WORKER_TIMEOUT_BUFFER_IN_SECONDS: int = 60 * 60
+
 PYTHON_VERSION = "3.9"  # Set default python version
 
 # rolling time window for accumulating build results for selecting notebooks
 MAX_RESULTS_AGE_SECONDS: int = (60 * 60) * 24 * 60  # 60 days
+# maximum time since last run to force a run on the current build
+MAX_AGE_BEFORE_FORCE_RUN: int = (60 * 60) * 24 * 30
 
 
 def format_timedelta(delta: datetime.timedelta) -> str:
@@ -118,8 +121,12 @@ def load_results(results_bucket: str,
                 if notebook in accumulative_results:
                     accumulative_results[notebook]['passed'] += build_results[notebook]['passed']
                     accumulative_results[notebook]['failed'] += build_results[notebook]['failed']
+                    if accumulative_results[notebook]['last_time_ran'] < time_created:
+                        accumulative_results[notebook]['last_time_ran'] = time_created
                 else:
                     accumulative_results[notebook] = build_results[notebook]
+                    accumulative_results[notebook]['failed_on_latest_run'] = build_results[notebook]['failed']
+                    accumulative_results[notebook]['last_time_ran'] = time_created
 
         print(accumulative_results)
     except Exception as e:
@@ -138,19 +145,37 @@ def select_notebook(changed_notebook: str,
     if changed_notebook in accumulative_results:
         pass_count = accumulative_results[changed_notebook]['passed']
         fail_count = accumulative_results[changed_notebook]['failed']
+        failed_on_latest_run = accumulative_results[changed_notebook]['failed_on_latest_run']
+        last_time_ran = accumulative_results[changed_notebook]['last_time_ran']
     else:
         pass_count = 1
         fail_count = 0
+        failed_on_latest_run = 0
+        last_time_ran = datetime.datetime.now().replace(tzinfo=None)
 
-    inferred_failure_rate = fail_count / (pass_count + fail_count)
+    # If notebook has not been ran in a long time, force running it
+    if (datetime.datetime.now().replace(tzinfo=None) - last_time_ran).total_seconds() > MAX_AGE_BEFORE_FORCE_RUN:
+        should_test_do_to_age = True
+    else:
+        should_test_do_to_age  = False
+
+
+    # if failed on the last time it was ran, select the notebook
+    if failed_on_latest_run:
+        inferred_failure_rate = 1
+    # otherwise, calculate the frequency of failure
+    else:
+        inferred_failure_rate = fail_count / (pass_count + fail_count)
 
     # If failure rate is high, the chance of testing should be higher
     should_test_due_to_failure = random.uniform(0, 1) <= inferred_failure_rate
 
+    #if accumulative_resultsi[changed_notebook]['latest_date_ran']
+
     # Additionally, only test a percentage of these
     should_test_due_to_random_subset = random.uniform(0, 1) <= (test_percent / 100)
 
-    if should_test_due_to_failure or should_test_due_to_random_subset:
+    if should_test_due_to_failure or should_test_due_to_random_subset or should_test_do_to_age:
         print(f"Selected: {changed_notebook}, {should_test_due_to_failure}, {should_test_due_to_random_subset}")
         return True
     else:
@@ -213,7 +238,7 @@ def _get_notebook_python_version(notebook_path: str) -> str:
 
             # Look for the python version specification pattern
             re_match = re.search(
-                "python version = (\d\.\d)", markdown, flags=re.IGNORECASE
+                "python version = (\d+\.\d+)", markdown, flags=re.IGNORECASE
             )
             if re_match:
                 # get the version number
@@ -246,7 +271,7 @@ def process_and_execute_notebook(
     private_pool_id: Optional[str],
     deadline: datetime.datetime,
     notebook: str,
-    should_get_tail_logs: bool = False,
+    should_get_tail_logs: bool = True,
 ) -> NotebookExecutionResult:
 
     print(f"Running notebook: {notebook}")
@@ -428,15 +453,39 @@ def _save_results(results: List[NotebookExecutionResult],
         else:
             pass_count = 0
             fail_count = 1
+        if result.error_message is None:
+            error_type = ''
+        elif '500 Internal' in result.error_message or 'INTERNAL' in result.error_message or 'internal error' in result.error_message:
+            error_type = 'INTERNAL'
+        elif 'context deadline exceeded' in result.error_message or 'TIMEOUT' in result.error_message:
+            error_type = 'TIMEOUT'
+        elif 'Quota' in result.error_message or 'quotas are exceeded' in result.error_message:
+            error_type = 'QUOTA'
+        elif 'ServiceUnavailable' in result.error_message:
+            error_type = 'SERVICEUNAVAILABLE'
+        elif 'ModuleNotFoundError' in result.error_message:
+            error_type = 'IMPORT'
+        elif result.is_pass:
+            error_type = ''
+        else:
+            error_type = 'undetermined'
+
+        if error_type != '':
+            log_url = result.log_url
+        else:
+            log_url = ''
+
         build_results[result.path] = {
                 'duration': result.duration.total_seconds(),
                 'start_time': str(result.start_time),
                 'passed': pass_count,
-                'failed': fail_count
+                'failed': fail_count,
+                'error_type': error_type,
+                'log_url': log_url
         }
         print(f"adding {result.path}")
 
-    print("Saving accumulative results ...")
+    print(f"Saving accumulative results to {results_file}, nentries {len(build_results)}")
     content = json.dumps(build_results)
 
     client = storage.Client()
@@ -459,6 +508,7 @@ def process_and_execute_notebooks(
     variable_vpc_network: Optional[str] = None,
     private_pool_id: Optional[str] = None,
     concurrent_notebooks: Optional[int] = 10,
+    aiplatform_whl: Optional[str] = None,
 ):
     """
     Run the notebooks that exist under the folders defined in the test_paths_file.
@@ -490,6 +540,7 @@ def process_and_execute_notebooks(
         timeout (str):
             Required. Timeout string according to https://cloud.google.com/build/docs/build-config-file-schema#timeout.
         concurrent_notebooks (int): Max number of notebooks per minute to run in parallel.
+        aiplatform_whl: alternate whl version of Vertex AI SDK to install
     """
 
     # Calculate deadline

.cloud-build/notebook-execution-test-cloudbuild.yaml

@@ -36,7 +36,7 @@ steps:
     - -c
     - |
       . workspace/env/bin/activate &&
-      python3 .cloud-build/execute_changed_notebooks_cli.py --test_paths_file "${_TEST_PATHS_FILE}" --base_branch "${_FORCED_BASE_BRANCH}" --container_uri ${_PYTHON_IMAGE} --staging_bucket ${_GCS_STAGING_BUCKET}  --artifacts_bucket ${_GCS_STAGING_BUCKET}/executed_notebooks/PR_${_PR_NUMBER}/BUILD_${BUILD_ID} --variable_project_id ${PROJECT_ID} --variable_region ${_GCP_REGION} --variable_service_account ${_GCP_SERVICE_ACCOUNT} --variable_vpc_network "${_GPC_VPC_NETWORK_NAME}" `if [ ! -z "${_PRIVATE_POOL_NAME}" ]; then echo "--private_pool_id ${_PRIVATE_POOL_NAME}"; fi` --build_id ${BUILD_ID} --test_percent=${_TEST_PERCENT} --concurrent_notebooks=${_CONCURRENT_NOTEBOOKS}
+      python3 .cloud-build/execute_changed_notebooks_cli.py --test_paths_file "${_TEST_PATHS_FILE}" --base_branch "${_FORCED_BASE_BRANCH}" --container_uri ${_PYTHON_IMAGE} --staging_bucket ${_GCS_STAGING_BUCKET}  --artifacts_bucket ${_GCS_STAGING_BUCKET}/executed_notebooks/PR_${_PR_NUMBER}/BUILD_${BUILD_ID} --variable_project_id ${PROJECT_ID} --variable_region ${_GCP_REGION} --variable_service_account ${_GCP_SERVICE_ACCOUNT} --variable_vpc_network "${_GCP_VPC_NETWORK_NAME}" `if [ ! -z "${_PRIVATE_POOL_NAME}" ]; then echo "--private_pool_id ${_PRIVATE_POOL_NAME}"; fi` --build_id ${BUILD_ID} --test_percent=${_TEST_PERCENT} --concurrent_notebooks=${_CONCURRENT_NOTEBOOKS} --run_first_file=${_RUN_FIRST_FILE}
     env:
     - 'IS_TESTING=1'
 timeout: 86400s

.cloud-build/run_first.csv

@@ -0,0 +1,8 @@
+notebooks/official/pipelines/google_cloud_pipeline_components_automl_text.ipynb
+notebooks/official/generative_ai/rlhf_tune_llm.ipynb
+notebooks/official/generative_ai/tune_peft.ipynb
+notebooks/official/prediction/llm_streaming_prediction.ipynb
+notebooks/official/migration/sdk-automl-text-classification-batch-prediction.ipynb
+notebooks/official/vizier/get_started_vertex_vizier.ipynb
+notebooks/official/workbench/sentiment_analysis/Sentiment_Analysis.ipynb
+notebooks/official/model_monitoring/get_started_with_model_monitoring_automl.ipynb

.cloud-build/test_failing.txt

@@ -1,40 +0,0 @@
-notebooks/official/training/pytorch_gcs_data_training.ipynb
-notebooks/official/custom/custom_training_tensorboard_profiler.ipynb
-notebooks/official/custom/custom-tabular-bq-managed-dataset.ipynb
-notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-locally-trained-models.ipynb
-notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb
-notebooks/official/tabnet/tabnet_vertex_tutorial.ipynb
-notebooks/official/tabnet/get_started_with_tabnet.ipynb
-notebooks/official/pipelines/google_cloud_pipeline_components_automl_text.ipynb
-notebooks/official/pipelines/multicontender_vs_champion_deployment_method.ipynb
-notebooks/official/pipelines/google_cloud_pipeline_components_automl_images.ipynb
-notebooks/official/pipelines/rapid_prototyping_bqml_automl.ipynb
-notebooks/official/pipelines/challenger_vs_blessed_deployment_method.ipynb
-notebooks/official/matching_engine/sdk_matching_engine_create_stack_overflow_embeddings.ipynb
-notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb
-notebooks/official/matching_engine/sdk_matching_engine_create_text_to_image_embeddings.ipynb
-notebooks/official/explainable_ai/sdk_custom_image_classification_online_explain.ipynb
-notebooks/official/explainable_ai/xai_image_classification_feature_attributions.ipynb
-notebooks/official/explainable_ai/sdk_custom_image_classification_batch_explain.ipynb
-notebooks/official/tabular_workflows/tabnet_on_vertex_pipelines.ipynb
-notebooks/official/model_registry/get_started_with_model_registry.ipynb
-notebooks/official/model_registry/bqml_vertexai_model_registry.ipynb
-notebooks/official/sdk/SDK_Custom_Training_Python_Package_Managed_Text_Dataset_Tensorflow_Serving_Container.ipynb
-notebooks/official/model_monitoring/batch_prediction_model_monitoring.ipynb
-notebooks/official/model_monitoring/get_started_with_model_monitoring_setup.ipynb
-notebooks/official/model_monitoring/get_started_with_model_monitoring_custom.ipynb
-notebooks/official/model_monitoring/get_started_with_model_monitoring_custom_tf_serving.ipynb
-notebooks/official/model_monitoring/model_monitoring.ipynb
-notebooks/official/tensorboard/tensorboard_profiler_custom_training_with_prebuilt_container.ipynb
-notebooks/official/tensorboard/tensorboard_hyperparameter_tuning_with_hparams.ipynb
-notebooks/official/tensorboard/tensorboard_profiler_custom_training.ipynb
-notebooks/official/model_evaluation/custom_tabular_regression_model_evaluation.ipynb
-notebooks/official/model_evaluation/custom_tabular_classification_model_evaluation.ipynb
-notebooks/official/model_evaluation/automl_video_classification_model_evaluation.ipynb
-notebooks/official/experiments/comparing_local_trained_models.ipynb
-notebooks/official/automl/automl_image_classification_online_online_prediction.ipynb
-notebooks/official/automl/automl-text-classification.ipynb
-notebooks/official/automl/sdk_automl_video_object_tracking_batch.ipynb
-notebooks/official/feature_store/sdk-feature-store-pandas.ipynb
-notebooks/official/prediction/custom_batch_prediction_feature_filter.ipynb
-notebooks/official/prediction/pytorch_image_classification_with_prebuilt_serving_containers.ipynb

.cloud-build/test_passing.txt

@@ -1,80 +0,0 @@
-notebooks/official/training/hyperparameter_tuning_tensorflow.ipynb
-notebooks/official/training/get_started_with_vertex_distributed_training.ipynb
-notebooks/official/training/hyperparameter_tuning_xgboost.ipynb
-notebooks/official/training/multi_node_ddp_gloo_vertex_training_with_custom_container.ipynb
-notebooks/official/training/distributed_hyperparameter_tuning.ipynb
-notebooks/official/training/pytorch-text-sentiment-classification-custom-train-deploy.ipynb
-notebooks/official/training/xgboost_data_parallel_training_on_cpu_using_dask.ipynb
-notebooks/official/training/multi_node_ddp_nccl_vertex_training_with_custom_container.ipynb
-notebooks/official/bigquery_ml/get_started_with_bqml_training.ipynb
-notebooks/official/bigquery_ml/bqml-online-prediction.ipynb
-notebooks/official/custom/custom_training_container_and_model_registry.ipynb
-notebooks/official/custom/sdk-custom-image-classification-online.ipynb
-notebooks/official/custom/sdk-custom-image-classification-batch.ipynb
-notebooks/official/custom/SDK_FBProphet_Forecasting_Online.ipynb
-notebooks/official/custom/get_started_vertex_training_xgboost.ipynb
-notebooks/official/custom/get_started_with_vertex_endpoint_and_shared_vm.ipynb
-notebooks/official/custom/SDK_Custom_Container_Prediction.ipynb
-notebooks/official/reduction_server/pytorch_distributed_training_reduction_server.ipynb
-notebooks/official/tabnet/ai-explanations-tabnet-algorithm.ipynb
-notebooks/official/vizier/get_started_vertex_vizier.ipynb
-notebooks/official/vizier/gapic-vizier-multi-objective-optimization.ipynb
-notebooks/official/pipelines/get_started_with_hpt_pipeline_components.ipynb
-notebooks/official/pipelines/google_cloud_pipeline_components_automl_tabular.ipynb
-notebooks/official/pipelines/custom_tabular_train_batch_pred_bq_pipeline.ipynb
-notebooks/official/pipelines/metrics_viz_run_compare_kfp.ipynb
-notebooks/official/pipelines/google_cloud_pipeline_components_model_upload_predict_evaluate.ipynb
-notebooks/official/pipelines/google_cloud_pipeline_components_model_train_upload_deploy.ipynb
-notebooks/official/pipelines/get_started_with_machine_management.ipynb
-notebooks/official/pipelines/custom_model_training_and_batch_prediction.ipynb
-notebooks/official/pipelines/control_flow_kfp.ipynb
-notebooks/official/pipelines/lightweight_functions_component_io_kfp.ipynb
-notebooks/official/pipelines/google_cloud_pipeline_components_bqml_text.ipynb
-notebooks/official/pipelines/pipelines_intro_kfp.ipynb
-notebooks/official/pipelines/automl_tabular_classification_beans.ipynb
-notebooks/official/pipelines/google_cloud_pipeline_components_dataproc_tabular.ipynb
-notebooks/official/explainable_ai/sdk_automl_tabular_classification_online_explain.ipynb
-notebooks/official/explainable_ai/sdk_custom_tabular_regression_online_explain.ipynb
-notebooks/official/explainable_ai/sdk_automl_tabular_binary_classification_batch_explain.ipynb
-notebooks/official/explainable_ai/sdk_custom_tabular_regression_online_explain_get_metadata.ipynb
-notebooks/official/explainable_ai/sdk_custom_tabular_regression_batch_explain.ipynb
-notebooks/official/tabular_workflows/prophet_on_vertex_pipelines.ipynb
-notebooks/official/tabular_workflows/wide_and_deep_on_vertex_pipelines.ipynb
-notebooks/official/sdk/SDK_AutoML_Video_Classification.ipynb
-notebooks/official/model_monitoring/get_started_with_model_monitoring_automl.ipynb
-notebooks/official/model_monitoring/get_started_with_model_monitoring_automl_image_batch.ipynb
-notebooks/official/model_monitoring/get_started_with_model_monitoring_automl_image_online.ipynb
-notebooks/official/model_monitoring/get_started_with_model_monitoring_xgboost.ipynb
-notebooks/official/tensorboard/tensorboard_custom_training_with_custom_container.ipynb
-notebooks/official/tensorboard/tensorboard_custom_training_with_prebuilt_container.ipynb
-notebooks/official/tensorboard/tensorboard_vertex_ai_pipelines_integration.ipynb
-notebooks/official/model_evaluation/automl_text_classification_model_evaluation.ipynb
-notebooks/official/model_evaluation/get_started_with_custom_model_evaluation_import.ipynb
-notebooks/official/model_evaluation/automl_tabular_classification_model_evaluation.ipynb
-notebooks/official/model_evaluation/automl_tabular_regression_model_evaluation.ipynb
-notebooks/official/experiments/get_started_with_vertex_experiments.ipynb
-notebooks/official/experiments/comparing_pipeline_runs.ipynb
-notebooks/official/experiments/get_started_with_vertex_experiments_autologging.ipynb
-notebooks/official/experiments/build_model_experimentation_lineage_with_prebuild_code.ipynb
-notebooks/official/experiments/delete_outdated_tensorboard_experiments.ipynb
-notebooks/official/automl/sdk_automl_tabular_regression_batch_bq.ipynb
-notebooks/official/automl/sdk_automl_text_sentiment_analysis_online.ipynb
-notebooks/official/automl/sdk_automl_text_entity_extraction_online.ipynb
-notebooks/official/automl/sdk_automl_forecasting_hierarchical_batch.ipynb
-notebooks/official/automl/automl_text_entity_extraction_batch_prediction.ipynb
-notebooks/official/automl/automl_image_classification_batch_prediction.ipynb
-notebooks/official/automl/automl_text_sentiment_analysis_batch_prediction.ipynb
-notebooks/official/automl/sdk_automl_tabular_regression_online_bq.ipynb
-notebooks/official/automl/get_started_automl_training.ipynb
-notebooks/official/automl/automl-tabular-classification.ipynb
-notebooks/official/automl/automl_image_object_detection_export_edge.ipynb
-notebooks/official/automl/sdk_automl_image_object_detection_batch.ipynb
-notebooks/official/automl/automl_tabular_on_vertex_pipelines.ipynb
-notebooks/official/automl/sdk_automl_video_classification_batch.ipynb
-notebooks/official/automl/sdk_automl_video_action_recognition_batch.ipynb
-notebooks/official/automl/sdk_automl_tabular_forecasting_batch.ipynb
-notebooks/official/automl/automl_image_object_detection_online_prediction.ipynb
-notebooks/official/automl/automl_forecasting_bqml_arima_plus_comparison.ipynb
-notebooks/official/datasets/get_started_bq_datasets.ipynb
-notebooks/official/datasets/get_started_with_data_labeling.ipynb
-notebooks/official/feature_store/feature_store_streaming_ingestion_sdk.ipynb

.cloud-build/utils/NotebookProcessors.py

@@ -35,7 +35,7 @@ class RemoveNoExecuteCells(Preprocessor):
 
 
 class UpdateVariablesPreprocessor(Preprocessor):
-    def __init__(self, replacement_map: Dict):
+    def __init__(self, replacement_map: Dict[str, str]):
         self._replacement_map = replacement_map
 
     @staticmethod
@@ -98,3 +98,28 @@ class UniqueStringsPreprocessor(Preprocessor):
             executable_cells.append(cell)
         notebook.cells = executable_cells
         return notebook, resources
+
+class VertexAIInstallProprocessor(Preprocessor):
+    def __init__(self, vertex_ai_wheel):
+        self.vertex_ai_wheel = vertex_ai_wheel
+
+    @staticmethod
+    def update_vertex_ai_install(content: str):
+        if "google-cloud-aiplatform" not in content:
+            return content
+        return (
+            f"gsutil cp {self.vertex_ai_wheel} google-cloud-aiplatform.whl\n" + 
+            content.replace("google-cloud-aiplatform\n", "google-cloud-aiplatform.whl\n")
+            .replace("google-cloud-aiplatform ", "google-cloud-aiplatform.whl ")
+        )
+
+    def preprocess(self, notebook, resources=None):
+        executable_cells = []
+        for cell in notebook.cells:
+            if cell.cell_type == "code":
+                cell.source = self.update_vertex_ai_install(
+                    content=cell.source,
+                )
+
+            executable_cells.append(cell)
+        notebook.cells = executable_cells

.cloud-build/utils/build_results_viewer.py

@@ -5,18 +5,69 @@ Cloud Storage location: gs://cloud-build-notebooks-presubmit/build_results/
 '''
 import argparse
 import json
+from  util import download_file
+import csv
+import datetime
+from google.cloud import storage
+
+BUILD_BUCKET = "cloud-build-notebooks-presubmit"
+BUILD_FOLDER = "build_results"
 
 
 parser = argparse.ArgumentParser()
 parser.add_argument('--file', dest='file',
-                    default='build.json', type=str, help='build results file')
-import json
+                    default=None, type=str, help='build results filei (local or GCS)')
+args = parser.parse_args()
 
-with open('build.json', 'r') as f:
+investigate = {}
+with open('investigate.csv', 'r') as csvfile:
+    reader = csv.reader(csvfile)
+    for row in reader:
+        investigate[row[0][:-6]] = row[1]
+
+if not args.file:
+    client = storage.Client()
+    blobs = client.list_blobs(BUILD_BUCKET, prefix=BUILD_FOLDER)
+    newest_time = datetime.datetime(2000, 1, 1)
+    for blob in blobs:
+        # individual PR
+        if blob.size < 2000:
+            continue
+        time_created = blob.time_created.replace(tzinfo=None)
+        if time_created > newest_time:
+            newest_time = time_created
+            args.file = f"gs://{BUILD_BUCKET}/{blob.name}"
+
+if args.file.startswith("gs://"):
+    path = args.file[5:]
+    bucket = path.split('/')[0]
+    file   = path[len(bucket)+1:]
+    download_file(bucket, file, "build.json")
+    args.file = "build.json"
+
+with open(args.file, 'r') as f:
     results = json.load(f)
 
 for item in results.items():
+    notebook = item[0][len("/notebooks/official/")-1:-6]
     if item[1]['passed']:
-        print(f"{item[0]},PASSED")
+        passed = "PASS"
     else:
-        print(f"{item[0]},FAILED")
+        if notebook in investigate:
+            passed = "INVG"
+        else:
+            passed = "FAIL"
+      
+    error = item[1]['error_type']
+  
+    if passed == "FAIL":
+        if error == '':
+            error = "undetermined"
+        if 'log_url' in item[1]:
+            log_url = item[1]['log_url']
+        else:
+            log_url = ''
+    else:
+        log_url = ''
+
+    print(f"{notebook:75} {passed} {error:10} {log_url}")

.cloud-build/utils/investigate.csv

@@ -0,0 +1,19 @@
+notebook,status
+prediction/llm_streaming_prediction.ipynb,wait_for_fix
+custom/get_started_with_vertex_endpoint_and_shared_vm.ipynb,issue 2527
+feature_store/online_feature_serving_and_fetching_bigquery_data_with_feature_store.ipynb,wait_for_reaper
+feature_store/online_feature_serving_and_vector_retrieval_bigquery_data_with_feature_store.ipynb,wait_for_reaper
+pipelines/google_cloud_pipeline_components_TPU_model_train_upload_deploy.ipynb,wait_for_fix
+explainable_ai/sdk_custom_image_classification_batch_explain.ipynb,issue 2528
+explainable_ai/sdk_custom_image_classification_online_explain.ipynb,issue 2528
+explainable_ai/sdk_custom_tabular_regression_online_explain.ipynb,issue 2528
+explainable_ai/sdk_automl_tabular_binary_classification_batch_explain.ipynb,issue 2528
+explainable_ai/xai_image_classification_feature_attributions.ipynb,issue 2528
+matching_engine,sdk_matching_engine_create_stack_overflow_embeddings.ipynb,issue 2530
+automl/automl_forecasting_bqml_arima_plus_comparison.ipynb,flaky
+model_evaluation/custom_tabular_regression_model_evaluation.ipynb,regr
+experiments/get_started_with_vertex_experiments.ipynb,regr
+experiments/comparing_local_trained_models.ipynb,regr
+generative_ai/tune_peft.ipynb,internal
+pipelines/custom_model_training_and_batch_prediction.ipynb,regr
+feature_store/online_feature_serving_and_fetching_bigquery_data_with_feature_store_optimized.ipynb,wait_for_reaper

.cloud-build/web-index/do_not_index.txt

@@ -0,0 +1,11 @@
+sdk2_remote_tabnet_training.ipynb
+remote_hyperparameter_tuning.ipynb
+remote_prediction.ipynb
+remote_training_bigframes_pytorch.ipynb
+remote_training_bigframes_sklearn.ipynb
+remote_training_bigframes_tensorflow.ipynb
+remote_training_lightning.ipynb
+remote_training_pytorch.ipynb
+remote_training_sklearn.ipynb
+remote_training_tensorflow_with_autologging.ipynb
+

.cloud-build/web-index/notebook-index-cloudbuild.yaml

@@ -15,7 +15,7 @@ steps:
     - -c
     - |
       . workspace/env/bin/activate &&
-      python3 notebooks/notebook_template_review.py --web --title --steps --desc --linkback --notebook-dir=notebooks/official >web.html
+      python3 notebooks/notebook_template_review.py --web --title --steps --desc --linkback --notebook-dir=notebooks/official --skip-file=${_DO_NOT_INDEX_FILE} >web.html
 artifacts:
   objects:
           location: gs://${_GCS_ARTIFACTS_BUCKET}/webdoc

.github/workflows/ci.yaml

@@ -7,11 +7,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: '3.x'      
     - name: Fetch pull request branch
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         fetch-depth: 0      
     - name: Fetch base main branch

.github/workflows/linter/Dockerfile

@@ -4,7 +4,7 @@
 # 2. To lint specific notebooks:
 # docker run -v ${PWD}:/setup/app gcr.io/python-docs-samples-tests/notebook_linter:latest notebooks/1.ipynb notebooks/2.ipynb
 
-FROM python:3.10
+FROM python:3.12
 
 WORKDIR setup
 

.github/workflows/linter/requirements.txt

@@ -2,9 +2,9 @@ git+https://github.com/tensorflow/docs
 ipython
 jupyter
 nbconvert
-black==23.3.0
-pyupgrade==3.7.0
-isort==5.12.0
-flake8==6.0.0
-nbqa==1.7.0
+black==24.4.2
+pyupgrade==3.15.2
+isort==5.13.2
+flake8==7.0.0
+nbqa==1.8.5
 

community-content/CODEOWNERS

@@ -10,14 +10,22 @@
 /pipeline_components @Ark-kun
 /pipeline_components/image_ml_model_training @lakeyk
 /prediction_featurestore_integration @googleapis/vertex-prediction-team
-/vertex_vision_model_garden/model_oss/util @weigary
-/vertex_vision_model_garden/model_oss/diffusers @weigary
-/vertex_vision_model_garden/model_oss/keras @dstnluong-google
-/vertex_vision_model_garden/model_oss/transformers @dstnluong-google
-/vertex_vision_model_garden/model_oss/pic2word @jismailyan-google
-/vertex_vision_model_garden/model_oss/open_clip @lydhr
-/vertex_vision_model_garden/model_oss/movinet @KCFindstr
-/vertex_vision_model_garden/model_oss/data_converter @KCFindstr
-/vertex_vision_model_garden/model_oss/peft @weigary
-/vertex_vision_model_garden/model_oss/lm-evaluation-harness @kathyyu-google
+/vertex_model_garden/model_oss/notebook_util @minwoo33park
+/vertex_model_garden/model_oss/util @weigary
+/vertex_model_garden/model_oss/diffusers @weigary
+/vertex_model_garden/model_oss/keras @dstnluong-google
+/vertex_model_garden/model_oss/transformers @dstnluong-google
+/vertex_model_garden/model_oss/pic2word @jismailyan-google
+/vertex_model_garden/model_oss/open_clip @lydhr
+/vertex_model_garden/model_oss/movinet @KCFindstr
+/vertex_model_garden/model_oss/data_converter @KCFindstr
+/vertex_model_garden/model_oss/peft @weigary
+/vertex_model_garden/model_oss/lm-evaluation-harness @kathyyu-google
+/vertex_model_garden/model_oss/tfvision @dstnluong-google
+/vertex_model_garden/model_oss/fvlm @minwoo33park
+/vertex_model_garden/model_oss/imagebind @kathyyu-google
+/vertex_model_garden/model_oss/llava @py4
+/vertex_model_garden/model_oss/vllm @kathyyu-google
+/vertex_model_garden/benchmarking_reports @lavraicse
+/vertex_model_garden/model_oss/autogluon @lavraicse
 

community-content/cpr-examples/timm_serving/requirements.txt

@@ -1,5 +1,5 @@
 absl-py==1.1.0
-fastapi==0.75.2
+fastapi==0.109.1
 uvicorn==0.18.2
 timm==0.5.4
 smart_open==6.0.0

community-content/tf_agents_bandits_movie_recommendation_with_kfp_and_vertex_sdk/mlops_pipeline_tf_agents_bandits_movie_recommendation/src/logger/requirements.txt

@@ -1,4 +1,4 @@
 google-cloud-bigquery==2.20.0
 tensorflow==2.7.2
-pillow==9.0.1
+pillow==10.3.0
 tf-agents==0.8.0

community-content/tf_agents_bandits_movie_recommendation_with_kfp_and_vertex_sdk/mlops_pipeline_tf_agents_bandits_movie_recommendation/src/prediction_container/requirements.txt

@@ -1,4 +1,4 @@
 google-cloud-pubsub==2.5.0
-pillow==9.0.1
+pillow==10.3.0
 tf-agents==0.8.0
 tensorflow==2.7.2

community-content/tf_agents_bandits_movie_recommendation_with_kfp_and_vertex_sdk/mlops_pipeline_tf_agents_bandits_movie_recommendation/src/simulator/requirements.txt

@@ -1,5 +1,5 @@
 dataclasses==0.6
 google-cloud-aiplatform==1.8.1
 tensorflow==2.7.2
-pillow==9.0.1
+pillow==10.3.0
 tf-agents==0.8.0

community-content/vertex_model_garden/benchmarking_reports/jax_vit_benchmarking_report.md

@@ -0,0 +1,253 @@
+# ViT PyTorch vs JAX training benchmarks on Vertex AI Training Platform
+
+Lav Rai, Software Engineer, Google Cloud
+
+Xiang Xu, Software Engineer, Google Cloud
+
+Andreas Steiner, Software Engineer, Google DeepMind
+
+Tao Wang, Software Engineer, Google DeepMind
+
+Alexander Kolesnikov, Research Engineer, Google DeepMind
+
+## Introduction
+
+Many repositories now offer both PyTorch and JAX versions of a model. For
+example, [Hugging Face offers many models such as GPT2, BERT][1]
+etc. Other examples are [OpenLLaMa][2] and [ViT][3]
+models which were first developed in JAX and then their corresponding PyTorch
+versions were made available. **Given both the PyTorch and JAX options for a
+model, it may not be obvious as to which option to choose**. To make such a
+decision, it is important for one to know about the training cost, effectiveness
+and efficiency for each choice.
+
+Apart from the framework choice, the other choice that one faces on Vertex AI
+training platform is the type and count of the accelerators. Although the
+[Vertex AI pricing table][4] lists the price per hour for each
+machine, **one may not know beforehand about the training speed of JAX and
+PyTorch frameworks for different types and count of the accelerators**.
+
+If one has access to some training benchmark numbers for the same model
+under (a) PyTorch and JAX frameworks and (b) for different types and count of
+the accelerators, then it will be easier for them to make a cost effective
+decision. Such a benchmark will also aid the developers in identifying strength
+and weakness of different choices and then figure out recipes to remove those
+weaknesses if possible.
+
+This blog uses the ViT [classification models][5] of varying sizes
+to benchmark the training performance of PyTorch and JAX versions on the Vertex
+AI Platform under different machine configurations. The goal is to:
+
+- Benchmark OSS ViT training for both PyTorch and JAX frameworks.
+- Benchmark OSS ViT L16, H14, g14, and G14 models.
+- Benchmark OSS ViT PyTorch training with A100 GPUs.
+- Benchmark OSS ViT JAX training with A100 GPUs and TPU V3 accelerators.
+
+## Benchmarking setup
+
+This section lays out the benchmarking set up for the [PyTorch][6] and [JAX][7]
+frameworks and provides a reasoning for choosing those settings.
+
+### PyTorch GPU
+
+#### Machine configuration
+
+We run training jobs on [Vertex AI Custom Training][8] using 1
+single node with 8 A100-40GB GPUs.
+
+- Machine type: [a2-highgpu-8g][9]
+- Machine count: 1
+- Accelerator type: [NVIDIA_TESLA_A100 (40GB)][10]
+- Accelerator count: 8
+
+#### Modeling
+
+We benchmark 4 variants of ViT model in different sizes:
+
+- [ViT-L16, 300M params][11]
+- [ViT-H14, 630M params][12]
+- [ViT-g14, 1B params][13]
+- [ViT-G14, 1.8B params][14]
+
+We use the Huggingface [transformers library][15] for ViT L16 and
+H14 variants, and the [TIMM library][16] for ViT g14 and G14
+variants.
+
+#### Dataset
+
+We run training against the [cifar10][17] dataset with 50K training
+images and 10K test images. To factor out network communication overhead for
+data loading, we copy the whole dataset to the local disk then load data from
+the local disk during training.
+
+#### Training parameters
+
+- Trainer
+  - We use [PyTorch Lightning][18] as the trainer for the
+    boilerplate data loading and train loop coding.
+- Precision
+  - Float16
+- Input resolution
+  - 224 x 224
+- Strategy
+  - We use [DDP][19] for models which can be entirely loaded to one
+    GPU, use [Deepspeed-ZeRO][20] otherwise:
+    - ViT-L16: DDP
+    - ViT-H14: DDP
+    - ViT-g14: DDP
+    - ViT-G14: Deepspeed-ZeRO stage-3
+- Batch size
+  - We use the max batch size as power of 2 without CUDA OOM for each model:
+    - ViT-L16: 64 per GPU
+    - ViT-H14: 16 per GPU
+    - ViT-g14: 16 per GPU
+    - ViT-G14: 32 per GPU
+- Compilation
+  - We apply [torch.compile][21] to model whenever it's applicable:
+    - ViT-L16: torch.compile
+    - ViT-H14: torch.compile
+    - ViT-g14: torch.compile
+    - ViT-G14: N/A
+
+### JAX TPU and GPU
+
+#### Machine configuration
+
+All the TPU and GPU training jobs are run on [Vertex AI Custom
+Training][8]. The following machine configurations were used for the
+TPU and GPU experiments:
+
+**Note**: TPU V3 POD requires multi-host supporting training code. For example,
+a 32 core POD runs on 4 hosts with each host using 8 cores.
+
+**Note**: 8 A100 are similar to TPU V3 32 cores in terms of [Vertex AI
+pricing][4].
+
+**Note**: [Each TPU v3 chip has 2 cores which can use 32 GB high-bandwidth
+memory][22] (16 GB per core) so total memory for 32 cores is 16x32 =
+512 GB. Therefore for the same price, TPUs offer more memory than 8 A100-40GB
+GPUs.
+
+#### Modeling
+
+We decided to use an OSS code repository for model implementation. Using an OSS
+repository helps anyone to independently verify the benchmarking results and
+also relate to the results well. For JAX, we selected the
+[Big Vision][23] code repository.
+
+Same as the PyTorch modeling, we benchmark 4 variants of ViT model in different
+sizes:
+
+- [ViT-L16, 300M params][24]
+- [ViT-H14, 630M params][24]
+- [ViT-g14, 1B params][24]
+- [ViT-G14, 1.8B params][24]
+
+**Note**: The [Big Vision code repo][23] has not made the
+checkpoints publicly available for the models larger than the ViT-L16. Therefore
+for the rest of the three variants, the experiments only used random
+initialization for benchmarking the training speed.
+
+#### Dataset
+
+We use training against the [cifar10 TensorFlow dataset][25] with
+50K training images and 10K test images. This dataset is the same as the one
+used for PyTorch experiments except that it is loaded as a TensorFlow dataset.
+Similar to the PyTorch experiments, we copy the whole dataset to the docker
+image to factor out network communication overhead for data loading.
+
+#### Training parameters
+
+- Precision
+  - "bfloat16" setting was used.
+- Input resolution
+  - 224 x 224 after resize (to 448x448) and random crop (to 224x224) before
+    training.
+    - This resolution for training was the same as the PyTorch settings.
+- Strategy
+  - Used DDP for all models except ViT-G14. ViT-G14 used the FSDP strategy.
+- Batch size
+  - We use the max batch size as power of 2 without OOM for each model. The
+    [Benchmarking results][26] section shows the final
+    batch size for each experiment.
+  - Once a maximum batch-size for TPU V3 8 cores was determined, we just scaled
+    it linearly for 32 cores.
+  - Once a maximum batch-size for 1 A100 GPU was determined, we just scaled it
+    linearly for 8 A100 GPUs.
+- Compilation
+  - [jax.jit() compilation][27] is used in JAX codes for efficient
+    execution in XLA.
+- GPU related flags
+  - The following flags are set in the dockerfile for the GPU runs.
+    - Note: _xla_gpu_enable_pipelined_collectives_ is set to false for the
+      ViT-G14 FSDP run.
+
+### Evaluation metric
+
+For both the PyTorch and JAX experiments, the following evaluation metrics are
+collected:
+
+- Throughput: Images-per-second observed for training.
+- Cost: The training-cost-per-epoch (USD).
+
+**Note**: The above metrics are not biased against any framework or machine
+configurations. In addition, these metrics will help one decide the most
+efficient training configurations on Vertex AI.
+
+## Benchmarking results
+
+The lowest cost experiment for each model is marked in **bold** in the last
+column.
+
+![vit_benchmarking_table](images/vit_benchmarking_table.png)
+
+The following bar charts summarize the performance visually:
+
+![vit_training_time](images/vit_training_time.png)
+
+![vit_training_cost](images/vit_training_cost.png)
+
+The following section provides observations and conclusions for these results.
+
+## Observation and Conclusions
+
+- Training with JAX TPU V3 POD with 32 cores costs 33% less than the PyTorch GPU
+  8 A100-40GBs runs.
+- Training with JAX GPU 8 A100-40GBs costs 23% less than the PyTorch GPU 8
+  A100-40GBs runs.
+- JAX TPU V3 POD with 32 cores was 4x faster and slightly more cost-effective
+  than the JAX TPU V3 8 core run for the ViT-large model. This indicates that it
+  might be better to use more cores. The JAX TPU V3 speed scales very well with
+  the number of cores.
+- Cloud TPU VM training speed numbers were the same as the Vertex AI for
+  TPU V3 8 cores. The dataset was copied to the docker in both the cases.
+- The training-cost-per-epoch increases with the model size irrespective of the
+  framework.
+
+[1]: https://github.com/huggingface/transformers/blob/main/examples/research_projects/jax-projects/README.md#quickstart-flax-and-jax-in-transformers
+[2]: https://github.com/openlm-research/open_llama
+[3]: https://github.com/google-research/vision_transformer
+[4]: https://cloud.google.com/vertex-ai/pricing#custom-trained_models
+[5]: https://arxiv.org/abs/2010.11929
+[6]: #pytorch-gpu
+[7]: #jax-tpu-and-gpu
+[8]: https://cloud.google.com/vertex-ai/docs/training/overview
+[9]: https://cloud.google.com/vertex-ai/docs/training/configure-compute#machine-types
+[10]: https://cloud.google.com/vertex-ai/docs/training/configure-compute#specifying_gpus
+[11]: https://huggingface.co/google/vit-large-patch16-224-in21k
+[12]: https://huggingface.co/google/vit-huge-patch14-224-in21k
+[13]: https://github.com/huggingface/pytorch-image-models/blob/v0.9.2/timm/models/vision_transformer.py#L1308
+[14]: https://github.com/huggingface/pytorch-image-models/blob/v0.9.2/timm/models/vision_transformer.py#L1312
+[15]: https://huggingface.co/docs/transformers/main/model_doc/vit#transformers.ViTModel
+[16]: https://github.com/huggingface/pytorch-image-models
+[17]: https://huggingface.co/datasets/cifar10
+[18]: https://lightning.ai/docs/pytorch/stable/
+[19]: https://pytorch.org/docs/stable/notes/ddp.html
+[20]: https://www.deepspeed.ai/tutorials/zero/
+[21]: https://pytorch.org/tutorials/intermediate/torch_compile_tutorial.html
+[22]: https://cloud.google.com/tpu/docs/system-architecture-tpu-vm#tpu_v3
+[23]: https://github.com/google-research/big_vision
+[24]: https://screenshot.googleplex.com/BximJgxsgvBVu38
+[25]: https://www.tensorflow.org/datasets/catalog/cifar10
+[26]: #benchmarking-results
+[27]: https://jax.readthedocs.io/en/latest/jax-101/02-jitting.html

community-content/vertex_model_garden/benchmarking_reports/pytorch_openllama_7b_finetune_benchmark_report.md

@@ -0,0 +1,227 @@
+# Benchmark report on fine tuning the OpenLLaMA 7B model on Google Cloud Vertex Model Garden
+
+Gary Wei, Software Engineer, Google Cloud
+Dustin Luong, Software Engineer, Google Cloud
+Changyu Zhu, Software Engineer, Google Cloud
+Genquan Duan, Software Engineer, Google Cloud
+
+## Introduction
+
+Fine-tuning of LLMs can be non-trivial to find an optimal configuration of
+machine types, training parameters, and other hyperparameters that achieves a
+good balance between cost efficiency and model performance. To facilitate users
+in conducting tuning experiments, this report benchmarks OpenLLaMA 7B
+fine-tuning on Google Cloud Vertex Model Garden, demonstrating both efficiency
+and effectiveness. The observations are general and can be applied to other LLM
+models.
+
+We benchmarked fine tuning algorithms [LoRA](https://arxiv.org/abs/2106.09685)
+and [QLoRA](https://arxiv.org/abs/2305.14314) supported by
+[huggingface PEFT libraries](https://github.com/huggingface/peft). LoRA, short
+for Low-Rank Adaptation of Large Language Models, is an improved fine tuning
+method where instead of fine tuning all the weights that constitute the weight
+matrix of the pre-trained large language model, two smaller matrices that
+approximate this larger matrix are fine-tuned. QLoRA is an even more
+memory-efficient version of LoRA, where the pretrained model is loaded to GPU
+memory as quantized 4-bit weights, while preserving similar effectiveness to
+LoRA. We also provide simple scripts and parameter settings to reproduce the
+results reported in this report.
+
+In general, there are many factors that affect the performance of fine-tuning
+experiments, such as hardware settings, parameters, cost, and accuracy. It is
+impractical to obtain benchmarks for all possible combinations of these factors.
+Instead, we focus on tuning a subset of related parameters and evaluating their
+impact on a set of chosen metrics. The evaluation metrics are GPU memory usage,
+percentage of parameters tuned, tuning speed, cost, and accuracy. The tuning
+parameters are batch size, lora rank, maximum sequence length, and maximum
+training steps.
+
+## Key takeaways
+
+-   **Use QLoRA to minimize the peak GPU requirements**: The QLoRA can
+    significantly reduce the peak GPU memory usage by ~75% compared to LoRA. For
+    OpenLLaMA7b, the peak memory is ~28G for LoRA and ~7G for QLoRA.
+-   **Use LoRA to maximize the tuning speed and minimize the tuning cost**: LoRA
+    is ~66% faster than QLoRA in fine tuning speed. LoRA/QLoRA tuning cost is
+    low generally, while LoRA is even ~40% cheaper than QLoRA with the same
+    parameters. Suggest to use QLoRA for limited GPU memories, and LoRA for
+    limited training budgets. For OpenLLaMA7b, the tuning speed for LoRA/QLoRA
+    ~5 samples / 3 samples per second, and the tuning cost for LoRA/QLoRA in 500
+    steps is ~$1/$1.7 on `a2-highgpu-1g` with 1 A100 40G GPU. The tuning cost
+    for QLoRA in 500 steps is $6.75 on n1-standard-8 with 1 V100 GPU, while LoRA
+    could not run because of OOM.
+-   **Use QLoRA to tune models with large sequence lengths**. For OpenLLaMA7b,
+    the max sequence length for QLoRA can be 2048 when consuming 16.3G GPU,
+    while the max sequence length for LoRA is 512 when consuming 28.2G GPU, and
+    encounter OOM when max sequence length is 1024.
+-   **Both LoRA and QLoRA give similar accuracy improvement after fine tuning.**
+    For OpenLLaMA7b, both LoRA/QLoRA can improve the average accuracy by ~4%
+    evaluating on 3 typical tasks (ARC challenge, HellaSwag and TruthfulQA),
+    after training 1875 steps on dataset
+    [timdettmers/openassistant-guanaco](https://huggingface.co/datasets/timdettmers/openassistant-guanaco).
+-   **Use a big batch size if GPU memory is not a constraint**. For OpenLLaMA7b
+    with other default parameters, we suggest using a batch size as 24 for
+    QLoRA, but 2 for LoRA when tuning with 1 A100 40G. We also suggest using a
+    batch size as 8 for QLoRA when tuning with 1 V100. Tuning with LoRA and
+    batch size as 1 got OOM and we don't recommend tuning LoRA with 1 V100.
+
+## Benchmark Details
+
+### Experiment Setup
+
+The benchmark dataset is
+[timdettmers/openassistant-guanaco](https://huggingface.co/datasets/timdettmers/openassistant-guanaco).
+The training dataset is directly downloaded from hugging face to the VM, before
+every experiment.
+
+The default tuning parameters during benchmark are:
+
+-   Host VM: a2-highgpu-1g
+-   Accelerator type: 1 A100 40G
+-   batch size: 2
+-   lora_rank: 16
+-   max_seq_length: 512
+-   precision_mode: float16
+-   max_train_steps: 500
+
+For simplicity, we set the precision mode to `float16` when tuning LoRA models,
+and set the precision to `4bit` for QLoRA.
+
+Sample script to start fine tuning dockers in a VM on GCP.
+
+```shell
+IMAGE_TAG=us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-peft-train:latest
+docker run  --runtime=nvidia  -e NVIDIA_VISIBLE_DEVICES=0 \
+--rm --name "test_gpu"  -it --pull=always ${IMAGE_TAG} \
+--task=instruct-lora \
+--pretrained_model_id=openlm-research/open_llama_7b \
+--dataset_name="timdettmers/openassistant-guanaco"  \
+--instruct_column_in_dataset="text" \
+--precision_mode="float16" \
+--output_dir=<OUTPUT DIR>  \
+--lora_rank=2 \
+--max_sequence_length=512 \
+--learning_rate=2e-4 \
+--max_steps=50
+```
+
+### GPU Memory
+
+In this benchmark, we investigated the impact of batch size, lora rank, and
+maximum sequence length on GPU memory, and then made recommendations on the
+maximum batch size for different GPUs.
+
+#### Peak GPU memory by batch size (GB)
+
+<img src="images/openllama_7b_fine_tune_benchmark_report/openllama-7b-peak-gpu-vs-batch-size.png" width="600">
+
+-   The QLoRA can significantly reduce the peak GPU memory usage by ~75%
+    compared to LoRA. The peak GPU memory is ~28G for LoRA and ~7G for QLoRA
+    when batch size is 2.
+-   QLoRA can support much larger batch sizes than LoRA
+    -   We can use a batch size as 32 for QLoRA, but only 2 for LoRA on 1 A100
+        40G.
+    -   We can use a batch size of 8 for QLoRA on 1 V100 GPU. LoRA will fail
+        with OOM even with a batch size of 1.
+
+#### Peak GPU memory by LoRA rank (GB)
+
+<img src="images/openllama_7b_fine_tune_benchmark_report/openllama-7b-peak-gpu-vs-lora-rank.png" width="600">
+
+-   Peak GPU memories are quite similar for different LoRA ranks for both
+    LoRA/QLoRA.
+-   The peak GPU memory increasing percentages are very small generally when
+    LoRA rank increases.
+    -   The peak GPU memory increases from 28G with LoRA rank 4 to 29.09G with
+        LoRA rank 64, and the increasing percentage is only ~3.9%.
+
+#### Peak GPU memory by max sequence length for LoRA/QLoRA (GB)
+
+<img src="images/openllama_7b_fine_tune_benchmark_report/openllama-7b-peak-gpu-vs-max-seq-length.png" width="600">
+
+-   The peak GPU increases quickly when max sequence length increases for both
+    LoRA/QLoRA, and the increasing rate of LoRA is much faster than QLoRA.
+    -   For LoRA tuning, the GPU memory increased from 20.5G (max sequence
+        length=256) to 28.2G (max sequence length=512), an increase of ~37%.
+    -   For QLoRA tuning, the GPU memory increased from 6.94G (max sequence
+        length=256) to 7.57G (max sequence length=512), an increase of ~9%.
+-   The max sequence length for QLoRA can be 2048 when consuming 16.3G GPU,
+    while the max sequence length for LoRA is 512 when consuming 28.2G GPU, and
+    encounter OOM when max sequence length is 1024.
+
+### Fine Tuning Parameters
+
+This section shows the number/percentage of trainable parameters, and the sizes
+of the fine tuned models. LoRA and QLoRA differ only in how they represent the
+precision of their parameters. The total number of parameters and the number of
+trainable parameters are the same for both methods.
+
+| LoRA Rank | Finetuned parameters | Total parameters | Trainable Parameter Percentage | Fine tuned model size (MB) |
+| --------- | -------------------- | ---------------- | ------------------------------ | -------------------------- |
+| 8         | 2.00E+07             | 6.76E+09         | 0.3%                           | 76.4                       |
+| 16        | 4.00E+07             | 6.78E+09         | 0.6%                           | 152.65                     |
+| 32        | 8.00E+07             | 6.82E+09         | 1.2%                           | 305.15                     |
+| 64        | 1.60E+08             | 6.90E+09         | 2.3%                           | 610.15                     |
+
+
+LoRA/QLoRA tunes quite a small fraction (only 0.3% with LoRA rank=8) of all
+parameters, and the tuned models are very small (only 76.4MB with LoRA rank=8).
+
+### Fine Tuning Speed And Costs
+
+The fine-tuning speed and cost are affected by various factors, such as the
+GPUs, LoRA ranks, and max sequence lengths.
+
+-   LoRA is ~66% faster than QLoRA in fine tuning speed. The tuning speed for
+    LoRA/QLoRA ~5 samples / 3 samples per second on 1 A100 40G GPU
+-   Higher LoRA ranks, slower tuning speed for both LoRA/QLoRA.
+    -   LoRA tuning speed reduces from ~5 samples per second with LoRA rank as 8
+        to ~4 samples per second with LoRA rank as 64, slowed down by 20%.
+    -   QLoRA tuning speed reduces from ~3 samples per second with LoRA rank as
+        8 to ~2.5 samples per second with LoRA rank as 64, slowed down by 17%.
+
+<img src="images/openllama_7b_fine_tune_benchmark_report/openllama-7b-tune-speed-vs-lora-rank.png" width="600">
+
+-   Longer sequence lengths, slower tuning speed.
+    -   LoRA tuning speed reduces from ~5.56 samples per second with max
+        sequence length as 256 to ~4.84 samples per second with max sequence
+        length as 512 slowed down by 13%.
+    -   LoRA tuning speed reduces from ~2.95 samples per second with max
+        sequence length as 256 to ~2.88 samples per second with max sequence
+        length as 512 slowed down by ~2.4%.
+
+<img src="images/openllama_7b_fine_tune_benchmark_report/openllama-7b-tune-speed-lora-qlora.png" width="600">
+
+-   LoRA/QLoRA tuning cost is low generally, while LoRA is even ~40% cheaper
+    than QLoRA with the same parameters.
+    -   The LoRA/QLoRA fine tuning cost for 500 steps is ~$1/$1.7 on 1 A100 40G.
+    -   The tuning cost for QLoRA in 500 steps is $6.75 on n1-standard-8 with 1
+        V100 GPU, while LoRA could not run because of OOM.
+
+<img src="images/openllama_7b_fine_tune_benchmark_report/openllama-7b-tune-cost-lora-qlora.png" width="600">
+
+### Accuracy
+
+We fine tuned Open Llama 7B model with
+[timdettmers/openassistant-guanaco](https://huggingface.co/datasets/timdettmers/openassistant-guanaco),
+and report accuracy similar to the
+[HuggingFace leaderboard](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard)
+using
+[Eleuther AI Language Model Evaluation Harness](https://github.com/EleutherAI/lm-evaluation-harness).
+[HuggingFace leaderboard](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard)
+mainly compares models on ARC, HellaSwag, MMLU, and TruthfulQA. The authors did
+not publish OpenLLaMA 7B on MMLU
+([link](https://huggingface.co/openlm-research/open_llama_7b)). Therefore, we
+only benchmark accuracies on ARC, HellaSwag, and TruthfulQA.
+
+|                                                              | Mean | ARC  | HellaSwag | TruthfulQA | Tuning Parameters                                            |
+| ------------------------------------------------------------ | ---- | ---- | --------- | ---------- | ------------------------------------------------------------ |
+| OpenLLaMA7B ([Original Report](https://huggingface.co/openlm-research/open_llama_7b)) | 0.49 | 0.41 | 0.73      | 0.34       | n/a                                                          |
+| OpenLLaMA7B ([Re-run with lm-evaluation-harness](https://github.com/EleutherAI/lm-evaluation-harness)) | 0.51 | 0.47 | 0.72      | 0.35       | n/a                                                          |
+| OpenLLaMA7B+LoRA                                             | 0.56 | 0.48 | 0.74      | 0.45       | LoRA Rank=16; Max Sequence Length=512;Learning Rate=1e-4; Train steps=1875 |
+| OpenLLaMA7B+QLoRA                                            | 0.53 | 0.45 | 0.73      | 0.42       | LoRA Rank=16; Max Sequence Length=512; Learning Rate=1e-4; Train steps=1875 |
+
+-   The base OpenLLaMA7B model gets better performance (2%) when using the
+    [Eleuther AI Language Model Evaluation Harness](https://github.com/EleutherAI/lm-evaluation-harness).
+-   LoRA/QLoRA can improve the performance by ~2-4% when trained for 1875 steps
+    with learning rate 1e-4.

community-content/vertex_model_garden/benchmarking_reports/pytorch_openllama_hyperparameter_tuning_benchmark_report.md

@@ -0,0 +1,188 @@
+# Benchmark report on hyperparameter tuning the OpenLLaMA models on Google Cloud Vertex Model Garden
+
+Changyu Zhu, Software Engineer, Google Cloud
+
+Dustin Luong, Software Engineer, Google Cloud
+
+Gary Wei, Software Engineer, Google Cloud
+
+Genquan Duan, Software Engineer, Google Cloud
+
+## Introduction
+
+Fine-tuning of LLMs can be non-trivial to find an optimal configuration of
+machine types, training parameters, and other hyperparameters that achieves a
+good balance between cost efficiency and model performance. To facilitate users
+in conducting tuning experiments, this report benchmarks fine-tuning OpenLLaMA
+models with [Vertex AI Hyperparameter Tuning Service](https://cloud.google.com/vertex-ai/docs/training/hyperparameter-tuning-overview), demonstrating both efficiency
+and effectiveness. Similar hyperparameter tuning techniques can apply to other models as well.
+
+## Key takeaways
+
+- **The hyperparameter tuning service finds good parameters**: The best model found by the hyperparameter tuning service has an average improvement of around 4% in accuracy in *ARC*, *HellaSwag*, and *TruthfulQA* datasets, while only tuning the learning rate.
+
+- **Hyperparameter tuning works with QLoRA on limited resources**: 4bit QLoRA is sufficient for hyperparameter tuning to find a set of good parameters. In this way, all OpenLLaMA models can run on 1 single `NVIDIA_L4` GPU. It is also possible to train for more steps on the good parameters discovered by hyperparameter tuning, avoiding the waste of computing resources on fine-tuning with suboptimal hyperparameters.
+
+- **Hyperparameter tuning is cost-effective**: While `NVIDIA_L4` is slower than `NVIDIA_TESLA_V100`, it costs less and avoids the overhead of multi-GPU training since it has more GPU memory. Finding a good 3B/7B/13B OpenLLaMA model costs $28.5671, $47.8016, and $87.9208, respectively.
+
+## Benchmarking setup
+
+This section describes the experiment setup of the hyperparameter tuning experiments. The default tuning parameters are:
+
+### Machine configuration
+
+- Machine type: g2-standard-8
+- Machine count: 1
+- Accelerator type: NVIDIA_L4
+- Accelerator count: 1
+
+### Modeling
+
+We benchmark all 3 OpenLLaMA models:
+
+- [open_llama_3b](https://huggingface.co/openlm-research/open_llama_3b)
+- [open_llama_7b](https://huggingface.co/openlm-research/open_llama_7b)
+- [open_llama_13b](https://huggingface.co/openlm-research/open_llama_13b)
+
+We use the Huggingface [PEFT](https://github.com/huggingface/peft) library for fine-tuning.
+
+### Training dataset
+
+We use the dataset [timdettmers/openassistant-guanaco](https://huggingface.co/datasets/timdettmers/openassistant-guanaco) loaded directly via Huggingface.
+
+### Training parameters
+
+The set of training parameters used during benchmarking:
+
+- Batch size: 4
+- Precision mode: 4bit QLoRA
+- LoRA rank: 32
+- LoRA alpha: 64
+- Max sequence length: 512
+- Max train steps: 1000
+
+### Evaluation dataset
+
+We use the [lm-evaluation-harness](https://github.com/EleutherAI/lm-evaluation-harness) library injected into the training loop for evaluation. The hyperparameter tuning job will pick the model according to the evaluation metrics.
+
+- Eval task: [ARC Challenge](https://huggingface.co/datasets/ai2_arc)
+- Eval metric: acc_norm
+- Max eval examples: 10000
+
+### Standalone evaluation dataset
+
+After finding the best model with Vertex hyperparameter tuning service, we run standalone evaluations with the model on the following datasets:
+
+- [ARC Challenge](https://huggingface.co/datasets/ai2_arc)
+- [HellaSwag](https://huggingface.co/datasets/Rowan/hellaswag)
+- [TruthfulQA](https://huggingface.co/datasets/EleutherAI/truthful_qa_mc)
+
+### Hyperparameter tuning
+
+We only tune the learning rate hyperparameter. It is considered a floating point value in the continuous range [1e-5, 1e-4]. We run 8 trials in total, with a parallelism of 1 or 2.
+
+### Code example
+
+The following code example launches an example hyperparameter tuning job of OpenLLaMA 7B model.
+
+```py
+from google.cloud import aiplatform
+from google.cloud.aiplatform import hyperparameter_tuning as hpt
+
+
+TRAIN_DOCKER_URI = 'us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-peft-train:20231130_0936_RC00'
+output_dir = "gs://path/to/output/dir"
+base_model_id = "openlm-research/open_llama_7b"
+dataset_name = "timdettmers/openassistant-guanaco"
+hpt_precision_mode = "4bit"
+machine_type = "g2-standard-8"
+accelerator_type = "NVIDIA_L4"
+accelerator_count = 1
+eval_task = "arc_challenge"
+eval_metric_name = "acc_norm"
+max_steps = 1000
+eval_limit = 10000
+
+flags = {
+    "learning_rate": 1e-5,
+    "precision_mode": hpt_precision_mode,
+    "task": "instruct-lora",
+    "pretrained_model_id": base_model_id,
+    "output_dir": output_dir,
+    "warmup_steps": 10,
+    "max_steps": max_steps,
+    "lora_rank": 32,
+    "lora_alpha": 64,
+    "lora_dropout": 0.05,
+    "dataset_name": dataset_name,
+    "eval_steps": max_steps + 1,  # Only evaluates at the end.
+    "eval_tasks": eval_task,
+    "eval_limit": eval_limit,
+    "eval_metric_name": eval_metric_name,
+}
+worker_pool_specs = [
+    {
+        "machine_spec": {
+            "machine_type": machine_type,
+            "accelerator_type": accelerator_type,
+            "accelerator_count": accelerator_count,
+        },
+        "replica_count": 1,
+        "container_spec": {
+            "image_uri": TRAIN_DOCKER_URI,
+            "args": ["--{}={}".format(k, v) for k, v in flags.items()],
+        },
+    }
+]
+metric_spec = {"model_performance": "maximize"}
+parameter_spec = {
+    "learning_rate": hpt.DoubleParameterSpec(
+        min=1e-5, max=1e-4, scale="linear"
+    ),
+}
+
+train_job = aiplatform.CustomJob(
+    display_name=job_name,
+    worker_pool_specs=worker_pool_specs,
+    staging_bucket=STAGING_BUCKET,
+)
+
+train_hpt_job = aiplatform.HyperparameterTuningJob(
+    display_name=f"{job_name}_hpt",
+    custom_job=train_job,
+    metric_spec=metric_spec,
+    parameter_spec=parameter_spec,
+    max_trial_count=8,
+    parallel_trial_count=2,
+)
+
+train_hpt_job.run()
+```
+
+## Benchmark results
+
+### Fine-tuning cost
+
+The fine-tuning cost is calculated from `us-central1` pricing and may be subject to changes.
+
+| Model         | Train time | Trials | Parallel Trials | Hourly cost | Cost     | Eval acc_norm (ARC-Challenge) |
+|---------------|------------|--------|-----------------|-------------|----------|-------------------------------|
+| OpenLLaMA 3B  | 16 hrs     | 8      | 2               | $1.7072     | $28.5671 | 39.9%                         |
+| OpenLLaMA 7B  | 28 hrs     | 8      | 2               | $1.7072     | $47.8016 | 45.8%                         |
+| OpenLLaMA 13B | 103 hrs    | 8      | 1               | $0.8536     | $87.9208 | 47.6%                         |
+
+### Fine-tuning performance
+
+Here are the evaluation results of the best model found by hyperparameter tuning, compared with the baseline model. The column `Eval acc_norm` is calculated during training, which is always lower than that during standalone evaluation, because the model is loaded and evaluated at a lower precision (4bit during training / float16 during standalone evaluation).
+
+| Model         | Eval acc_norm (ARC-Challenge) | ARC    | hellaswag | Truthfulqa_mc | ∆ARC   | ∆Hellaswag | ∆Truthfulqa_mc | ∆Average |
+|---------------|-------------------------------|--------|-----------|---------------|--------|------------|----------------|----------|
+| OpenLLaMA 3B  | 39.9%                         | 41.47% | 69.97%    | 38.31%        | +1.62% | +7.32%     | +3.34%         | +4.09%   |
+| OpenLLaMA 7B  | 45.8%                         | 49.83% | 75.53%    | 41.53%        | +2.82% | +3.55%     | +6.68%         | +4.35%   |
+| OpenLLaMA 13B | 47.6%                         | 52.20% | 78.90%    | 44.27%        | +1.01% | +3.67%     | +6.19%         | +3.62%   |
+
+## Related documents
+
+1. [Benchmark report on fine tuning the OpenLLaMA 7B model on Google Cloud Vertex Model Garden
+](
+https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/community-content/vertex_model_garden/benchmarking_reports/pytorch_openllama_7b_finetune_benchmark_report.md)

community-content/vertex_model_garden/benchmarking_reports/stable_diffusion_v1-5_benchmarking_report.md

@@ -0,0 +1,218 @@
+# Benchmark Stable Diffusion v1-5 Fine Tuning and Serving With Google Cloud Vertex Model Garden
+
+Dustin Luong, Software Engineer, Google Cloud
+Gary Wei, Software Engineer, Google Cloud
+Changyu Zhu, Software Engineer, Google Cloud
+Genquan Duan, Software Engineer, Google Cloud
+
+## Introduction
+[The public notebook][1] shows the full examples of fine tuning and serving of Stable diffusion v1-5. [The github repo][2] contains examples of building training and serving dockers for Google Cloud Vertex Model Garden. This report benchmarks Stable diffusion v1-5 fine tuning and serving in Google Cloud Vertex AI, showing both efficiencies and effectiveness.
+
+### Benchmark Highlights
+- Fine tuning
+  - Stable diffusion v1-5 with LoRA and Gradient checkpointing only requires ~10G GPU memory. Larger batch sizes, or larger resolutions require more GPU memories, but not does not change much for different LoRA ranks.
+  - The fine tuning speed is fast in ~11 minutes for 1k steps, and costs less than $1 in 1 A100. The fine tuning speed increases with batch sizes, decreases with resolution, but is not affected much by LoRA ranks.
+  - LoRA tunes a few percent (only 0.1% with LoRA rank=8) of all parameters, and the tuned models are very small (only 3.1MB with LoRA rank=8).
+  - Dreambooth+LoRA and Dreambooth can achieve similar performances, but Dreambooth LoRA can require much less GPU.
+  - Increasing batch size, reducing training steps, and increasing learning rate can result in models with the same performance for less cost.
+- Inference
+  - The optimized serving docker pytorch-peft-serve can speed up inference by 2x than current pytorch-diffuser-serve, and support both base models and fine tuned lora models.
+  - The optimized serving docker pytorch-peft-serve can generate 4 512*512 images in 4.1 seconds on 1 V100 and 1.7 seconds on 1 A100.
+
+Benchmark details are below.
+
+## Fine Tuning Benchmarks
+
+### Experiment Setup
+We mainly compare two tuning algorithms:
+- parameter efficient finetuning based on [dreambooth][3] and [LoRA][4] (shorten as Dreambooth+LoRA below)
+- full parameter fine tuning based on [dreambooth][3] (shorten as Dreambooth below)
+
+And then  report benchmark results on GPU memories, tuning parameters, tuning speeds, costs and accuracy, using the public oxford flowers dataset: [train][5] and [test][6], where the column blip_caption as texts, and column image as images. We also benchmark subject and prompt fidelity using the [dataset][7] from the Dreambooth paper.
+
+The default tuning parameters during benchmark are:
+- Hardware: 1 A100 40G
+- batch size: 4
+- lora_rank: 8
+- resolution: 512
+- max_train_steps: 10
+- use_lora: False
+- gradient_checkpointing: False
+
+```
+# Examples to start finetuning dockers.
+MODEL_NAME="runwayml/stable-diffusion-v1-5"
+OUTPUT_DIR=<OUTPUT_DIR>
+INSTANCE_DATA_DIR=<INSTANCE_DATA_DIR>
+INSTANCE_PROMPT=<INSTANCE_PROMPT>
+IMAGE="us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-peft-train"
+docker run \
+  --runtime=nvidia -e NVIDIA_VISIBLE_DEVICES=0 \
+  --rm --name "test_gpu" \
+  -it ${IMAGE} \
+  --task=text-to-image-dreambooth-lora-peft \
+  --pretrained_model_name_or_path=$MODEL_NAME \
+  --resolution=512 \
+  --instance_data_dir=$INSTANCE_DATA_DIR \
+  --instance_prompt=$INSTANCE_PROMPT \
+  --train_batch_size=4 \
+  --max_train_steps=10 \
+  --output_dir=${OUTPUT_DIR} \
+  --use_lora \
+  --lora_r=8 \
+  --gradient_checkpointing
+```
+
+### GPU Memories
+Many various factors will impact GPU memory usages. In this benchmark, we mainly benchmark with different finetuning algorithms, batch sizes, lora rank, resolution, and then recommended max batch size on different GPUs.
+
+
+![sd_v1-5_peak_gpu_algorithm](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_peak_gpu_algorithm.png)
+
+![sd_v1-5_peak_gpu_batch_size](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_peak_gpu_batch_size.png)
+
+![sd_v1-5_peak_gpu_lora_rank](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_peak_gpu_lora_rank.png)
+
+![sd_v1-5_peak_gpu_resolution](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_peak_gpu_resolution.png)
+
+- LoRA tuning reduced about 47% peak RAM and 42% peak VRAM for GPU memory, compared to full parameter fine tuning.
+- Gradient checkpointing decreases about 1% peak RAM and 31% peak VRAM for GPU memory further, compared without gradient checkpointing.
+- The GPU memory does not change much for different LoRA ranks.
+- Larger batch sizes require more GPU memories.
+- Larger resolutions require more GPU memories.
+- Dreambooth+LoRA+Gradient_Checkpointing can support max batch size as 32, or max resolution as 2048, but Dreambooth can only support max batch size as 8, or max resolution as 1024.
+
+### Fine Tuning Parameters
+This section shows the percentage of trainable parameters, and tuned model sizes.
+
+- LoRA tunes quite a few percent (only 0.1% with LoRA rank=8) of all parameters, and the tuned models are very small (only 3.1MB with LoRA rank=8).
+
+| LoRA Rank | Trainable parameters | Total parameters | Trainable Parameter Percentage | Fine tuned model size (MB) |
+|---|---|---|---|---|
+| 4 | 398592 | 859919556 | 0.05% | 1.57 |
+|8 | 797184 | 860318148 | 0.09% | 3.09 |
+| 16 | 1594368| 861115332| 0.19%| 6.13|
+| 32| 3188736| 862709700| 0.37%| 12.21|
+### Fine Tuning Speed And Costs
+Fine tuning speeds and costs are affected by many different factors, such as batch size, tuning parameters, image resolutions, GPUs, and datasets. In order to make the report easy to understand, we set the following values in this section:
+- Hardware: 1 A100 40G
+- use_lora: True
+- gradient_checkpointing: True
+
+![sd_v1-5_training_speed_batch_size](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_training_speed_batch_size.png)
+
+![sd_v1-5_training_speed_lora_rank](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_training_speed_lora_rank.png)
+
+![sd_v1-5_training_speed_resolution](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_training_speed_resolution.png)
+
+![sd_v1-5_training_cost_max_steps](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_training_cost_max_steps.png)
+
+- The fine tuning speed increases with batch sizes, decreases with resolution, but is not affected much by LoRA ranks.
+- The fine tuning speed is about 11 minutes for 1k steps, and costs less than $1 in 1 A100.
+
+### Fine Tuning Quality
+In this benchmark, we mainly benchmark Dreambooth and Dreambooth+LoRA to compare fine tuning quality. We compare [subject fidelity scored (DINO)][8], how well the subject is represented in the generated images, and [prompt fidelity scores (CoCa)][9], how well the generated images match the given prompt, for a single subject, a [dog][10] from the dataset released with the original Dreambooth paper. In practice, we recommend saving checkpoints periodically and inspecting validation prompts visually. We fine tuned the unet without fine tuning the text encoder and used the following hyperparameters:
+
+Dreambooth
+- Learning rate: 5e-6
+- Batch size: 1
+
+Dreambooth+LoRA
+- Learning rate: 1e-4
+- Batch size: 1
+
+![sd_v1-5_finetuning_quality_subject_fidelity](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_finetuning_quality_subject_fidelity.png)
+
+![sd_v1-5_finetuning_quality_prompt_fidelity](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_finetuning_quality_prompt_fidelity.png)
+
+- Fine tuning with Dreambooth or Dreambooth+LoRA can result in models with comparable performance. The base model produced images of the class rather than the instance.
+- Dreambooth+LoRA is able to achieve the same subject fidelity score as Dreambooth if trained for more epochs.
+- Increasing the number of training steps results in better subject fidelity but at the cost of prompt fidelity.
+
+### Suggested Max Batch Sizes By Resolutions
+We benchmarked and suggested max batch sizes by resolutions on 1 A100 and 1 V100 as below. This is with LoRA and gradient checkpointing enabled.
+
+![sd_v1-5_batch_size_by_resolution](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_batch_size_by_resolution.png)
+
+### Fine Tuning Cost Optimization
+Increasing batch size allows for more images to be considered at each training step for fine tuning. This allows models to be trained in fewer training steps. In this benchmark, we aim to show how batch size can be increased to reduce training costs while still preserving subject and prompt fidelity.
+
+Since the training dataset consists of 5 images, we train with a batch size of 5 and reduce the number of training steps from 400 to 80. Doing so results in a model that has not learned the subject since we’ve decreased the number of training steps. Conceptually, the model is taking a more precise step at each iteration, but it is taking fewer steps. To compensate for this, we increased the learning rate from 5e-6 and observed the best results at 1e-5 for full parameter finetuning.
+
+![sd_v1-5_subject_fidelity_batch_size_5_learning_rate](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_subject_fidelity_batch_size_5_learning_rate.png)
+
+![sd_v1-5_prompt_fidelity_batch_size_5_learning_rate](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_prompt_fidelity_batch_size_5_learning_rate.png)
+
+Comparing cost of training the “best” model for batch size 1 vs. batch size 5
+
+![sd_v1-5_cost_batch_size](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_cost_batch_size.png)
+
+
+| Train method| Training parameters| Sample image| CoCa (prompt fidelity)| DINO (subject fidelity) | Cost of training on A100 |
+|---|---|---|---|---|---|
+| dreambooth| dreambooth, num_train_steps=400, batch_size=1, lr=5e-6| ![dog1](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_dog1.png) | 0.12215| 0.76531| $0.26 |
+| dreambooth | dreambooth, num_train_steps=80, batch_size=5,lr=1e-5| ![dog2](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_dog2.png)| 0.12644| 0.74697 | $0.15 |
+| dreambooth-lora| num_train_steps=500, batch_size=1, lr=1e-4, gc|![dog3](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_dog3.png)| 0.12856| 0.78148 | $0.26|
+| dreambooth-lora | num_train_steps=50, batch_size=5, lr=1e-3, gc | ![dog4](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_dog4.png) | 0.12566 | 0.75479 | $0.09 |
+
+
+
+A followup question is that since finetuning can be run on a single GPU, should finetuning be run on 1 V100 or A100?
+
+Setup:
+- num_train_steps=800 / batch_size
+- Resolution=512
+
+![sd_v1-5_cost_training_method_batch_size](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_cost_training_method_batch_size.png)
+- Although V100 has a lower $/hr cost than an A100, the same training setup takes longer. Even given the longer training time, the cost on V100 is still lower.
+- Dreambooth+LoRA enables training with larger batch sizes, however, larger batch sizes will not necessarily mean faster training time.
+- It is possible to fine tune with 1 V100 on 512 resolution with Dreambooth+LoRA.
+- Dreambooth fine tuning must be run on 1 A100 at 512 resolution.
+
+## Inference Benchmarks
+We provide two serving dockers in vertex model garden for stable diffusion:
+- pytorch-diffuser-serve:
+  - us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-diffusers-serve
+  - This serving docker only serves base stable diffusion models and does not contain any optimizations yet.
+- pytorch-peft-serve:
+  - us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-peft-serve
+  - This serving docker can serve base stable diffusion models, and base stable diffusion models with fine tuned lora models, and contains optimization for serving.
+
+We run the two serving dockers on T4/V100/A100 to generate 4 512*512 images, and compare the inference speed without network considerations as:
+
+![sd_v1-5_inference_speed_gpu](images/stable_diffusion_v1-5_benchmarking_report/sd_v1-5_inference_speed_gpu.png)
+The speed up of optimized pytorch-peft-serve is about 2x than current pytorch-diffuser-serve.
+
+### Serving cost comparison
+
+Pytorch-diffuser-serve (without any optimizations)
+
+| GPU type|  Time required to generate 4 512x512 images | Machine unit price ($ / hour) | Cost per image ($) |
+|---|---|---|---|
+| T4 | 28.6 | 0.4025| 0.00080 |
+| V100 | 8.8 | 2.852| 0.00174|
+| A100 | 4.2 | 4.2245 | 0.00123 |
+
+Pytorch-peft-serve (with optimizations)
+
+| GPU type | Time required to generate 4 512x512 images | Machine unit price ($ / hour) | Cost per image ($) |
+|--- |---|---|---|
+| T4 | 12.6 | 0.4025 | 0.00035 |
+| V100 | 4.1 | 2.852 | 0.00081 |
+|  A100 | 1.7 | 4.2245 | 0.00050 |
+
+- The optimized pytorch-peft-serve has approximately half the price per image, compared with the un-optimized pytorch-diffuser-serve.
+- Serving the model with a T4 is most cost effective, however, serving with an A100 still has the best throughput and fastest predictions.
+
+
+
+[1]: https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion.ipynb
+[2]: https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/community-content/vertex_model_garden/model_oss
+[3]: https://arxiv.org/abs/2208.12242
+[4]: https://arxiv.org/abs/2106.09685
+[5]: https://huggingface.co/datasets/Multimodal-Fatima/OxfordFlowers_train
+[6]: https://huggingface.co/datasets/Multimodal-Fatima/OxfordFlowers_test_facebook_opt_6.7b_Attributes_ns_6149
+[7]: https://github.com/google/dreambooth
+[8]: https://arxiv.org/abs/2104.14294
+[9]: https://arxiv.org/abs/2205.01917
+[10]: https://github.com/google/dreambooth/tree/main/dataset/dog6

community-content/vertex_model_garden/model_oss/autogluon/dockerfile/autogluon_serve.Dockerfile

@@ -0,0 +1,50 @@
+# Dockerfile for serving dockers with AutoGluon.
+#
+# To build:
+# docker build -f model_oss/autogluon/dockerfile/serve.Dockerfile . -t ${YOUR_IMAGE_TAG}
+#
+# To push to gcr:
+# docker tag ${YOUR_IMAGE_TAG} gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
+# docker push gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
+
+FROM pytorch/pytorch:2.1.2-cuda11.8-cudnn8-runtime
+
+USER root
+
+# AutoGluon might require libgomp for some dependencies.
+RUN apt-get update && apt-get -y upgrade && apt-get install -y --no-install-recommends \
+        curl \
+        wget \
+        vim \
+        libgomp1
+
+# Install AutoGluon and other dependencies.
+RUN pip install --upgrade pip
+RUN pip install autogluon==1.0.0
+RUN pip install flask==3.0.0
+
+# Dependencies needed to work with GCS.
+RUN pip install absl-py==2.0.0
+RUN pip install google-cloud-storage==2.7.0
+
+# Copy scripts into the container.
+COPY model_oss/autogluon /autogluon
+COPY model_oss/util /autogluon/util
+WORKDIR /autogluon
+
+# Copy license.
+RUN wget https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/LICENSE
+RUN wget https://github.com/pallets/flask/blob/main/LICENSE.rst
+
+# Expose the port the app runs on.
+EXPOSE 8501
+
+# Set the working directory to a specific path for consistency.
+WORKDIR /autogluon
+
+# Change to a non-root user for security purposes.
+RUN useradd -m autogluonuser
+USER autogluonuser
+
+# Run Flask application.
+CMD ["python", "serve.py"]

community-content/vertex_model_garden/model_oss/autogluon/dockerfile/autogluon_train.Dockerfile

@@ -0,0 +1,36 @@
+# Dockerfile for training dockers with Autogluon.
+#
+# To build:
+# docker build -f model_oss/autogluon/dockerfile/train.Dockerfile . -t ${YOUR_IMAGE_TAG}
+#
+# To push to gcr:
+# docker tag ${YOUR_IMAGE_TAG} gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
+# docker push gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
+
+FROM pytorch/pytorch:2.1.2-cuda11.8-cudnn8-runtime
+
+# Install tools.
+ENV DEBIAN_FRONTEND=noninteractive
+ENV PIP_ROOT_USER_ACTION=ignore
+
+RUN apt-get update && apt-get -y upgrade && apt-get install -y --no-install-recommends \
+apt-utils \
+curl \
+wget \
+git \
+jq \
+gnupg \
+build-essential \
+tesseract-ocr \
+vim
+
+# Install libraries.
+RUN pip install autogluon==1.0.0
+
+COPY model_oss/autogluon /autogluon
+WORKDIR /autogluon
+
+# Copy license.
+RUN wget https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/LICENSE
+
+ENTRYPOINT ["python", "train.py"]

community-content/vertex_model_garden/model_oss/autogluon/serve.py

@@ -0,0 +1,87 @@
+r"""AutoGluon serving binary.
+
+This module sets up a Flask web server for serving predictions from a
+trained AutoGluon model. The server exposes two endpoints:
+
+1.  `/ping`: A health check endpoint that returns "pong" to
+    indicate that the server is running.
+2.  `/predict`: An endpoint that accepts POST requests with JSON content.
+    Each request should contain one or more instances for which the
+    predictions are desired. The endpoint returns the predictions and
+    associated probabilities in a JSON response.
+
+The server expects an environment variable `model_path` that points to
+the directory where the AutoGluon model artifacts are
+stored. If `model_path` is not provided, it defaults to '/autogluon/models'.
+"""
+
+import json
+import logging
+import os
+
+from autogluon.tabular import TabularPredictor
+import flask
+import pandas as pd
+
+from util import constants
+from util import fileutils
+
+_SUCCESS_STATUS = 200
+_ERROR_STATUS = 500
+_PORT = 8501
+
+app = flask.Flask(__name__)
+# Check the environment variables.
+model_dir = os.getenv('model_path', '/autogluon/models')
+logging.info('Model directory passed by the user is: %s', model_dir)
+# If the model is on GCS then copy it to a local folder first.
+if model_dir.startswith(constants.GCS_URI_PREFIX):
+  gcs_path = model_dir[len(constants.GCS_URI_PREFIX) :]
+  local_model_dir = os.path.join(constants.LOCAL_MODEL_DIR, gcs_path)
+  logging.info('Download %s to %s', model_dir, local_model_dir)
+  fileutils.download_gcs_dir_to_local(model_dir, local_model_dir)
+  model_dir = local_model_dir
+  logging.info('Local model directory is: %s', model_dir)
+
+
+# Load the predictor at startup.
+predictor = TabularPredictor.load(model_dir)
+
+
+@app.route('/ping', methods=['GET'])
+def ping() -> flask.Response:
+  """Health check route."""
+  return flask.Response('pong', status=_SUCCESS_STATUS)
+
+
+@app.route('/predict', methods=['POST'])
+def predict() -> flask.Response:
+  """Prediction route."""
+  try:
+    # Extract JSON content from the POST request.
+    data = flask.request.get_json(force=True)
+    instances = data.get('instances', [])
+
+    # Convert instances to DataFrame.
+    df_to_predict = pd.DataFrame(instances)
+
+    # Perform prediction.
+    predictions = predictor.predict(df_to_predict).tolist()
+    response = {'predictions': predictions}
+
+    return flask.Response(
+        json.dumps(response),
+        status=_SUCCESS_STATUS,
+        mimetype='application/json',
+    )
+
+  except Exception as e:  # pylint: disable=broad-exception-caught
+    return flask.Response(
+        json.dumps({'error': str(e)}),
+        status=_ERROR_STATUS,
+        mimetype='application/json',
+    )
+
+
+if __name__ == '__main__':
+  app.run(host='0.0.0.0', port=_PORT)

community-content/vertex_model_garden/model_oss/autogluon/train.py

@@ -0,0 +1,144 @@
+"""AutoGluon training binary. """
+
+import argparse
+import json
+from typing import Any
+
+from autogluon.tabular import TabularPredictor
+import pandas as pd
+
+
+class BaseConfig:
+
+  def to_dict(self) -> dict[str, Any]:
+    return {
+        key: value for key, value in self.__dict__.items() if value is not None
+    }
+
+
+class DataConfig(BaseConfig):
+
+  def __init__(self, train_data_path: Any) -> None:
+    self.train_data_path = train_data_path
+
+
+class ProblemConfig(BaseConfig):
+
+  def __init__(self, label: Any, problem_type: Any) -> None:
+    self.label = label
+    self.problem_type = problem_type
+
+
+class EvaluationConfig(BaseConfig):
+
+  def __init__(self, eval_metric: Any) -> None:
+    self.eval_metric = eval_metric
+
+
+class TrainingConfig(BaseConfig):
+  """Config for training."""
+
+  def __init__(
+      self,
+      time_limit: Any,
+      presets: Any,
+      hyperparameters: Any,
+      model_save_path: str,
+  ) -> None:
+    self.time_limit = time_limit
+    self.hyperparameters = hyperparameters
+    self.presets = presets
+    self.model_save_path = model_save_path
+
+
+def parse_args() -> (
+    tuple[DataConfig, ProblemConfig, EvaluationConfig, TrainingConfig]
+):
+  """Parse command line arguments."""
+  parser = argparse.ArgumentParser(description="AutoGluon Tabular Predictor")
+  # Add arguments for each config class
+  parser.add_argument(
+      "--train_data_path",
+      type=str,
+      required=True,
+      help="Path to the input data CSV file.",
+  )
+  parser.add_argument(
+      "--label", type=str, required=True, help="Target variable column name."
+  )
+  parser.add_argument(
+      "--problem_type",
+      type=str,
+      choices=["binary", "multiclass", "regression", "quantile"],
+      default=None,
+      help="Problem type.",
+  )
+  parser.add_argument(
+      "--eval_metric", type=str, default=None, help="Evaluation metric to use."
+  )
+  # Add arguments for TrainingConfig if needed
+  parser.add_argument(
+      "--time_limit",
+      type=int,
+      default=None,
+      help="Time limit in seconds for training.",
+  )
+  parser.add_argument(
+      "--presets",
+      type=str,
+      default="medium_quality",
+      help="Presets used for training ",
+  )
+  parser.add_argument(
+      "--hyperparameters",
+      type=json.loads,
+      default=None,
+      help="Hyperparameter dictionary in JSON format.",
+  )
+  parser.add_argument(
+      "--model_save_path",
+      type=str,
+      default=None,
+      help="Path to save the trained model.",
+  )
+  args = parser.parse_args()
+
+  data_config = DataConfig(train_data_path=args.train_data_path)
+  problem_config = ProblemConfig(
+      label=args.label, problem_type=args.problem_type
+  )
+  eval_config = EvaluationConfig(eval_metric=args.eval_metric)
+  training_config = TrainingConfig(
+      time_limit=args.time_limit,
+      presets=args.presets,
+      hyperparameters=args.hyperparameters,
+      model_save_path=args.model_save_path,
+  )
+
+  return data_config, problem_config, eval_config, training_config
+
+
+def main() -> None:
+  data_config, problem_config, eval_config, training_config = parse_args()
+
+  # Load the training data.
+  data = pd.read_csv(data_config.train_data_path)
+
+  # Create a TabularPredictor.
+  predictor = TabularPredictor(
+      label=problem_config.label,
+      eval_metric=eval_config.eval_metric,
+      path=training_config.model_save_path,
+  )
+
+  # Fit the model
+  predictor.fit(
+      data,
+      presets=training_config.presets,
+      time_limit=training_config.time_limit,
+      hyperparameters=training_config.hyperparameters,
+  )
+
+
+if __name__ == "__main__":
+  main()

community-content/vertex_model_garden/model_oss/cloudnerf/configs/360.gin

@@ -0,0 +1,25 @@
+# The provided content is a configuration file for the ZipNeRF
+# PyTorch implementation.
+
+# Sets the name of the experiment to 'test'.
+Config.exp_name = 'test'
+# Specifies the dataset loader, in this case, 'llff' for light field.
+Config.dataset_loader = 'llff'
+# Defines the near and far clipping planes for the camera view.
+Config.near = 0.2
+Config.far = 1e6
+# Image downsampling.
+Config.factor = 4
+
+# For the model configurations.
+Model.raydist_fn = 'power_transformation'
+Model.opaque_background = True
+
+# Disables the computation of density normals and RGB values, and sets
+# the grid level dimension to 1 for PropMLP.
+PropMLP.disable_density_normals = True
+PropMLP.disable_rgb = True
+PropMLP.grid_level_dim = 1
+
+# Disable density normals for NerfMLP
+NerfMLP.disable_density_normals = True

community-content/vertex_model_garden/model_oss/cloudnerf/configs/360_glo.gin

@@ -0,0 +1,21 @@
+# The provided content is a configuration file for Generative
+# Latent Optimization (GLO) vectors in the Pytorch implemnetation of ZipNeRF.
+
+# Specifies the dataset loader, in this case, 'llff' for light field.
+Config.dataset_loader = 'llff'
+# Defines the near and far clipping planes for the camera view.
+Config.near = 0.2
+Config.far = 1e6
+# Image downsampling.
+Config.factor = 4
+
+# For the model configurations.
+Model.raydist_fn = 'power_transformation'
+Model.num_glo_features = 128
+Model.opaque_background = True
+
+PropMLP.disable_density_normals = True
+PropMLP.disable_rgb = True
+PropMLP.grid_level_dim = 1
+
+NerfMLP.disable_density_normals = True

community-content/vertex_model_garden/model_oss/cloudnerf/configs/accelerate_config.yaml

@@ -0,0 +1,18 @@
+# The provided content is a configuration file running ZipNeRF
+# training on 8 gpu machine.
+compute_environment: LOCAL_MACHINE
+debug: false
+distributed_type: MULTI_GPU
+downcast_bf16: 'no'
+gpu_ids: all
+machine_rank: 0
+main_training_function: main
+mixed_precision: fp16
+num_machines: 1
+num_processes: 8
+rdzv_backend: static
+same_network: true
+tpu_env: []
+tpu_use_cluster: false
+tpu_use_sudo: false
+use_cpu: false

community-content/vertex_model_garden/model_oss/cloudnerf/dockerfile/pytorch_cloudnerf_base.Dockerfile

@@ -0,0 +1,120 @@
+# Dockerfile for ZipNeRF base image.
+#
+# To build:
+# docker build -f model_oss/cloudnerf/dockerfile/pytorch_cloudnerf_base.Dockerfile . -t ${YOUR_IMAGE_TAG}
+#
+# To push to gcr:
+# docker tag ${YOUR_IMAGE_TAG} gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
+# docker push gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
+
+FROM pytorch/pytorch:2.1.0-cuda11.8-cudnn8-devel
+
+USER root
+
+ARG COLMAP_GIT_COMMIT=main
+ARG CUDA_ARCHITECTURES=60;70;75;80;86
+
+# Prevent stop building ubuntu at time zone selection.
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update -y --allow-releaseinfo-change && apt-get -y upgrade && apt-get install -y --no-install-recommends \
+        curl \
+        g++ \
+        wget \
+        vim \
+        bash \
+        cmake \
+        imagemagick \
+        ninja-build \
+        build-essential \
+        libboost-program-options-dev \
+        libboost-filesystem-dev \
+        libboost-graph-dev \
+        libboost-system-dev \
+        libeigen3-dev \
+        libflann-dev \
+        libfreeimage-dev \
+        libmetis-dev \
+        libgoogle-glog-dev \
+        libgtest-dev \
+        libsqlite3-dev \
+        libglew-dev \
+        qtbase5-dev \
+        libqt5opengl5-dev \
+        libcgal-dev \
+        libceres-dev \
+        git \
+        git-lfs \
+        python3-cffi \
+        python3-cryptography \
+        libffi-dev \
+        python-dev
+
+# Copy license.
+RUN wget https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/LICENSE
+
+# Install google cloud CLI.
+RUN wget -q https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-430.0.0-linux-x86.tar.gz
+RUN tar xzf google-cloud-cli-430.0.0-linux-x86.tar.gz
+RUN ./google-cloud-sdk/install.sh -q
+# Make sure gsutil will use the default service account.
+RUN echo '[GoogleCompute]\nservice_account = default' > /etc/boto.cfg
+
+# Install deps and install gsutil.
+RUN pip install gsutil==5.27
+
+# When building colmap in colab, the link error "undefined reference.
+# to '_glapi_tls_Current'" happens. A solution is to install "libglvnd"
+# as described in this page https://github.com/colmap/colmap/issues/1271.
+RUN git clone --depth 1 --branch v1.7.0 https://github.com/NVIDIA/libglvnd && \
+    apt-get install -y libxext-dev libx11-dev x11proto-gl-dev && \
+    cd libglvnd/  && \
+    apt-get install -y autoconf automake libtool && \
+    apt-get install -y libffi-dev && \
+    ./autogen.sh && \
+    ./configure && \
+    make  -j4 && \
+    make install
+
+RUN apt remove nvidia-cuda-toolkit -y \
+    nvidia-cuda-toolkit \
+    nvidia-cuda-toolkit-gcc
+
+# Install libraries.
+ENV PIP_ROOT_USER_ACTION=ignore
+RUN python3 -m pip install --upgrade pip
+
+ENV CUDA_HOME=/usr/local/cuda
+
+RUN git clone --branch main https://github.com/SuLvXiangXin/zipnerf-pytorch.git
+# Set current directory to the downloaded 'zipnerf-pytorch' repository.
+WORKDIR ./zipnerf-pytorch
+# Using git reset command to pin it down to a specific version.
+RUN git reset --hard 4de3d21ebb9e15412d36951b56e2d713fddd812b
+COPY model_oss/cloudnerf/requirements.txt requirements.txt
+RUN pip install -r requirements.txt
+
+# Install gridencoder extensions and nvdiffrast (for textured mesh).
+RUN cd .. && \
+    TORCH_CUDA_ARCH_LIST="6.0 7.0 7.5 8.0 8.6+PTX" CXX=g++ pip install ./zipnerf-pytorch/gridencoder
+
+# Install cuda version of torch_scatter.
+RUN pip install torch-scatter==2.1.2 -f https://data.pyg.org/whl/torch-2.0.1+cu118.html
+RUN pip install google-cloud-aiplatform==1.25.0
+RUN pip install google-cloud-storage==2.9.0
+
+# Build and install COLMAP.
+RUN git clone  --depth 1 --branch 3.8 https://github.com/colmap/colmap.git
+RUN cd colmap && \
+    git fetch https://github.com/colmap/colmap.git ${COLMAP_GIT_COMMIT} && \
+    mkdir build && \
+    cd build && \
+    cmake .. -GNinja -DCMAKE_CUDA_ARCHITECTURES=${CUDA_ARCHITECTURES} && \
+    ninja && \
+    ninja install && \
+    cd .. && rm -rf colmap
+
+RUN git clone  --depth 1 --branch v1.0.2 https://github.com/dranjan/python-plyfile.git
+
+RUN sed -i "20 i\sys.path.append('/workspace/zipnerf-pytorch/internal/pycolmap')" /workspace/zipnerf-pytorch/internal/datasets.py
+RUN sed -i "21 i\sys.path.append('/workspace/zipnerf-pytorch/internal/pycolmap/pycolmap')" /workspace/zipnerf-pytorch/internal/datasets.py

community-content/vertex_model_garden/model_oss/cloudnerf/dockerfile/pytorch_cloudnerf_calibrate.Dockerfile

@@ -0,0 +1,16 @@
+# Dockerfile for ZipNeRF COLMAP image calibration.
+#
+# To build:
+# docker build -f model_oss/cloudnerf/dockerfile/cloudnerf_pytorch_calibrate.Dockerfile . -t ${YOUR_IMAGE_TAG}
+#
+# To push to gcr:
+# docker tag ${YOUR_IMAGE_TAG} gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
+# docker push gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
+
+FROM us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-cloudnerf-base:20231206_0923_RC00
+
+COPY model_oss/cloudnerf/local_colmap_and_resize.sh /workspace/zipnerf-pytorch/scripts/local_colmap_and_resize.sh
+
+WORKDIR /workspace/zipnerf-pytorch/
+
+ENTRYPOINT ["bash","scripts/local_colmap_and_resize.sh"]

community-content/vertex_model_garden/model_oss/cloudnerf/dockerfile/pytorch_cloudnerf_render.Dockerfile

@@ -0,0 +1,22 @@
+# Dockerfile for ZipNeRF rendering.
+#
+# To build:
+# docker build -f model_oss/cloudnerf/dockerfile/pytorch_cloudnerf_render.Dockerfile . -t ${YOUR_IMAGE_TAG}
+#
+# To push to gcr:
+# docker tag ${YOUR_IMAGE_TAG} gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
+# docker push gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
+
+FROM us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-cloudnerf-base:20231206_0923_RC00
+
+COPY model_oss/cloudnerf/render.sh /workspace/zipnerf-pytorch/scripts/render.sh
+COPY model_oss/cloudnerf/configs/360.gin /workspace/zipnerf-pytorch/configs/360.gin
+COPY model_oss/cloudnerf/configs/360_glo.gin /workspace/zipnerf-pytorch/configs/360_glo.gin
+COPY model_oss/cloudnerf/configs/accelerate_config.yaml /root/.cache/huggingface/accelerate/default_config.yaml
+RUN sed -i '324s/.*/            keyframe_names = fp.read().splitlines()/' /workspace/zipnerf-pytorch/internal/camera_utils.py
+
+ENV PYTHONPATH "${PYTHONPATH}:/workspace/zipnerf-pytorch/util"
+
+WORKDIR /workspace/zipnerf-pytorch/
+
+ENTRYPOINT ["bash", "scripts/render.sh"]

community-content/vertex_model_garden/model_oss/cloudnerf/dockerfile/pytorch_cloudnerf_train.Dockerfile

@@ -0,0 +1,21 @@
+# Dockerfile for ZipNeRF training.
+#
+# To build:
+# docker build -f model_oss/cloudnerf/dockerfile/pytorch_cloudnerf_train.Dockerfile . -t ${YOUR_IMAGE_TAG}
+#
+# To push to gcr:
+# docker tag ${YOUR_IMAGE_TAG} gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
+# docker push gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
+
+FROM us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-cloudnerf-base:20231206_0923_RC00
+
+COPY model_oss/cloudnerf/train.sh /workspace/zipnerf-pytorch/scripts/train.sh
+COPY model_oss/cloudnerf/configs/360.gin /workspace/zipnerf-pytorch/configs/360.gin
+COPY model_oss/cloudnerf/configs/360_glo.gin /workspace/zipnerf-pytorch/configs/360_glo.gin
+COPY model_oss/cloudnerf/configs/accelerate_config.yaml /root/.cache/huggingface/accelerate/default_config.yaml
+
+ENV PYTHONPATH "${PYTHONPATH}:/workspace/zipnerf-pytorch/util"
+
+WORKDIR /workspace/zipnerf-pytorch/
+
+ENTRYPOINT ["bash", "scripts/train.sh"]

community-content/vertex_model_garden/model_oss/cloudnerf/local_colmap_and_resize.sh

@@ -0,0 +1,144 @@
+#!/bin/bash
+# This script runs colmap for scale invariant feature (SIFT) extraction and
+# matching to map camera extrinsics and intrinsics values for ZipNeRF,
+# given a folder of images and videos
+# from a GCS bucket. It uses ffmepg to extract an image from a video at
+# 1fps. The folder can contain images or videos. If both images and videos
+# are present, the extracted frames from the videos is added to the images
+# to create the final combined image dataset.
+# vv-docker:google3-begin(internal)
+# TODO(b/314042136): Specify cloudnerf colmap fps.
+# vv-docker:google3-end
+
+# Initialize variables.
+use_gpu=1  # Default to 1 (assuming the docker is run on a machine with GPU)
+gcs_dataset_path=""
+gcs_experiment_path=""
+camera=""
+
+# This loop processes command-line arguments for configuring the container.
+# It supports arguments for GPU usage, dataset and experiment paths,
+# and camera type.
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    -use_gpu)
+      use_gpu="$2"
+      if ! [[ $use_gpu =~ ^[0-9]+$ ]]; then
+        echo "Error: -use_gpu must be an integer."
+        exit 1
+      fi
+      shift # past argument
+      shift # past value
+      ;;
+    -gcs_dataset_path)
+      gcs_dataset_path="$2"
+      shift # past argument
+      shift # past value
+      ;;
+    -gcs_experiment_path)
+      gcs_experiment_path="$2"
+      shift # past argument
+      shift # past value
+      ;;
+    -camera)
+      camera="$2"
+      if [[ $camera != "OPENCV" && $camera != "OPENCV_FISHEYE" ]]; then
+        echo "Error: -camera must be either 'OPENCV' or 'OPENCV_FISHEYE'."
+        exit 1
+      fi
+      shift # past argument
+      shift # past value
+      ;;
+    *)    # unknown option
+      echo "Unknown option: $1" >&2
+      exit 1
+      ;;
+  esac
+done
+
+local_folder="dataset_content"
+images_folder="dataset_images"
+images_subfolder="images"
+output_folder="$images_folder/$images_subfolder"
+
+# Create the local folder if it doesn't exist
+mkdir -p "$local_folder"
+mkdir -p "$output_folder"
+
+# Download the content from the GCS URI
+gsutil -m cp -r "$gcs_dataset_path"/* "$local_folder/"
+
+# Process files in the local folder
+for file in "$local_folder"/*; do
+  if [[ -f "$file" ]]; then
+    # Check if the file is an image (e.g., jpg, png, etc.)
+    if file --mime-type "$file" | grep -q "image"; then
+      # Copy the image to the "images" subfolder within the "dataset_images" folder
+      cp "$file" "$output_folder/$(basename "$file")"
+    elif file --mime-type "$file" | grep -q "video"; then
+      # Use FFmpeg to extract an image every 30 frames from the video
+      ffmpeg -i "$file" -vf "select='not(mod(n,30))'" "$output_folder/$(basename "$file" ."${file##*.}")_%03d.jpg"
+    else
+      echo "Skipping unsupported file: $file"
+    fi
+  fi
+done
+
+# Run COLMAP Feature extraction
+colmap feature_extractor \
+    --database_path "$local_folder"/database.db \
+    --image_path "$output_folder" \
+    --ImageReader.single_camera 1 \
+    --ImageReader.camera_model "$camera" \
+    --SiftExtraction.use_gpu "$use_gpu"
+
+# Run COLMAP Feature matching
+colmap exhaustive_matcher \
+    --database_path "$local_folder"/database.db \
+    --SiftMatching.use_gpu "$use_gpu"
+
+# Bundle adjustment. The default Mapper tolerance is unnecessarily large,
+# decreasing it speeds up bundle adjustment steps.
+mkdir -p "$local_folder"/sparse
+colmap mapper \
+    --database_path "$local_folder"/database.db \
+    --image_path "$output_folder" \
+    --output_path "$local_folder"/sparse \
+    --Mapper.ba_global_function_tolerance=0.000001
+
+# Downsample images at 1/2, 1/4, 1/8 scales. Save feature matching to
+# sqlite database.
+# All input and output images:
+#   $gcs_dataset_path
+#   $gcs_experiment_path/data/images
+# Downsampled output images:
+#   $gcs_experiment_path/data/images_2/
+#   $gcs_experiment_path/data/images_4/
+#   $gcs_experiment_path/data/images_8/
+# COLMAP sparse reconstruction files: project.ini, images.bin,
+# cameras.bin, points3D.bin
+#   $gcs_experiment_path/data/sparse/0/
+cp -r "$output_folder" "$images_folder"/images_2
+pushd "$images_folder"/images_2
+ls | xargs -P 8 -I {} mogrify -resize 50% {}
+popd
+gsutil -m cp -r "$images_folder"/images_2/*  "$gcs_experiment_path"/data/images_2
+
+cp -r "$output_folder" "$images_folder"/images_4
+pushd "$images_folder"/images_4
+ls | xargs -P 8 -I {} mogrify -resize 25% {}
+popd
+gsutil -m cp -r "$images_folder"/images_4/*  "$gcs_experiment_path"/data/images_4
+
+cp -r "$output_folder" "$images_folder"/images_8
+pushd "$images_folder"/images_8
+ls | xargs -P 8 -I {} mogrify -resize 12.5% {}
+popd
+gsutil -m cp "$images_folder"/images_8/*  "$gcs_experiment_path"/data/images_8
+
+# Copy images and sparse reconstruction files to gcs experiment folder.
+gsutil -m cp "$images_folder"/images/*  "$gcs_experiment_path"/data/images
+gsutil -m cp -r "$local_folder"/sparse  "$gcs_experiment_path"/data
+gsutil -m cp "$local_folder"/database.db  "$gcs_experiment_path"/data
+
+echo "Processing complete."

community-content/vertex_model_garden/model_oss/cloudnerf/render.sh

@@ -0,0 +1,117 @@
+#!/bin/bash
+# This script runs rendering for ZipNeRF given an experiment folder
+# from a GCS bucket with colmap dataset.
+
+# Initialize associative array for arguments.
+declare -A args
+
+# vv-docker:google3-begin(internal)
+# TODO(b/311468174): Pass gin config file from gcs bucket.
+# vv-docker:google3-end
+# Function to parse named arguments.
+parse_args() {
+  while [[ $# -gt 0 ]]; do
+    key="$1"
+    case $key in
+      -gcs_experiment_path|-gin_config_file|-gcs_keyframes_file)
+        args[$key]="$2"
+        shift # past argument
+        shift # past value
+        ;;
+      -training_job_name)
+        training_job_name="$2"
+        shift # past argument
+        shift # past value
+        ;;
+      -rendering_job_name)
+        rendering_job_name="$2"
+        shift # past argument
+        shift # past value
+        ;;
+      -render_path_frames|-factor|-render_video_fps)
+        args[$key]="$2"
+        if ! [[ ${args[$key]} =~ ^[0-9]+$ ]]; then
+          echo "Error: $key must be an integer."
+          exit 1
+        fi
+        shift # past argument
+        shift # past value
+        ;;
+      *)
+        echo "Unknown option: $1" >&2
+        exit 1
+        ;;
+    esac
+  done
+}
+
+# Function to create a directory if it doesn't exist.
+create_dir_if_not_exists() {
+  local dir_path=$1
+  if [[ ! -d "$dir_path" ]]; then
+    echo "Creating folder: $dir_path"
+    mkdir "$dir_path"
+  else
+    echo "Folder $dir_path already exists."
+  fi
+}
+
+# Function to launch rendering.
+launch_rendering() {
+  local keyframes_file=$1
+  local render_bindings=(
+    "--gin_configs=${args[-gin_config_file]}"
+    "--gin_bindings=Config.data_dir='${DATASET_PATH}'"
+    "--gin_bindings=Config.exp_name='${EXPERIMENT}'"
+    "--gin_bindings=Config.render_path=True"
+    "--gin_bindings=Config.render_path_frames=${args[-render_path_frames]}"
+    "--gin_bindings=Config.render_video_fps=${args[-render_video_fps]}"
+    "--gin_bindings=Config.factor=${args[-factor]}"
+  )
+
+  if [[ -n $keyframes_file ]]; then
+    render_bindings+=("--gin_bindings=Config.render_spline_keyframes='${keyframes_file}'")
+  fi
+
+  accelerate launch render.py "${render_bindings[@]}"
+}
+
+# Parse arguments.
+parse_args "$@"
+
+# Extract folder names and paths.
+scene_folder_name=$(basename "${args[-gcs_experiment_path]}")
+local_dataset_path="local_dataset"
+local_experiment_path="exp"
+exp_folder_name=$(basename "${args[-gcs_experiment_path]}")
+DATASET_PATH="$local_experiment_path/$exp_folder_name/data"
+CHECKPOINTS_PATH="$local_experiment_path/$exp_folder_name/checkpoints"
+OUTPUT_RENDER_PATH="$local_experiment_path/$scene_folder_name/render"
+EXPERIMENT=$exp_folder_name
+
+# Create necessary directories.
+create_dir_if_not_exists "$local_dataset_path"
+create_dir_if_not_exists "$local_experiment_path"
+create_dir_if_not_exists "$local_experiment_path/$exp_folder_name"
+create_dir_if_not_exists "$CHECKPOINTS_PATH"
+
+# Create the file log_render.txt in the exp folder.
+touch "$local_experiment_path/$exp_folder_name/log_render.txt"
+
+# Copy experiment from GCS bucket to local
+gsutil -m cp -r "${args[-gcs_experiment_path]}/data" "$local_experiment_path/$exp_folder_name" || exit 1
+gsutil -m cp -r "${args[-gcs_experiment_path]}/checkpoints/${training_job_name}/*" "$CHECKPOINTS_PATH" || exit 1
+
+# Check and copy keyframes file.
+if [[ -n ${args[-gcs_keyframes_file]} ]]; then
+  keyframes_file_basename=$(basename "${args[-gcs_keyframes_file]}")
+  local_keyframes_file="$local_dataset_path/$keyframes_file_basename"
+  gsutil cp "${args[-gcs_keyframes_file]}" "$local_keyframes_file" || exit 1
+  echo "Local keyframe file: $local_keyframes_file"
+  launch_rendering "$local_keyframes_file"
+else
+  launch_rendering ""
+fi
+
+# Copy rendered data back to GCS.
+gsutil -m cp -r "$OUTPUT_RENDER_PATH" "${args[-gcs_experiment_path]}/render/${rendering_job_name}"

community-content/vertex_model_garden/model_oss/cloudnerf/requirements.txt

@@ -0,0 +1,24 @@
+--find-links https://download.pytorch.org/whl/torch_stable.html
+
+torch==2.0.1+cu118
+numpy==1.26.1
+absl_py==2.0.0
+accelerate==0.24.0
+gin_config==0.5.0
+imageio==2.31.6
+imageio-ffmpeg==0.4.9
+matplotlib==3.8.0
+mediapy==1.1.9
+ninja==1.11.1.1
+opencv_contrib_python==4.8.1.78
+opencv_python==4.8.1.78
+Pillow==10.3.0
+rawpy==0.18.1
+scipy==1.11.3
+scikit-image==0.22.0
+scikit-learn==1.3.2
+tensorboard==2.15.0
+tensorboardX==2.6.2.2
+tqdm==4.66.3
+trimesh==4.0.1
+xatlas==0.0.8

community-content/vertex_model_garden/model_oss/cloudnerf/train.sh

@@ -0,0 +1,94 @@
+#!/bin/bash
+
+# Initialize variables.
+training_job_name=""
+gcs_experiment_path=""
+gin_config_file="configs/360.gin"
+factor=4
+max_training_steps=25000
+
+# Parse named arguments.
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    -training_job_name)
+      training_job_name="$2"
+      shift # past argument
+      shift # past value
+      ;;
+    -gcs_experiment_path)
+      gcs_experiment_path="$2"
+      shift # past argument
+      shift # past value
+      ;;
+    -gin_config_file)
+      gin_config_file="$2"
+      shift # past argument
+      shift # past value
+      ;;
+    -factor)
+      factor="$2"
+      if ! [[ $factor =~ ^[0-9]+$ ]]; then
+        echo "Error: -factor must be an integer."
+        exit 1
+      fi
+      shift # past argument
+      shift # past value
+      ;;
+    -max_training_steps)
+      max_training_steps="$2"
+      if ! [[ $max_training_steps =~ ^[0-9]+$ ]]; then
+        echo "Error: -max_training_steps must be an integer."
+        exit 1
+      fi
+      shift # past argument
+      shift # past value
+      ;;
+    *)    # unknown option
+      echo "Unknown option: $1" >&2
+      exit 1
+      ;;
+  esac
+done
+
+# Function to create a directory if it doesn't exist.
+create_dir_if_not_exists() {
+  local dir_path=$1
+  if [[ ! -d "$dir_path" ]]; then
+    echo "Creating folder: $dir_path"
+    mkdir "$dir_path"
+  else
+    echo "Folder $dir_path already exists."
+  fi
+}
+
+# Extract folder names and paths.
+scene_folder_name=$(basename "${gcs_experiment_path}")
+local_dataset_path="local_dataset"
+local_experiment_path="exp"
+DATASET_PATH="$local_experiment_path/$scene_folder_name/data"
+EXPERIMENT=$scene_folder_name
+
+# Create necessary directories.
+create_dir_if_not_exists "$local_dataset_path"
+create_dir_if_not_exists "$local_experiment_path"
+create_dir_if_not_exists "$local_experiment_path/$scene_folder_name"
+
+# Copy experiment from GCS bucket to local.
+gsutil -m cp -r "${gcs_experiment_path}/data" "$local_experiment_path/$scene_folder_name" || exit 1
+
+echo "GCS Experiment: $gcs_experiment_path"
+echo "Gin Config File: $gin_config_file"
+echo "Factor: $factor"
+echo "Scene: $scene_folder_name"
+echo "Local Dataset: $DATASET_PATH"
+echo "Local Experiment: $EXPERIMENT"
+
+accelerate launch train.py --gin_configs="$gin_config_file" \
+  --gin_bindings="Config.data_dir = '${DATASET_PATH}'" \
+  --gin_bindings="Config.exp_name = '${EXPERIMENT}'" \
+  --gin_bindings="Config.factor = ${factor}" \
+  --gin_bindings="Config.max_steps = ${max_training_steps}"
+
+gsutil -m rm -r "${gcs_experiment_path}/checkpoints/${training_job_name}"
+gsutil -m cp -r "$local_experiment_path/$scene_folder_name/config.gin" "${gcs_experiment_path}/${training_job_name}_config.gin"
+gsutil -m cp -r "$local_experiment_path/$scene_folder_name/checkpoints/*/*" "${gcs_experiment_path}/checkpoints/${training_job_name}"

community-content/vertex_model_garden/model_oss/detectron2/dockerfile/serving.Dockerfile

@@ -0,0 +1,90 @@
+# Dockerfile for Detectron2 serving.
+#
+# To build:
+# docker build -f model_oss/detectron2/dockerfile/serving.Dockerfile . -t ${YOUR_IMAGE_TAG}
+#
+# To push to gcr:
+# docker tag ${YOUR_IMAGE_TAG} gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
+# docker push gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
+FROM pytorch/torchserve:0.7.0-cpu
+
+USER root
+
+# Install tools.
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        curl \
+        wget \
+        vim
+
+# run and update some basic packages software packages, including security libs
+RUN apt-get update &&  apt-get install -y \
+    software-properties-common && \
+    add-apt-repository -y ppa:ubuntu-toolchain-r/test && \
+    apt-get update && apt-get install -y \
+    gcc-9 g++-9 apt-transport-https ca-certificates gnupg curl
+
+# Install gcloud tools for gsutil as well as debugging
+RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | \
+    tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \
+    curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | \
+    apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - && \
+    apt-get update -y && apt-get install google-cloud-sdk -y
+
+USER model-server
+
+# install detectron2 dependencies
+RUN python3 -m pip install --upgrade pip
+RUN python3 -m pip install --user numpy==1.24.2
+RUN python3 -m pip install --user opencv-python==4.7.0.72
+RUN python3 -m pip install --user 'git+https://github.com/facebookresearch/detectron2.git@v0.6'
+
+# Install GCS storage library.
+RUN pip install google-cloud-storage==2.6.0
+
+# For mask encoding.
+RUN pip install --upgrade pycocotools==2.0.6
+
+ARG MODEL_NAME=detectron2_serving
+ENV MODEL_NAME="${MODEL_NAME}"
+
+# health and prediction listener ports
+ARG AIP_HTTP_PORT=7080
+ENV AIP_HTTP_PORT="${AIP_HTTP_PORT}"
+
+ARG MODEL_MGMT_PORT=7081
+
+# expose health and prediction listener ports from the image
+EXPOSE "${AIP_HTTP_PORT}"
+EXPOSE "${MODEL_MGMT_PORT}"
+EXPOSE 8080 8081 8082 7070 7071
+
+# create torchserve configuration file
+USER root
+RUN echo "service_envelope=json\n" \
+    "inference_address=http://0.0.0.0:${AIP_HTTP_PORT}\n"  \
+    "management_address=http://0.0.0.0:${MODEL_MGMT_PORT}" >> /home/model-server/config.properties
+USER model-server
+
+# Copy license.
+RUN wget https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/LICENSE
+
+# Copy model artifacts.
+COPY ./model_oss/detectron2/handler.py /home/model-server/handler.py
+WORKDIR /home/model-server/
+
+# Create model archive file packaging model artifacts and dependencies.
+# Note(lavrai): The model `.pth` file and `cfg.yaml` file will be set by the
+# customer as an environment variable and will be later loaded by the
+# `handler.py` file.
+RUN torch-model-archiver \
+  --model-name="${MODEL_NAME}" \
+  --version=1.0 \
+  --handler=/home/model-server/handler.py \
+  --export-path=/home/model-server/model-store \
+  -f
+
+# run Torchserve HTTP serve to respond to prediction requests
+CMD ["ls", "-ltr", "/home/model-server/model-store/", ";", \
+    "torchserve", "--start", "--ts-config=/home/model-server/config.properties", \
+    "--models", "${MODEL_NAME}=${MODEL_NAME}.mar", \
+    "--model-store", "/home/model-server/model-store"]

community-content/vertex_model_garden/model_oss/detectron2/dockerfile/train.Dockerfile

@@ -0,0 +1,100 @@
+# Dockerfile for Detectron2 training.
+#
+# To build:
+# docker build -f model_oss/detectron2/dockerfile/train.Dockerfile . -t ${YOUR_IMAGE_TAG}
+#
+# To push to gcr:
+# docker tag ${YOUR_IMAGE_TAG} gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
+# docker push gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
+
+FROM nvidia/cuda:11.1.1-cudnn8-devel-ubuntu18.04
+# Using an older system (18.04) to avoid opencv incompatibility (issue#3524).
+
+ENV DEBIAN_FRONTEND noninteractive
+RUN apt-get update && apt-get install -y \
+    python3.7 python3.7-dev python3.7-distutils \
+    python3-opencv ca-certificates git wget sudo ninja-build \
+    curl wget vim
+
+# Make python3 available for python3.7.
+RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.6 1
+RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 2
+RUN update-alternatives --config python3
+# Make python available for python3.7.
+RUN ln -sv /usr/bin/python3.7 /usr/bin/python
+
+# Create a non-root user.
+ARG USER_ID=1000
+RUN useradd -m --no-log-init --system  --uid ${USER_ID} appuser -g sudo
+RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+USER appuser
+WORKDIR /home/appuser
+
+ENV PATH="/home/appuser/.local/bin:${PATH}"
+RUN wget https://bootstrap.pypa.io/pip/get-pip.py && \
+    python3.7 get-pip.py --user && \
+    rm get-pip.py
+
+# Important! Otherwise, it uses existing numpy from host-modules
+# which throws error.
+RUN pip install --user numpy==1.20.3
+
+# Install dependencies:
+# See https://pytorch.org/ for other options if you use
+# a different version of CUDA.
+RUN pip install --user tensorboard==2.11.0
+# cmake from apt-get is too old.
+RUN pip install --user cmake==3.25.2
+RUN pip install --user torch==1.10.0+cu111 torchvision==0.11.0+cu111 -f https://download.pytorch.org/whl/torch_stable.html
+RUN pip install --user setuptools==59.5.0
+RUN pip install --user opencv-python==4.7.0.72
+RUN pip install --user cloudml-hypertune==0.1.0.dev6
+RUN pip install --user fvcore==0.1.5.post20221221
+# Install detectron2.
+RUN git clone -b v0.6 https://github.com/facebookresearch/detectron2 detectron2_repo
+# Set FORCE_CUDA because during `docker build` cuda is not accessible.
+ENV FORCE_CUDA="1"
+# This will by default build detectron2 for all common cuda
+# architectures and take a lot more time,
+# because inside `docker build`, there is no way to tell
+# which architecture will be used.
+ARG TORCH_CUDA_ARCH_LIST="Kepler;Kepler+Tesla;Maxwell;Maxwell+Tegra;Pascal;Volta;Turing"
+ENV TORCH_CUDA_ARCH_LIST="${TORCH_CUDA_ARCH_LIST}"
+RUN pip install --user -e detectron2_repo
+
+# Set a fixed model cache directory.
+ENV FVCORE_CACHE="/tmp"
+
+# Copy license.
+RUN wget https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/LICENSE
+
+# Copy model-garden detectron2 files to '/home/appuser/trainer' folder.
+ADD ./model_oss/detectron2 /home/appuser/trainer
+
+################ Copy plain_train_net.py to task.py and
+# then modify it using sed commands. ###################
+# Src: https://github.com/facebookresearch/detectron2/blob/v0.6/tools/plain_train_net.py
+RUN sudo cp /home/appuser/detectron2_repo/tools/plain_train_net.py /home/appuser/trainer/task.py
+# Make additional changes to task.py.
+# Note(lavrai): Start adding SED commands from end of file towards the top
+# so that the line numbers do not keep changing for the source file.
+# For entry-point:
+RUN sudo sed -i "214 d" /home/appuser/trainer/task.py
+RUN sudo sed -i "213 a\    default_arg_parser = default_argument_parser()" /home/appuser/trainer/task.py
+RUN sudo sed -i "214 a\    extended_parser = trainer_utils.extend_parser_arguments(default_arg_parser)" /home/appuser/trainer/task.py
+RUN sudo sed -i "215 a\    args = extended_parser.parse_args()" /home/appuser/trainer/task.py
+# For main() function:
+RUN sudo sed -i "192 a\    trainer_utils.register_dataset(args)" /home/appuser/trainer/task.py
+# For setup() function:
+RUN sudo sed -i "184 a\    cfg.SOLVER.BASE_LR = args.lr" /home/appuser/trainer/task.py
+RUN sudo sed -i "185 a\    cfg.OUTPUT_DIR = args.output_dir" /home/appuser/trainer/task.py
+RUN sudo sed -i "186 a\    cfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url(config_file_copy)" /home/appuser/trainer/task.py
+RUN sudo sed -i "182 a\    config_file_copy = args.config_file" /home/appuser/trainer/task.py
+RUN sudo sed -i "183 a\    args.config_file = model_zoo.get_config_file(args.config_file)" /home/appuser/trainer/task.py
+# For new import:
+RUN sudo sed -i "27 a\from detectron2 import model_zoo" /home/appuser/trainer/task.py
+RUN sudo sed -i "21 a\import trainer_utils" /home/appuser/trainer/task.py
+
+ENV PYTHONPATH /home/appuser/trainer
+
+ENTRYPOINT ["python", "-m", "trainer.task"]

community-content/vertex_model_garden/model_oss/detectron2/handler.py

@@ -0,0 +1,154 @@
+"""Custom handler for Detectron2 serving."""
+
+import io
+import json
+import os
+from typing import Any, List, Tuple
+
+import cv2
+from detectron2.config import get_cfg
+from detectron2.engine import DefaultPredictor
+from google.cloud import storage
+import numpy as np
+import pycocotools.mask as mask_util
+import torch
+
+
+def get_bucket_and_blob_name(gcs_filepath: str) -> Tuple[str, str]:
+  """Gets bucket and blob name from gcs path."""
+  # The gcs path is of the form gs://<bucket-name>/<blob-name>
+  gs_suffix = gcs_filepath.split("gs://", 1)[1]
+  return tuple(gs_suffix.split("/", 1))
+
+
+def download_gcs_file(src_file_path: str, dst_file_path: str):
+  """Downloads gcs-file to local folder."""
+  src_bucket_name, src_blob_name = get_bucket_and_blob_name(src_file_path)
+  client = storage.Client()
+  src_bucket = client.get_bucket(src_bucket_name)
+  src_blob = src_bucket.blob(src_blob_name)
+  src_blob.download_to_filename(dst_file_path)
+
+
+class ModelHandler:
+  """Custom model handler for Detectron2."""
+
+  def __init__(self):
+    self.error = None
+    self._batch_size = 0
+    self.initialized = False
+    self.predictor = None
+    self.test_threshold = 0.5
+
+  def initialize(self, context: Any):
+    """Initialize."""
+    print("context.system_properties: ", context.system_properties)
+    print("context.manifest: ", context.manifest)
+    self.manifest = context.manifest
+    properties = context.system_properties
+    # Get threshold from environment variable.
+    # This will be set by customer.
+    self.test_threshold = float(os.environ.get("TEST_THRESHOLD"))
+    print("test_threshold: ", self.test_threshold)
+    # Get model and config file location from environment variables.
+    # These will be set by customer when doing model upload.
+    gcs_model_file = os.environ["MODEL_PTH_FILE"]
+    gcs_config_file = os.environ["CONFIG_YAML_FILE"]
+    print("Copying gcs_model_file: ", gcs_model_file)
+    print("Copying gcs_config_file: ", gcs_config_file)
+    # Copy these files from GCS location to local file.
+    # Note(lavrai): GCSFuse path does not seem to work here for now.
+    model_file = "./model.pth"
+    config_file = "./cfg.yaml"
+    download_gcs_file(src_file_path=gcs_model_file, dst_file_path=model_file)
+    if not os.path.exists(model_file):
+      raise RuntimeError("Missing model_file: %s" % model_file)
+    download_gcs_file(src_file_path=gcs_config_file, dst_file_path=config_file)
+    if not os.path.exists(config_file):
+      raise RuntimeError("Missing config_file: %s" % config_file)
+
+    # Set up config file.
+    cfg = get_cfg()
+    cfg.merge_from_file(config_file)
+    cfg.MODEL.WEIGHTS = model_file
+    cfg.MODEL.DEVICE = (
+        cfg.MODEL.DEVICE + str(properties.get("gpu_id"))
+        if torch.cuda.is_available()
+        else "cpu"
+    )
+    cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = self.test_threshold
+
+    # Build predictor from config.
+    self.predictor = DefaultPredictor(cfg)
+    self._batch_size = context.system_properties["batch_size"]
+    self.initialized = True
+
+  def preprocess(self, batch: List[Any]) -> List[Any]:
+    """Preprocess raw input and return as list of images."""
+    print("Running pre-processing.")
+    images = []
+    for request in batch:
+      request_data = request.get("data")
+      input_bytes = io.BytesIO(request_data)
+      img = cv2.imdecode(np.fromstring(input_bytes.read(), np.uint8), 1)
+      images.append(img)
+    return images
+
+  def inference(self, model_input: List[Any]) -> List[Any]:
+    """Runs inference."""
+    print("Running model-inference.")
+    return [self.predictor(image) for image in model_input]
+
+  def postprocess(self, inference_result: List[Any]) -> List[Any]:
+    """Post process inference result."""
+    response_list = []
+    print("Num inference_items are:", len(inference_result))
+    for inference_item in inference_result:
+      predictions = inference_item["instances"].to("cpu")
+      print("Predictions are:", predictions)
+      boxes = None
+      if predictions.has("pred_boxes"):
+        boxes = predictions.pred_boxes.tensor.numpy().tolist()
+      scores = None
+      if predictions.has("scores"):
+        scores = predictions.scores.numpy().tolist()
+      classes = None
+      if predictions.has("pred_classes"):
+        classes = predictions.pred_classes.numpy().tolist()
+      masks_rle = None
+      if predictions.has("pred_masks"):
+        # Do run length encoding, else the mask output becomes huge.
+        masks_rle = [
+            mask_util.encode(np.asfortranarray(mask))
+            for mask in predictions.pred_masks
+        ]
+        for rle in masks_rle:
+          rle["counts"] = rle["counts"].decode("utf-8")
+      response = {
+          "classes": classes,
+          "scores": scores,
+          "boxes": boxes,
+          "masks_rle": masks_rle,
+      }
+      response_list.append(json.dumps(response))
+    print("response_list: ", response_list)
+    return response_list
+
+  def handle(self, data: Any, context: Any) -> List[Any]:  # pylint: disable=unused-argument
+    """Runs preprocess, inference, and post-processing."""
+    model_input = self.preprocess(data)
+    model_out = self.inference(model_input)
+    output = self.postprocess(model_out)
+    print("Done handling input.")
+    return output
+
+
+_service = ModelHandler()
+
+
+def handle(data: Any, context: Any) -> List[Any]:
+  if not _service.initialized:
+    _service.initialize(context)
+  if data is None:
+    return None
+  return _service.handle(data, context)

community-content/vertex_model_garden/model_oss/detectron2/trainer_utils.py

@@ -0,0 +1,97 @@
+"""Detectron2 trainer helper functions."""
+
+import argparse
+from detectron2.data.datasets import register_coco_instances
+
+
+def extend_parser_arguments(
+    parser: argparse.ArgumentParser,
+) -> argparse.ArgumentParser:
+  """Adds additional model-garden related arguments."""
+  parser.add_argument(
+      "--train_dataset_name",
+      required=False,
+      default="",
+      type=str,
+      help=(
+          "The training dataset name for registration. "
+          "For example: 'balloon_train'."
+      ),
+  )
+  parser.add_argument(
+      "--train_coco_json_file",
+      required=False,
+      default="",
+      type=str,
+      help="The path to the training coco-json format file.",
+  )
+  parser.add_argument(
+      "--train_image_root",
+      required=False,
+      default="",
+      type=str,
+      help="The path to the root folder containing the training images.",
+  )
+  parser.add_argument(
+      "--val_dataset_name",
+      required=False,
+      default="",
+      type=str,
+      help=(
+          "The validation dataset name for registration. "
+          "For example: 'balloon_val'."
+      ),
+  )
+  parser.add_argument(
+      "--val_coco_json_file",
+      required=False,
+      default="",
+      type=str,
+      help="The path to the validation coco-json format file.",
+  )
+  parser.add_argument(
+      "--val_image_root",
+      required=False,
+      default="",
+      type=str,
+      help="The path to the root folder containing the validation images.",
+  )
+  parser.add_argument(
+      "--output_dir",
+      required=True,
+      type=str,
+      help="The path to the output directory.",
+  )
+  # Add hyper-parameter tuning related variables.
+  parser.add_argument(
+      "--lr",
+      type=float,
+      default=0.00025,
+      help="The learning rate to be tuned.",
+  )
+  parser.add_argument(
+      "--hp_eval_task",
+      type=str,
+      choices=["bbox", "segm"],
+      default="bbox",
+      help="The task choice for HP tuning.",
+  )
+  return parser
+
+
+def register_dataset(args: argparse.Namespace):
+  """Register the input dataset in Detectron2 Coco format."""
+  if args.train_dataset_name:
+    register_coco_instances(
+        name=args.train_dataset_name,
+        metadata={},
+        json_file=args.train_coco_json_file,
+        image_root=args.train_image_root,
+    )
+  if args.val_dataset_name:
+    register_coco_instances(
+        name=args.val_dataset_name,
+        metadata={},
+        json_file=args.val_coco_json_file,
+        image_root=args.val_image_root,
+    )

community-content/vertex_model_garden/model_oss/fvlm/dockerfile/jax_fvlm_model_conversion.Dockerfile

@@ -0,0 +1,83 @@
+# This Dockerfile converts JAX vision transformer model to
+# tensorflow saved model format.
+# Here is an example to build this dockerfile:
+# PROJECT="your gcp project"
+# IMAGE_TAG="jax-f-vlm-model-conversion:${USER}-test"
+# docker build -f model_oss/fvlm/dockerfile/jax_fvlm_model_conversion.Dockerfile . -t "${IMAGE_TAG}"
+# docker tag "${IMAGE_TAG}" "gcr.io/${PROJECT}/${IMAGE_TAG}"
+# docker push "gcr.io/${PROJECT}/${IMAGE_TAG}"
+
+
+# See https://cloud.google.com/tensorflow-enterprise/docs/overview for details.
+FROM gcr.io/deeplearning-platform-release/tf2-gpu.2-12.py310:m110
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Install basic libs
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        cmake \
+        curl \
+        wget \
+        sudo \
+        gnupg \
+        libsm6 \
+        libxext6 \
+        libxrender-dev \
+        lsb-release \
+        ca-certificates \
+        build-essential \
+        git \
+        libgl1
+
+# Copy Apache license.
+RUN wget https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/LICENSE
+
+# Install required libs
+RUN pip install --upgrade pip
+
+# Using the commit 6712c224985c694001ba8ee68697bbf4dcb32edb on Jan 4th, 2024.
+ARG COMMIT_ID=6712c224985c694001ba8ee68697bbf4dcb32edb
+RUN git clone -c \
+remote.origin.fetch=+${COMMIT_ID}:refs/remotes/origin/${COMMIT_ID} \
+https://github.com/google-research/google-research --no-checkout --progress \
+--depth 1
+WORKDIR ./google-research
+RUN git sparse-checkout init --cone
+RUN git sparse-checkout set fvlm
+RUN git checkout ${COMMIT_ID}
+
+# The following pip installs are pinned down versions satisfying
+# fvlm/requirements.txt file.
+# NOTE: Using `no-deps` flag to avoid overwriting of dependent library
+# versions. For example, both `chex` and `jax` can overwrite each other's
+# `jax-lib` version.
+# Note: The following libraries are pinned down versions of:
+# https://github.com/google-research/google-research/blob/master/fvlm/requirements.txt
+RUN pip install --no-cache-dir tensorflow==2.12.0
+RUN pip install --no-cache-dir tensorflow-datasets==4.9.2
+RUN pip install --no-cache-dir numpy==1.23.5
+RUN pip install --no-cache-dir torch==2.0.1
+RUN pip install --no-cache-dir torchvision==0.15.2
+RUN pip install --no-cache-dir opencv-python==4.7.0.72
+RUN pip install --no-cache-dir tqdm==4.65.0
+RUN pip install --no-cache-dir git+https://github.com/openai/CLIP.git@a1d071733d7111c9c014f024669f959182114e33
+RUN pip install --no-cache-dir Pillow==9.5.0
+RUN pip install --no-cache-dir orbax-checkpoint==0.3.3
+RUN pip install --no-cache-dir gin-config==0.5.0
+RUN pip install --no-cache-dir pycocotools==2.0.6
+RUN pip install --no-cache-dir contextlib2==21.6.0
+RUN pip install --no-cache-dir ml-collections==0.1.1
+RUN pip install --no-cache-dir chex==0.1.7
+RUN pip install --no-cache-dir optax==0.1.5
+# Dependencies already included. Use no-deps to not update numpy.
+RUN pip install --no-cache-dir --no-deps flax==0.7.2
+RUN pip install --no-cache-dir --no-deps clu==0.0.9
+RUN pip install --no-cache-dir jax[cuda11_cudnn86]==0.4.9 \
+--find-links https://storage.googleapis.com/jax-releases/jax_cuda_releases.html
+RUN pip install --no-cache-dir ml-dtypes==0.2.0
+RUN pip install --no-cache-dir tensorflow_text==2.12.0
+
+WORKDIR ./fvlm
+ENV PYTHONPATH ./
+
+ENTRYPOINT ["python", "export_saved_model.py"]

community-content/vertex_model_garden/model_oss/fvlm/dockerfile/jax_fvlm_train_gpu.Dockerfile

@@ -0,0 +1,78 @@
+# This Dockerfile trains the F-VLM model on GPU.
+# Here is an example to build this dockerfile:
+# PROJECT="your gcp project"
+# IMAGE_TAG="jax-f-vlm-train:${USER}-test"
+# docker build -f model_oss/fvlm/dockerfile/jax_fvlm_train_gpu.Dockerfile . -t "${IMAGE_TAG}"
+# docker tag "${IMAGE_TAG}" "gcr.io/${PROJECT}/${IMAGE_TAG}"
+# docker push "gcr.io/${PROJECT}/${IMAGE_TAG}"
+
+# See https://cloud.google.com/tensorflow-enterprise/docs/overview for details.
+FROM gcr.io/deeplearning-platform-release/tf2-gpu.2-12.py310:m110
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Install basic libs
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        cmake \
+        curl \
+        wget \
+        sudo \
+        gnupg \
+        libsm6 \
+        libxext6 \
+        libxrender-dev \
+        lsb-release \
+        ca-certificates \
+        build-essential \
+        git
+
+# Copy Apache license.
+RUN wget https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/LICENSE
+
+# Install required libs
+RUN pip install --upgrade pip
+# The following pip installs are pinned down versions satisfying
+# fvlm/requirements.txt file.
+# Get F-VLM repository by using git sparse-checkout to avoid downloading entire
+# google-research repository.
+# Using the commit 6712c224985c694001ba8ee68697bbf4dcb32edb on Jan 4th, 2024.
+ARG COMMIT_ID=6712c224985c694001ba8ee68697bbf4dcb32edb
+RUN git clone -c \
+remote.origin.fetch=+${COMMIT_ID}:refs/remotes/origin/${COMMIT_ID} \
+https://github.com/google-research/google-research --no-checkout --progress \
+--depth 1
+WORKDIR ./google-research
+RUN git sparse-checkout init --cone
+RUN git sparse-checkout set fvlm
+RUN git checkout ${COMMIT_ID}
+
+# Note: The following libraries are pinned down versions of:
+# https://github.com/google-research/google-research/blob/master/fvlm/requirements.txt
+RUN pip install --no-cache-dir tensorflow==2.12.0
+RUN pip install --no-cache-dir tensorflow-datasets==4.9.2
+RUN pip install --no-cache-dir numpy==1.23.5
+RUN pip install --no-cache-dir torch==2.0.1
+RUN pip install --no-cache-dir torchvision==0.15.2
+RUN pip install --no-cache-dir opencv-python==4.7.0.72
+RUN pip install --no-cache-dir tqdm==4.65.0
+RUN pip install --no-cache-dir git+https://github.com/openai/CLIP.git@a1d071733d7111c9c014f024669f959182114e33
+RUN pip install --no-cache-dir Pillow==9.5.0
+RUN pip install --no-cache-dir orbax-checkpoint==0.3.3
+RUN pip install --no-cache-dir gin-config==0.5.0
+RUN pip install --no-cache-dir pycocotools==2.0.6
+RUN pip install --no-cache-dir contextlib2==21.6.0
+RUN pip install --no-cache-dir ml-collections==0.1.1
+RUN pip install --no-cache-dir chex==0.1.7
+RUN pip install --no-cache-dir optax==0.1.5
+# Dependencies already included. Use no-deps to not update numpy.
+RUN pip install --no-cache-dir --no-deps flax==0.7.2
+RUN pip install --no-cache-dir --no-deps clu==0.0.9
+# Installing jax at the very end with GPU support.
+# NOTE: Not using `no-deps` flag here because we need CUDA support.
+RUN pip install --no-cache-dir jax[cuda11_cudnn86]==0.4.9 \
+--find-links https://storage.googleapis.com/jax-releases/jax_cuda_releases.html
+
+WORKDIR ./fvlm
+ENV PYTHONPATH ./
+
+ENTRYPOINT ["python", "train_and_eval.py"]

community-content/vertex_model_garden/model_oss/fvlm/dockerfile/jax_fvlm_train_tpu.Dockerfile

@@ -0,0 +1,138 @@
+# This Dockerfile trains the F-VLM model on TPU.
+# Here is an example to build this dockerfile:
+# PROJECT="your gcp project"
+# IMAGE_TAG="jax-f-vlm-train-tpu:${USER}-test"
+# docker build -f model_oss/fvlm/dockerfile/jax_fvlm_train_tpu.Dockerfile . -t "${IMAGE_TAG}"
+# docker tag "${IMAGE_TAG}" "gcr.io/${PROJECT}/${IMAGE_TAG}"
+# docker push "gcr.io/${PROJECT}/${IMAGE_TAG}"
+
+FROM python:3.11
+
+# Get libtpu shared library. See go/what-is-libtpu.
+RUN curl -L https://storage.googleapis.com/cloud-tpu-tpuvm-artifacts/libtpu/1.6.0/libtpu.so -o /lib/libtpu.so
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Install basic libs
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        cmake \
+        curl \
+        wget \
+        sudo \
+        gnupg \
+        libsm6 \
+        libxext6 \
+        libxrender-dev \
+        lsb-release \
+        ca-certificates \
+        build-essential \
+        git \
+        libgl1
+
+# Copy Apache license.
+RUN wget https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/LICENSE
+
+# Install required libs
+RUN pip install --upgrade pip
+
+# Get F-VLM repository by using git sparse-checkout to avoid downloading entire
+# google-research repository.
+# Using the commit 05ece4b1c97285b48b51fa44321ccb2cb347406a on Dec 11th, 2023.
+ARG COMMIT_ID=05ece4b1c97285b48b51fa44321ccb2cb347406a
+RUN git clone -c \
+remote.origin.fetch=+${COMMIT_ID}:refs/remotes/origin/${COMMIT_ID} \
+https://github.com/google-research/google-research --no-checkout --progress \
+--depth 1
+WORKDIR ./google-research
+RUN git sparse-checkout init --cone
+RUN git sparse-checkout set fvlm
+RUN git checkout ${COMMIT_ID}
+
+# Note: The following libraries are pinned down versions of:
+# https://github.com/google-research/google-research/blob/master/fvlm/requirements.txt
+RUN pip install --no-cache-dir ml_dtypes==0.3.1
+RUN pip install --no-cache-dir tensorstore==0.1.51
+RUN pip install --no-cache-dir MarkupSafe==2.1.3
+RUN pip install --no-cache-dir Pillow==9.5.0
+RUN pip install --no-cache-dir PyYAML==6.0.1
+RUN pip install --no-cache-dir absl_py==1.4.0
+RUN pip install --no-cache-dir array_record==0.4.1
+RUN pip install --no-cache-dir astunparse==1.6.3
+RUN pip install --no-cache-dir cachetools==5.3.1
+RUN pip install --no-cache-dir certifi==2023.7.22
+RUN pip install --no-cache-dir charset_normalizer==3.3.0
+RUN pip install --no-cache-dir chex==0.1.83
+RUN pip install --no-cache-dir click==8.1.7
+RUN pip install --no-cache-dir clip==0.2.0
+RUN pip install --no-cache-dir clu==0.0.9
+RUN pip install --no-cache-dir contourpy==1.1.1
+RUN pip install --no-cache-dir cycler==0.12.1
+RUN pip install --no-cache-dir dm_tree==0.1.8
+RUN pip install --no-cache-dir etils==1.5.1
+RUN pip install --no-cache-dir filelock==3.12.4
+RUN pip install --no-cache-dir flatbuffers==23.5.26
+RUN pip install --no-cache-dir flax==0.7.4
+RUN pip install --no-cache-dir fonttools==4.43.1
+RUN pip install --no-cache-dir fsspec==2023.9.2
+RUN pip install --no-cache-dir ftfy==6.1.1
+RUN pip install --no-cache-dir gast==0.5.4
+RUN pip install --no-cache-dir gin_config==0.5.0
+RUN pip install --no-cache-dir google_auth==2.23.3
+RUN pip install --no-cache-dir google_auth_oauthlib==1.0.0
+RUN pip install --no-cache-dir google_pasta==0.2.0
+RUN pip install --no-cache-dir googleapis_common_protos==1.61.0
+RUN pip install --no-cache-dir grpcio==1.59.0
+RUN pip install --no-cache-dir h5py==3.10.0
+RUN pip install --no-cache-dir importlib_resources==6.1.0
+RUN pip install --no-cache-dir 'jax[tpu]==0.4.18' \
+-f https://storage.googleapis.com/jax-releases/libtpu_releases.html
+RUN pip install --no-cache-dir jaxlib==0.4.18
+RUN pip install --no-cache-dir jinja2==3.1.2
+RUN pip install --no-cache-dir keras==2.14.0
+RUN pip install --no-cache-dir kiwisolver==1.4.5
+RUN pip install --no-cache-dir libclang==16.0.6
+RUN pip install --no-cache-dir markdown==3.5
+RUN pip install --no-cache-dir matplotlib==3.8.0
+RUN pip install --no-cache-dir mpmath==1.3.0
+RUN pip install --no-cache-dir networkx==3.1
+RUN pip install --no-cache-dir numpy==1.26.0
+RUN pip install --no-cache-dir nvidia_cublas_cu12==12.1.3.1
+RUN pip install --no-cache-dir nvidia_cuda_cupti_cu12==12.1.105
+RUN pip install --no-cache-dir nvidia_cuda_nvrtc_cu12==12.1.105
+RUN pip install --no-cache-dir nvidia_cuda_runtime_cu12==12.1.105
+RUN pip install --no-cache-dir nvidia_cudnn_cu12==8.9.2.26
+RUN pip install --no-cache-dir nvidia_cufft_cu12==11.0.2.54
+RUN pip install --no-cache-dir nvidia_curand_cu12==10.3.2.106
+RUN pip install --no-cache-dir nvidia_cusolver_cu12==11.4.5.107
+RUN pip install --no-cache-dir nvidia_cusparse_cu12==12.1.0.106
+RUN pip install --no-cache-dir nvidia_nccl_cu12==2.18.1
+RUN pip install --no-cache-dir nvidia_nvjitlink_cu12==12.2.140
+RUN pip install --no-cache-dir nvidia_nvtx_cu12==12.1.105
+RUN pip install --no-cache-dir opencv_python==4.8.1.78
+RUN pip install --no-cache-dir orbax_checkpoint==0.4.1
+RUN pip install --no-cache-dir promise==2.3
+RUN pip install --no-cache-dir protobuf==3.20.3
+RUN pip install --no-cache-dir psutil==5.9.5
+RUN pip install --no-cache-dir pyasn1==0.5.0
+RUN pip install --no-cache-dir pycocotools==2.0.7
+RUN pip install --no-cache-dir pygments==2.16.1
+RUN pip install --no-cache-dir regex==2023.10.3
+RUN pip install --no-cache-dir rich==13.6.0
+RUN pip install --no-cache-dir scipy==1.11.3
+RUN pip install --no-cache-dir sympy==1.12
+RUN pip install --no-cache-dir tensorboard==2.14.1
+RUN pip install --no-cache-dir tensorboard_data_server==0.7.1
+RUN pip install --no-cache-dir tensorflow==2.14.0
+RUN pip install --no-cache-dir tensorflow_datasets==4.9.3
+RUN pip install --no-cache-dir torch==2.1.0
+RUN pip install --no-cache-dir torchvision==0.16.0
+RUN pip install --no-cache-dir urllib3==2.0.6
+RUN pip install --no-cache-dir wcwidth==0.2.8
+RUN pip install --no-cache-dir werkzeug==3.0.0
+RUN pip install --no-cache-dir wheel==0.41.2
+RUN pip install --no-cache-dir tensorflow_text==2.14.0
+
+WORKDIR ./fvlm
+ENV PYTHONPATH ./
+
+ENTRYPOINT ["python", "train_and_eval.py"]

community-content/vertex_model_garden/model_oss/imagebind/config.properties

@@ -0,0 +1,21 @@
+number_of_netty_threads=32
+job_queue_size=1000
+model_store=/home/model-server/model-store
+workflow_store=/home/model-server/wf-store
+default_response_timeout=1800
+service_envelope=json
+inference_address=http://0.0.0.0:7080
+management_address=http://0.0.0.0:7081
+metrics_address=http://0.0.0.0:7082
+
+models={\
+  "imagebind_serving": {\
+    "1.0": {\
+        "defaultVersion": true,\
+        "marName": "imagebind_serving.mar",\
+        "minWorkers": 1,\
+        "maxWorkers": 1,\
+        "batchSize": 1\
+    }\
+  }\
+}

community-content/vertex_model_garden/model_oss/imagebind/dockerfile/serve.Dockerfile

@@ -0,0 +1,74 @@
+# Dockerfile for the serving docker for ImageBind.
+#
+# To build:
+# docker build -f model_oss/imagebind/dockerfile/serve.Dockerfile . -t ${YOUR_IMAGE_TAG}
+#
+# To push to gcr:
+# docker tag ${YOUR_IMAGE_TAG} gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
+# docker push gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
+
+FROM pytorch/torchserve:0.7.0-gpu
+
+USER root
+
+ENV infer_port=7080
+ENV mng_port=7081
+ENV model_name="imagebind_serving"
+ENV PATH="/home/model-server/:${PATH}"
+
+# Install tools.
+RUN apt-get update && apt-get -y upgrade && apt-get install -y --no-install-recommends \
+        curl \
+        wget \
+        vim \
+        git \
+        libgeos-dev
+
+# Install libraries.
+ENV PIP_ROOT_USER_ACTION=ignore
+RUN python3 -m pip install --upgrade pip
+RUN pip install absl-py==1.4.0
+RUN pip install google-cloud-storage==2.7.0
+
+# Install ImageBind and dependencies.
+RUN git clone https://github.com/facebookresearch/ImageBind.git
+WORKDIR ImageBind
+# Pin the commit at 07/14/2023.
+RUN git reset --hard 95d27c7fd5a8362f3527e176c3a80ae5a4d880c0
+# Modify tokenizer file path from ImageBind repo to work with the server.
+RUN sed -i '25d' imagebind/data.py
+RUN sed -i '25 i\BPE_PATH = "/home/model-server/ImageBind/bpe/bpe_simple_vocab_16e6.txt.gz"' imagebind/data.py
+RUN pip install .
+
+WORKDIR /home/model-server
+
+# Copy license.
+RUN wget https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/LICENSE
+
+# Copy model artifacts.
+COPY model_oss/imagebind/handler.py /home/model-server/handler.py
+COPY model_oss/imagebind/config.properties /home/model-server/config.properties
+COPY model_oss/util/ /home/model-server/util/
+ENV PYTHONPATH /home/model-server/
+
+# Expose ports.
+EXPOSE ${infer_port}
+EXPOSE ${mng_port}
+
+# Archive model artifacts and dependencies.
+# Do not set --model-file and --serialized-file because model and checkpoint
+# will be dynamically loaded in handler.py.
+RUN torch-model-archiver \
+--model-name=${model_name} \
+--version=1.0 \
+--handler=/home/model-server/handler.py \
+--runtime=python3 \
+--export-path=/home/model-server/model-store \
+--archive-format=default \
+--force
+
+# Run Torchserve HTTP serve to respond to prediction requests.
+CMD ["torchserve", "--start", \
+     "--ts-config", "/home/model-server/config.properties", \
+     "--models", "${model_name}=${model_name}.mar", \
+     "--model-store", "/home/model-server/model-store"]

community-content/vertex_model_garden/model_oss/imagebind/handler.py

@@ -0,0 +1,277 @@
+"""Custom handler for the ImageBind model."""
+
+import logging
+import os
+from typing import Any, Dict, List
+
+from imagebind import data as data_util
+from imagebind.models import imagebind_model
+from imagebind.models.imagebind_model import ModalityType
+from PIL import Image
+import torch
+from torchvision import transforms
+from ts.torch_handler import base_handler
+
+from util import constants
+from util import fileutils
+
+
+_VIDEO_KEY_TO_AVOID_CONFLICT_WITH_IMAGE = "video"
+
+
+class ImageBindHandler(base_handler.BaseHandler):
+  """Custom handler for the ImageBind model.
+
+  Attributes:
+    map_location: Mapping storage location.
+    device: Device on which to run inference.
+    manifest: TorchServe manifest.
+    task: Task for which to run the ImageBind model.
+    model: ImageBind model instance.
+  """
+
+  def initialize(self, context: Any) -> None:
+    """Initializes the ImageBind model handler.
+
+    Args:
+      context: TorchServe context, which contains system information and the
+        manifest.
+
+    Raises:
+      ValueError: A task that is unsupported by the handler.
+    """
+    properties = context.system_properties
+    self.map_location = (
+        "cuda"
+        if torch.cuda.is_available() and properties.get("gpu_id") is not None
+        else "cpu"
+    )
+    self.device = torch.device(
+        self.map_location + ":" + str(properties.get("gpu_id"))
+        if torch.cuda.is_available() and properties.get("gpu_id") is not None
+        else self.map_location
+    )
+    self.manifest = context.manifest
+
+    self.task = os.environ.get("TASK", constants.FEATURE_EMBEDDING_GENERATION)
+    if self.task not in [
+        constants.FEATURE_EMBEDDING_GENERATION,
+        constants.ZERO_SHOT_CLASSIFICATION,
+    ]:
+      raise ValueError(f"Invalid task: {self.task}.")
+    logging.info(
+        "Handler initializing ImageBind pretrained model for task %s.",
+        self.task,
+    )
+
+    self.model = imagebind_model.imagebind_huge(pretrained=True)
+    self.model.eval()
+    self.model.to(self.device)
+
+    logging.info("Initialized ImageBind pretrained model.")
+    self.initialized = True
+
+  def preprocess(self, data: Any) -> List[Dict[str, Any]]:
+    """Preprocesses input data, including text, image, audio and video data.
+
+    Args:
+      data: Input data.
+
+    Returns:
+      A list of processed data samples, with each sample being a dictionary of
+      modality (key): input (value) pairs.
+    """
+    logging.info("Preprocessing: %d instances received.", len(data))
+    preprocessed_sample_list = []
+    for item in data:
+      preprocessed_sample = {}
+      if ModalityType.TEXT in item:
+        preprocessed_sample[ModalityType.TEXT] = (
+            data_util.load_and_transform_text(
+                item[ModalityType.TEXT], self.device
+            )
+        )
+      for image_modality in [
+          ModalityType.VISION,
+          ModalityType.DEPTH,
+          ModalityType.THERMAL,
+      ]:
+        if image_modality in item:
+          image_paths = item[image_modality]
+          local_image_paths = fileutils.download_gcs_file_list_to_local(
+              image_paths, constants.LOCAL_DATA_DIR
+          )
+          is_depth_or_thermal = image_modality in [
+              ModalityType.DEPTH,
+              ModalityType.THERMAL,
+          ]
+          preprocessed_sample[image_modality] = (
+              self._load_and_transform_image_data(
+                  local_image_paths,
+                  self.device,
+                  is_depth_or_thermal=is_depth_or_thermal,
+              )
+          )
+      if ModalityType.AUDIO in item:
+        audio_paths = item[ModalityType.AUDIO]
+        local_audio_paths = fileutils.download_gcs_file_list_to_local(
+            audio_paths, constants.LOCAL_DATA_DIR
+        )
+        preprocessed_sample[ModalityType.AUDIO] = (
+            data_util.load_and_transform_audio_data(
+                local_audio_paths, self.device
+            )
+        )
+      if _VIDEO_KEY_TO_AVOID_CONFLICT_WITH_IMAGE in item:
+        video_paths = item[_VIDEO_KEY_TO_AVOID_CONFLICT_WITH_IMAGE]
+        local_video_paths = fileutils.download_gcs_file_list_to_local(
+            video_paths, constants.LOCAL_DATA_DIR
+        )
+        preprocessed_sample[_VIDEO_KEY_TO_AVOID_CONFLICT_WITH_IMAGE] = (
+            data_util.load_and_transform_video_data(
+                local_video_paths, self.device
+            )
+        )
+      if ModalityType.IMU in item:
+        # Input data in the IMU modality are expected in shape [B, 6, 2000].
+        preprocessed_sample[ModalityType.IMU] = torch.tensor(
+            item[ModalityType.IMU], dtype=torch.float32, device=self.device
+        )
+      if preprocessed_sample:
+        preprocessed_sample_list.append(preprocessed_sample)
+    return preprocessed_sample_list
+
+  def _load_and_transform_image_data(
+      self,
+      image_paths: List[str],
+      device: torch.device,
+      is_depth_or_thermal: bool = False,
+  ) -> torch.Tensor:
+    """Loads and transforms 3-channel images, depth images and thermal images.
+
+    Args:
+      image_paths: A list of image paths.
+      device: Device onto which to load images.
+      is_depth_or_thermal: Whether the images are depth or thermal images.
+
+    Returns:
+      A list of processed tensors corresponding to the input images.
+
+    Raises:
+      ValueError: The input image_paths is None.
+    """
+    if image_paths is None:
+      raise ValueError("image_paths must not be None.")
+
+    image_outputs = []
+    for image_path in image_paths:
+      transforms_list = [
+          transforms.Resize(
+              224, interpolation=transforms.InterpolationMode.BICUBIC
+          ),
+          transforms.CenterCrop(224),
+          transforms.ToTensor(),
+      ]
+      if not is_depth_or_thermal:
+        transforms_list.append(
+            transforms.Normalize(
+                mean=(0.48145466, 0.4578275, 0.40821073),
+                std=(0.26862954, 0.26130258, 0.27577711),
+            )
+        )
+      data_transform = transforms.Compose(transforms_list)
+      with open(image_path, "rb") as fopen:
+        if is_depth_or_thermal:
+          image = Image.open(fopen).convert("L")
+        else:
+          image = Image.open(fopen).convert("RGB")
+
+      image = data_transform(image).to(device)
+      image_outputs.append(image)
+    return torch.stack(image_outputs, dim=0)
+
+  def inference(
+      self, data: List[Dict[str, Any]], *args, **kwargs
+  ) -> List[Dict[str, Any]]:
+    """Runs inference using the ImageBind model.
+
+    Args:
+      data: A list of processed data samples, with each sample being a
+        dictionary of modality (key): input (value) pairs.
+      *args: Additional inference args.
+      **kwargs: Additional inference kwargs.
+
+    Returns:
+      A list of model outputs, with each output being a dictionary of
+      modality (key): embedding (value) pairs.
+    """
+    output_list = []
+    with torch.no_grad():
+      for inputs in data:
+        if _VIDEO_KEY_TO_AVOID_CONFLICT_WITH_IMAGE in inputs:
+          # Allows inference on both image and video data, which both fall under
+          # ModalityType.VISION.
+          video_inputs = {
+              ModalityType.VISION: inputs[
+                  _VIDEO_KEY_TO_AVOID_CONFLICT_WITH_IMAGE
+              ]
+          }
+          video_embeddings = self.model(video_inputs)
+          video_embeddings[_VIDEO_KEY_TO_AVOID_CONFLICT_WITH_IMAGE] = (
+              video_embeddings[ModalityType.VISION]
+          )
+          del video_embeddings[ModalityType.VISION]
+          del inputs[_VIDEO_KEY_TO_AVOID_CONFLICT_WITH_IMAGE]
+        else:
+          video_embeddings = {}
+        embeddings = self.model(inputs)
+        embeddings.update(video_embeddings)
+        output_list.append(embeddings)
+    return output_list
+
+  def postprocess(self, output_list: List[Dict[str, Any]]) -> List[Any]:
+    """Postprocesses model outputs for the task of interest.
+
+    For feature embedding generation, returns the embeddings for each modality
+    for each input.
+    For zero-shot classification, generates classification probabilities
+    between the inputs of a pair of modalities for all possible pairings.
+
+    Args:
+      output_list: A list of model outputs, with each output being a dictionary
+        of modality (key): embedding (value) pairs.
+
+    Returns:
+      A list of postprocessed model outputs for the task of interest, with each
+      output corresponding to an input.
+
+    Raises:
+      ValueError: Fewer than two modalities are provided for zero-shot
+      classification, or the task is not supported.
+    """
+    preds = []
+    if self.task == constants.FEATURE_EMBEDDING_GENERATION:
+      for item in output_list:
+        preds.append({k: v.tolist() for k, v in item.items()})
+    elif self.task == constants.ZERO_SHOT_CLASSIFICATION:
+      for item in output_list:
+        modalities = list(item.keys())
+        if len(modalities) < 2:
+          raise ValueError(
+              "Two or more modalities are needed for task"
+              f" {constants.ZERO_SHOT_CLASSIFICATION}."
+          )
+        pairwise_probs = {}
+        for m1 in modalities:
+          for m2 in modalities:
+            if m1 == m2:
+              continue
+            probs = torch.softmax(item[m1] @ item[m2].T, dim=-1)
+            pairwise_probs[
+                f"Classify each input in {m1} (row) against inputs in"
+                f" {m2} (column)"
+            ] = probs.tolist()
+        preds.append(pairwise_probs)
+    else:
+      raise ValueError(f"Task {self.task} is not supported by the handler.")
+    return preds

community-content/vertex_model_garden/model_oss/jax_vision_transformer/dockerfile/jax_vit_model_conversion.Dockerfile

@@ -0,0 +1,151 @@
+# This Dockerfile converts JAX vision transformer model to
+# tensorflow saved model format.
+# Here is an example to build this dockerfile:
+# PROJECT="your gcp project"
+# IMAGE_TAG="jax-vit-model-conversion:${USER}-test"
+# docker build -f model_oss/jax_vision_transformer/dockerfile/jax_vit_model_conversion.Dockerfile . -t "${IMAGE_TAG}"
+# docker tag "${IMAGE_TAG}" "gcr.io/${PROJECT}/${IMAGE_TAG}"
+# docker push "gcr.io/${PROJECT}/${IMAGE_TAG}"
+
+
+FROM tensorflow/tensorflow:2.12.0-gpu
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Install basic libs
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        cmake \
+        curl \
+        wget \
+        sudo \
+        gnupg \
+        libsm6 \
+        libxext6 \
+        libxrender-dev \
+        lsb-release \
+        ca-certificates \
+        build-essential \
+        git
+
+# Copy Apache license.
+RUN wget https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/LICENSE
+
+# Get 'vision_transformer' repository from github.
+RUN git clone https://github.com/google-research/vision_transformer
+# Set current directory to the downloaded 'vision_transformer' repository.
+WORKDIR ./vision_transformer
+# Using git reset command to pin it down to a specific version.
+RUN git reset --hard e66b4732d44504251197a3da3f5949f3f3ce9ca6
+
+# Install required libs
+RUN pip install --upgrade pip
+# The following pip installs are pinned down versions of those inside
+# vit_jax/requirements.txt file.
+# NOTE: Using `no-deps` flag to avoid overwriting of
+# dependent library versions. For example,
+# both `chex` and `jax` can overwrite each others
+# `jax-lib` version.
+RUN pip install --no-deps absl-py==1.4.0
+RUN pip install --no-deps aqtp==0.0.10
+RUN pip install --no-deps array-record==0.2.0
+RUN pip install --no-deps astunparse==1.6.3
+RUN pip install --no-deps cached-property==1.5.2
+RUN pip install --no-deps cachetools==5.3.0
+RUN pip install --no-deps certifi==2019.11.28
+RUN pip install --no-deps chardet==3.0.4
+RUN pip install --no-deps chex==0.1.7
+RUN pip install --no-deps click==8.1.3
+RUN pip install --no-deps cloudpickle==2.2.1
+RUN pip install --no-deps clu==0.0.9
+RUN pip install --no-deps contextlib2==21.6.0
+RUN pip install --no-deps dacite==1.8.1
+RUN pip install --no-deps dbus-python==1.2.16
+RUN pip install --no-deps decorator==5.1.1
+RUN pip install --no-deps dm-tree==0.1.8
+RUN pip install --no-deps einops==0.6.1
+RUN pip install --no-deps etils==1.3.0
+RUN pip install --no-deps flatbuffers==23.3.3
+RUN pip install --no-deps flax==0.6.10
+RUN pip install --no-deps git+https://github.com/google/flaxformer@9adaa4467cf17703949b9f537c3566b99de1b416
+RUN pip install --no-deps gast==0.4.0
+RUN pip install --no-deps google-auth==2.16.2
+RUN pip install --no-deps google-auth-oauthlib==0.4.6
+RUN pip install --no-deps google-pasta==0.2.0
+RUN pip install --no-deps googleapis-common-protos==1.59.0
+RUN pip install --no-deps grpcio==1.51.3
+RUN pip install --no-deps h5py==3.8.0
+RUN pip install --no-deps idna==2.8
+RUN pip install --no-deps importlib-metadata==6.1.0
+RUN pip install --no-deps importlib-resources==5.12.0
+RUN pip install --no-deps keras==2.12.0
+RUN pip install --no-deps libclang==16.0.0
+RUN pip install --no-deps Markdown==3.4.3
+RUN pip install --no-deps markdown-it-py==2.2.0
+RUN pip install --no-deps MarkupSafe==2.1.2
+RUN pip install --no-deps mdurl==0.1.2
+RUN pip install --no-deps ml-collections==0.1.1
+RUN pip install --no-deps msgpack==1.0.5
+RUN pip install --no-deps nest-asyncio==1.5.6
+RUN pip install --no-deps numpy==1.23.5
+RUN pip install --no-deps oauthlib==3.2.2
+RUN pip install --no-deps opt-einsum==3.3.0
+RUN pip install --no-deps optax==0.1.5
+RUN pip install --no-deps orbax-checkpoint==0.1.6
+RUN pip install --no-deps packaging==23.0
+RUN pip install --no-deps pandas==2.0.1
+RUN pip install --no-deps pip==23.1.2
+RUN pip install --no-deps promise==2.3
+RUN pip install --no-deps protobuf==4.22.1
+RUN pip install --no-deps psutil==5.9.5
+RUN pip install --no-deps pyasn1==0.4.8
+RUN pip install --no-deps pyasn1-modules==0.2.8
+RUN pip install --no-deps Pygments==2.15.1
+RUN pip install --no-deps PyGObject==3.36.0
+RUN pip install --no-deps python-apt==2.0.1+ubuntu0.20.4.1
+RUN pip install --no-deps python-dateutil==2.8.2
+RUN pip install --no-deps pytz==2023.3
+RUN pip install --no-deps PyYAML==6.0
+RUN pip install --no-deps requests==2.22.0
+RUN pip install --no-deps requests-oauthlib==1.3.1
+RUN pip install --no-deps requests-unixsocket==0.2.0
+RUN pip install --no-deps rich==13.3.5
+RUN pip install --no-deps rsa==4.9
+RUN pip install --no-deps scipy==1.10.1
+RUN pip install --no-deps setuptools==67.6.0
+RUN pip install --no-deps six==1.14.0
+RUN pip install --no-deps tensorboard==2.12.0
+RUN pip install --no-deps tensorboard-data-server==0.7.0
+RUN pip install --no-deps tensorboard-plugin-wit==1.8.1
+RUN pip install --no-deps tensorflow==2.12.0
+RUN pip install --no-deps tensorflow-cpu==2.12.0
+RUN pip install --no-deps tensorflow-datasets==4.9.2
+RUN pip install --no-deps tensorflow-estimator==2.12.0
+RUN pip install --no-deps tensorflow-hub==0.13.0
+RUN pip install --no-deps tensorflow-io-gcs-filesystem==0.31.0
+RUN pip install --no-deps tensorflow-metadata==1.13.1
+RUN pip install --no-deps tensorflow-probability==0.20.0
+RUN pip install --no-deps tensorflow-text==2.12.1
+RUN pip install --no-deps tensorstore==0.1.36
+RUN pip install --no-deps termcolor==2.2.0
+RUN pip install --no-deps toml==0.10.2
+RUN pip install --no-deps toolz==0.12.0
+RUN pip install --no-deps tqdm==4.65.0
+RUN pip install --no-deps typing_extensions==4.5.0
+RUN pip install --no-deps tzdata==2023.3
+RUN pip install --no-deps urllib3==1.25.8
+RUN pip install --no-deps Werkzeug==2.2.3
+RUN pip install --no-deps wheel==0.40.0
+RUN pip install --no-deps wrapt==1.14.1
+RUN pip install --no-deps zipp==3.15.0
+# Installing jax at the very end with GPU support.
+# NOTE: Not using `no-deps` flag here because
+# we need CUDA support.
+RUN pip install jax[cuda11_cudnn82]==0.4.6 \
+--find-links https://storage.googleapis.com/jax-releases/jax_cuda_releases.html
+
+ENV PYTHONPATH ./vit_jax
+
+COPY ./model_oss/jax_vision_transformer/vit_jax2tf.py ./
+COPY ./model_oss/jax_vision_transformer/vit_config_without_data.py vit_jax/configs/vit.py
+
+ENTRYPOINT ["python", "vit_jax2tf.py"]

community-content/vertex_model_garden/model_oss/jax_vision_transformer/dockerfile/train_vit_gpu.Dockerfile

@@ -0,0 +1,149 @@
+# This Dockerfile runs the JAX based Vision transformer training on GPU.
+# See https://github.com/google-research/vision_transformer#running-on-cloud
+# for more details.
+# Here is an example to build this dockerfile:
+# PROJECT="your gcp project"
+# IMAGE_TAG="trainn_vit_gpu:${USER}-test"
+# docker build -f model_oss/jax_vision_transformer/dockerfile/train_vit_gpu.Dockerfile . -t "${IMAGE_TAG}"
+# docker tag "${IMAGE_TAG}" "gcr.io/${PROJECT}/${IMAGE_TAG}"
+# docker push "gcr.io/${PROJECT}/${IMAGE_TAG}"
+
+FROM tensorflow/tensorflow:2.12.0-gpu
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Install basic libs
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        cmake \
+        curl \
+        wget \
+        sudo \
+        gnupg \
+        libsm6 \
+        libxext6 \
+        libxrender-dev \
+        lsb-release \
+        ca-certificates \
+        build-essential \
+        git
+
+# Copy Apache license.
+RUN wget https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/LICENSE
+
+# Get 'vision_transformer' repository from github.
+RUN git clone https://github.com/google-research/vision_transformer
+# Ser current directory to the downloaded 'vision_transformer' repository.
+WORKDIR ./vision_transformer
+# Using git reset command to pin it down to a specific version.
+RUN git reset --hard e66b4732d44504251197a3da3f5949f3f3ce9ca6
+
+# Install required libs
+RUN pip install --upgrade pip
+# The following pip installs are pinned down versions of those inside
+# vit_jax/requirements.txt file.
+# NOTE: Using `no-deps` flag to avoid overwriting of
+# dependent library versions. For example,
+# both `chex` and `jax` can overwrite each others
+# `jax-lib` version.
+RUN pip install --no-deps absl-py==1.4.0
+RUN pip install --no-deps aqtp==0.0.10
+RUN pip install --no-deps array-record==0.2.0
+RUN pip install --no-deps astunparse==1.6.3
+RUN pip install --no-deps cached-property==1.5.2
+RUN pip install --no-deps cachetools==5.3.0
+RUN pip install --no-deps certifi==2019.11.28
+RUN pip install --no-deps chardet==3.0.4
+RUN pip install --no-deps chex==0.1.7
+RUN pip install --no-deps click==8.1.3
+RUN pip install --no-deps cloudpickle==2.2.1
+RUN pip install --no-deps clu==0.0.9
+RUN pip install --no-deps contextlib2==21.6.0
+RUN pip install --no-deps dacite==1.8.1
+RUN pip install --no-deps dbus-python==1.2.16
+RUN pip install --no-deps decorator==5.1.1
+RUN pip install --no-deps dm-tree==0.1.8
+RUN pip install --no-deps einops==0.6.1
+RUN pip install --no-deps etils==1.3.0
+RUN pip install --no-deps flatbuffers==23.3.3
+RUN pip install --no-deps flax==0.6.10
+RUN pip install --no-deps git+https://github.com/google/flaxformer@9adaa4467cf17703949b9f537c3566b99de1b416
+RUN pip install --no-deps gast==0.4.0
+RUN pip install --no-deps google-auth==2.16.2
+RUN pip install --no-deps google-auth-oauthlib==0.4.6
+RUN pip install --no-deps google-pasta==0.2.0
+RUN pip install --no-deps googleapis-common-protos==1.59.0
+RUN pip install --no-deps grpcio==1.51.3
+RUN pip install --no-deps h5py==3.8.0
+RUN pip install --no-deps idna==2.8
+RUN pip install --no-deps importlib-metadata==6.1.0
+RUN pip install --no-deps importlib-resources==5.12.0
+RUN pip install --no-deps keras==2.12.0
+RUN pip install --no-deps libclang==16.0.0
+RUN pip install --no-deps Markdown==3.4.3
+RUN pip install --no-deps markdown-it-py==2.2.0
+RUN pip install --no-deps MarkupSafe==2.1.2
+RUN pip install --no-deps mdurl==0.1.2
+RUN pip install --no-deps ml-collections==0.1.1
+RUN pip install --no-deps msgpack==1.0.5
+RUN pip install --no-deps nest-asyncio==1.5.6
+RUN pip install --no-deps numpy==1.23.5
+RUN pip install --no-deps oauthlib==3.2.2
+RUN pip install --no-deps opt-einsum==3.3.0
+RUN pip install --no-deps optax==0.1.5
+RUN pip install --no-deps orbax-checkpoint==0.1.6
+RUN pip install --no-deps packaging==23.0
+RUN pip install --no-deps pandas==2.0.1
+RUN pip install --no-deps pip==23.1.2
+RUN pip install --no-deps promise==2.3
+RUN pip install --no-deps protobuf==4.22.1
+RUN pip install --no-deps psutil==5.9.5
+RUN pip install --no-deps pyasn1==0.4.8
+RUN pip install --no-deps pyasn1-modules==0.2.8
+RUN pip install --no-deps Pygments==2.15.1
+RUN pip install --no-deps PyGObject==3.36.0
+RUN pip install --no-deps python-apt==2.0.1+ubuntu0.20.4.1
+RUN pip install --no-deps python-dateutil==2.8.2
+RUN pip install --no-deps pytz==2023.3
+RUN pip install --no-deps PyYAML==6.0
+RUN pip install --no-deps requests==2.22.0
+RUN pip install --no-deps requests-oauthlib==1.3.1
+RUN pip install --no-deps requests-unixsocket==0.2.0
+RUN pip install --no-deps rich==13.3.5
+RUN pip install --no-deps rsa==4.9
+RUN pip install --no-deps scipy==1.10.1
+RUN pip install --no-deps setuptools==67.6.0
+RUN pip install --no-deps six==1.14.0
+RUN pip install --no-deps tensorboard==2.12.0
+RUN pip install --no-deps tensorboard-data-server==0.7.0
+RUN pip install --no-deps tensorboard-plugin-wit==1.8.1
+RUN pip install --no-deps tensorflow==2.12.0
+RUN pip install --no-deps tensorflow-cpu==2.12.0
+RUN pip install --no-deps tensorflow-datasets==4.9.2
+RUN pip install --no-deps tensorflow-estimator==2.12.0
+RUN pip install --no-deps tensorflow-hub==0.13.0
+RUN pip install --no-deps tensorflow-io-gcs-filesystem==0.31.0
+RUN pip install --no-deps tensorflow-metadata==1.13.1
+RUN pip install --no-deps tensorflow-probability==0.20.0
+RUN pip install --no-deps tensorflow-text==2.12.1
+RUN pip install --no-deps tensorstore==0.1.36
+RUN pip install --no-deps termcolor==2.2.0
+RUN pip install --no-deps toml==0.10.2
+RUN pip install --no-deps toolz==0.12.0
+RUN pip install --no-deps tqdm==4.65.0
+RUN pip install --no-deps typing_extensions==4.5.0
+RUN pip install --no-deps tzdata==2023.3
+RUN pip install --no-deps urllib3==1.25.8
+RUN pip install --no-deps Werkzeug==2.2.3
+RUN pip install --no-deps wheel==0.40.0
+RUN pip install --no-deps wrapt==1.14.1
+RUN pip install --no-deps zipp==3.15.0
+# Installing jax at the very end with GPU support.
+# NOTE: Not using `no-deps` flag here because
+# we need CUDA support.
+RUN pip install jax[cuda11_cudnn82]==0.4.6 \
+--find-links https://storage.googleapis.com/jax-releases/jax_cuda_releases.html
+
+COPY ./model_oss/jax_vision_transformer/vit_config_without_data.py vit_jax/configs/vit.py
+
+ENV PYTHONPATH ./vit_jax
+ENTRYPOINT ["python", "-m", "vit_jax.main"]

community-content/vertex_model_garden/model_oss/jax_vision_transformer/vit_config_without_data.py

@@ -0,0 +1,27 @@
+"""Returns a config for a Vision Transformer model without asking for data."""
+import ml_collections
+from vit_jax.configs import common
+from vit_jax.configs import models
+
+
+def get_config(model: str) -> ml_collections.ConfigDict:
+  """Returns default parameters for finetuning ViT `model`."""
+  config = common.get_config()
+
+  get_model_config = getattr(models, f'get_{model}_config')
+  config.model = get_model_config()
+
+  # These values are often overridden on the command line.
+  config.base_lr = 0.03
+  config.total_steps = 500
+  config.warmup_steps = 100
+  config.pp = ml_collections.ConfigDict()
+  config.pp.train = 'train'
+  config.pp.test = 'test'
+  config.pp.resize = 448
+  config.pp.crop = 384
+
+  # This value MUST be overridden on the command line.
+  config.dataset = ''
+
+  return config