fix: broken links

* Add OpenCLIP notebook

* Resolve PR comments.

.cloud-build/cleanup/cleanup.py

@@ -1,5 +1,14 @@
 from typing import List
 from ratemate import RateLimit
+import argparse
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--dry_run",
+                    type=bool,
+                    default=False)
+args = parser.parse_args()
+
+
 from resource_cleanup_manager import (
     DatasetResourceCleanupManager,
     ModelResourceCleanupManager,
@@ -7,6 +16,14 @@ from resource_cleanup_manager import (
     ResourceCleanupManager,
     MatchingEngineIndexEndpointResourceCleanupManager,
     MatchingEngineIndexResourceCleanupManager,
+    FeatureStoreCleanupManager,
+    PipelineJobCleanupManager,
+    TrainingJobCleanupManager,
+    HyperparameterTuningCleanupManager,
+    BatchPredictionJobCleanupManager,
+    ExperimentCleanupManager,
+    BucketCleanupManager,
+    ArtifactRegistryCleanupManager
 )
 
 rate_limit = RateLimit(max_count=25, per=60, greedy=False)
@@ -23,7 +40,6 @@ def run_cleanup_managers(managers: List[ResourceCleanupManager], is_dry_run: boo
             try:
                 if not manager.is_deletable(resource):
                     continue
-
                 if is_dry_run:
                     resource_name = manager.resource_name(resource)
                     print(f"Will delete '{type_name}': {resource_name}")
@@ -36,9 +52,7 @@ def run_cleanup_managers(managers: List[ResourceCleanupManager], is_dry_run: boo
         print("")
 
 
-is_dry_run = False
-
-if is_dry_run:
+if args.dry_run:
     print("Starting cleanup in dry run mode...")
 
 # List of all cleanup managers
@@ -48,6 +62,14 @@ managers: List[ResourceCleanupManager] = [
     ModelResourceCleanupManager(),  # ModelResourceCleanupManager must follow EndpointResourceCleanupManager due to deployed models blocking model deletion.
     MatchingEngineIndexEndpointResourceCleanupManager(),
     MatchingEngineIndexResourceCleanupManager(),
+    FeatureStoreCleanupManager(),
+    PipelineJobCleanupManager(),
+    TrainingJobCleanupManager(),
+    HyperparameterTuningCleanupManager(),
+    BatchPredictionJobCleanupManager(),
+    ExperimentCleanupManager(), # Experiment missing _resource_noun
+    BucketCleanupManager(),
+    ArtifactRegistryCleanupManager()
 ]
 
-run_cleanup_managers(managers=managers, is_dry_run=is_dry_run)
+run_cleanup_managers(managers=managers, is_dry_run=args.dry_run)

.cloud-build/cleanup/resource_cleanup_manager.py

@@ -1,8 +1,18 @@
+'''
+READ FIRST BEFORE MAKING CHANGES
+-  Create a convention for resources created from vertex-ai-samples GH. We already have one IIRC
+- Only delete those objects as part of our clean-up script.
+- Don't run any tests on python-docs-samples-tests project, especially ones that affect resources created outside of our purview
+- Add --dry-run option to the clean-up script. This option will just output the list of resources the script will delete instead of actually deleting the resources.
+- Have a larger conversation in DEE before touching any resources that were not created as part of vertex-ai-samples
+'''
+import os
 import abc
 from typing import Any, Type
 
 from google.cloud import aiplatform
 from google.cloud.aiplatform import base
+from google.cloud import storage
 from proto.datetime_helpers import DatetimeWithNanoseconds
 
 # If a resource was updated within this number of seconds, do not delete.
@@ -69,7 +79,7 @@ class VertexAIResourceCleanupManager(ResourceCleanupManager):
     def delete(self, resource):
         resource.delete()
 
-    def get_seconds_since_modification(self, resource: Any) -> bool:
+    def get_seconds_since_modification(self, resource: Any) -> float:
         update_time = resource.update_time
         current_time = DatetimeWithNanoseconds.now(tz=update_time.tzinfo)
         return (current_time - update_time).total_seconds()
@@ -97,13 +107,10 @@ class EndpointResourceCleanupManager(VertexAIResourceCleanupManager):
     vertex_ai_resource = aiplatform.Endpoint
 
     def delete(self, resource):
-        # TODO: Remove this once https://github.com/googleapis/python-aiplatform/issues/1441 is fixed
-        resource._sync_gca_resource()
         for deployed_model_id in [
             models.id for models in resource._gca_resource.deployed_models
         ]:
             resource._undeploy(deployed_model_id=deployed_model_id)
-
         resource.delete(force=True)
 
 
@@ -117,3 +124,137 @@ class MatchingEngineIndexResourceCleanupManager(VertexAIResourceCleanupManager):
 
 class MatchingEngineIndexEndpointResourceCleanupManager(VertexAIResourceCleanupManager):
     vertex_ai_resource = aiplatform.MatchingEngineIndexEndpoint
+
+    def delete(self, resource):
+        resource.undeploy_all()
+        resource.delete(force=True)
+
+class FeatureStoreCleanupManager(VertexAIResourceCleanupManager):
+    vertex_ai_resource = aiplatform.Featurestore
+
+    def resource_name(self, resource: Any) -> str:
+        return resource.name
+
+class PipelineJobCleanupManager(VertexAIResourceCleanupManager):
+    vertex_ai_resource = aiplatform.PipelineJob
+
+class TrainingJobCleanupManager(VertexAIResourceCleanupManager):
+    vertex_ai_resource = aiplatform.training_jobs._CustomTrainingJob
+
+    job_types = [
+            aiplatform.AutoMLImageTrainingJob,
+            aiplatform.AutoMLTextTrainingJob,
+            aiplatform.AutoMLTabularTrainingJob,
+            aiplatform.AutoMLVideoTrainingJob,
+            aiplatform.AutoMLForecastingTrainingJob,
+            aiplatform.CustomJob,
+            aiplatform.CustomTrainingJob,
+            aiplatform.CustomContainerTrainingJob,
+            aiplatform.CustomPythonPackageTrainingJob
+    ]
+
+    def list(self) -> Any:
+        return [
+            job
+            for job_type in self.job_types
+            for job in job_type.list()
+        ]
+
+class HyperparameterTuningCleanupManager(VertexAIResourceCleanupManager):
+    vertex_ai_resource = aiplatform.HyperparameterTuningJob
+
+
+class BatchPredictionJobCleanupManager(VertexAIResourceCleanupManager):
+    vertex_ai_resource = aiplatform.BatchPredictionJob
+
+class ExperimentCleanupManager(VertexAIResourceCleanupManager):
+    vertex_ai_resource = aiplatform.Experiment
+
+    @property
+    def type_name(self) -> str:
+        return "Experiment"
+
+    def resource_name(self, resource: Any) -> str:
+        return resource.name
+
+    def get_seconds_since_modification(self, resource: Any) -> float:
+        update_time = resource._metadata_context.update_time
+        current_time = DatetimeWithNanoseconds.now()
+        return float(current_time.timestamp() - update_time.timestamp())
+
+class BucketCleanupManager(ResourceCleanupManager):
+    vertex_ai_resource = storage.bucket.Bucket
+
+    def list(self) -> Any:
+        storage_client = storage.Client()
+        return list(storage_client.list_buckets())
+
+    def delete(self, resource):
+        try:
+            resource.delete(force=True)
+        except Exception as e:
+            print(e)
+
+    @property
+    def type_name(self) -> str:
+        return "Bucket"
+
+    def get_seconds_since_modification(self, resource: Any) -> float:
+        # Bucket has no last_update property, only time created
+        created_time = resource.time_created
+        current_time = DatetimeWithNanoseconds.now()
+        return float(current_time.timestamp() - created_time.timestamp())
+
+    def resource_name(self, resource: Any) -> str:
+        return resource.name
+
+    def is_deletable(self, resource: Any) -> bool:
+        time_difference = self.get_seconds_since_modification(resource)
+
+        if not self.resource_name(resource).startswith('your-bucket-name'):
+            print(f"Skipping '{resource}' not a Vertex AI notebook bucket")
+            return False
+
+        # Check that it wasn't created too recently, to prevent race conditions
+        if time_difference <= RESOURCE_UPDATE_BUFFER_IN_SECONDS:
+            print(
+                f"Skipping '{resource}' due to update_time being '{time_difference}', which is less than '{RESOURCE_UPDATE_BUFFER_IN_SECONDS}'."
+            )
+            return False
+        return True
+
+class ArtifactRegistryCleanupManager(ResourceCleanupManager):
+    vertex_ai_resource = "Artifact Registry"
+
+    def list(self) -> Any:
+        import subprocess
+
+        result = subprocess.run(["gcloud artifacts repositories list --location=us-central1"], 
+                                shell=True, capture_output=True, text=True)
+
+        ret = []
+        lines = result.stdout.split('\n')[2:]
+        for line in lines:
+            repo = line.split(' ')[0]
+            if repo.startswith("my-docker-repo"):
+                ret.append(repo)
+
+        return ret
+
+    def delete(self, resource):
+        os.system(f"! gcloud artifacts repositories delete {resource} --location=us-central1")
+
+    @property
+    def type_name(self) -> str:
+        return "ArtifactRepository"
+
+    def resource_name(self, resource: Any) -> str:
+        return resource
+
+    # delete repository regardless of age
+    def get_seconds_since_modification(self, resource: Any) -> float:
+        return RESOURCE_UPDATE_BUFFER_IN_SECONDS + 1
+    
+    def is_deleteable(self, resource: Any) -> bool:
+        return True
+

.cloud-build/execute_changed_notebooks_cli.py

@@ -17,6 +17,7 @@
 
 import argparse
 import pathlib
+import os
 
 import execute_changed_notebooks_helper
 
@@ -39,6 +40,19 @@ parser.add_argument(
     help="The path to the file that has newline-limited folders of notebooks that should be tested.",
     required=True,
 )
+parser.add_argument(
+    "--test_percent",
+    type=int,
+    help="The percent of notebooks to be tested (between 1 and 100).",
+    required=False,
+    default=100,
+)
+parser.add_argument(
+    "--build_id",
+    type=str,
+    help="The build id (which may be a Cloud Build job specific or user explicit.",
+    required=True
+)
 parser.add_argument(
     "--base_branch",
     help="The base git branch to diff against to find changed files.",
@@ -107,24 +121,60 @@ parser.add_argument(
     default=True,
     help="Should run notebooks in parallel.",
 )
+parser.add_argument(
+    "--concurrent_notebooks",
+    type=int,
+    help="Maximum number of parallel notebook executions per minute",
+    default=10,
+    required=False,
+)
+parser.add_argument(
+    "--dry_run",
+    type=str2bool,
+    default=False,
+    help="Dry run for testing - no execution",
+)
 
 args = parser.parse_args()
 
-notebooks = execute_changed_notebooks_helper.get_changed_notebooks(
+changed_notebooks = execute_changed_notebooks_helper.get_changed_notebooks(
     test_paths_file=args.test_paths_file,
     base_branch=args.base_branch,
 )
 
-execute_changed_notebooks_helper.process_and_execute_notebooks(
-    notebooks=notebooks,
-    container_uri=args.container_uri,
-    staging_bucket=args.staging_bucket,
-    artifacts_bucket=args.artifacts_bucket,
-    should_parallelize=args.should_parallelize,
-    timeout=args.timeout,
-    variable_project_id=args.variable_project_id,
-    variable_region=args.variable_region,
-    variable_service_account=args.variable_service_account,
-    variable_vpc_network=args.variable_vpc_network,
-    private_pool_id=args.private_pool_id,
+
+results_bucket = f"{args.artifacts_bucket}"
+# artifacts_bucket may get set by trigger to a full gs:// folder path
+if results_bucket.startswith("gs://"):
+    results_bucket = results_bucket[5:]
+results_bucket = results_bucket.split('/')[0]
+results_file = f"build_results/{args.build_id}.json"
+
+if args.test_percent == 100:
+    notebooks = changed_notebooks
+    accumulative_results = {}
+else:
+    accumulative_results = execute_changed_notebooks_helper.load_results(results_bucket, results_file)
+
+    notebooks = [changed_notebook for changed_notebook in changed_notebooks if execute_changed_notebooks_helper.select_notebook(changed_notebook, accumulative_results, args.test_percent)]
+
+if args.dry_run:
+    print("Dry run ...\n")
+    for notebook in notebooks:
+        print(f"Would execute: {notebook}")
+else:
+    execute_changed_notebooks_helper.process_and_execute_notebooks(
+        notebooks=notebooks,
+        container_uri=args.container_uri,
+        staging_bucket=args.staging_bucket,
+        artifacts_bucket=args.artifacts_bucket,
+        results_file=results_file,
+        should_parallelize=args.should_parallelize,
+        timeout=args.timeout,
+        variable_project_id=args.variable_project_id,
+        variable_region=args.variable_region,
+        variable_service_account=args.variable_service_account,
+        variable_vpc_network=args.variable_vpc_network,
+        private_pool_id=args.private_pool_id,
+        concurrent_notebooks=args.concurrent_notebooks,
 )

.cloud-build/execute_changed_notebooks_helper.py

@@ -21,18 +21,21 @@ import json
 import git
 import operator
 import os
+import io
+import json
 import pathlib
 import re
 import subprocess
+import random
+from google.cloud import storage
 import utils
-from typing import List, Optional
+from typing import List, Optional, Dict, Any
 from utils import util
 
 import execute_notebook_helper
 import execute_notebook_remote
 import nbformat
 from google.cloud.devtools.cloudbuild_v1.types import BuildOperationMetadata
-from ratemate import RateLimit
 from tabulate import tabulate
 from utils import NotebookProcessors, util
 
@@ -40,6 +43,9 @@ from utils import NotebookProcessors, util
 WORKER_TIMEOUT_BUFFER_IN_SECONDS: int = 60 * 60
 PYTHON_VERSION = "3.9"  # Set default python version
 
+# rolling time window for accumulating build results for selecting notebooks
+MAX_RESULTS_AGE_SECONDS: int = (60 * 60) * 24 * 60  # 60 days
+
 
 def format_timedelta(delta: datetime.timedelta) -> str:
     """Formats a timedelta duration to [N days] %H:%M:%S format"""
@@ -65,7 +71,9 @@ def format_timedelta(delta: datetime.timedelta) -> str:
 @dataclasses.dataclass
 class NotebookExecutionResult:
     name: str
+    path: str
     duration: datetime.timedelta
+    start_time: datetime.datetime
     is_pass: bool
     log_url: str
     output_uri: str
@@ -81,6 +89,75 @@ class NotebookExecutionResult:
             return None
 
 
+def load_results(results_bucket: str,
+                 results_file: str) -> Dict[str, Any]:
+    '''
+    Load accumulated notebook test results
+    '''
+
+    print("Loading existing accumulative results ...")
+    accumulative_results = {}
+    try:
+        client = storage.Client()
+        bucket = client.bucket(results_bucket)
+
+        build_results_dir = os.path.dirname(results_file)
+        blobs = client.list_blobs(results_bucket, prefix=build_results_dir)
+        for blob in blobs:
+            time_created = blob.time_created.replace(tzinfo=None)
+            if (datetime.datetime.now().replace(tzinfo=None) - time_created).total_seconds() > MAX_RESULTS_AGE_SECONDS:
+                continue
+
+            content = util.download_blob_into_memory(results_bucket, blob.name, download_as_text=True)
+
+            try:
+                build_results = json.loads(content)
+            except:
+                continue  # skip corrupted build results files
+            for notebook in build_results:
+                if notebook in accumulative_results:
+                    accumulative_results[notebook]['passed'] += build_results[notebook]['passed']
+                    accumulative_results[notebook]['failed'] += build_results[notebook]['failed']
+                else:
+                    accumulative_results[notebook] = build_results[notebook]
+
+        print(accumulative_results)
+    except Exception as e:
+        print(e)
+
+    # If there are no accumulative results, an empty dict is returned
+    return accumulative_results
+
+def select_notebook(changed_notebook: str,
+                    accumulative_results: Dict[str, Any],
+                    test_percent: int) -> bool:
+    '''
+    Algorithm to randomly select a notebook, but weight the propbability of selected based on past failures
+    '''
+
+    if changed_notebook in accumulative_results:
+        pass_count = accumulative_results[changed_notebook]['passed']
+        fail_count = accumulative_results[changed_notebook]['failed']
+    else:
+        pass_count = 1
+        fail_count = 0
+
+    inferred_failure_rate = fail_count / (pass_count + fail_count)
+
+    # If failure rate is high, the chance of testing should be higher
+    should_test_due_to_failure = random.uniform(0, 1) <= inferred_failure_rate
+
+    # Additionally, only test a percentage of these
+    should_test_due_to_random_subset = random.uniform(0, 1) <= (test_percent / 100)
+
+    if should_test_due_to_failure or should_test_due_to_random_subset:
+        print(f"Selected: {changed_notebook}, {should_test_due_to_failure}, {should_test_due_to_random_subset}")
+        return True
+    else:
+        print(f"Not Selected: {changed_notebook}, pass {pass_count}, fail {fail_count}")
+        return False
+
+
 def _process_notebook(
     notebook_path: str,
     variable_project_id: str,
@@ -156,7 +233,6 @@ def _create_tag(filepath: str) -> str:
     return tag
 
 
-rate_limit = RateLimit(max_count=50, per=60, greedy=True)
 
 
 def process_and_execute_notebook(
@@ -172,7 +248,6 @@ def process_and_execute_notebook(
     notebook: str,
     should_get_tail_logs: bool = False,
 ) -> NotebookExecutionResult:
-    rate_limit.wait()  # wait before creating the task
 
     print(f"Running notebook: {notebook}")
 
@@ -191,7 +266,9 @@ def process_and_execute_notebook(
 
     result = NotebookExecutionResult(
         name=tag,
+        path=notebook,
         duration=datetime.timedelta(seconds=0),
+        start_time=datetime.datetime.now(),
         is_pass=False,
         output_uri=notebook_output_uri,
         log_url="",
@@ -201,7 +278,6 @@ def process_and_execute_notebook(
     )
 
     # TODO: Handle cases where multiple notebooks have the same name
-    time_start = datetime.datetime.now()
     operation = None
     try:
         # Get the python version for running the notebook if specified
@@ -247,9 +323,10 @@ def process_and_execute_notebook(
         # Block and wait for the result
         operation_result = operation.result(timeout=timeout_in_seconds)
 
-        result.duration = datetime.datetime.now() - time_start
+        result.duration = datetime.datetime.now() - result.start_time
         result.is_pass = True
         print(f"{notebook} PASSED in {format_timedelta(result.duration)}.")
+
     except Exception as error:
         result.error_message = str(error)
 
@@ -268,7 +345,7 @@ def process_and_execute_notebook(
             except Exception as error:
                 result.error_message = str(error)
 
-        result.duration = datetime.datetime.now() - time_start
+        result.duration = datetime.datetime.now() - result.start_time
         result.is_pass = False
 
         print(
@@ -336,12 +413,44 @@ def get_changed_notebooks(
 
     return notebooks
 
+def _save_results(results: List[NotebookExecutionResult],
+                  artifacts_bucket: str,
+                  results_file: str):
+
+    artifacts_bucket = artifacts_bucket.replace("gs://", "").split('/')[0]
+
+    print("Updating build results ...")
+    build_results = {}
+    for result in results:
+        if result.is_pass:
+            pass_count = 1
+            fail_count = 0
+        else:
+            pass_count = 0
+            fail_count = 1
+        build_results[result.path] = {
+                'duration': result.duration.total_seconds(),
+                'start_time': str(result.start_time),
+                'passed': pass_count,
+                'failed': fail_count
+        }
+        print(f"adding {result.path}")
+
+    print("Saving accumulative results ...")
+    content = json.dumps(build_results)
+
+    client = storage.Client()
+    bucket = client.get_bucket(artifacts_bucket)
+    bucket.blob(str(results_file)).upload_from_string(content, 'text/json')
+
+
 
 def process_and_execute_notebooks(
     notebooks: List[str],
     container_uri: str,
     staging_bucket: str,
     artifacts_bucket: str,
+    results_file: str,
     should_parallelize: bool,
     timeout: int,
     variable_project_id: str,
@@ -349,6 +458,7 @@ def process_and_execute_notebooks(
     variable_service_account: str,
     variable_vpc_network: Optional[str] = None,
     private_pool_id: Optional[str] = None,
+    concurrent_notebooks: Optional[int] = 10,
 ):
     """
     Run the notebooks that exist under the folders defined in the test_paths_file.
@@ -369,6 +479,8 @@ def process_and_execute_notebooks(
             Required. The GCS staging bucket to write source code to.
         artifacts_bucket (str):
             Required. The GCS staging bucket to write executed notebooks to.
+        results_file (str):
+            Required: The path to the artifacts bucket to save results 
         variable_project_id (str):
             Required. The value for PROJECT_ID to inject into notebooks.
         variable_region (str):
@@ -377,6 +489,7 @@ def process_and_execute_notebooks(
             Required. Should run notebooks in parallel using a thread pool as opposed to in sequence.
         timeout (str):
             Required. Timeout string according to https://cloud.google.com/build/docs/build-config-file-schema#timeout.
+        concurrent_notebooks (int): Max number of notebooks per minute to run in parallel.
     """
 
     # Calculate deadline
@@ -393,7 +506,9 @@ def process_and_execute_notebooks(
             print(
                 "Running notebooks in parallel, so no logs will be displayed. Please wait..."
             )
-            with concurrent.futures.ThreadPoolExecutor(max_workers=100) as executor:
+
+
+            with concurrent.futures.ThreadPoolExecutor(max_workers=concurrent_notebooks) as executor:
                 print(f"Max workers: {executor._max_workers}")
 
                 notebook_execution_results = list(
@@ -471,7 +586,7 @@ def process_and_execute_notebooks(
             print("=" * 100)
 
             build_id = results_sorted[0].build_id
-            logs_bucket_name = (results_sorted[0].logs_bucket).removeprefix("gs://")
+            logs_bucket_name = (results_sorted[0].logs_bucket).replace("gs://", "")
             log_file_name = f"log-{build_id}.txt"
 
             log_contents = util.download_blob_into_memory(
@@ -489,6 +604,10 @@ def process_and_execute_notebooks(
             else:
                 print(log_contents)
 
+        _save_results(results_sorted, 
+                      artifacts_bucket, 
+                      results_file)
+
         print("\n=== END RESULTS===\n")
 
         total_notebook_duration = functools.reduce(

.cloud-build/notebook-execution-test-cloudbuild.yaml

@@ -36,7 +36,7 @@ steps:
     - -c
     - |
       . workspace/env/bin/activate &&
-      python3 .cloud-build/execute_changed_notebooks_cli.py --test_paths_file "${_TEST_PATHS_FILE}" --base_branch "${_FORCED_BASE_BRANCH}" --container_uri ${_PYTHON_IMAGE} --staging_bucket ${_GCS_STAGING_BUCKET}  --artifacts_bucket ${_GCS_STAGING_BUCKET}/executed_notebooks/PR_${_PR_NUMBER}/BUILD_${BUILD_ID} --variable_project_id ${PROJECT_ID} --variable_region ${_GCP_REGION} --variable_service_account ${_GCP_SERVICE_ACCOUNT} --variable_vpc_network "${_GPC_VPC_NETWORK_NAME}" `if [ ! -z "${_PRIVATE_POOL_NAME}" ]; then echo "--private_pool_id ${_PRIVATE_POOL_NAME}"; fi`
+      python3 .cloud-build/execute_changed_notebooks_cli.py --test_paths_file "${_TEST_PATHS_FILE}" --base_branch "${_FORCED_BASE_BRANCH}" --container_uri ${_PYTHON_IMAGE} --staging_bucket ${_GCS_STAGING_BUCKET}  --artifacts_bucket ${_GCS_STAGING_BUCKET}/executed_notebooks/PR_${_PR_NUMBER}/BUILD_${BUILD_ID} --variable_project_id ${PROJECT_ID} --variable_region ${_GCP_REGION} --variable_service_account ${_GCP_SERVICE_ACCOUNT} --variable_vpc_network "${_GPC_VPC_NETWORK_NAME}" `if [ ! -z "${_PRIVATE_POOL_NAME}" ]; then echo "--private_pool_id ${_PRIVATE_POOL_NAME}"; fi` --build_id ${BUILD_ID} --test_percent=${_TEST_PERCENT} --concurrent_notebooks=${_CONCURRENT_NOTEBOOKS}
     env:
     - 'IS_TESTING=1'
 timeout: 86400s

.cloud-build/requirements.txt

@@ -3,11 +3,15 @@ numpy
 jupyter
 nbconvert
 papermill
-pandas
 matplotlib
 tabulate
 google-cloud-aiplatform
 google-cloud-storage
 google-cloud-build
+google-cloud-storage
 ratemate
 GitPython
+tqdm
+fsspec
+pandas
+

.cloud-build/test_failing.txt

@@ -0,0 +1,40 @@
+notebooks/official/training/pytorch_gcs_data_training.ipynb
+notebooks/official/custom/custom_training_tensorboard_profiler.ipynb
+notebooks/official/custom/custom-tabular-bq-managed-dataset.ipynb
+notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-locally-trained-models.ipynb
+notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb
+notebooks/official/tabnet/tabnet_vertex_tutorial.ipynb
+notebooks/official/tabnet/get_started_with_tabnet.ipynb
+notebooks/official/pipelines/google_cloud_pipeline_components_automl_text.ipynb
+notebooks/official/pipelines/multicontender_vs_champion_deployment_method.ipynb
+notebooks/official/pipelines/google_cloud_pipeline_components_automl_images.ipynb
+notebooks/official/pipelines/rapid_prototyping_bqml_automl.ipynb
+notebooks/official/pipelines/challenger_vs_blessed_deployment_method.ipynb
+notebooks/official/matching_engine/sdk_matching_engine_create_stack_overflow_embeddings.ipynb
+notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb
+notebooks/official/matching_engine/sdk_matching_engine_create_text_to_image_embeddings.ipynb
+notebooks/official/explainable_ai/sdk_custom_image_classification_online_explain.ipynb
+notebooks/official/explainable_ai/xai_image_classification_feature_attributions.ipynb
+notebooks/official/explainable_ai/sdk_custom_image_classification_batch_explain.ipynb
+notebooks/official/tabular_workflows/tabnet_on_vertex_pipelines.ipynb
+notebooks/official/model_registry/get_started_with_model_registry.ipynb
+notebooks/official/model_registry/bqml_vertexai_model_registry.ipynb
+notebooks/official/sdk/SDK_Custom_Training_Python_Package_Managed_Text_Dataset_Tensorflow_Serving_Container.ipynb
+notebooks/official/model_monitoring/batch_prediction_model_monitoring.ipynb
+notebooks/official/model_monitoring/get_started_with_model_monitoring_setup.ipynb
+notebooks/official/model_monitoring/get_started_with_model_monitoring_custom.ipynb
+notebooks/official/model_monitoring/get_started_with_model_monitoring_custom_tf_serving.ipynb
+notebooks/official/model_monitoring/model_monitoring.ipynb
+notebooks/official/tensorboard/tensorboard_profiler_custom_training_with_prebuilt_container.ipynb
+notebooks/official/tensorboard/tensorboard_hyperparameter_tuning_with_hparams.ipynb
+notebooks/official/tensorboard/tensorboard_profiler_custom_training.ipynb
+notebooks/official/model_evaluation/custom_tabular_regression_model_evaluation.ipynb
+notebooks/official/model_evaluation/custom_tabular_classification_model_evaluation.ipynb
+notebooks/official/model_evaluation/automl_video_classification_model_evaluation.ipynb
+notebooks/official/experiments/comparing_local_trained_models.ipynb
+notebooks/official/automl/automl_image_classification_online_online_prediction.ipynb
+notebooks/official/automl/automl-text-classification.ipynb
+notebooks/official/automl/sdk_automl_video_object_tracking_batch.ipynb
+notebooks/official/feature_store/sdk-feature-store-pandas.ipynb
+notebooks/official/prediction/custom_batch_prediction_feature_filter.ipynb
+notebooks/official/prediction/pytorch_image_classification_with_prebuilt_serving_containers.ipynb

.cloud-build/test_passing.txt

@@ -0,0 +1,80 @@
+notebooks/official/training/hyperparameter_tuning_tensorflow.ipynb
+notebooks/official/training/get_started_with_vertex_distributed_training.ipynb
+notebooks/official/training/hyperparameter_tuning_xgboost.ipynb
+notebooks/official/training/multi_node_ddp_gloo_vertex_training_with_custom_container.ipynb
+notebooks/official/training/distributed_hyperparameter_tuning.ipynb
+notebooks/official/training/pytorch-text-sentiment-classification-custom-train-deploy.ipynb
+notebooks/official/training/xgboost_data_parallel_training_on_cpu_using_dask.ipynb
+notebooks/official/training/multi_node_ddp_nccl_vertex_training_with_custom_container.ipynb
+notebooks/official/bigquery_ml/get_started_with_bqml_training.ipynb
+notebooks/official/bigquery_ml/bqml-online-prediction.ipynb
+notebooks/official/custom/custom_training_container_and_model_registry.ipynb
+notebooks/official/custom/sdk-custom-image-classification-online.ipynb
+notebooks/official/custom/sdk-custom-image-classification-batch.ipynb
+notebooks/official/custom/SDK_FBProphet_Forecasting_Online.ipynb
+notebooks/official/custom/get_started_vertex_training_xgboost.ipynb
+notebooks/official/custom/get_started_with_vertex_endpoint_and_shared_vm.ipynb
+notebooks/official/custom/SDK_Custom_Container_Prediction.ipynb
+notebooks/official/reduction_server/pytorch_distributed_training_reduction_server.ipynb
+notebooks/official/tabnet/ai-explanations-tabnet-algorithm.ipynb
+notebooks/official/vizier/get_started_vertex_vizier.ipynb
+notebooks/official/vizier/gapic-vizier-multi-objective-optimization.ipynb
+notebooks/official/pipelines/get_started_with_hpt_pipeline_components.ipynb
+notebooks/official/pipelines/google_cloud_pipeline_components_automl_tabular.ipynb
+notebooks/official/pipelines/custom_tabular_train_batch_pred_bq_pipeline.ipynb
+notebooks/official/pipelines/metrics_viz_run_compare_kfp.ipynb
+notebooks/official/pipelines/google_cloud_pipeline_components_model_upload_predict_evaluate.ipynb
+notebooks/official/pipelines/google_cloud_pipeline_components_model_train_upload_deploy.ipynb
+notebooks/official/pipelines/get_started_with_machine_management.ipynb
+notebooks/official/pipelines/custom_model_training_and_batch_prediction.ipynb
+notebooks/official/pipelines/control_flow_kfp.ipynb
+notebooks/official/pipelines/lightweight_functions_component_io_kfp.ipynb
+notebooks/official/pipelines/google_cloud_pipeline_components_bqml_text.ipynb
+notebooks/official/pipelines/pipelines_intro_kfp.ipynb
+notebooks/official/pipelines/automl_tabular_classification_beans.ipynb
+notebooks/official/pipelines/google_cloud_pipeline_components_dataproc_tabular.ipynb
+notebooks/official/explainable_ai/sdk_automl_tabular_classification_online_explain.ipynb
+notebooks/official/explainable_ai/sdk_custom_tabular_regression_online_explain.ipynb
+notebooks/official/explainable_ai/sdk_automl_tabular_binary_classification_batch_explain.ipynb
+notebooks/official/explainable_ai/sdk_custom_tabular_regression_online_explain_get_metadata.ipynb
+notebooks/official/explainable_ai/sdk_custom_tabular_regression_batch_explain.ipynb
+notebooks/official/tabular_workflows/prophet_on_vertex_pipelines.ipynb
+notebooks/official/tabular_workflows/wide_and_deep_on_vertex_pipelines.ipynb
+notebooks/official/sdk/SDK_AutoML_Video_Classification.ipynb
+notebooks/official/model_monitoring/get_started_with_model_monitoring_automl.ipynb
+notebooks/official/model_monitoring/get_started_with_model_monitoring_automl_image_batch.ipynb
+notebooks/official/model_monitoring/get_started_with_model_monitoring_automl_image_online.ipynb
+notebooks/official/model_monitoring/get_started_with_model_monitoring_xgboost.ipynb
+notebooks/official/tensorboard/tensorboard_custom_training_with_custom_container.ipynb
+notebooks/official/tensorboard/tensorboard_custom_training_with_prebuilt_container.ipynb
+notebooks/official/tensorboard/tensorboard_vertex_ai_pipelines_integration.ipynb
+notebooks/official/model_evaluation/automl_text_classification_model_evaluation.ipynb
+notebooks/official/model_evaluation/get_started_with_custom_model_evaluation_import.ipynb
+notebooks/official/model_evaluation/automl_tabular_classification_model_evaluation.ipynb
+notebooks/official/model_evaluation/automl_tabular_regression_model_evaluation.ipynb
+notebooks/official/experiments/get_started_with_vertex_experiments.ipynb
+notebooks/official/experiments/comparing_pipeline_runs.ipynb
+notebooks/official/experiments/get_started_with_vertex_experiments_autologging.ipynb
+notebooks/official/experiments/build_model_experimentation_lineage_with_prebuild_code.ipynb
+notebooks/official/experiments/delete_outdated_tensorboard_experiments.ipynb
+notebooks/official/automl/sdk_automl_tabular_regression_batch_bq.ipynb
+notebooks/official/automl/sdk_automl_text_sentiment_analysis_online.ipynb
+notebooks/official/automl/sdk_automl_text_entity_extraction_online.ipynb
+notebooks/official/automl/sdk_automl_forecasting_hierarchical_batch.ipynb
+notebooks/official/automl/automl_text_entity_extraction_batch_prediction.ipynb
+notebooks/official/automl/automl_image_classification_batch_prediction.ipynb
+notebooks/official/automl/automl_text_sentiment_analysis_batch_prediction.ipynb
+notebooks/official/automl/sdk_automl_tabular_regression_online_bq.ipynb
+notebooks/official/automl/get_started_automl_training.ipynb
+notebooks/official/automl/automl-tabular-classification.ipynb
+notebooks/official/automl/automl_image_object_detection_export_edge.ipynb
+notebooks/official/automl/sdk_automl_image_object_detection_batch.ipynb
+notebooks/official/automl/automl_tabular_on_vertex_pipelines.ipynb
+notebooks/official/automl/sdk_automl_video_classification_batch.ipynb
+notebooks/official/automl/sdk_automl_video_action_recognition_batch.ipynb
+notebooks/official/automl/sdk_automl_tabular_forecasting_batch.ipynb
+notebooks/official/automl/automl_image_object_detection_online_prediction.ipynb
+notebooks/official/automl/automl_forecasting_bqml_arima_plus_comparison.ipynb
+notebooks/official/datasets/get_started_bq_datasets.ipynb
+notebooks/official/datasets/get_started_with_data_labeling.ipynb
+notebooks/official/feature_store/feature_store_streaming_ingestion_sdk.ipynb

.cloud-build/tests/flaky.py

@@ -0,0 +1,46 @@
+# grep PASSED tests.txt | cut -c 10-100 >passed.txt
+
+import os
+
+repo_dir = '/home/jupyter/vertex-ai-samples/'
+repo_dir_len = len(repo_dir)
+official_dir = repo_dir + 'notebooks/official'
+
+entries = os.scandir(official_dir)
+folders = []
+for entry in entries:
+    if entry.is_dir():
+        folders.append(entry.path)
+
+# Passing
+with open('passed.txt', 'r') as pass_file:
+    notebook_names = pass_file.readlines()
+        
+notebooks = []
+for folder in folders:
+    entries = os.scandir(folder)
+    for entry in entries:
+        for notebook in notebook_names:
+            if entry.name == notebook.rstrip():
+                notebooks.append(entry.path[repo_dir_len:])
+                
+with open('passing_tests.txt', 'w') as f:
+    for notebook in notebooks:
+        f.write(notebook + '\n')
+        
+        
+# Failing
+with open('failed.txt', 'r') as fail_file:
+    notebook_names = fail_file.readlines()
+        
+notebooks = []
+for folder in folders:
+    entries = os.scandir(folder)
+    for entry in entries:
+        for notebook in notebook_names:
+            if entry.name == notebook.rstrip():
+                notebooks.append(entry.path[repo_dir_len:])
+                
+with open('failing_tests.txt', 'w') as f:
+    for notebook in notebooks:
+        f.write(notebook + '\n')

.cloud-build/tests/select_notebooks_test.py

@@ -0,0 +1,33 @@
+import sys
+
+from execute_changed_notebooks_helper import (load_results, select_notebook)
+
+
+def test_load_results():
+    bucket: str = "cloud-build-notebooks-presubmit"
+    bucket_file: str = "build_results"
+
+    accum = load_results(bucket, bucket_file)
+
+    print(accum)
+
+    assert len(accum) > 0
+
+def test_select_notebook():
+    bucket: str = "cloud-build-notebooks-presubmit"
+    bucket_file: str = "build_results"
+
+    accum = load_results(bucket, bucket_file)
+
+    n_select = 0
+    n_notselect = 0
+    for notebook in accum:
+        if select_notebook(notebook, accum, 50):
+            n_select += 1
+        else:
+            n_notselect += 1
+
+    print(f"SELECTED {n_select}, NOT SELECTED {n_notselect}")
+
+    assert n_select > 0
+    assert n_notselect > 0

.cloud-build/utils/build_results_viewer.py

@@ -0,0 +1,22 @@
+'''
+Viewer for the weekly regression testing of the official notebooks
+
+Cloud Storage location: gs://cloud-build-notebooks-presubmit/build_results/
+'''
+import argparse
+import json
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--file', dest='file',
+                    default='build.json', type=str, help='build results file')
+import json
+
+with open('build.json', 'r') as f:
+    results = json.load(f)
+
+for item in results.items():
+    if item[1]['passed']:
+        print(f"{item[0]},PASSED")
+    else:
+        print(f"{item[0]},FAILED")

.cloud-build/web-index/notebook-index-cloudbuild.yaml

@@ -0,0 +1,23 @@
+steps:
+  # Fetch full repo for diff purposes
+  - name: gcr.io/cloud-builders/git
+    args: [fetch, --unshallow, --quiet]
+  # Create a virtual environment
+  - name: ${_PYTHON_IMAGE}
+    entrypoint: /bin/sh
+    args:
+    - -c
+    - python3 -m venv workspace/env
+  # Install Python dependencies and run testing script
+  - name: ${_PYTHON_IMAGE}
+    entrypoint: /bin/sh
+    args:
+    - -c
+    - |
+      . workspace/env/bin/activate &&
+      python3 notebooks/notebook_template_review.py --web --title --steps --desc --linkback --notebook-dir=notebooks/official >web.html
+artifacts:
+  objects:
+          location: gs://${_GCS_ARTIFACTS_BUCKET}/webdoc
+          paths: ['web.html']
+timeout: 86400s

.github/workflows/linter/requirements.txt

@@ -2,9 +2,9 @@ git+https://github.com/tensorflow/docs
 ipython
 jupyter
 nbconvert
-black==22.10.0
-pyupgrade==2.38.4
-isort==5.10.1
-flake8==4.0.1
-nbqa==1.5.3
+black==23.3.0
+pyupgrade==3.7.0
+isort==5.12.0
+flake8==6.0.0
+nbqa==1.7.0
 

.github/workflows/linter/run_linter.sh

@@ -84,7 +84,7 @@ if [ ${#notebooks[@]} -gt 0 ]; then
                 # python3 -m nbqa black "$notebook" --check
                 # BLACK_RTN=$?
                 echo "Running pyupgrade..."
-                python3 -m nbqa pyupgrade "$notebook"
+                python3 -m nbqa pyupgrade --exit-zero-even-if-changed "$notebook"
                 PYUPGRADE_RTN=$?
                 echo "Running isort..."
                 python3 -m nbqa isort "$notebook" --check
@@ -97,7 +97,7 @@ if [ ${#notebooks[@]} -gt 0 ]; then
                 python3 -m nbqa black "$notebook"
                 BLACK_RTN=$?
                 echo "Running pyupgrade..."
-                python3 -m nbqa pyupgrade "$notebook"
+                python3 -m nbqa pyupgrade --exit-zero-even-if-changed "$notebook"
                 PYUPGRADE_RTN=$?
                 echo "Running isort..."
                 python3 -m nbqa isort "$notebook"

CONTRIBUTING.md

@@ -44,12 +44,10 @@ Finally, run this code block to check for errors. Each step will attempt to
 automatically fix any issues. If the fixes can't be performed automatically,
 then you will need to manually address them before submitting your PR.
 
+Note: For official, only submit one notebook per PR.
+
 ```shell
-nbqa black "$notebook"
-nbqa pyupgrade "$notebook"
-nbqa isort "$notebook"
-nbqa flake8 "$notebook" --extend-ignore=W391,E501,F821,E402,F404,W503,E203,E722,W293,W291
-python3 -m tensorflow_docs.tools.nbfmt --remove_outputs "$notebook"
+docker run -v ${PWD}:/setup/app gcr.io/cloud-devrel-public-resources/notebook_linter:latest your_notebook
 ```
 
 ## Code Reviews

community-content/CODEOWNERS

@@ -9,3 +9,13 @@
 /Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines @Ark-kun
 /pipeline_components @Ark-kun
 /pipeline_components/image_ml_model_training @lakeyk
+/prediction_featurestore_integration @googleapis/vertex-prediction-team
+/vertex_vision_model_garden/model_oss/util @weigary
+/vertex_vision_model_garden/model_oss/diffusers @weigary
+/vertex_vision_model_garden/model_oss/keras @dstnluong-google
+/vertex_vision_model_garden/model_oss/transformers @dstnluong-google
+/vertex_vision_model_garden/model_oss/pic2word @jismailyan-google
+/vertex_vision_model_garden/model_oss/open_clip @lydhr
+/vertex_vision_model_garden/model_oss/movinet @KCFindstr
+/vertex_vision_model_garden/model_oss/data_converter @KCFindstr
+

community-content/pytorch_image_classification_distributed_data_parallel_training_with_vertex_sdk/trainer/requirements.txt

@@ -1,3 +1,3 @@
-torch==1.8.1
+torch==1.13.1
 torchvision==0.9.1
 tensorboard==2.5.0

community-content/pytorch_image_classification_single_gpu_with_vertex_sdk_and_torchserve/trainer/requirements.txt

@@ -1,3 +1,3 @@
-torch==1.8.1
+torch==1.13.1
 torchvision==0.9.1
 tensorboard==2.5.0

community-content/pytorch_pre_built_images_deployment/pytorch-text-classification-vertex-ai-deploy.ipynb

@@ -31,17 +31,7 @@
       "source": [
         "# Deploying a PyTorch Text Classification Model on [Vertex AI](https://cloud.google.com/vertex-ai)\n",
         "\n",
-        "**This is an Experimental release**, covered by the Pre-GA Offerings Terms of your Google Cloud Platform [Terms of Service](https://cloud.google.com/terms).\n",
-        "\n",
-        "Experiments are focused on validating a prototype and are not guaranteed to be released. They are not intended for production use or covered by any SLA, support obligation, or deprecation policy and might be subject to backward-incompatible changes.\n",
-        "\n",
-        "**Kindly drop us a note before you run any scale tests.**\n",
-        "\n",
-        "**Do not hesitate to contact vertexai-prediction-preview-feedback@google.com if you have any questions or run into any issues.**\n",
-        "\n",
-        "The usage of the product is free during the Experimental release period: you will still incur charges for other GCP products usage, such as storage.\n",
-        "\n",
-        "The projects need to be added to the allowlist in order to deploy PyTorch models using Vertex AI Prediction pre-built PyTorch images. If you are interested in the feature, please send an email to vertexai-prediction-preview-feedback@google.com to provide your project numbers OR project ids."
+        "**Kindly reach out to Vertex AI before you run any scale tests or you have any questions.**\n"
       ]
     },
     {

community-content/vertex_vision_model_garden/.pylintrc

@@ -0,0 +1,4 @@
+[MASTER]
+
+generated-members=get_concrete_function,cv2.*
+ignored-modules=tensorflow,google.cloud

community-content/vertex_vision_model_garden/model_oss/data_converter/common_lib.py

@@ -0,0 +1,623 @@
+"""Library with functions to use for data conversion."""
+
+import json
+import os
+import random
+from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple, Union
+import uuid
+
+from absl import logging
+import apache_beam as beam
+import cv2
+import numpy as np
+import pandas as pd
+import PIL
+from PIL import Image
+import tensorflow as tf
+import yaml
+
+from util import constants
+from util import fileutils
+from apache_beam.options import pipeline_options
+
+REFORMATTED_CSV_SUFFIX = '-reformatted.csv'
+
+LABEL_MAP_NAME = 'label_map.yaml'
+
+_SPLIT_RATIO_ERROR_THRESHOLD = 1e-5
+# Internal constant. Only for distinguishing rows without ML use.
+ML_USE_UNASSIGNED = 'unassigned'
+ALL_ML_USES = (
+    constants.ML_USE_TRAINING,
+    constants.ML_USE_VALIDATION,
+    constants.ML_USE_TEST,
+    ML_USE_UNASSIGNED,
+)
+COLUMN_NAME_ML_USE = 'ml_use'
+COLUMN_NAME_GCS_FILE_PATH = 'gcs_file_path'
+COLUMN_NAME_LABEL = 'label'
+COLUMN_NAME_START_SEC = 'start_sec'
+COLUMN_NAME_END_SEC = 'end_sec'
+# Output filenames
+TRAIN_TFRECORD_NAME = 'train.tfrecord'
+VALIDATION_TFRECORD_NAME = 'val.tfrecord'
+TEST_TFRECORD_NAME = 'test.tfrecord'
+# Jsonl keys
+JSON_GCS_URI_KEY = 'imageGcsUri'
+JSON_RESOURCE_LABEL_KEY = 'dataItemResourceLabels'
+JSON_ML_USE_KEY = 'aiplatform.googleapis.com/ml_use'
+# I/O parameters
+READ_CHUNK_SIZE = 1024 * 1024 * 1024  # 1GB
+
+
+class WriteToTFRecord(beam.DoFn):
+  """DoFn to write TF examples to sharded TF record files."""
+
+  def __init__(
+      self,
+      output_prefix: str,
+      num_shards: int,
+      convert_fn: Callable[[Dict[str, Any]], tf.train.Example],
+  ):
+    self.output_prefix = output_prefix
+    self.num_shards = num_shards
+    self.writer: list[tf.io.TFRecordWriter] = []
+    self.sharded_files: list[str] = []
+    self.convert_fn = convert_fn
+    self.success_counter = beam.metrics.Metrics.counter(
+        self.__class__.__name__, 'Success'
+    )
+    self.failure_counter = beam.metrics.Metrics.counter(
+        self.__class__.__name__, 'Failure'
+    )
+
+  def start_bundle(self):
+    logging.info('Start writing TF Record to %s.', self.output_prefix)
+    unique_str = uuid.uuid4().hex
+    for i in range(self.num_shards):
+      uri = f'{self.output_prefix}-{i}-{unique_str}'
+      self.sharded_files.append(uri)
+      self.writer.append(tf.io.TFRecordWriter(uri))
+
+  def process(self, data: Dict[str, Any]) -> Iterable[Tuple[int, str]]:
+    try:
+      example = self.convert_fn(data)
+      data = example.SerializeToString()
+      idx = hash(data) % self.num_shards
+      self.writer[idx].write(data)
+      self.success_counter.inc()
+      yield (idx, self.sharded_files[idx])
+    # pylint: disable-next=broad-exception-caught
+    except Exception as err:
+      logging.error('Failed to process %s', data)
+      logging.exception(err)
+      self.failure_counter.inc()
+
+  def finish_bundle(self):
+    logging.info('Finish writing TF Record to %s.', self.output_prefix)
+    for writer in self.writer:
+      writer.close()
+    self.writer = []
+
+
+def convert_to_feature(
+    value: Union[List[Union[int, float, bytes]], int, float, bytes],
+    value_type: Optional[str] = None,
+) -> tf.train.Feature:
+  """Converts the given python object to a tf.train.Feature.
+
+  This is copied from tensorflow_models/official/vision/data/tfrecord_lib.py.
+
+  Args:
+    value: int, float, bytes or a list of them.
+    value_type: optional, if specified, forces the feature to be of the given
+      type. Otherwise, type is inferred automatically. Can be one of ['bytes',
+      'int64', 'float', 'bytes_list', 'int64_list', 'float_list']
+
+  Returns:
+    feature: A tf.train.Feature object.
+  """
+
+  if value_type is None:
+    element = value[0] if isinstance(value, list) else value
+
+    if isinstance(element, bytes):
+      value_type = 'bytes'
+
+    elif isinstance(element, (int, np.integer)):
+      value_type = 'int64'
+
+    elif isinstance(element, (float, np.floating)):
+      value_type = 'float'
+
+    else:
+      raise ValueError(
+          'Cannot convert type {} to feature'.format(type(element))
+      )
+
+    if isinstance(value, list):
+      value_type = value_type + '_list'
+
+  if value_type == 'int64':
+    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))
+
+  elif value_type == 'int64_list':
+    value = np.asarray(value).astype(np.int64).reshape(-1)
+    return tf.train.Feature(int64_list=tf.train.Int64List(value=value))
+
+  elif value_type == 'float':
+    return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))
+
+  elif value_type == 'float_list':
+    value = np.asarray(value).astype(np.float32).reshape(-1)
+    return tf.train.Feature(float_list=tf.train.FloatList(value=value))
+
+  elif value_type == 'bytes':
+    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
+
+  elif value_type == 'bytes_list':
+    return tf.train.Feature(bytes_list=tf.train.BytesList(value=value))
+
+  else:
+    raise ValueError('Unknown value_type parameter - {}'.format(value_type))
+
+
+def convert_to_string_feature(
+    value: str, encoding: str = 'utf-8'
+) -> tf.train.Feature:
+  """Returns a bytes_list from an encoded string."""
+  return convert_to_feature(value.encode(encoding))
+
+
+def convert_to_list_string_feature(
+    lst: list[str], encoding: str = 'utf-8'
+) -> tf.train.Feature:
+  """Returns a bytes_list from a list of encoded strings."""
+  return convert_to_feature([value.encode(encoding) for value in lst])
+
+
+def create_ml_use_array_with_split(
+    total_size: int,
+    split_ratio: Sequence[float],
+) -> list[str]:
+  """Create randomized list of 'training', 'validation', 'test'.
+
+  The list of will be of length total_size with ratios according to train_size,
+  validation_size, and test_size.
+
+  Args:
+    total_size: Length of sequence to return
+    split_ratio: Proportions to split into 'training', 'validation', and 'test'
+
+  Returns:
+    List containing 'training', 'validation', and 'test'
+  """
+  train_size, validation_size, _ = split_ratio
+  num_train = round(train_size * total_size)
+  num_validation = round(validation_size * total_size)
+  num_test = total_size - num_train - num_validation
+  ml_use_row = (
+      [constants.ML_USE_TRAINING] * num_train
+      + [constants.ML_USE_VALIDATION] * num_validation
+      + [constants.ML_USE_TEST] * num_test
+  )
+  random.shuffle(ml_use_row)
+  return ml_use_row
+
+
+def format_ml_use_column(df: pd.DataFrame):
+  df[COLUMN_NAME_ML_USE].replace(
+      # We need to support non-standard ML uses other than documented ones,
+      # since they are used by some existing datasets.
+      [r'(?i)^train(ing)?$', r'(?i)^test$', r'(?i)^validat(ion|e)$'],
+      [
+          constants.ML_USE_TRAINING,
+          constants.ML_USE_TEST,
+          constants.ML_USE_VALIDATION,
+      ],
+      inplace=True,
+      regex=True,
+  )
+
+
+def insert_missing_ml_use(df: pd.DataFrame) -> None:
+  """For every row that does not have ml_use as the first column, insert a column containing 'unassigned' to the front.
+
+  Args:
+    df: The DataFrame to process. The first column should be 'ml_use'.
+  """
+  df[COLUMN_NAME_ML_USE].fillna(ML_USE_UNASSIGNED, inplace=True)
+  rows_to_fill = ~df[COLUMN_NAME_ML_USE].isin(ALL_ML_USES)
+  df.loc[rows_to_fill] = df[rows_to_fill].shift(
+      axis=1, fill_value=ML_USE_UNASSIGNED
+  )
+
+
+def replace_unassigned_ml_use(
+    ml_uses: List[str],
+    split_ratio: Sequence[float],
+):
+  """Replace `unassigned` in ml_uses with `training`, `validation`, and `test` with ratios according to split_ratio.
+
+  Args:
+    ml_uses: List of ml_use string values.
+    split_ratio: Proportions to split into `training`, `validation`, and `test`.
+  """
+  unassigned_indices = [
+      i for i, ml_use in enumerate(ml_uses) if ml_use == ML_USE_UNASSIGNED
+  ]
+  ml_use_arr = create_ml_use_array_with_split(
+      len(unassigned_indices), split_ratio
+  )
+  for unassigned_index, ml_use in zip(unassigned_indices, ml_use_arr):
+    ml_uses[unassigned_index] = ml_use
+
+
+def merge_seq_into_dicts(
+    key: str, values: Sequence[Any], dicts: Sequence[Dict[Any, Any]]
+):
+  """Merges a list of values into a list of dicts, inserted with the given key.
+
+  Args:
+    key: Key to insert or overwrite in the dictionary.
+    values: A list of values to insert.
+    dicts: A list of dictionaries. Each value will be inserted into the
+      corresponding dictionary. The original value will be overwritten if the
+      key already existed.
+
+  Raises:
+    ValueError: The values and dicts have different lengths.
+  """
+  if len(values) != len(dicts):
+    raise ValueError(
+        f'Length of values and dicts must match, got {len(values)} and'
+        f' {len(dicts)}'
+    )
+  for val, d in zip(values, dicts):
+    d[key] = val
+
+
+def drop_invalid_rows(df: pd.DataFrame) -> int:
+  """Drops DataFrame rows missing the gcs_file_path column or the label column.
+
+  Args:
+    df: The DataFrame to process in place.
+
+  Returns:
+    The number of rows dropped.
+  """
+  original_rows = df.shape[0]
+  df.dropna(subset=[COLUMN_NAME_GCS_FILE_PATH, COLUMN_NAME_LABEL], inplace=True)
+  dropped_num = original_rows - df.shape[0]
+  if dropped_num > 0:
+    df.reset_index(drop=True, inplace=True)
+  return dropped_num
+
+
+def check_split_ratio(split_ratio: Sequence[float]):
+  """Checks if the give split ratio is valid.
+
+  Args:
+    split_ratio: Proportions to split into 'training', 'validation', and 'test'
+
+  Raises:
+    ValueError: Must have valid entries, correct length, and sum to 1.
+  """
+  if len(split_ratio) != 3:
+    raise ValueError('split_ratio must contain exactly 3 values.')
+  if abs(sum(split_ratio) - 1) > _SPLIT_RATIO_ERROR_THRESHOLD:
+    raise ValueError('split_ratio must sum to 1.')
+  if not all([0 <= val <= 1 for val in split_ratio]):
+    raise ValueError('Entries of split_ratio must be in the range [0, 1].')
+
+
+def check_num_shard(num_shard: Sequence[int]):
+  """Checks if the number of shards is valid.
+
+  Args:
+    num_shard: The number of shards for each tfrecord.
+
+  Raises:
+    ValueError: Must have valid entries and correct length.
+  """
+  if len(num_shard) != 3:
+    raise ValueError('num_shard must contain exactly 3 values.')
+  if not all([val >= 1 for val in num_shard]):
+    raise ValueError('Shards must be at least 1.')
+
+
+def create_label_map_yaml(meta_data_path: str, output_dir: str) -> None:
+  """Generate label_map.yaml from meta_data.yaml.
+
+  Args:
+    meta_data_path: Path to a meta_data.yaml file.
+    output_dir: Directory to output label_map.yaml.
+  """
+  tf.io.gfile.copy(
+      meta_data_path, os.path.join(output_dir, LABEL_MAP_NAME), overwrite=True
+  )
+
+
+def reformat_bbox(
+    bbox: Sequence[int], img_width: int, img_height: int
+) -> Tuple[float, float, float, float]:
+  """Converts XYWH unnormalized bounding box with to a normalized XYXY bounding box.
+
+  Args:
+    bbox: Relative bounding box with unnormalized coordinates as [x, y, width,
+      height].
+    img_width: Image's pixel width.
+    img_height: Image's pixel height.
+
+  Returns:
+    Absolute bounding box with normalized coordinates as
+      [xmin, ymin, xmax, ymax].
+  """
+  x, y, width, height = bbox
+  xmin = x / img_width
+  ymin = y / img_height
+  xmax = (x + width) / img_width
+  ymax = (y + height) / img_height
+  return xmin, ymin, xmax, ymax
+
+
+def encode_image(
+    filepath: str,
+    output_shape: Optional[Sequence[int]] = None,
+    image_format: str = 'png',
+) -> Tuple[bytes, Sequence[int]]:
+  """Encodes an image at the given path.
+
+  Args:
+    filepath: Path to the image.
+    output_shape: The output shape of the image, (height, width).
+    image_format: The format of the output image.
+
+  Returns:
+    The encoded image data in bytes and the shape of the image, (height, width).
+
+  Raises:
+    IOError: The image file is corrupt.
+  """
+  filepath = fileutils.force_gcs_fuse_path(filepath)
+  with open(filepath, 'rb') as f:
+    # If an output_shape is specified, resize the image and set data to the new
+    # bytes.
+    try:
+      img = Image.open(f)
+    except PIL.UnidentifiedImageError as e:
+      raise IOError(f'Failed to open {filepath}') from e
+
+    try:
+      if output_shape is not None:
+        rgb_img = img.resize((output_shape[1], output_shape[0])).convert('RGB')
+      else:
+        rgb_img = img.convert('RGB')
+      rgb_img = np.array(rgb_img)
+
+      _, data = cv2.imencode(f'.{image_format}', rgb_img)
+      data = data.tobytes()
+      return data, rgb_img.shape
+    except cv2.error as e:
+      raise IOError(f'Failed to encode {filepath}') from e
+    finally:
+      img.close()
+
+
+def encode_video(
+    filepath: str,
+    start_sec: float,
+    end_sec: float,
+    output_fps: int = 5,
+    output_shape: Optional[Sequence[int]] = None,
+    image_format: str = 'jpg',
+) -> Sequence[bytes]:
+  """Encodes a video clip at the given path with start and end timestamps.
+
+  Args:
+    filepath: Path to the video.
+    start_sec: Start timestamp of the video clip in seconds.
+    end_sec: End timestamp of the video clip in seconds.
+    output_fps: The output frame rate per second.
+    output_shape: The output shape of each frame, (height, width).
+    image_format: The format of the encoded frames.
+
+  Returns:
+    A list of the encoded frames data in bytes.
+
+  Raises:
+    IOError if the video file is corrupt.
+  """
+  filepath = fileutils.force_gcs_fuse_path(filepath)
+  video = None
+
+  try:
+    video = cv2.VideoCapture(filepath)
+    frames = []
+    frame_interval = 1 / output_fps
+    total_frames = video.get(cv2.CAP_PROP_FRAME_COUNT)
+    original_fps = video.get(cv2.CAP_PROP_FPS)
+    if not original_fps:
+      # 0 or None indicates the video is invalid
+      raise IOError(f'Failed to load {filepath}')
+    video_length = total_frames / original_fps
+    start_sec = max(start_sec, 0)
+    end_sec = min(end_sec, video_length)
+    for t in np.arange(start_sec, end_sec, frame_interval):
+      frame_idx = min(total_frames - 1, round(t * original_fps))
+      video.set(cv2.CAP_PROP_POS_FRAMES, frame_idx)
+      ret, frame = video.read()
+      if not ret:
+        raise IOError(f'Failed to load {filepath} at frame {frame_idx}')
+      if output_shape is not None:
+        frame = cv2.resize(frame, (output_shape[1], output_shape[0]))
+      _, data = cv2.imencode(f'.{image_format}', frame)
+      frames.append(data.tobytes())
+  except cv2.error as e:
+    raise IOError(f'Failed to load {filepath}') from e
+  finally:
+    if video:
+      video.release()
+  return frames
+
+
+def create_label_map(
+    labels: Sequence[str],
+) -> Tuple[Sequence[int], Dict[int, str]]:
+  """Creates a label map from a sequence of label strings.
+
+  Args:
+    labels: The sequence of labels to create label map from. Must not contain
+      invalid values, which means data without labels should be filtered first.
+
+  Returns:
+    The integer labels and the mapping from integers to the original strings.
+  """
+  inverse_label_map: Dict[str, int] = dict()
+  num_labels = 0
+  for label in labels:
+    if label not in inverse_label_map:
+      num_labels += 1
+      inverse_label_map[label] = num_labels
+  int_labels = [inverse_label_map[label] for label in labels]
+  label_map = {value: key for key, value in inverse_label_map.items()}
+  return int_labels, label_map
+
+
+def write_label_map(output_file: str, label_map: Dict[int, str]) -> None:
+  """Writes a label map to the output file, which can be a GCS uri."""
+  with tf.io.gfile.GFile(output_file, 'w') as f:
+    yaml.dump({'label_map': label_map}, f)
+
+
+def detectron_json_to_image_rows(input_json: str) -> list[Dict[str, Any]]:
+  """Converts a Detectron JSON file to a list of image rows.
+
+  Args:
+    input_json: A path to a Detectron JSON or JSONL file.
+
+  Returns:
+    A list of dictionaries, where each dictionary contains Detectron format
+    entry.
+
+  Raises:
+    ValueError: If the input JSON is invalid.
+  """
+
+  image_rows = []
+  with tf.io.gfile.GFile(input_json, 'r') as f:
+    for line in f:
+      json_data = json.loads(line)
+      if isinstance(json_data, dict):
+        image_rows.append(json_data)
+      elif isinstance(json_data, list):
+        image_rows.extend(json_data)
+      else:
+        raise ValueError(
+            'The input JSON is invalid. Dict or list is expected, but got '
+            f'{type(json_data)}.'
+        )
+  return image_rows
+
+
+def coco_json_to_image_rows(
+    input_json: str,
+) -> List[Dict[str, Any]]:
+  """Converts a COCO JSON file to a list of image rows.
+
+  Args:
+    input_json: A path to a COCO JSON or JSONL file.
+
+  Returns:
+    A list of dictionaries, where each dictionary contains COCO format entry.
+
+  Raises:
+    ValueError: If the input JSON is invalid.
+  """
+
+  with tf.io.gfile.GFile(input_json, 'r') as f:
+    coco_json = json.load(f)
+  if 'annotations' not in coco_json:
+    raise ValueError('"annotations" is not in the dataset.')
+  if 'images' not in coco_json:
+    raise ValueError('"images" is not in the dataset.')
+
+  images = coco_json['images']
+  return images
+
+
+def partition_by_ml_use(element: Dict[str, Any], num_partitions: int) -> int:
+  """Beam partition function to split data by ml_use."""
+  del num_partitions
+  try:
+    partition = ALL_ML_USES.index(element[COLUMN_NAME_ML_USE])
+  except Exception as e:
+    raise ValueError(f'Invalid ML use: {element[COLUMN_NAME_ML_USE]}') from e
+  return partition
+
+
+def run_beam_pipeline(pipeline: Any) -> None:
+  """Runs a beam pipeline. Works in both internal and docker environment."""
+  options = pipeline_options.PipelineOptions([
+      '--runner=FlinkRunner',
+      '--faster_copy',
+      '--max_parallelism', '8',
+  ])
+  p = beam.Pipeline(options=options)
+  pipeline(p)
+  result = p.run()
+  result.wait_until_finish()
+  for counter in result.metrics().query()['counters']:
+    logging.info('%s counter: %s.', counter.key.metric.name, counter)
+  logging.info('Completing beam pipeline.')
+
+
+def beam_convert_tfexamples(
+    root: beam.Pipeline,
+    data_list: Sequence[Dict[str, Any]],
+    convert_fn: Callable[[Dict[str, Any]], tf.train.Example],
+    output_dir: str,
+    num_shards: Sequence[int],
+) -> None:
+  """Constructs beam pipelines to convert train, val, test TF Examples."""
+  names = [TRAIN_TFRECORD_NAME, VALIDATION_TFRECORD_NAME, TEST_TFRECORD_NAME]
+  split_data = (
+      root
+      | 'Create PCollection' >> beam.Create(data_list)
+      | 'Data split' >> beam.Partition(partition_by_ml_use, 3)
+  )
+  for i in range(3):
+    ml_use: str = ALL_ML_USES[i]
+    num_shard = num_shards[i]
+    output_prefix = os.path.join(output_dir, names[i])
+    _ = (
+        split_data[i]
+        | f'Convert {ml_use} TF Examples'
+        >> beam.ParDo(WriteToTFRecord(output_prefix, num_shard, convert_fn))
+        | f'Group {ml_use} TF Record files' >> beam.GroupBy(lambda x: x[0])
+        | f'Merge {ml_use} TF Record files'
+        >> beam.Map(merge_tfrecords_func(output_prefix, num_shard))
+    )
+
+
+def merge_tfrecords_func(output_prefix: str, num_shard: int) -> ...:
+  """Returns a function to merge sharded worker output into expected shards."""
+  output_prefix = fileutils.force_gcs_fuse_path(output_prefix)
+
+  def merge_tfrecords(worker_output: Tuple[int, Sequence[Tuple[int, str]]]):
+    idx = worker_output[0]
+    files: Sequence[str] = np.unique([x[1] for x in worker_output[1]])
+    output_file = f'{output_prefix}-{idx:05d}-of-{num_shard:05d}'
+    with open(output_file, 'wb') as f:
+      for file in files:
+        logging.info('Merging %s.', file)
+        file = fileutils.force_gcs_fuse_path(file)
+        with open(file, 'rb') as fin:
+          while True:
+            data = fin.read(READ_CHUNK_SIZE)
+            if not data:
+              break
+            f.write(data)
+        os.remove(file)
+
+  return merge_tfrecords

community-content/vertex_vision_model_garden/model_oss/data_converter/convert_coco_label_map_iod.py

@@ -0,0 +1,111 @@
+r"""Converts COCO labels as yamls for model garden playground (IOD).
+"""
+
+import os
+import urllib.request
+
+from absl import app
+from absl import flags
+import tensorflow as tf
+import yaml
+
+from object_detection.utils import label_map_util
+
+_CONVERT_LABEL_TYPE_COCO_80 = 'coco_80'
+_CONVERT_LABEL_TYPE_COCO_91 = 'coco_91'
+
+_CONVERT_LABEL_TYPE = flags.DEFINE_enum(
+    'convert_label_type',
+    None,
+    [
+        _CONVERT_LABEL_TYPE_COCO_80,
+        _CONVERT_LABEL_TYPE_COCO_91,
+    ],
+    'Different types of label type conversion.',
+    required=True,
+)
+
+_TEMPORARY_PATH = flags.DEFINE_string(
+    'temporary_path',
+    None,
+    'The tempory path.',
+    required=True,
+)
+
+_OUTPUT_YAML_FILEPATH = flags.DEFINE_string(
+    'output_yaml_filepath',
+    None,
+    'The output yaml filepath.',
+    required=True,
+)
+
+
+def convert_coco_label_map_91(
+    output_yaml_filepath: str,
+) -> None:
+  """Converts coco label map 91."""
+  input_proto_filepath = 'https://raw.githubusercontent.com/tensorflow/models/master/research/object_detection/data/mscoco_label_map.pbtxt'
+  local_input_proto_filepath = os.path.join(
+      _TEMPORARY_PATH.value, 'mscoco_label_map.pbtxt'
+  )
+  with open(local_input_proto_filepath, 'w') as writer:
+    contents = (
+        urllib.request.urlopen(input_proto_filepath).read().decode('utf-8')
+    )
+    writer.write(contents)
+
+  label_map = label_map_util.load_labelmap(local_input_proto_filepath)
+  label_map_dict = label_map_util.get_label_map_dict(
+      label_map, use_display_name=True
+  )
+  swapped_label_map_dict = {v: k for k, v in label_map_dict.items()}
+  print(swapped_label_map_dict)
+
+  # Saves new label maps as yamls.
+  with tf.io.gfile.GFile(output_yaml_filepath, 'w') as writer:
+    writer.write(yaml.dump(swapped_label_map_dict))
+
+
+def convert_coco_label_map_80(
+    output_yaml_filepath: str,
+) -> None:
+  """Converts coco label map 80."""
+  # Loads label maps from texts.
+  input_text_filepath = 'https://gist.githubusercontent.com/AruniRC/7b3dadd004da04c80198557db5da4bda/raw/2f10965ace1e36c4a9dca76ead19b744f5eb7e88/ms_coco_classnames.txt'
+  local_input_text_filepath = os.path.join(
+      _TEMPORARY_PATH.value, 'ms_coco_classnames.txt'
+  )
+  with open(local_input_text_filepath, 'w') as writer:
+    contents = (
+        urllib.request.urlopen(input_text_filepath).read().decode('utf-8')
+    )
+    writer.write(contents)
+  with open(local_input_text_filepath, 'r') as file:
+    content = file.read()
+    label_map = yaml.safe_load(content)
+
+  # Removes background in label maps.
+  new_label_map = {}
+  for k, v in label_map.items():
+    if k == 0:
+      continue
+    new_label_map[k - 1] = v
+  print(new_label_map)
+  # Saves new label maps as yamls.
+  with tf.io.gfile.GFile(output_yaml_filepath, 'w') as writer:
+    writer.write(yaml.dump(new_label_map))
+
+
+def main(_) -> None:
+  if _CONVERT_LABEL_TYPE.value == _CONVERT_LABEL_TYPE_COCO_80:
+    convert_coco_label_map_80(_OUTPUT_YAML_FILEPATH.value)
+  elif _CONVERT_LABEL_TYPE.value == _CONVERT_LABEL_TYPE_COCO_91:
+    convert_coco_label_map_91(
+        _OUTPUT_YAML_FILEPATH.value,
+    )
+  else:
+    print('Not supported convert label type: ', _CONVERT_LABEL_TYPE.value)
+
+
+if __name__ == '__main__':
+  app.run(main)

community-content/vertex_vision_model_garden/model_oss/data_converter/convert_imagenet_label_map.py

@@ -0,0 +1,86 @@
+r"""Converts ImageNet label texts as yamls for model garden playground.
+
+# ImageNet1K will have label maps with background.
+"""
+
+import urllib.request
+from absl import app
+from absl import flags
+import tensorflow as tf
+import yaml
+
+
+_INPUT_TEXT_FILEPATH = flags.DEFINE_string(
+    'input_text_filepath',
+    None,
+    'The input text filepath.',
+    required=True,
+)
+
+_ADD_BACKGROUND_LABEL = flags.DEFINE_boolean(
+    'add_background_label',
+    None,
+    'Whether or not add background labels.',
+    required=True,
+)
+
+_ADD_IDS = flags.DEFINE_boolean(
+    'add_ids',
+    None,
+    'Whether or not add ids.',
+    required=True,
+)
+
+_OUTPUT_YAML_FILEPATH = flags.DEFINE_string(
+    'output_yaml_filepath',
+    None,
+    'The output yaml filepath.',
+    required=True,
+)
+
+
+def convert_imagenet_label_map_from_text_to_yaml(
+    input_text_filepath: str,
+    add_background_label: bool,
+    add_ids: bool,
+    output_yaml_filepath: str,
+) -> None:
+  """Converts imagenet label map from text to yamls."""
+  label_map = {}
+
+  # Shifts all keys by 1, and add 0 as 'background'.
+  if add_background_label:
+    label_map = yaml.safe_load(
+        urllib.request.urlopen(input_text_filepath).read()
+    )
+    new_label_map = {}
+    for key, value in label_map.items():
+      new_label_map[key + 1] = value
+    new_label_map[0] = 'background'
+    label_map = new_label_map
+
+  # Adds maps from id to each line.
+  if add_ids:
+    lines = urllib.request.urlopen(input_text_filepath).readlines()
+    current_id = 0
+    for line in lines:
+      label_map[current_id] = line.decode('ascii').strip()
+      print(label_map[current_id])
+      current_id += 1
+
+  # Saves new label maps as yamls.
+  with tf.io.gfile.GFile(output_yaml_filepath, 'w') as writer:
+    writer.write(yaml.dump(label_map))
+
+
+def main(_) -> None:
+  convert_imagenet_label_map_from_text_to_yaml(
+      _INPUT_TEXT_FILEPATH.value,
+      _ADD_BACKGROUND_LABEL.value,
+      _ADD_IDS.value,
+      _OUTPUT_YAML_FILEPATH.value,
+  )
+
+
+if __name__ == '__main__':
+  app.run(main)

community-content/vertex_vision_model_garden/model_oss/data_converter/data_converter_icn_lib.py

@@ -0,0 +1,199 @@
+"""Converts ICN CSV/JSONL files to TFRecord with apache beam."""
+
+import json
+from os import path
+from typing import Any, Dict, Sequence, Union, cast
+
+from absl import logging
+import apache_beam as beam
+import pandas as pd
+import tensorflow as tf
+
+from data_converter import common_lib
+
+
+_COLUMN_NAMES = [
+    common_lib.COLUMN_NAME_ML_USE,
+    common_lib.COLUMN_NAME_GCS_FILE_PATH,
+    common_lib.COLUMN_NAME_LABEL,
+]
+_JSON_GCS_URI_KEY = 'imageGcsUri'
+_JSON_CLASS_ANNOTATION_KEY = 'classificationAnnotation'
+_JSON_RESOURCE_LABEL_KEY = 'dataItemResourceLabels'
+_JSON_CLASS_NAME_KEY = 'displayName'
+_JSON_ML_USE_KEY = 'aiplatform.googleapis.com/ml_use'
+
+
+def build_tf_example(element: Dict[str, Union[str, int]]) -> tf.train.Example:
+  """Builds a TF Example from an image uri and label.
+
+  Args:
+    element: A dict with the keys gcs_file_path and label.
+
+  Returns:
+    The created TF Example.
+  """
+  image_uri = cast(str, element[common_lib.COLUMN_NAME_GCS_FILE_PATH])
+  label = cast(int, element[common_lib.COLUMN_NAME_LABEL])
+  image_bytes, shape = common_lib.encode_image(image_uri, image_format='jpeg')
+  features = tf.train.Features(
+      feature={
+          'image/encoded': common_lib.convert_to_feature(image_bytes),
+          'image/format': common_lib.convert_to_string_feature('jpeg'),
+          'image/height': common_lib.convert_to_feature(shape[0]),
+          'image/width': common_lib.convert_to_feature(shape[1]),
+          'image/class/label': common_lib.convert_to_feature(label),
+      },
+  )
+  return tf.train.Example(features=features)
+
+
+def _run_convert_pipeline(
+    output_dir: str, df: pd.DataFrame, num_shards: Sequence[int]
+) -> None:
+  """Starts a Beam pipeline to write DataFrame as TF Records.
+
+  Args:
+    output_dir: TF Records output directory.
+    df: DataFrame to convert from.
+    num_shards: Number of shards for train/validation/test TFRecord files.
+  """
+  images_list = df.to_dict('records')
+
+  def pipeline(root: beam.Pipeline):
+    common_lib.beam_convert_tfexamples(
+        root,
+        images_list,
+        build_tf_example,
+        output_dir,
+        num_shards,
+    )
+
+  common_lib.run_beam_pipeline(pipeline)
+
+
+def _convert_df_to_tfrecord(
+    df: pd.DataFrame,
+    output_dir: str,
+    split_ratio: Sequence[float],
+    num_shard: Sequence[int],
+) -> None:
+  """Converts a DataFrame into three separate tfrecords for training, validation, and testing into output_dir.
+
+  Args:
+    df: DataFrame to convert.
+    output_dir: The directory to save TFRecords and label_map.yaml.
+    split_ratio: List specifying the training, validation, and testing splits
+      for unassigned TFRecords.
+    num_shard: Number of shards for train/validation/test TFRecord files.
+  """
+  # Replaces ml_use with common_lib string constants for consistency.
+  common_lib.format_ml_use_column(df)
+  common_lib.insert_missing_ml_use(df)
+
+  # Ignores invalid rows.
+  dropped_row_num = common_lib.drop_invalid_rows(df)
+  if dropped_row_num > 0:
+    logging.warning('Ignored %d invalid rows.', dropped_row_num)
+
+  common_lib.replace_unassigned_ml_use(
+      df[common_lib.COLUMN_NAME_ML_USE], split_ratio
+  )
+
+  # Converts labels to integers as required by training.
+  new_labels, label_map = common_lib.create_label_map(
+      df[common_lib.COLUMN_NAME_LABEL]
+  )
+  df[common_lib.COLUMN_NAME_LABEL] = new_labels
+  label_map_path = path.join(output_dir, common_lib.LABEL_MAP_NAME)
+  logging.info('Writing label map to %s.', label_map_path)
+  common_lib.write_label_map(label_map_path, label_map)
+
+  _run_convert_pipeline(output_dir, df, num_shard)
+
+
+def convert_csv_to_tfrecord(
+    input_csv: str,
+    output_dir: str,
+    split_ratio: Sequence[float] = (0.8, 0.1, 0.1),
+    num_shard: Sequence[int] = (10, 10, 10),
+) -> None:
+  """Parses input_csv file into three separate tfrecords for training, validation, and testing into output_dir.
+
+  The csv format is shown in
+    https://cloud.google.com/vertex-ai/docs/image-data/classification/prepare-data#csv.
+
+  If an ml_use column is not provided, one will be created.
+
+  label_map.yaml containing the label map will be placed in output_dir.
+
+  Args:
+    input_csv: Name of the csv file.
+    output_dir: The directory to save TFRecords and label_map.yaml.
+    split_ratio: List specifying the training, validation, and testing splits
+      for unassigned TFRecords.
+    num_shard: Number of shards for train/validation/test TFRecord files.
+  """
+  with tf.io.gfile.GFile(input_csv, 'r') as f:
+    df: pd.DataFrame = pd.read_csv(
+        f, header=None, names=_COLUMN_NAMES, on_bad_lines='warn'
+    )
+
+  _convert_df_to_tfrecord(df, output_dir, split_ratio, num_shard)
+
+
+def convert_jsonl_to_tfrecord(
+    input_jsonl: str,
+    output_dir: str,
+    split_ratio: Sequence[float] = (0.8, 0.1, 0.1),
+    num_shard: Sequence[int] = (10, 10, 10),
+) -> None:
+  """Parses input_jsonl file into three separate tfrecords for training, validation, and testing into output_dir.
+
+  The JSONL format is shown in
+    https://cloud.google.com/vertex-ai/docs/image-data/classification/prepare-data#json-lines.
+
+  If an ml_use column is not provided, one will be created.
+
+  label_map.yaml containing the label map will be placed in output_dir.
+
+  Args:
+    input_jsonl: Name of the JSONL file.
+    output_dir: The directory to save TFRecords and label_map.yaml.
+    split_ratio: List specifying the training, validation, and testing splits
+      for unassigned TFRecords.
+    num_shard: Number of shards for train/validation/test TFRecord files.
+  """
+  df_rows = []
+  with tf.io.gfile.GFile(input_jsonl, 'r') as f:
+    lines = f.read().rstrip().splitlines()
+
+  for i, line in enumerate(lines, 1):
+    try:
+      item: Dict[str, Any] = json.loads(line)
+
+      gcs_uri = item.get(_JSON_GCS_URI_KEY)
+      label = item.get(_JSON_CLASS_ANNOTATION_KEY, {}).get(_JSON_CLASS_NAME_KEY)
+      if not gcs_uri or not label:
+        logging.warning('Invalid JSON at line %d, skipped.', i)
+        continue
+
+      ml_use = item.get(_JSON_RESOURCE_LABEL_KEY, {}).get(
+          _JSON_ML_USE_KEY, common_lib.ML_USE_UNASSIGNED
+      )
+    except (json.JSONDecodeError, AttributeError):
+      logging.warning('Invalid JSON at line %d, skipped.', i)
+      continue
+
+    df_rows.append([ml_use, gcs_uri, label])
+
+  df = pd.DataFrame(
+      data=df_rows,
+      columns=[
+          common_lib.COLUMN_NAME_ML_USE,
+          common_lib.COLUMN_NAME_GCS_FILE_PATH,
+          common_lib.COLUMN_NAME_LABEL,
+      ],
+  )
+
+  _convert_df_to_tfrecord(df, output_dir, split_ratio, num_shard)

community-content/vertex_vision_model_garden/model_oss/data_converter/data_converter_iod_lib.py

@@ -0,0 +1,430 @@
+"""Converts IOD dataset files to TFRecord with apache beam."""
+
+import collections
+import json
+from os import path
+from typing import Any, Dict, Sequence
+
+from absl import logging
+import apache_beam as beam
+import pandas as pd
+import tensorflow as tf
+
+from data_converter import common_lib
+from util import constants
+
+COLUMN_NAME_LABEL_INT = 'label_int'
+_COLUMN_NAME_XMIN = 'X_MIN'
+_COLUMN_NAME_YMIN = 'Y_MIN'
+_COLUMN_NAME_XMAX = 'X_MAX'
+_COLUMN_NAME_YMAX = 'Y_MAX'
+COLUMN_NAMES = [
+    common_lib.COLUMN_NAME_ML_USE,
+    common_lib.COLUMN_NAME_GCS_FILE_PATH,
+    common_lib.COLUMN_NAME_LABEL,
+    _COLUMN_NAME_XMIN,
+    _COLUMN_NAME_YMIN,
+    'XMAX_NOT_USED',
+    'YMIN_NOT_USED',
+    _COLUMN_NAME_XMAX,
+    _COLUMN_NAME_YMAX,
+    'XMIN_NOT_USED',
+    'YMAX_NOT_USED',
+]
+_BOUNDING_BOX_COLUMNS = [
+    _COLUMN_NAME_XMIN,
+    _COLUMN_NAME_YMIN,
+    _COLUMN_NAME_XMAX,
+    _COLUMN_NAME_YMAX,
+]
+_JSON_BBOX_ANNOTATIONS_KEY = 'boundingBoxAnnotations'
+_JSON_DISPLAY_NAME_KEY = 'displayName'
+_JSON_X_MIN_KEY = 'xMin'
+_JSON_X_MAX_KEY = 'xMax'
+_JSON_Y_MIN_KEY = 'yMin'
+_JSON_Y_MAX_KEY = 'yMax'
+
+
+def build_tf_example(image_row: Dict[str, Any]) -> tf.train.Example:
+  """Builds a TF Example from an image row.
+
+  Args:
+    image_row: A dictionary containing information about the image, such as its
+      GCS uri, labels, and bounding box coordinates.
+
+  Returns:
+    A tf.train.Example containing the encoded image and optionally a
+      bounding box and label.
+  """
+  image_uri = image_row[common_lib.COLUMN_NAME_GCS_FILE_PATH]
+  image_bytes, shape = common_lib.encode_image(image_uri, image_format='jpeg')
+  feature = {
+      'image/encoded': common_lib.convert_to_feature(image_bytes),
+      'image/format': common_lib.convert_to_string_feature('jpeg'),
+      'image/height': common_lib.convert_to_feature(shape[0]),
+      'image/width': common_lib.convert_to_feature(shape[1]),
+      'image/source_id': common_lib.convert_to_string_feature(image_uri),
+      'image/object/bbox/xmin': common_lib.convert_to_feature(
+          image_row[_COLUMN_NAME_XMIN]
+      ),
+      'image/object/bbox/ymin': common_lib.convert_to_feature(
+          image_row[_COLUMN_NAME_YMIN]
+      ),
+      'image/object/bbox/xmax': common_lib.convert_to_feature(
+          image_row[_COLUMN_NAME_XMAX]
+      ),
+      'image/object/bbox/ymax': common_lib.convert_to_feature(
+          image_row[_COLUMN_NAME_YMAX]
+      ),
+      'image/object/class/text': common_lib.convert_to_list_string_feature(
+          image_row[common_lib.COLUMN_NAME_LABEL]
+      ),
+      'image/object/class/label': common_lib.convert_to_feature(
+          image_row[COLUMN_NAME_LABEL_INT]
+      ),
+  }
+  return tf.train.Example(features=tf.train.Features(feature=feature))
+
+
+def _run_convert_pipeline(
+    output_dir: str,
+    image_rows: Sequence[Dict[str, Any]],
+    num_shards: Sequence[int],
+) -> None:
+  """Starts a Beam pipeline to write DataFrame as TF Records.
+
+  Args:
+    output_dir: TF Records output directory.
+    image_rows: Contains all necessary information to create a TF Example.
+    num_shards: Number of shards for train/validation/test TFRecord files.
+  """
+
+  def pipeline(root: beam.Pipeline):
+    common_lib.beam_convert_tfexamples(
+        root,
+        image_rows,
+        build_tf_example,
+        output_dir,
+        num_shards,
+    )
+
+  common_lib.run_beam_pipeline(pipeline)
+
+
+def _convert_df_to_tfrecord(
+    df: pd.DataFrame,
+    output_dir: str,
+    split_ratio: Sequence[float],
+    num_shard: Sequence[int],
+) -> None:
+  """Converts a DataFrame into three separate tfrecords for training, validation, and testing into output_dir.
+
+  Args:
+    df: DataFrame to convert.
+    output_dir: The directory to save TFRecords and label_map.yaml.
+    split_ratio: List specifying the training, validation, and testing splits
+      for unassigned TFRecords.
+    num_shard: Number of shards for train/validation/test TFRecord files.
+  """
+  # Replaces ml_use with common_lib string constants for consistency.
+  common_lib.format_ml_use_column(df)
+  common_lib.insert_missing_ml_use(df)
+
+  # Specify bounding box columns to be numeric.
+  df[_BOUNDING_BOX_COLUMNS] = df[_BOUNDING_BOX_COLUMNS].apply(pd.to_numeric)
+
+  # Ignores invalid rows.
+  dropped_row_num = common_lib.drop_invalid_rows(df)
+  dropped_row_num += drop_rows_without_bbox(df)
+  if dropped_row_num > 0:
+    logging.warning('Ignored %d invalid rows.', dropped_row_num)
+
+  # Converts labels to integers as required by training.
+  int_labels, label_map = common_lib.create_label_map(
+      df[common_lib.COLUMN_NAME_LABEL]
+  )
+  df[COLUMN_NAME_LABEL_INT] = int_labels
+  label_map_path = path.join(output_dir, common_lib.LABEL_MAP_NAME)
+  logging.info('Writing label map to %s.', label_map_path)
+  common_lib.write_label_map(label_map_path, label_map)
+
+  image_rows = _condense_bounding_boxes(df.to_dict(orient='records'))
+  ml_uses = [row[common_lib.COLUMN_NAME_ML_USE] for row in image_rows]
+  common_lib.replace_unassigned_ml_use(ml_uses, split_ratio)
+  common_lib.merge_seq_into_dicts(
+      common_lib.COLUMN_NAME_ML_USE, ml_uses, image_rows
+  )
+
+  _run_convert_pipeline(output_dir, image_rows, num_shard)
+
+
+def _condense_bounding_boxes(
+    image_rows: Sequence[Dict[str, Any]]
+) -> Sequence[Dict[str, Any]]:
+  """Gather all the bounding boxes in an image and put them in the same dictionary.
+
+  Args:
+    image_rows: List of dictionaries, each containing information about the
+      image, such as its GCS uri, labels, and bounding box coordinates.
+
+  Returns:
+    List of dictionaries such that each contains all the bounding boxes for a
+    given gcs_file_path.
+
+  Raises:
+    RuntimeError: This is raised when the input data contains images that have
+      annotations in different ml_use classes.
+  """
+  output = {}
+  for image_row in image_rows:
+    ml_use = image_row[common_lib.COLUMN_NAME_ML_USE]
+    gcs_file_path = image_row[common_lib.COLUMN_NAME_GCS_FILE_PATH]
+    label = image_row[common_lib.COLUMN_NAME_LABEL]
+    xmin = image_row[_COLUMN_NAME_XMIN]
+    ymin = image_row[_COLUMN_NAME_YMIN]
+    xmax = image_row[_COLUMN_NAME_XMAX]
+    ymax = image_row[_COLUMN_NAME_YMAX]
+    label_int = image_row[COLUMN_NAME_LABEL_INT]
+    if gcs_file_path in output:
+      d = output[gcs_file_path]
+      if ml_use != common_lib.ML_USE_UNASSIGNED:
+        if d[common_lib.COLUMN_NAME_ML_USE] == common_lib.ML_USE_UNASSIGNED:
+          d[common_lib.COLUMN_NAME_ML_USE] = ml_use
+        elif ml_use != d[common_lib.COLUMN_NAME_ML_USE]:
+          raise RuntimeError(
+              f'Image {gcs_file_path} can only be placed in one of'
+              f' training/validation/test. It is currently in {ml_use} and'
+              f' {d[common_lib.COLUMN_NAME_ML_USE]}.'
+          )
+      d[common_lib.COLUMN_NAME_LABEL].append(label)
+      d[_COLUMN_NAME_XMIN].append(xmin)
+      d[_COLUMN_NAME_YMIN].append(ymin)
+      d[_COLUMN_NAME_XMAX].append(xmax)
+      d[_COLUMN_NAME_YMAX].append(ymax)
+      d[COLUMN_NAME_LABEL_INT].append(label_int)
+    else:
+      output[gcs_file_path] = {
+          common_lib.COLUMN_NAME_ML_USE: ml_use,
+          common_lib.COLUMN_NAME_GCS_FILE_PATH: gcs_file_path,
+          common_lib.COLUMN_NAME_LABEL: [label],
+          _COLUMN_NAME_XMIN: [xmin],
+          _COLUMN_NAME_YMIN: [ymin],
+          _COLUMN_NAME_XMAX: [xmax],
+          _COLUMN_NAME_YMAX: [ymax],
+          COLUMN_NAME_LABEL_INT: [label_int],
+      }
+  return list(output.values())
+
+
+def convert_csv_to_tfrecord(
+    input_csv: str,
+    output_dir: str,
+    split_ratio: Sequence[float] = (0.8, 0.1, 0.1),
+    num_shard: Sequence[int] = (10, 10, 10),
+) -> None:
+  """Parses input_csv file into three separate tfrecords for training, validation, and testing into output_dir.
+
+  The csv format is shown in
+    https://cloud.google.com/vertex-ai/docs/image-data/object-detection/prepare-data#csv.
+
+  If an ml_use column is not provided, one will be created.
+
+  label_map.yaml containing the label map will be placed in output_dir.
+
+  Args:
+    input_csv: Name of the csv file.
+    output_dir: The directory to save TFRecords and label_map.yaml.
+    split_ratio: List specifying the train, validation, and test splits for
+      unassigned TFRecords.
+    num_shard: Number of shards for train/validation/test TFRecord files.
+  """
+  with tf.io.gfile.GFile(input_csv, 'r') as f:
+    df: pd.DataFrame = pd.read_csv(
+        f, header=None, names=COLUMN_NAMES, on_bad_lines='warn'
+    )
+
+  _convert_df_to_tfrecord(df, output_dir, split_ratio, num_shard)
+
+
+def drop_rows_without_bbox(df: pd.DataFrame) -> int:
+  """Drops DataFrame rows without bounding_boxes.
+
+  Args:
+    df: The DataFrame to process in place.
+
+  Returns:
+    The number of rows dropped.
+  """
+  invalid_rows = df.index[~(df[_BOUNDING_BOX_COLUMNS].notnull().all(axis=1))]
+  dropped_num = len(invalid_rows)
+  if dropped_num > 0:
+    invalid_df = df.loc[invalid_rows].to_dict(orient='records')
+    for entry in invalid_df:
+      logging.warning('Skipping entry due to missing bounding box: %s.', entry)
+    df.drop(invalid_rows, inplace=True)
+    df.reset_index(drop=True, inplace=True)
+  return dropped_num
+
+
+def convert_coco_json_categories_to_label_map(
+    categories: Sequence[Dict[str, Any]]
+) -> Dict[int, str]:
+  return {category['id']: category['name'] for category in categories}
+
+
+def convert_coco_json_to_tfrecord(
+    input_coco_json: str,
+    output_dir: str,
+    split_ratio: Sequence[float] = (0.8, 0.1, 0.1),
+    num_shard: Sequence[int] = (10, 10, 10),
+) -> None:
+  """Parses input_csv file into three separate tfrecords for training, validation, and testing into output_dir.
+
+  The COCO json format is shown here: https://cocodataset.org/#format-data.
+
+  label_map.yaml containing the label map will be placed in output_dir.
+
+  Args:
+    input_coco_json: Name of coco json file.
+    output_dir: The directory to save TFRecords and label_map.yaml.
+    split_ratio: List specifying the train, validation, and test splits for
+      dataset.
+    num_shard: Number of shards for train/validation/test TFRecord files.
+  """
+  with tf.io.gfile.GFile(input_coco_json, 'r') as f:
+    coco_json = json.load(f)
+  # Writes label map from coco json categories.
+  label_map = convert_coco_json_categories_to_label_map(
+      coco_json[constants.COCO_JSON_CATEGORIES]
+  )
+  label_map_path = path.join(output_dir, common_lib.LABEL_MAP_NAME)
+  logging.info('Writes label map to %s.', label_map_path)
+  common_lib.write_label_map(label_map_path, label_map)
+
+  img_to_anns = collections.defaultdict(list)
+  imgs = {}
+  if constants.COCO_JSON_ANNOTATIONS in coco_json:
+    for ann in coco_json[constants.COCO_JSON_ANNOTATIONS]:
+      img_to_anns[ann[constants.COCO_JSON_ANNOTATION_IMAGE_ID]].append(ann)
+
+  if constants.COCO_JSON_IMAGES in coco_json:
+    for img in coco_json[constants.COCO_JSON_IMAGES]:
+      imgs[img[constants.COCO_JSON_IMAGE_ID]] = img
+
+  df_rows = []
+
+  for image_id, annotations in img_to_anns.items():
+    img = imgs[image_id]
+    for ann in annotations:
+      xmin, ymin, xmax, ymax = common_lib.reformat_bbox(
+          ann[constants.COCO_ANNOTATION_BBOX],
+          img[constants.COCO_JSON_IMAGE_WIDTH],
+          img[constants.COCO_JSON_IMAGE_HEIGHT],
+      )
+      df_rows.append([
+          common_lib.ML_USE_UNASSIGNED,
+          img[constants.COCO_JSON_IMAGE_COCO_URL],
+          label_map[ann[constants.COCO_JSON_ANNOTATION_CATEGORY_ID]],
+          xmin,
+          ymin,
+          xmax,
+          ymin,
+          xmax,
+          ymax,
+          xmin,
+          ymax,
+          ann[constants.COCO_JSON_ANNOTATION_CATEGORY_ID],
+      ])
+  df = pd.DataFrame(
+      data=df_rows,
+      columns=COLUMN_NAMES + [COLUMN_NAME_LABEL_INT],
+  )
+
+  # Replaces ml_use with common_lib string constants for consistency.
+  common_lib.format_ml_use_column(df)
+  common_lib.insert_missing_ml_use(df)
+
+  # Species bounding box columns to be numeric.
+  df[_BOUNDING_BOX_COLUMNS] = df[_BOUNDING_BOX_COLUMNS].apply(pd.to_numeric)
+
+  # Ignores invalid rows.
+  dropped_row_num = common_lib.drop_invalid_rows(df)
+  dropped_row_num += drop_rows_without_bbox(df)
+  if dropped_row_num > 0:
+    logging.warning('Ignored %d invalid rows.', dropped_row_num)
+
+  image_rows = _condense_bounding_boxes(df.to_dict(orient='records'))
+  ml_uses = [row[common_lib.COLUMN_NAME_ML_USE] for row in image_rows]
+  common_lib.replace_unassigned_ml_use(ml_uses, split_ratio)
+  common_lib.merge_seq_into_dicts(
+      common_lib.COLUMN_NAME_ML_USE, ml_uses, image_rows
+  )
+
+  _run_convert_pipeline(output_dir, image_rows, num_shard)
+
+
+def convert_jsonl_to_tfrecord(
+    input_jsonl: str,
+    output_dir: str,
+    split_ratio: Sequence[float] = (0.8, 0.1, 0.1),
+    num_shard: Sequence[int] = (10, 10, 10),
+) -> None:
+  """Parses input_jsonl file into three separate tfrecords for training, validation, and testing into output_dir.
+
+  The JSONL format is shown in
+    https://cloud.google.com/vertex-ai/docs/image-data/object-detection/prepare-data#json-lines.
+
+  If an ml_use column is not provided, one will be created.
+
+  label_map.yaml containing the label map will be placed in output_dir.
+
+  Args:
+    input_jsonl: Name of the JSONL file.
+    output_dir: The directory to save TFRecords and label_map.yaml.
+    split_ratio: List specifying the training, validation, and testing splits
+      for unassigned TFRecords.
+    num_shard: Number of shards for train/validation/test TFRecord files.
+  """
+  df_rows = []
+  with tf.io.gfile.GFile(input_jsonl, 'r') as f:
+    lines = f.read().rstrip().splitlines()
+
+  for i, line in enumerate(lines, start=1):
+    try:
+      item: Dict[str, Any] = json.loads(line)
+    except (json.JSONDecodeError, AttributeError):
+      logging.warning('Invalid JSON at line %d skipped.', i)
+      continue
+
+    gcs_uri = item.get(common_lib.JSON_GCS_URI_KEY)
+    if not gcs_uri:
+      logging.warning(
+          'Invalid JSON at line %d skipped. Missing gcs_uri_key.', i
+      )
+      continue
+    ml_use = item.get(common_lib.JSON_RESOURCE_LABEL_KEY, {}).get(
+        common_lib.JSON_ML_USE_KEY, common_lib.ML_USE_UNASSIGNED
+    )
+
+    for bbox in item.get(_JSON_BBOX_ANNOTATIONS_KEY, []):
+      label = bbox.get(_JSON_DISPLAY_NAME_KEY)
+      xmin = bbox.get(_JSON_X_MIN_KEY)
+      ymin = bbox.get(_JSON_Y_MIN_KEY)
+      xmax = bbox.get(_JSON_X_MAX_KEY)
+      ymax = bbox.get(_JSON_Y_MAX_KEY)
+
+      df_rows.append([ml_use, gcs_uri, label, xmin, ymin, xmax, ymax])
+
+  df = pd.DataFrame(
+      data=df_rows,
+      columns=[
+          common_lib.COLUMN_NAME_ML_USE,
+          common_lib.COLUMN_NAME_GCS_FILE_PATH,
+          common_lib.COLUMN_NAME_LABEL,
+          _COLUMN_NAME_XMIN,
+          _COLUMN_NAME_YMIN,
+          _COLUMN_NAME_XMAX,
+          _COLUMN_NAME_YMAX,
+      ],
+  )
+  _convert_df_to_tfrecord(df, output_dir, split_ratio, num_shard)

community-content/vertex_vision_model_garden/model_oss/data_converter/data_converter_isg_lib.py

@@ -0,0 +1,328 @@
+"""Python script to convert different file formats for ISG to tfrecords."""
+
+import hashlib
+import os
+from typing import Any, Dict, Iterator, List, Optional, Tuple, Union
+
+from absl import logging
+import apache_beam as beam
+from apache_beam.io import tfrecordio
+import cv2
+import numpy as np
+from pycocotools import coco
+import tensorflow as tf
+import yaml
+
+from data_converter import common_lib
+from util import constants
+from util import fileutils
+
+_IMAGE_FORMAT = 'PNG'
+
+
+def build_tf_example(
+    image_info: dict[str, Union[str, int]],
+    segmentation_image: List[List[int]],
+    output_shape: Optional[Tuple[int, int]] = None,
+) -> tf.train.Example:
+  """Encodes an image and its segmentation mask into a tf.train.Example.
+
+  Args:
+    image_info: A dictionary containing information about the image, such as its
+      file name, height, and width.
+    segmentation_image: 2D image in list of lists having category ids.
+    output_shape: The desired output shape of the image. If None, the original
+      image shape will be used.
+
+  Returns:
+    A tf.train.Example containing the encoded image and segmentation mask.
+
+  Raises:
+    IOError: If image cannot be found in the path.
+  """
+  file_name = image_info[constants.COCO_JSON_FILE_NAME]
+  height = int(image_info[constants.COCO_JSON_IMAGE_HEIGHT])
+  width = int(image_info[constants.COCO_JSON_IMAGE_WIDTH])
+
+  segmentation_image = np.expand_dims(
+      np.asarray(segmentation_image, dtype=np.int32), axis=-1
+  )
+  _, encoded_seg = cv2.imencode(f'.{_IMAGE_FORMAT.lower()}', segmentation_image)
+  encoded_seg = encoded_seg.tobytes()
+
+  encoded_img, _ = common_lib.encode_image(
+      image_info[constants.COCO_JSON_IMAGE_COCO_URL],
+      output_shape=output_shape,
+      image_format=_IMAGE_FORMAT.lower(),
+  )
+
+  key = hashlib.sha256(encoded_img).hexdigest()
+
+  return tf.train.Example(
+      features=tf.train.Features(
+          feature={
+              'image/height': common_lib.convert_to_feature(height),
+              'image/width': common_lib.convert_to_feature(width),
+              'image/filename': common_lib.convert_to_string_feature(file_name),
+              'image/sha256': common_lib.convert_to_string_feature(key),
+              'image/encoded': common_lib.convert_to_feature(encoded_img),
+              'image/format': common_lib.convert_to_string_feature(
+                  _IMAGE_FORMAT
+              ),
+              'image/segmentation/class/encoded': common_lib.convert_to_feature(
+                  encoded_seg
+              ),
+              'image/segmentation/class/format': (
+                  common_lib.convert_to_string_feature(_IMAGE_FORMAT)
+              ),
+              'image/segmentation/class/height': common_lib.convert_to_feature(
+                  height
+              ),
+              'image/segmentation/class/width': common_lib.convert_to_feature(
+                  width
+              ),
+          }
+      )
+  )
+
+
+class AcquireTFExampleDoFn(beam.DoFn):
+  """Beam DoFn to build TF Examples from a single row of image_info data."""
+
+  # These tags will be used to tag the outputs of this DoFn.
+  output_tag_train = constants.ML_USE_TRAINING
+  output_tag_validation = constants.ML_USE_VALIDATION
+  output_tag_test = constants.ML_USE_TEST
+
+  valid_ml_use_set = set(
+      [output_tag_train, output_tag_validation, output_tag_test]
+  )
+
+  def __init__(self, output_shape: Optional[Tuple[int, int]] = None):
+    self.acquired_examples_counter = beam.metrics.Metrics.counter(
+        self.__class__.__name__, 'Success'
+    )
+    self.failure_counter = beam.metrics.Metrics.counter(
+        self.__class__.__name__, 'Failure'
+    )
+    self.output_shape = output_shape
+
+  def process(
+      self,
+      row: Tuple[str, Dict[str, Union[str, int]], List[List[int]]],
+  ) -> Iterator[tf.train.Example]:
+    ml_use, image_info, annotation_info = row
+    if ml_use not in self.valid_ml_use_set:
+      logging.warning('ml_use invalid: %s', ml_use)
+      self.failure_counter.inc()
+      return
+
+    try:
+      tf_example = build_tf_example(
+          image_info, annotation_info, self.output_shape
+      )
+    except IOError as e:
+      logging.warning('Failed to build TF Example: %s', e)
+      self.failure_counter.inc()
+    else:
+      self.acquired_examples_counter.inc()
+      yield beam.pvalue.TaggedOutput(ml_use, tf_example)
+
+
+def _define_data_conversion_pipeline(
+    root: beam.Pipeline,
+    ml_use_rows: List[str],
+    image_rows: List[Dict[str, Union[str, int]]],
+    segmentation_rows: List[List[List[int]]],
+    output_dir: str,
+    output_shape: Optional[Tuple[int, int]],
+    num_shard_list: List[int],
+):
+  """Define a data conversion pipeline.
+
+  Args:
+    root: A Beam pipeline.
+    ml_use_rows: List containing the ml_use.
+    image_rows: List of dictionaries containing information about the image,
+      such as its file name, height, and width.
+    segmentation_rows: List of 2D images of integers representing segmentation
+      masks.
+    output_dir: Directory where the output TFRecords will be written.
+    output_shape: Desired output shape of the image. If None, the original image
+      shape will be used.
+    num_shard_list: Number of shards to write to each output TFRecord.
+
+  Returns:
+    A Beam pipeline.
+  """
+  train, validation, test = (
+      root
+      | 'Load ml use and image rows to beam'
+      >> beam.Create(zip(ml_use_rows, image_rows, segmentation_rows))
+      | 'Build TF Examples'
+      >> beam.ParDo(AcquireTFExampleDoFn(output_shape)).with_outputs(
+          AcquireTFExampleDoFn.output_tag_train,
+          AcquireTFExampleDoFn.output_tag_validation,
+          AcquireTFExampleDoFn.output_tag_test,
+      )
+  )
+
+  # Save each split to TFRecord.
+  _ = train | 'Save train split to TFRecord' >> tfrecordio.WriteToTFRecord(
+      os.path.join(output_dir, common_lib.TRAIN_TFRECORD_NAME),
+      coder=beam.coders.ProtoCoder(tf.train.Example),
+      num_shards=num_shard_list[0],
+  )
+  _ = (
+      validation
+      | 'Save validation split to TFRecord'
+      >> tfrecordio.WriteToTFRecord(
+          os.path.join(output_dir, common_lib.VALIDATION_TFRECORD_NAME),
+          coder=beam.coders.ProtoCoder(tf.train.Example),
+          num_shards=num_shard_list[1],
+      )
+  )
+  _ = test | 'Save test split to TFRecord' >> tfrecordio.WriteToTFRecord(
+      os.path.join(output_dir, common_lib.TEST_TFRECORD_NAME),
+      coder=beam.coders.ProtoCoder(tf.train.Example),
+      num_shards=num_shard_list[2],
+  )
+
+
+def _image_info_to_segmentation_image(
+    img: Dict[str, Any],
+    coco_dataset: coco.COCO,
+    label_id_by_category_id: Dict[int, int],
+) -> List[List[int]]:
+  """Convert image information to a segmentation image.
+
+  Args:
+      img: The image information.
+      coco_dataset: The COCO dataset.
+      label_id_by_category_id: The mapping from label id used for training to
+        category_id defined in dataset.
+
+  Returns:
+      The segmentation image.
+
+  Raises:
+    ValueError: If the mask size does not match the image or if a pixel has
+      multiple labels.
+  """
+  seg_img = np.zeros(
+      shape=(
+          img[constants.COCO_JSON_IMAGE_HEIGHT],
+          img[constants.COCO_JSON_IMAGE_WIDTH],
+      ),
+      dtype=np.int32,
+  )
+  for ann in coco_dataset.imgToAnns[img[constants.COCO_JSON_IMAGE_ID]]:
+    new_category_id = ann[constants.COCO_JSON_ANNOTATION_CATEGORY_ID]
+    binary_mask = coco_dataset.annToMask(ann)
+    if seg_img.shape != binary_mask.shape:
+      raise ValueError(
+          'Binary mask does not have the same shape as image. image_id:'
+          f' {img["id"]}'
+      )
+    boolean_mask = binary_mask == 1
+    if (seg_img[boolean_mask] != 0).any():
+      raise ValueError(
+          'Error: Some pixels have more than one label in image_id:'
+          f' {img["id"]}.'
+      )
+    seg_img[boolean_mask] = label_id_by_category_id[new_category_id]
+
+  return seg_img.tolist()
+
+
+def get_input_rows(
+    coco_dataset: coco.COCO,
+    split_ratio: List[float],
+    label_id_by_category_id: Dict[int, int],
+) -> Tuple[List[str], List[Dict[str, Union[str, int]]], List[List[List[int]]]]:
+  """Get input rows for training and validation.
+
+  Args:
+      coco_dataset: The COCO dataset.
+      split_ratio: The split ratio for training and validation.
+      label_id_by_category_id: The mapping from label id used for training to
+        category_id defined in dataset.
+
+  Returns:
+      - A list of ml_use strings.
+      - A list of image informations.
+      - A list of segmentation images for the corresponding images.
+  """
+  image_rows = coco_dataset.dataset[constants.COCO_JSON_IMAGES]
+
+  segmentation_rows = [
+      _image_info_to_segmentation_image(
+          img, coco_dataset, label_id_by_category_id
+      )
+      for img in image_rows
+  ]
+
+  ml_use_rows = common_lib.create_ml_use_array_with_split(
+      len(image_rows), split_ratio
+  )
+  return ml_use_rows, image_rows, segmentation_rows
+
+
+def beam_build_tfrecord_from_coco_json(
+    input_json: str,
+    output_dir: str,
+    split_ratio: List[float],
+    num_shard_list: List[int],
+    output_shape: Optional[Tuple[int, int]] = None,
+) -> None:
+  """Builds TFRecord files from COCO dataset.
+
+  The output file names are `_TRAIN_TFRECORD_NAME`, `_VALIDATION_TFRECORD_NAME`,
+  and `_TEST_TFRECORD_NAME`.
+
+  Args:
+    input_json: Path to a COCO JSON or JSONL file.
+    output_dir: Directory to output the TFRecord files.
+    split_ratio: List of how to split entries to train, validation, and test
+      TFRecords.
+    num_shard_list: List of the number of shards for each TFRecord file.
+    output_shape: The desired output shape of the image. If None, the original
+      image shape will be used.
+  """
+  # `coco` cannot access gcs uri. Use gcsfuse, it is faster.
+  input_json = fileutils.force_gcs_fuse_path(input_json)
+  coco_dataset = coco.COCO(input_json)
+
+  label_map = {}
+  label_id_by_category_id = {}
+  for idx, category in enumerate(
+      coco_dataset.dataset[constants.COCO_JSON_CATEGORIES], start=1
+  ):
+    label_map[idx] = category[constants.COCO_JSON_CATEGORY_NAME]
+    label_id_by_category_id[category[constants.COCO_JSON_CATEGORY_ID]] = idx
+  label_map_path = os.path.join(output_dir, common_lib.LABEL_MAP_NAME)
+  logging.info('Writing label map to %s.', label_map_path)
+  common_lib.write_label_map(label_map_path, label_map)
+
+  with tf.io.gfile.GFile(
+      os.path.join(output_dir, 'label_id_by_category_id.yaml'), 'w'
+  ) as f:
+    yaml.dump(label_id_by_category_id, f)
+
+  ml_use_rows, image_rows, segmentation_rows = get_input_rows(
+      coco_dataset, split_ratio, label_id_by_category_id
+  )
+
+  def pipeline(root):
+    _define_data_conversion_pipeline(
+        root,
+        ml_use_rows,
+        image_rows,
+        segmentation_rows,
+        output_dir,
+        output_shape,
+        num_shard_list,
+    )
+
+  logging.info('Beginning beam pipeline to acquire tfrecords.')
+  common_lib.run_beam_pipeline(pipeline)

community-content/vertex_vision_model_garden/model_oss/data_converter/data_converter_main.py

@@ -0,0 +1,166 @@
+r"""Python script to convert user input data to training docker format.
+
+
+Note: the training format is designed to be tfrecord as in the design doc.
+If there are training efficiency issues for pytorch algorithms, we will also
+support pytorch formats as well.
+"""
+
+from absl import app
+from absl import flags
+from absl import logging
+from data_converter import common_lib
+from data_converter import data_converter_icn_lib
+from data_converter import data_converter_iod_lib
+from data_converter import data_converter_isg_lib
+from data_converter import data_converter_vcn_lib
+from util import constants
+
+
+_INPUT_FILE_PATH = flags.DEFINE_string(
+    'input_file_path',
+    None,
+    'Input file path.',
+    required=True,
+)
+_INPUT_FILE_TYPE = flags.DEFINE_enum(
+    'input_file_type',
+    None,
+    [
+        constants.INPUT_FILE_TYPE_CSV,
+        constants.INPUT_FILE_TYPE_JSONL,
+        constants.INPUT_FILE_TYPE_COCO_JSON,
+    ],
+    'Input file type.',
+    required=True,
+)
+_OBJECTIVE = flags.DEFINE_enum(
+    'objective',
+    None,
+    [
+        constants.OBJECTIVE_IMAGE_CLASSIFICATION,
+        constants.OBJECTIVE_IMAGE_OBJECT_DETECTION,
+        constants.OBJECTIVE_IMAGE_SEGMENTATION,
+        constants.OBJECTIVE_VIDEO_CLASSIFICATION,
+    ],
+    'The objective of this training job.',
+    required=True,
+)
+_OUTPUT_DIR = flags.DEFINE_string(
+    'output_dir',
+    None,
+    'The output directory for converted data and label map files.',
+    required=True,
+)
+_SPLIT_RATIO = flags.DEFINE_list(
+    'split_ratio',
+    '0.8,0.1,0.1',
+    'Proportion of data to split into train/validation/test.',
+)
+_NUM_SHARD = flags.DEFINE_list(
+    'num_shard', '10,10,10', 'The number of shards for train/validation/test.'
+)
+_OUTPUT_FPS = flags.DEFINE_integer(
+    'output_fps', 5, 'For videos only. The output frames rate per second.'
+)
+
+
+def main(_) -> None:
+  logging.info(
+      (
+          'Start data converter on: %s (type: %s) with split: %s for %s'
+          ' (shard=%s), and output to %s.'
+      ),
+      _INPUT_FILE_PATH.value,
+      _INPUT_FILE_TYPE.value,
+      _SPLIT_RATIO.value,
+      _OBJECTIVE.value,
+      _NUM_SHARD.value,
+      _OUTPUT_DIR.value,
+  )
+  split_ratio = list(map(float, _SPLIT_RATIO.value))
+  num_shard = list(map(int, _NUM_SHARD.value))
+  common_lib.check_split_ratio(split_ratio)
+  common_lib.check_num_shard(num_shard)
+  if (
+      _OBJECTIVE.value == constants.OBJECTIVE_IMAGE_OBJECT_DETECTION
+      and _INPUT_FILE_TYPE.value == constants.INPUT_FILE_TYPE_CSV
+  ):
+    data_converter_iod_lib.convert_csv_to_tfrecord(
+        _INPUT_FILE_PATH.value,
+        _OUTPUT_DIR.value,
+        split_ratio,
+        num_shard,
+    )
+  elif (
+      _OBJECTIVE.value == constants.OBJECTIVE_IMAGE_OBJECT_DETECTION
+      and _INPUT_FILE_TYPE.value == constants.INPUT_FILE_TYPE_JSONL
+  ):
+    data_converter_iod_lib.convert_jsonl_to_tfrecord(
+        _INPUT_FILE_PATH.value, _OUTPUT_DIR.value, split_ratio, num_shard
+    )
+  elif (
+      _OBJECTIVE.value == constants.OBJECTIVE_IMAGE_OBJECT_DETECTION
+      and _INPUT_FILE_TYPE.value == constants.INPUT_FILE_TYPE_COCO_JSON
+  ):
+    data_converter_iod_lib.convert_coco_json_to_tfrecord(
+        _INPUT_FILE_PATH.value,
+        _OUTPUT_DIR.value,
+        split_ratio,
+        num_shard,
+    )
+  elif _OBJECTIVE.value == constants.OBJECTIVE_IMAGE_SEGMENTATION:
+    data_converter_isg_lib.beam_build_tfrecord_from_coco_json(
+        _INPUT_FILE_PATH.value,
+        _OUTPUT_DIR.value,
+        split_ratio,
+        num_shard,
+    )
+  elif (
+      _OBJECTIVE.value == constants.OBJECTIVE_IMAGE_CLASSIFICATION
+      and _INPUT_FILE_TYPE.value == constants.INPUT_FILE_TYPE_CSV
+  ):
+    data_converter_icn_lib.convert_csv_to_tfrecord(
+        _INPUT_FILE_PATH.value,
+        _OUTPUT_DIR.value,
+        split_ratio,
+        num_shard,
+    )
+  elif (
+      _OBJECTIVE.value == constants.OBJECTIVE_IMAGE_CLASSIFICATION
+      and _INPUT_FILE_TYPE.value == constants.INPUT_FILE_TYPE_JSONL
+  ):
+    data_converter_icn_lib.convert_jsonl_to_tfrecord(
+        _INPUT_FILE_PATH.value, _OUTPUT_DIR.value, split_ratio, num_shard
+    )
+  elif (
+      _OBJECTIVE.value == constants.OBJECTIVE_VIDEO_CLASSIFICATION
+      and _INPUT_FILE_TYPE.value == constants.INPUT_FILE_TYPE_CSV
+  ):
+    data_converter_vcn_lib.convert_csv_to_tfrecord(
+        _INPUT_FILE_PATH.value,
+        _OUTPUT_DIR.value,
+        _OUTPUT_FPS.value,
+        split_ratio,
+        num_shard,
+    )
+  elif (
+      _OBJECTIVE.value == constants.OBJECTIVE_VIDEO_CLASSIFICATION
+      and _INPUT_FILE_TYPE.value == constants.INPUT_FILE_TYPE_JSONL
+  ):
+    data_converter_vcn_lib.convert_jsonl_to_tfrecord(
+        _INPUT_FILE_PATH.value,
+        _OUTPUT_DIR.value,
+        _OUTPUT_FPS.value,
+        split_ratio,
+        num_shard,
+    )
+  else:
+    raise NotImplementedError(
+        f'File format {_INPUT_FILE_TYPE.value} is not supported for'
+        f' {_OBJECTIVE.value}.'
+    )
+
+
+if __name__ == '__main__':
+  app.run(main)

community-content/vertex_vision_model_garden/model_oss/data_converter/data_converter_vcn_lib.py

@@ -0,0 +1,289 @@
+"""Converts VCN CSV/JSONL files to TFRecord with apache beam."""
+
+import json
+from os import path
+from typing import Any, Dict, Iterator, Sequence, Union, cast
+
+from absl import logging
+import apache_beam as beam
+from apache_beam.io import tfrecordio
+import numpy as np
+import pandas as pd
+import tensorflow as tf
+
+from data_converter import common_lib
+from util import constants
+
+
+_COLUMN_NAMES = [
+    common_lib.COLUMN_NAME_ML_USE,
+    common_lib.COLUMN_NAME_GCS_FILE_PATH,
+    common_lib.COLUMN_NAME_LABEL,
+    common_lib.COLUMN_NAME_START_SEC,
+    common_lib.COLUMN_NAME_END_SEC,
+]
+_JSON_GCS_URI_KEY = 'videoGcsUri'
+_JSON_CLASS_ANNOTATION_KEY = 'timeSegmentAnnotations'
+_JSON_CLASS_NAME_KEY = 'displayName'
+_JSON_START_TIME_KEY = 'startTime'
+_JSON_END_TIME_KEY = 'endTime'
+_JSON_RESOURCE_LABEL_KEY = 'dataItemResourceLabels'
+_JSON_ML_USE_KEY = 'aiplatform.googleapis.com/ml_use'
+
+
+def build_tf_example(
+    video_uri: str,
+    label: int,
+    start_sec: float,
+    end_sec: float,
+    output_fps: int,
+) -> tf.train.SequenceExample:
+  """Builds a TF Example from a video clip.
+
+  Args:
+    video_uri: GCS URI to the video file.
+    label: Class label as an integer.
+    start_sec: Start timestamp of the video clip in seconds.
+    end_sec: End timestamp of the video clip in seconds.
+    output_fps: The output frame rate per second.
+
+  Returns:
+    The created TF Example.
+  """
+  frame_bytes = common_lib.encode_video(
+      video_uri, start_sec, end_sec, output_fps, image_format='jpg'
+  )
+  seq_example = tf.train.SequenceExample()
+  seq_example.context.feature['clip/label/index'].int64_list.value[:] = [label]
+  for frame in frame_bytes:
+    seq_example.feature_lists.feature_list.get_or_create(
+        'image/encoded'
+    ).feature.add().bytes_list.value[:] = [frame]
+
+  return seq_example
+
+
+class AcquireTFExampleDoFn(beam.DoFn):
+  """Beam DoFn to build TF Examples from a DataFrame row dict for VCN."""
+
+  def __init__(self, output_fps: int):
+    self._success_counter = beam.metrics.Metrics.counter(
+        self.__class__.__name__, 'Success'
+    )
+    self._failure_counter = beam.metrics.Metrics.counter(
+        self.__class__.__name__, 'Failure'
+    )
+    self._output_fps = output_fps
+
+  def process(
+      self, element: Dict[str, Union[float, int, str]]
+  ) -> Iterator[tf.train.SequenceExample]:
+    ml_use: str = cast(str, element[common_lib.COLUMN_NAME_ML_USE])
+    video_uri: str = cast(str, element[common_lib.COLUMN_NAME_GCS_FILE_PATH])
+
+    try:
+      label: int = int(element[common_lib.COLUMN_NAME_LABEL])
+      start_sec: float = float(element[common_lib.COLUMN_NAME_START_SEC])
+      end_sec: float = float(element[common_lib.COLUMN_NAME_END_SEC])
+
+      tf_example = build_tf_example(
+          video_uri,
+          label,
+          start_sec,
+          end_sec,
+          self._output_fps,
+      )
+      self._success_counter.inc()
+      yield beam.pvalue.TaggedOutput(ml_use, tf_example)
+    except (ValueError, IOError) as err:
+      logging.error('Failed to process %s', video_uri)
+      logging.exception(err)
+      self._failure_counter.inc()
+
+
+def _run_convert_pipeline(
+    output_dir: str,
+    df: pd.DataFrame,
+    num_shards: Sequence[int],
+    output_fps: int,
+) -> None:
+  """Starts a Beam pipeline to write DataFrame as TF Records.
+
+  Args:
+    output_dir: TF Records output directory.
+    df: DataFrame to convert from.
+    num_shards: Number of shards for train/validation/test TFRecord files.
+    output_fps: The output frame rate per second.
+  """
+  clip_list = df.to_dict('records')
+
+  def pipeline(root):
+    train, val, test = (
+        root
+        | 'Create PCollection' >> beam.Create(clip_list)
+        | 'Convert to TF Example'
+        >> beam.ParDo(AcquireTFExampleDoFn(output_fps)).with_outputs(
+            constants.ML_USE_TRAINING,
+            constants.ML_USE_VALIDATION,
+            constants.ML_USE_TEST,
+        )
+    )
+    _ = train | 'Save train TF Record' >> tfrecordio.WriteToTFRecord(
+        path.join(output_dir, common_lib.TRAIN_TFRECORD_NAME),
+        coder=beam.coders.ProtoCoder(tf.train.Example),
+        num_shards=num_shards[0],
+    )
+    _ = val | 'Save val TF Record' >> tfrecordio.WriteToTFRecord(
+        path.join(output_dir, common_lib.VALIDATION_TFRECORD_NAME),
+        coder=beam.coders.ProtoCoder(tf.train.Example),
+        num_shards=num_shards[1],
+    )
+    _ = test | 'Save test TF Record' >> tfrecordio.WriteToTFRecord(
+        path.join(output_dir, common_lib.TEST_TFRECORD_NAME),
+        coder=beam.coders.ProtoCoder(tf.train.Example),
+        num_shards=num_shards[2],
+    )
+
+  common_lib.run_beam_pipeline(pipeline)
+
+
+def _convert_df_to_tfrecord(
+    df: pd.DataFrame,
+    output_dir: str,
+    split_ratio: Sequence[float],
+    num_shard: Sequence[int],
+    output_fps: int,
+) -> None:
+  """Converts a DataFrame into three separate tfrecords for training, validation, and testing into output_dir.
+
+  Args:
+    df: DataFrame to convert.
+    output_dir: The directory to save TFRecords and label_map.yaml.
+    split_ratio: List specifying the training, validation, and testing splits
+      for unassigned TFRecords.
+    num_shard: Number of shards for train/validation/test TFRecord files.
+    output_fps: The output frame rate per second.
+  """
+  # Replaces ml_use with common_lib string constants for consistency.
+  common_lib.format_ml_use_column(df)
+  common_lib.insert_missing_ml_use(df)
+
+  # Ignores invalid rows.
+  dropped_row_num = common_lib.drop_invalid_rows(df)
+  if dropped_row_num > 0:
+    logging.warning('Ignored %d invalid rows.', dropped_row_num)
+
+  common_lib.replace_unassigned_ml_use(
+      df[common_lib.COLUMN_NAME_ML_USE], split_ratio
+  )
+
+  # Converts labels to integers as required by training.
+  new_labels, label_map = common_lib.create_label_map(
+      df[common_lib.COLUMN_NAME_LABEL]
+  )
+  df[common_lib.COLUMN_NAME_LABEL] = new_labels
+  label_map_path = path.join(output_dir, common_lib.LABEL_MAP_NAME)
+  logging.info('Writing label map to %s.', label_map_path)
+  common_lib.write_label_map(label_map_path, label_map)
+
+  # Missing start / end times are treated as 0, inf, respectively.
+  df[common_lib.COLUMN_NAME_START_SEC].fillna(0, inplace=True)
+  df[common_lib.COLUMN_NAME_END_SEC].fillna(np.inf, inplace=True)
+
+  _run_convert_pipeline(output_dir, df, num_shard, output_fps)
+
+
+def convert_csv_to_tfrecord(
+    input_csv: str,
+    output_dir: str,
+    output_fps: int,
+    split_ratio: Sequence[float] = (0.8, 0.1, 0.1),
+    num_shard: Sequence[int] = (10, 10, 10),
+) -> None:
+  """Parses input_csv file into three separate tfrecords for training, validation, and testing into output_dir.
+
+  The csv format is shown in
+    https://cloud.google.com/vertex-ai/docs/video-data/classification/prepare-data#csv
+
+  If an ml_use column is not provided, one will be created.
+
+  label_map.yaml containing the label map will be placed in output_dir.
+
+  Args:
+    input_csv: Name of the csv file.
+    output_dir: The directory to save TFRecords and label_map.yaml.
+    output_fps: The output frame rate per second.
+    split_ratio: List specifying the training, validation, and testing splits
+      for unassigned TFRecords.
+    num_shard: Number of shards for train/validation/test TFRecord files.
+  """
+  with tf.io.gfile.GFile(input_csv, 'r') as f:
+    df: pd.DataFrame = pd.read_csv(
+        f, header=None, names=_COLUMN_NAMES, on_bad_lines='warn'
+    )
+
+  _convert_df_to_tfrecord(df, output_dir, split_ratio, num_shard, output_fps)
+
+
+def convert_jsonl_to_tfrecord(
+    input_jsonl: str,
+    output_dir: str,
+    output_fps: int,
+    split_ratio: Sequence[float] = (0.8, 0.1, 0.1),
+    num_shard: Sequence[int] = (10, 10, 10),
+) -> None:
+  """Parses input_jsonl file into three separate tfrecords for training, validation, and testing into output_dir.
+
+  The JSONL format is shown in
+    https://cloud.google.com/vertex-ai/docs/video-data/classification/prepare-data#jsonl.
+
+  If an ml_use column is not provided, one will be created.
+
+  label_map.yaml containing the label map will be placed in output_dir.
+
+  Args:
+    input_jsonl: Name of the JSONL file.
+    output_dir: The directory to save TFRecords and label_map.yaml.
+    output_fps: The output frame rate per second.
+    split_ratio: List specifying the training, validation, and testing splits
+      for unassigned TFRecords.
+    num_shard: Number of shards for train/validation/test TFRecord files.
+  """
+  df_rows = []
+  with tf.io.gfile.GFile(input_jsonl, 'r') as f:
+    lines = f.read().rstrip().splitlines()
+
+  for i, line in enumerate(lines, 1):
+    try:
+      item: Dict[str, Any] = json.loads(line)
+
+      gcs_uri = item.get(_JSON_GCS_URI_KEY)
+      if not gcs_uri:
+        logging.warning('Invalid JSON at line %d, skipped.', i)
+        continue
+
+      annotations = item.get(_JSON_CLASS_ANNOTATION_KEY, [])
+      ml_use = item.get(_JSON_RESOURCE_LABEL_KEY, {}).get(
+          _JSON_ML_USE_KEY, common_lib.ML_USE_UNASSIGNED
+      )
+
+      for j, annotation in enumerate(annotations):
+        label = annotation.get(_JSON_CLASS_NAME_KEY)
+        if not label:
+          logging.warning('Invalid annotation #%d at line %d, skipped.', j, i)
+          continue
+        # The example in external documentation uses strings like "1.0s", so we
+        # need to remove the "s" suffix.
+        start_time = annotation.get(_JSON_START_TIME_KEY, '0').removesuffix('s')
+        end_time = annotation.get(_JSON_END_TIME_KEY, 'inf').removesuffix('s')
+        df_rows.append([ml_use, gcs_uri, label, start_time, end_time])
+    except (json.JSONDecodeError, AttributeError):
+      logging.warning('Invalid JSON at line %d, skipped.', i)
+      continue
+
+  df = pd.DataFrame(
+      data=df_rows,
+      columns=_COLUMN_NAMES,
+  )
+
+  _convert_df_to_tfrecord(df, output_dir, split_ratio, num_shard, output_fps)

community-content/vertex_vision_model_garden/model_oss/data_converter/dockerfile/base.Dockerfile

@@ -0,0 +1,50 @@
+FROM python:3.9
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Install basic libs.
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        cmake \
+        curl \
+        wget \
+        sudo \
+        gnupg \
+        python3-opencv \
+        lsb-release \
+        ca-certificates \
+        build-essential \
+        git \
+        vim \
+        screen \
+        libportaudio2 \
+        libusb-1.0-0-dev \
+        openjdk-17-jre
+
+# Add gcsfuse distribution URL as a package source and import its public key.
+RUN echo "deb https://packages.cloud.google.com/apt gcsfuse-`lsb_release -c -s` main" | sudo tee /etc/apt/sources.list.d/gcsfuse.list
+RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
+
+# Install gcsfuse.
+RUN apt-get update && apt-get install -y --no-install-recommends gcsfuse
+
+# Install google cloud SDK.
+RUN wget -q https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-359.0.0-linux-x86_64.tar.gz
+RUN tar xzf google-cloud-sdk-359.0.0-linux-x86_64.tar.gz
+RUN ./google-cloud-sdk/install.sh -q
+# Make sure gsutil will use the default service account.
+RUN echo '[GoogleCompute]\nservice_account = default' > /etc/boto.cfg
+
+
+# Install required libs.
+RUN pip install --upgrade pip
+RUN pip install pyyaml==5.4.1
+RUN pip install pycocotools==2.0.6
+RUN pip install opencv-python-headless==4.7.0.72
+RUN pip install numpy==1.24.2
+RUN pip install pandas==1.5.3
+RUN pip install Pillow==9.4.0
+RUN pip install apache-beam[gcp]==2.45.0
+RUN pip install object-detection==0.0.3
+RUN pip install google-cloud-storage==1.42.3
+RUN pip install gcsfs==2021.10.1
+RUN pip install pylint==2.17.2

community-content/vertex_vision_model_garden/model_oss/data_converter/dockerfile/run.Dockerfile

@@ -0,0 +1,23 @@
+FROM gcr.io/automl-migration-test/automl-vision-data-converter-base:latest
+
+# Copy license.
+RUN wget https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/LICENSE
+
+COPY model_oss/data_converter /automl_vision/data_converter
+COPY model_oss/util /automl_vision/util
+
+WORKDIR /automl_vision
+ENV PYTHONPATH "${PYTHONPATH}:/automl_vision"
+
+# Run pylint to validate code.
+COPY .pylintrc /automl_vision/.pylintrc
+RUN find . -type f -name "*.py" | xargs pylint --rcfile=./.pylintrc --errors-only
+
+ENTRYPOINT ["python3","data_converter/data_converter_main.py"]
+
+CMD ["--input_file_path=YOUR_INPUT_FILE",\
+"--input_file_type=csv",\
+"--objective=iod",\
+"--output_dir=YOUR_OUTPUT_DIR",\
+"--num_shard=10,10,10",\
+"--split_ratio=0.8,0.1,0.1"]

community-content/vertex_vision_model_garden/model_oss/diffusers/dockerfile/serve.Dockerfile

@@ -0,0 +1,74 @@
+# Dockerfile for Diffuser Serving.
+#
+# To build:
+# docker build -f model_oss/diffusers/dockerfile/serve.Dockerfile . -t ${YOUR_IMAGE_TAG}
+#
+# To push to gcr:
+# docker tag ${YOUR_IMAGE_TAG} gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
+# docker push gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
+
+FROM pytorch/torchserve:0.7.0-gpu
+
+USER root
+
+ENV infer_port=7080
+ENV mng_port=7081
+ENV model_name="diffusers_serving"
+ENV PATH="/home/model-server/:${PATH}"
+
+# Install libraries.
+ENV PIP_ROOT_USER_ACTION=ignore
+RUN python3 -m pip install --upgrade pip
+RUN pip install torch==1.13.1
+RUN pip install torchvision==0.14.1
+RUN pip install transformers==4.27.4
+RUN pip install datasets==2.9.0
+RUN pip install accelerate==0.17.0
+RUN pip install triton==2.0.0.dev20221120
+RUN pip install xformers==0.0.16
+RUN pip install google-cloud-storage==2.7.0
+RUN pip install imageio[ffmpeg]==2.31.0
+RUN pip install absl-py==1.4.0
+
+# Copy LICENSE file
+RUN apt-get update && apt-get install wget
+RUN wget https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/LICENSE
+
+# Install diffusers from main branch source code with a pinned commit.
+RUN git clone --depth 1 --branch v0.18.1 https://github.com/huggingface/diffusers.git
+WORKDIR diffusers
+RUN pip install -e .
+
+# Copy model artifacts.
+COPY model_oss/diffusers/handler.py /home/model-server/handler.py
+COPY model_oss/util/ /home/model-server/util/
+ENV PYTHONPATH /home/model-server/
+
+# Create torchserve configuration file.
+RUN echo \
+    "default_response_timeout=1800\n" \
+    "service_envelope=json\n" \
+    "inference_address=http://0.0.0.0:${infer_port}\n"  \
+    "management_address=http://0.0.0.0:${mng_port}" >> /home/model-server/config.properties
+
+# Expose ports.
+EXPOSE ${infer_port}
+EXPOSE ${mng_port}
+
+# Archive model artifacts and dependencies.
+# Do not set --model-file and --serialized-file because model and checkpoint
+# will be dynamically loaded in handler.py.
+RUN torch-model-archiver \
+--model-name=${model_name} \
+--version=1.0 \
+--handler=/home/model-server/handler.py \
+--runtime=python3 \
+--export-path=/home/model-server/model-store \
+--archive-format=default \
+--force
+
+# Run Torchserve HTTP serve to respond to prediction requests.
+CMD ["torchserve", "--start", \
+     "--ts-config", "/home/model-server/config.properties", \
+     "--models", "${model_name}=${model_name}.mar", \
+     "--model-store", "/home/model-server/model-store"]

community-content/vertex_vision_model_garden/model_oss/diffusers/dockerfile/train.Dockerfile

@@ -0,0 +1,47 @@
+# Dockerfile for Diffuser Training.
+#
+# To build:
+# docker build -f model_oss/diffusers/dockerfile/train.Dockerfile . -t ${YOUR_IMAGE_TAG}
+#
+# To push to gcr:
+# docker tag ${YOUR_IMAGE_TAG} gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
+# docker push gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
+
+# Base on pytorch-cuda image.
+FROM pytorch/pytorch:1.13.0-cuda11.6-cudnn8-runtime
+
+# Install tools.
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        curl \
+        wget \
+        git \
+        vim
+
+# Copy license.
+RUN wget https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/LICENSE
+
+# Install libraries.
+RUN pip install torchvision==0.14.1
+RUN pip install transformers==4.26.1
+RUN pip install datasets==2.9.0
+RUN pip install accelerate==0.17.0
+RUN pip install triton==2.0.0.dev20221120
+RUN pip install xformers==0.0.16
+RUN pip install Jinja2==3.1.2
+RUN pip install ftfy==6.1.1
+RUN pip install cloudml-hypertune==0.1.0.dev6
+RUN pip install tensorboard==2.12.0
+
+# Install diffusers from main branch source code with a pinned commit.
+RUN git clone --depth 1 --branch v0.18.1 https://github.com/huggingface/diffusers.git
+WORKDIR diffusers
+RUN pip install -e .
+
+# Switch to diffusers examples folder.
+WORKDIR examples
+
+# Config accelerate.
+COPY model_oss/diffusers/train.sh train.sh
+
+# Generate accelerate config at the beginning of docker run.
+ENTRYPOINT ["/bin/bash", "train.sh"]

community-content/vertex_vision_model_garden/model_oss/diffusers/handler.py

@@ -0,0 +1,256 @@
+"""Custom handler for huggingface/diffusers models."""
+
+# pylint: disable=g-importing-member
+# pylint: disable=logging-fstring-interpolation
+
+import base64
+import io
+import logging
+import os
+from typing import Any, List, Sequence, Tuple
+
+from diffusers import ControlNetModel
+from diffusers import DiffusionPipeline
+from diffusers import DPMSolverMultistepScheduler
+from diffusers import EulerAncestralDiscreteScheduler
+from diffusers import StableDiffusionControlNetPipeline
+from diffusers import StableDiffusionImg2ImgPipeline
+from diffusers import StableDiffusionInpaintPipeline
+from diffusers import StableDiffusionInstructPix2PixPipeline
+from diffusers import StableDiffusionPipeline
+from diffusers import StableDiffusionUpscalePipeline
+from diffusers import TextToVideoZeroPipeline
+from diffusers import UniPCMultistepScheduler
+import imageio
+import numpy as np
+from PIL import Image
+import torch
+from ts.torch_handler.base_handler import BaseHandler
+
+from util import constants
+from util import fileutils
+from util import image_format_converter
+from video_util import video_format_converter
+
+STABLE_DIFFUSION_MODEL = "runwayml/stable-diffusion-v1-5"
+
+# Tasks
+TEXT_TO_IMAGE = "text-to-image"
+IMAGE_TO_IMAGE = "image-to-image"
+IMAGE_INPAINTING = "image-inpainting"
+INSTRUCT_PIX2PIX = "instruct-pix2pix"
+CONTROLNET = "controlnet"
+CONDITIONED_SUPER_RES = "conditioned-super-res"
+TEXT_TO_VIDEO_ZERO_SHOT = "text-to-video-zero-shot"
+TEXT_TO_VIDEO = "text-to-video"
+
+
+def frames_to_video_bytes(frames: Sequence[np.ndarray], fps: int) -> bytes:
+  images = [Image.fromarray(array) for array in frames]
+  io_obj = io.BytesIO()
+  imageio.mimsave(io_obj, images, format=".mp4", fps=fps)
+  return io_obj.getvalue()
+
+
+class DiffusersHandler(BaseHandler):
+  """Custom handler for TIMM models."""
+
+  def initialize(self, context: Any):
+    """Custom initialize."""
+
+    properties = context.system_properties
+    self.map_location = (
+        "cuda"
+        if torch.cuda.is_available() and properties.get("gpu_id") is not None
+        else "cpu"
+    )
+    self.device = torch.device(
+        self.map_location + ":" + str(properties.get("gpu_id"))
+        if torch.cuda.is_available() and properties.get("gpu_id") is not None
+        else self.map_location
+    )
+    self.manifest = context.manifest
+
+    self.model_id = os.environ["MODEL_ID"]
+    if self.model_id.startswith(constants.GCS_URI_PREFIX):
+      gcs_path = self.model_id[len(constants.GCS_URI_PREFIX) :]
+      local_model_dir = os.path.join(constants.LOCAL_MODEL_DIR, gcs_path)
+      logging.info(f"Download {self.model_id} to {local_model_dir}")
+      fileutils.download_gcs_dir_to_local(self.model_id, local_model_dir)
+      self.model_id = local_model_dir
+
+    self.task = os.environ.get("TASK", TEXT_TO_IMAGE)
+    logging.info(f"Using task:{self.task}, model:{self.model_id}")
+
+    if self.task == TEXT_TO_IMAGE:
+      pipeline = StableDiffusionPipeline.from_pretrained(
+          self.model_id, torch_dtype=torch.float16
+      )
+      pipeline.scheduler = EulerAncestralDiscreteScheduler.from_config(
+          pipeline.scheduler.config
+      )
+      pipeline = pipeline.to(self.map_location)
+      # Reduce memory footprint.
+      pipeline.enable_attention_slicing()
+    elif self.task == IMAGE_TO_IMAGE:
+      pipeline = StableDiffusionImg2ImgPipeline.from_pretrained(
+          self.model_id, torch_dtype=torch.float16
+      )
+      pipeline.scheduler = EulerAncestralDiscreteScheduler.from_config(
+          pipeline.scheduler.config
+      )
+      pipeline = pipeline.to(self.map_location)
+      # Reduce memory footprint.
+      pipeline.enable_attention_slicing()
+    elif self.task == IMAGE_INPAINTING:
+      pipeline = StableDiffusionInpaintPipeline.from_pretrained(
+          self.model_id, torch_dtype=torch.float16
+      )
+      pipeline.scheduler = EulerAncestralDiscreteScheduler.from_config(
+          pipeline.scheduler.config
+      )
+      pipeline = pipeline.to(self.map_location)
+      # Reduce memory footprint.
+      pipeline.enable_attention_slicing()
+    elif self.task == INSTRUCT_PIX2PIX:
+      pipeline = StableDiffusionInstructPix2PixPipeline.from_pretrained(
+          self.model_id, torch_dtype=torch.float16
+      )
+      pipeline.scheduler = EulerAncestralDiscreteScheduler.from_config(
+          pipeline.scheduler.config
+      )
+      pipeline = pipeline.to(self.map_location)
+      # Reduce memory footprint.
+      pipeline.enable_attention_slicing()
+    elif self.task == CONTROLNET:
+      controlnet = ControlNetModel.from_pretrained(
+          self.model_id, torch_dtype=torch.float16
+      )
+      pipeline = StableDiffusionControlNetPipeline.from_pretrained(
+          STABLE_DIFFUSION_MODEL,
+          controlnet=controlnet,
+          torch_dtype=torch.float16,
+      )
+      pipeline.scheduler = UniPCMultistepScheduler.from_config(
+          pipeline.scheduler.config
+      )
+      pipeline.enable_xformers_memory_efficient_attention()
+      pipeline.enable_model_cpu_offload()
+      pipeline = pipeline.to(self.map_location)
+      # Reduce memory footprint.
+      pipeline.enable_attention_slicing()
+    elif self.task == CONDITIONED_SUPER_RES:
+      pipeline = StableDiffusionUpscalePipeline.from_pretrained(
+          self.model_id, torch_dtype=torch.float16
+      )
+      pipeline.scheduler = EulerAncestralDiscreteScheduler.from_config(
+          pipeline.scheduler.config
+      )
+      # This is necessary to 4x upscale >=256x256 input images with V100.
+      logging.info("Enable xformers memory efficient attention for inference.")
+      pipeline.enable_xformers_memory_efficient_attention()
+      pipeline = pipeline.to(self.map_location)
+      # Reduce memory footprint.
+      pipeline.enable_attention_slicing()
+    elif self.task == TEXT_TO_VIDEO_ZERO_SHOT:
+      pipeline = TextToVideoZeroPipeline.from_pretrained(
+          STABLE_DIFFUSION_MODEL, torch_dtype=torch.float16
+      )
+      # Memory optimization.
+      pipeline.enable_xformers_memory_efficient_attention()
+      pipeline.enable_model_cpu_offload()
+      pipeline = pipeline.to(self.map_location)
+    elif self.task == TEXT_TO_VIDEO:
+      pipeline = DiffusionPipeline.from_pretrained(
+          self.model_id, torch_dtype=torch.float16, variant="fp16"
+      )
+      pipeline.enable_model_cpu_offload()
+      # Memory optimization.
+      pipeline.enable_vae_slicing()
+      pipeline.scheduler = DPMSolverMultistepScheduler.from_config(
+          pipeline.scheduler.config
+      )
+    else:
+      raise ValueError(f"Invalid TASK: {self.task}")
+
+    self.pipeline = pipeline
+    self.initialized = True
+    logging.info("Handler initialization done.")
+
+  def preprocess(self, data: Any) -> Tuple[Any, Any, Any]:
+    """Preprocess input data."""
+    prompts = [item["prompt"] for item in data]
+    images = None
+    mask_images = None
+
+    if "image" in data[0]:
+      images = [
+          image_format_converter.base64_to_image(item["image"]) for item in data
+      ]
+    if "mask_image" in data[0]:
+      mask_images = [
+          image_format_converter.base64_to_image(item["mask_image"])
+          for item in data
+      ]
+    return prompts, images, mask_images
+
+  def inference(self, data: Any, *args, **kwargs) -> List[Image.Image]:
+    """Run the inference."""
+    prompts, images, mask_images = data
+    if self.task == TEXT_TO_IMAGE:
+      predicted_images = self.pipeline(prompt=prompts).images
+    elif self.task == IMAGE_TO_IMAGE:
+      predicted_images = self.pipeline(prompt=prompts, image=images).images
+    elif self.task == IMAGE_INPAINTING:
+      predicted_images = self.pipeline(
+          prompt=prompts, image=images, mask_image=mask_images
+      ).images
+    elif self.task == INSTRUCT_PIX2PIX:
+      predicted_images = self.pipeline(prompt=prompts, image=images).images
+    elif self.task == CONTROLNET:
+      predicted_images = self.pipeline(
+          prompt=prompts, image=images, num_inference_steps=20
+      ).images
+    elif self.task == CONDITIONED_SUPER_RES:
+      predicted_images = self.pipeline(
+          prompt=prompts, image=images, num_inference_steps=20
+      ).images
+    elif self.task == TEXT_TO_VIDEO_ZERO_SHOT:
+      # For each given prompt, generate a short video.
+      # The pipeline doesn't support multiple prompts in one run yet.
+      videos = []
+      for prompt in prompts:
+        numpy_arrays = self.pipeline(prompt=prompt).images
+        numpy_arrays = [(i * 255).astype("uint8") for i in numpy_arrays]
+        videos.append(
+            frames_to_video_bytes(numpy_arrays, fps=4)
+        )
+      return videos
+    elif self.task == TEXT_TO_VIDEO:
+      predicted_images = np.asarray(self.pipeline(prompt=prompts).frames)
+      # For multiple prompts, the model concatenates video frames, i.e. the
+      # output shape is (num_frames, height, width * len(prompts), channels).
+      # Therefore we need to split the output into different videos.
+      predicted_images = np.array_split(predicted_images, len(prompts), axis=2)
+      videos = [
+          frames_to_video_bytes(images, fps=8)
+          for images in predicted_images
+      ]
+      return videos
+    else:
+      raise ValueError(f"Invalid TASK: {self.task}")
+    return predicted_images
+
+  def postprocess(self, data: Any) -> List[str]:
+    """Convert the images to base64 string."""
+    outputs = []
+    for prediction in data:
+      if isinstance(prediction, bytes):
+        # This is the video bytes.
+        outputs.append(base64.b64encode(prediction).decode("utf-8"))
+      else:
+        outputs.append(image_format_converter.image_to_base64(prediction))
+    return outputs
+
+
+# pylint: enable=logging-fstring-interpolation

community-content/vertex_vision_model_garden/model_oss/diffusers/train.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+# Setup accelerate config before running trainer.
+python -c "from accelerate.utils import write_basic_config; write_basic_config(mixed_precision='fp16')"
+
+accelerate launch "$@"

community-content/vertex_vision_model_garden/model_oss/keras/dockerfile/serve.Dockerfile

@@ -0,0 +1,118 @@
+# Dockerfile for basic serving dockers with Keras.
+#
+# To build:
+# docker build -f model_oss/keras/dockerfile/serve.Dockerfile . -t ${YOUR_IMAGE_TAG}
+#
+# To push to gcr:
+# docker tag ${YOUR_IMAGE_TAG} gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
+# docker push gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
+
+FROM tensorflow/tensorflow:2.12.0-gpu
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# This is added to fix docker build error related to Nvidia key update.
+RUN rm -f /etc/apt/sources.list.d/cuda.list
+RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
+
+# Install basic libs.
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        cmake \
+        curl \
+        wget \
+        sudo \
+        gnupg \
+        libsm6 \
+        libxext6 \
+        libxrender-dev \
+        lsb-release \
+        ca-certificates \
+        build-essential \
+        git \
+        vim \
+        screen \
+        libtcmalloc-minimal4
+
+
+# Install google cloud SDK.
+RUN wget -q https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-359.0.0-linux-x86_64.tar.gz
+RUN tar xzf google-cloud-sdk-359.0.0-linux-x86_64.tar.gz
+RUN ./google-cloud-sdk/install.sh -q
+# Make sure gsutil will use the default service account.
+RUN echo '[GoogleCompute]\nservice_account = default' > /etc/boto.cfg
+
+
+# Install required libs.
+RUN pip install --upgrade pip
+RUN pip install cloud-tpu-client==0.10
+RUN pip install pyyaml==5.4.1
+RUN pip install fsspec==2021.10.1
+RUN pip install gcsfs==2021.10.1
+RUN pip install tensorflow-text==2.11.0
+RUN pip install pyglove==0.1.0
+RUN pip install cloudml-hypertune==0.1.0.dev6
+RUN pip install pylint==2.17.2
+RUN pip install keras-cv==0.4.0
+RUN pip install tensorflow-datasets==4.8.3
+RUN pip install protobuf==3.20.3
+RUN pip install Pillow==9.5.0
+RUN pip install flask==2.3.2
+RUN pip install waitress==2.1.2
+
+# Installs Reduction Server NCCL plugin.
+RUN echo "deb https://packages.cloud.google.com/apt google-fast-socket main" | tee /etc/apt/sources.list.d/google-fast-socket.list \
+&&  curl -s -L https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - \
+&&  apt update && apt install -y google-reduction-server
+
+# Downloading gcloud package
+RUN curl https://dl.google.com/dl/cloudsdk/release/google-cloud-sdk.tar.gz > /tmp/google-cloud-sdk.tar.gz
+
+# Installing the package
+RUN mkdir -p /usr/local/gcloud \
+  && tar -C /usr/local/gcloud -xvf /tmp/google-cloud-sdk.tar.gz \
+  && /usr/local/gcloud/google-cloud-sdk/install.sh
+
+RUN wget https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/LICENSE
+
+# Adding the package path to local
+ENV PATH $PATH:/usr/local/gcloud/google-cloud-sdk/bin
+
+
+ENV PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp
+
+# Lower the memory fragmentation, and speed up the training.
+# https://github.com/tensorflow/tensorflow/issues/44176#issuecomment-783768033
+ENV LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libtcmalloc_minimal.so.4
+
+# Enable userspace DNS cache
+ENV GCS_RESOLVE_REFRESH_SECS=60
+ENV GCS_REQUEST_CONNECTION_TIMEOUT_SECS=300
+ENV GCS_METADATA_REQUEST_TIMEOUT_SECS=300
+ENV GCS_READ_REQUEST_TIMEOUT_SECS=300
+ENV GCS_WRITE_REQUEST_TIMEOUT_SECS=600
+# Each opened GCS file takes GCS_READ_CACHE_BLOCK_SIZE_MB of RAM, reduce the
+# value from the default 64MB to 8MB to decrease memory footprint.
+ENV GCS_READ_CACHE_BLOCK_SIZE_MB=8
+
+EXPOSE 8501
+
+WORKDIR /usr/local/lib/python3.8/dist-packages/official/vision
+
+COPY model_oss/keras /automl_vision/keras
+COPY model_oss/util /automl_vision/util
+
+WORKDIR /automl_vision
+
+ENV PYTHONPATH "${PYTHONPATH}:/automl_vision/util"
+ENV MODEL_PATH ""
+ENV IMAGE_WIDTH "512"
+ENV IMAGE_HEIGHT "512"
+
+COPY model_oss/keras/serve.py ./app.py
+
+# Run pylint to validate code.
+COPY .pylintrc /automl_vision/.pylintrc
+RUN find . -type f -name "*.py" | xargs pylint --rcfile=./.pylintrc --errors-only
+
+ENTRYPOINT ["flask","run"]
+CMD ["--host=0.0.0.0", "--port=8501"]

community-content/vertex_vision_model_garden/model_oss/keras/dockerfile/train.Dockerfile

@@ -0,0 +1,111 @@
+# Dockerfile for basic training dockers with Keras.
+#
+# To build:
+# docker build -f model_oss/keras/dockerfile/train.Dockerfile . -t ${YOUR_IMAGE_TAG}
+#
+# To push to gcr:
+# docker tag ${YOUR_IMAGE_TAG} gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
+# docker push gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
+
+FROM tensorflow/tensorflow:2.12.0-gpu
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# This is added to fix docker build error related to Nvidia key update.
+RUN rm -f /etc/apt/sources.list.d/cuda.list
+RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
+
+# Install basic libs.
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        cmake \
+        curl \
+        wget \
+        sudo \
+        gnupg \
+        libsm6 \
+        libxext6 \
+        libxrender-dev \
+        lsb-release \
+        ca-certificates \
+        build-essential \
+        git \
+        vim \
+        screen \
+        libtcmalloc-minimal4
+
+
+# Install google cloud SDK.
+RUN wget -q https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-359.0.0-linux-x86_64.tar.gz
+RUN tar xzf google-cloud-sdk-359.0.0-linux-x86_64.tar.gz
+RUN ./google-cloud-sdk/install.sh -q
+# Make sure gsutil will use the default service account.
+RUN echo '[GoogleCompute]\nservice_account = default' > /etc/boto.cfg
+
+
+# Install required libs.
+RUN pip install --upgrade pip
+RUN pip install cloud-tpu-client==0.10
+RUN pip install pyyaml==5.4.1
+RUN pip install fsspec==2021.10.1
+RUN pip install gcsfs==2021.10.1
+RUN pip install tensorflow-text==2.11.0
+RUN pip install pyglove==0.1.0
+RUN pip install cloudml-hypertune==0.1.0.dev6
+RUN pip install pylint==2.17.2
+RUN pip install keras-cv==0.4.0
+RUN pip install tensorflow-datasets==4.8.3
+RUN pip install tensorflow-estimator==2.12.0
+RUN pip install tensorflow-gcs-config==2.12.0
+RUN pip install tensorflow-hub==0.13.0
+RUN pip install tensorflow-io-gcs-filesystem==0.32.0
+RUN pip install tensorflow-metadata==1.13.1
+RUN pip install tensorflow-probability==0.19.0
+RUN pip install tensorboard==2.12.2
+RUN pip install tensorboard-data-server==0.7.0
+RUN pip install tensorboard-plugin-wit==1.8.1
+RUN pip install protobuf==3.20.3
+RUN pip install pandas==1.5.3
+RUN pip install pandas-datareader==0.10.0
+RUN pip install pandas-gbq==0.17.9
+RUN pip install pycocotools==2.0.6
+
+# Installs Reduction Server NCCL plugin.
+RUN echo "deb https://packages.cloud.google.com/apt google-fast-socket main" | tee /etc/apt/sources.list.d/google-fast-socket.list \
+&&  curl -s -L https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - \
+&&  apt update && apt install -y google-reduction-server
+
+RUN wget https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/LICENSE
+
+ENV PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp
+
+# Lower the memory fragmentation, and speed up the training.
+# https://github.com/tensorflow/tensorflow/issues/44176#issuecomment-783768033
+ENV LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libtcmalloc_minimal.so.4
+
+# Enable userspace DNS cache
+ENV GCS_RESOLVE_REFRESH_SECS=60
+ENV GCS_REQUEST_CONNECTION_TIMEOUT_SECS=300
+ENV GCS_METADATA_REQUEST_TIMEOUT_SECS=300
+ENV GCS_READ_REQUEST_TIMEOUT_SECS=300
+ENV GCS_WRITE_REQUEST_TIMEOUT_SECS=600
+# Each opened GCS file takes GCS_READ_CACHE_BLOCK_SIZE_MB of RAM, reduce the
+# value from the default 64MB to 8MB to decrease memory footprint.
+ENV GCS_READ_CACHE_BLOCK_SIZE_MB=8
+
+WORKDIR /usr/local/lib/python3.8/dist-packages/official/vision
+
+COPY model_oss/keras /automl_vision/keras
+COPY model_oss/util /automl_vision/util
+
+WORKDIR /automl_vision
+
+# Keras stable diffusion training codes set width and height as RESOLUTION.
+ENV RESOLUTION "512"
+
+ENV PYTHONPATH "${PYTHONPATH}:/automl_vision/util"
+
+# Run pylint to validate code.
+COPY .pylintrc /automl_vision/.pylintrc
+RUN find . -type f -name "*.py" | xargs pylint --rcfile=./.pylintrc --errors-only
+
+ENTRYPOINT ["python3","keras/train.py"]

community-content/vertex_vision_model_garden/model_oss/keras/serve.py

@@ -0,0 +1,184 @@
+r"""Servers Keras Stable Diffusion models.
+
+python serve.py --model_path=<model path in gcs>
+
+curl -d \
+'{"prompt":"Hello Kitty"}' \
+-H "Content-Type: application/json" \
+-X POST http://localhost:8501/predict
+"""
+
+import base64
+import io
+import json
+import os
+from typing import List, Tuple
+
+from absl import app
+# The docker builds could not find flask and waitress.
+# pylint: disable=import-error
+from flask import Flask
+from flask import request
+from flask import Response
+import keras_cv
+from PIL import Image
+from waitress import serve
+
+from util import constants
+from util import fileutils
+
+
+flask_app = Flask(__name__)
+
+stable_diffusion_model = None
+
+
+model_path = os.environ.get('MODEL_PATH', '')
+if model_path.startswith(constants.GCS_URI_PREFIX):
+  print('Downloading models from gcs to local.')
+  os.makedirs(constants.LOCAL_MODEL_DIR, exist_ok=True)
+  fileutils.download_gcs_dir_to_local(
+      os.path.dirname(model_path), constants.LOCAL_MODEL_DIR
+  )
+  model_path = os.path.join(
+      constants.LOCAL_MODEL_DIR, os.path.basename(model_path)
+  )
+
+image_width = int(os.environ.get('IMAGE_WIDTH', 512))
+image_height = int(os.environ.get('IMAGE_HEIGHT', 512))
+
+print('image_width=', image_width, 'image_height=', image_height)
+print('Create Keras stable diffusion models.')
+stable_diffusion_model = keras_cv.models.StableDiffusion(
+    img_width=image_width,
+    img_height=image_height,
+    jit_compile=True,
+)
+
+if model_path:
+  # We just reload the weights of the fine-tuned diffusion model.
+  print('Initialize finetuned models from: ', model_path)
+  stable_diffusion_model.diffusion_model.load_weights(model_path)
+
+
+def error(message: str) -> str:
+  """Returns a JSON representing an error response."""
+  return json.dumps({
+      'success': False,
+      'error': message,
+  })
+
+
+def check_key_in_json(content: str, keys: List[str]) -> str:
+  for key in keys:
+    if key not in content:
+      return error('No {} in request {}.'.format(key, content))
+  return None
+
+
+def validate_json_key(json_key_string: str) -> Tuple[str, bool]:
+  try:
+    json_key = json.loads(json_key_string)
+  except (ValueError, TypeError):
+    return (error('Invalid key found in request'), False)
+  return (json_key, True)
+
+
+# The health check route is required for docker deployment in google cloud.
+@flask_app.route('/ping')
+def ping() -> Response:
+  """Health checks."""
+  return Response(status=200)
+
+
+# The return should be `Response` for docker deployment in google cloud.
+@flask_app.route('/predict', methods=['GET', 'POST'])
+def predict_model() -> Response:
+  """Predictions."""
+  if request.method == 'POST':
+    contents = request.get_json(force=True)
+
+    print('The input contents are:', contents)
+    batch_size = 1
+    num_steps = 25
+    seed = 1234
+    if 'parameters' in contents:
+      parameters = contents['parameters']
+      if 'batch_size' in parameters:
+        batch_size = int(parameters['batch_size'])
+      if 'num_steps' in parameters:
+        num_steps = int(parameters['num_steps'])
+      if 'seed' in parameters:
+        seed = int(parameters['seed'])
+    print('batch_size=', batch_size, 'num_steps=', num_steps, 'seed=', seed)
+    if batch_size < 1:
+      return Response(
+          response=error('The batch size must be a positive integar.'),
+          status=200,
+          mimetype='text/plain',
+      )
+    if num_steps < 1:
+      return Response(
+          response=error('The num steps must be a positive integar.'),
+          status=200,
+          mimetype='text/plain',
+      )
+    predictions = []
+    for content in contents['instances']:
+      print('Processing:', content)
+      prompt = content['prompt']
+      generated_image_array = stable_diffusion_model.text_to_image(
+          prompt=prompt,
+          batch_size=batch_size,
+          num_steps=num_steps,
+          seed=seed,
+      )
+
+      generated_image_bytes_array = []
+      for i in range(batch_size):
+        generated_image = Image.fromarray(generated_image_array[i])
+        # Converts the image to a base64-encoded string.
+        buffered_image = io.BytesIO()
+        generated_image.save(buffered_image, format='JPEG')
+        generated_image_bytes = base64.b64encode(
+            buffered_image.getvalue()
+        ).decode('utf-8')
+        generated_image_bytes_array.append(generated_image_bytes)
+      prediction = {
+          'prompt': prompt,
+          'predicted_image': generated_image_bytes_array,
+      }
+      predictions.append(prediction)
+
+    return Response(
+        response=json.dumps({
+            'success': True,
+            'predictions': predictions,
+        }),
+        status=200,
+        mimetype='text/plain',
+    )
+  else:
+    return Response(
+        response=json.dumps({
+            'success': True,
+            'isalive': stable_diffusion_model is not None,
+        }),
+        status=200,
+        mimetype='text/plain',
+    )
+
+
+def serve_main(unused_argv):
+  """The main function to serve Keras models."""
+  del unused_argv
+  # This is used when running locally only. When deploying to Google App
+  # Engine, a webserver process such as Gunicorn will serve the app.
+  # # Debug deployment.
+  # flask_app.run(host='0.0.0.0', port=8501, debug=True)
+  # Prod deployment.
+  serve(flask_app, host='0.0.0.0', port=8501)
+
+
+if __name__ == '__main__':
+  app.run(serve_main)

community-content/vertex_vision_model_garden/model_oss/keras/train.py

@@ -0,0 +1,363 @@
+"""Train Keras Stable Diffusion.
+
+Most the codes below are from
+https://keras.io/examples/generative/finetune_stable_diffusion/.
+"""
+import os
+
+from absl import app
+from absl import flags
+from absl import logging
+import keras_cv
+# pylint: disable=g-importing-member
+from keras_cv.models.stable_diffusion.clip_tokenizer import SimpleTokenizer
+from keras_cv.models.stable_diffusion.diffusion_model import DiffusionModel
+from keras_cv.models.stable_diffusion.image_encoder import ImageEncoder
+from keras_cv.models.stable_diffusion.noise_scheduler import NoiseScheduler
+from keras_cv.models.stable_diffusion.text_encoder import TextEncoder
+import numpy as np
+# The docker builds could not find pandas.
+# pylint: disable=import-error
+import pandas as pd
+import tensorflow as tf
+from tensorflow import keras
+import tensorflow.experimental.numpy as tnp
+
+from util import constants
+from util import fileutils
+
+_INPUT_CSV_PATH = flags.DEFINE_string(
+    'input_csv_path',
+    None,
+    'The input csv path.',
+    required=True,
+)
+
+_USE_MP = flags.DEFINE_bool(
+    'use_mp',
+    True,
+    'Enable mixed-precision training if the underlying GPU has tensor cores.',
+)
+
+_EPOCHS = flags.DEFINE_integer('epochs', 1, 'The number of epochs.')
+
+_OUTPUT_MODEL_DIR = flags.DEFINE_string(
+    'output_model_dir',
+    None,
+    'The output model dir.',
+    required=True,
+)
+
+# These hyperparameters defaults come from this tutorial by Hugging Face:
+# https://huggingface.co/docs/diffusers/training/text2image
+_LEARNING_RATE = flags.DEFINE_float(
+    'learning_rate', 1e-5, 'The learning rate parameter for AdamW optimizer.'
+)
+
+_BETA_1 = flags.DEFINE_float(
+    'beta_1', 0.9, 'The beta_1 parameter for AdamW optimizer.'
+)
+
+_BETA_2 = flags.DEFINE_float(
+    'beta_2', 0.999, 'The beta_2 parameter for AdamW optimizer.'
+)
+
+_WEIGHT_DECAY = flags.DEFINE_float(
+    'weight_decay', 1e-2, 'The weight decay parameter for AdamW optimizer.'
+)
+
+_EPSILON = flags.DEFINE_float(
+    'epsilon', 1e-08, 'The epsilon parameter for AdamW optimizer.'
+)
+
+RESOLUTION = int(os.environ.get('RESOLUTION', 512))
+
+# The padding token and maximum prompt length are specific to the text encoder.
+# If you're using a different text encoder be sure to change them accordingly.
+PADDING_TOKEN = 49407
+MAX_PROMPT_LENGTH = 77
+
+AUTO = tf.data.AUTOTUNE
+POS_IDS = tf.convert_to_tensor([list(range(MAX_PROMPT_LENGTH))], dtype=tf.int32)
+
+
+augmenter = keras.Sequential(
+    layers=[
+        keras_cv.layers.CenterCrop(RESOLUTION, RESOLUTION),
+        keras_cv.layers.RandomFlip(),
+        tf.keras.layers.Rescaling(scale=1.0 / 127.5, offset=-1),
+    ]
+)
+text_encoder = TextEncoder(MAX_PROMPT_LENGTH)
+
+
+def process_image(image_path, tokenized_text):
+  image = tf.io.read_file(image_path)
+  image = tf.io.decode_png(image, 3)
+  image = tf.image.resize(image, (RESOLUTION, RESOLUTION))
+  return image, tokenized_text
+
+
+def apply_augmentation(image_batch, token_batch):
+  return augmenter(image_batch), token_batch
+
+
+def run_text_encoder(image_batch, token_batch):
+  return (
+      image_batch,
+      token_batch,
+      text_encoder([token_batch, POS_IDS], training=False),
+  )
+
+
+def prepare_dict(image_batch, token_batch, encoded_text_batch):
+  return {
+      'images': image_batch,
+      'tokens': token_batch,
+      'encoded_text': encoded_text_batch,
+  }
+
+
+def prepare_dataset(image_paths, tokenized_texts, batch_size=1):
+  dataset = tf.data.Dataset.from_tensor_slices((image_paths, tokenized_texts))
+  dataset = dataset.shuffle(batch_size * 10)
+  dataset = dataset.map(process_image, num_parallel_calls=AUTO).batch(
+      batch_size
+  )
+  dataset = dataset.map(apply_augmentation, num_parallel_calls=AUTO)
+  dataset = dataset.map(run_text_encoder, num_parallel_calls=AUTO)
+  dataset = dataset.map(prepare_dict, num_parallel_calls=AUTO)
+  return dataset.prefetch(AUTO)
+
+
+def prepare_training_dataset(dataset_csv):
+  """Prepares training datasets."""
+  if dataset_csv.startswith(constants.GCS_URI_PREFIX):
+    if not os.path.exists(constants.LOCAL_DATA_DIR):
+      os.makedirs(constants.LOCAL_DATA_DIR)
+    logging.info(
+        'Start to download data from %s to %s.',
+        os.path.dirname(dataset_csv),
+        constants.LOCAL_DATA_DIR,
+    )
+    fileutils.download_gcs_dir_to_local(
+        os.path.dirname(dataset_csv), constants.LOCAL_DATA_DIR
+    )
+    data_frame = pd.read_csv(
+        os.path.join(constants.LOCAL_DATA_DIR, os.path.basename(dataset_csv))
+    )
+    data_frame['image_path'] = data_frame['image_path'].apply(
+        lambda x: os.path.join(constants.LOCAL_DATA_DIR, x)
+    )
+  else:
+    # Keeps the following codes for experiments with
+    # https://keras.io/examples/generative/finetune_stable_diffusion/.
+    data_path = tf.keras.utils.get_file(origin=dataset_csv, untar=True)
+    data_frame = pd.read_csv(os.path.join(data_path, 'data.csv'))
+    data_frame['image_path'] = data_frame['image_path'].apply(
+        lambda x: os.path.join(data_path, x)
+    )
+  data_frame.head()
+
+  # Load the tokenizer.
+  tokenizer = SimpleTokenizer()
+
+  #  Method to tokenize and pad the tokens.
+  def process_text(caption):
+    tokens = tokenizer.encode(caption)
+    tokens = tokens + [PADDING_TOKEN] * (MAX_PROMPT_LENGTH - len(tokens))
+    return np.array(tokens)
+
+  # Collate the tokenized captions into an array.
+  tokenized_texts = np.empty((len(data_frame), MAX_PROMPT_LENGTH))
+
+  all_captions = list(data_frame['caption'].values)
+  for i, caption in enumerate(all_captions):
+    tokenized_texts[i] = process_text(caption)
+
+  # Prepare the dataset.
+  training_dataset = prepare_dataset(
+      np.array(data_frame['image_path']), tokenized_texts, batch_size=4
+  )
+
+  return training_dataset
+
+
+class Trainer(tf.keras.Model):
+  """The trainer for Keras Stable Diffusion."""
+
+  # Reference:
+  # https://github.com/huggingface/diffusers/blob/main/examples/text_to_image/train_text_to_image.py
+
+  def __init__(
+      self,
+      diffusion_model,
+      vae,
+      noise_scheduler,
+      use_mixed_precision=False,
+      max_grad_norm=1.0,
+      **kwargs,
+  ):
+    super().__init__(**kwargs)
+
+    self.diffusion_model = diffusion_model
+    self.vae = vae
+    self.noise_scheduler = noise_scheduler
+    self.max_grad_norm = max_grad_norm
+
+    self.use_mixed_precision = use_mixed_precision
+    self.vae.trainable = False
+
+  def train_step(self, inputs):
+    images = inputs['images']
+    encoded_text = inputs['encoded_text']
+    batch_size = tf.shape(images)[0]
+
+    with tf.GradientTape() as tape:
+      # Project image into the latent space and sample from it.
+      latents = self.sample_from_encoder_outputs(
+          self.vae(images, training=False)
+      )
+      # Know more about the magic number here:
+      # https://keras.io/examples/generative/fine_tune_via_textual_inversion/
+      latents = latents * 0.18215
+
+      # Sample noise that we'll add to the latents.
+      noise = tf.random.normal(tf.shape(latents))
+
+      # Sample a random timestep for each image.
+      timesteps = tnp.random.randint(
+          0, self.noise_scheduler.train_timesteps, (batch_size,)
+      )
+
+      # Add noise to the latents according to the noise magnitude at each
+      # timestep (this is the forward diffusion process).
+      noisy_latents = self.noise_scheduler.add_noise(
+          tf.cast(latents, noise.dtype), noise, timesteps
+      )
+
+      # Get the target for loss depending on the prediction type
+      # just the sampled noise for now.
+      target = noise  # noise_schedule.predict_epsilon == True
+
+      # Predict the noise residual and compute loss.
+      # pylint: disable=unnecessary-lambda
+      timestep_embedding = tf.map_fn(
+          lambda t: self.get_timestep_embedding(t), timesteps, dtype=tf.float32
+      )
+      timestep_embedding = tf.squeeze(timestep_embedding, 1)
+      model_pred = self.diffusion_model(
+          [noisy_latents, timestep_embedding, encoded_text], training=True
+      )
+      loss = self.compiled_loss(target, model_pred)
+      if self.use_mixed_precision:
+        loss = self.optimizer.get_scaled_loss(loss)
+
+    # Update parameters of the diffusion model.
+    trainable_vars = self.diffusion_model.trainable_variables
+    gradients = tape.gradient(loss, trainable_vars)
+    if self.use_mixed_precision:
+      gradients = self.optimizer.get_unscaled_gradients(gradients)
+    gradients = [tf.clip_by_norm(g, self.max_grad_norm) for g in gradients]
+    self.optimizer.apply_gradients(zip(gradients, trainable_vars))
+
+    return {m.name: m.result() for m in self.metrics}
+
+  def get_timestep_embedding(self, timestep, dim=320, max_period=10000):
+    half = dim // 2
+    log_max_preiod = tf.math.log(tf.cast(max_period, tf.float32))
+    # The docker builds could not support unary `-`.
+    # pylint: disable=invalid-unary-operand-type
+    freqs = tf.math.exp(
+        -log_max_preiod * tf.range(0, half, dtype=tf.float32) / half
+    )
+    args = tf.convert_to_tensor([timestep], dtype=tf.float32) * freqs
+    embedding = tf.concat([tf.math.cos(args), tf.math.sin(args)], 0)
+    embedding = tf.reshape(embedding, [1, -1])
+    return embedding
+
+  def sample_from_encoder_outputs(self, outputs):
+    mean, logvar = tf.split(outputs, 2, axis=-1)
+    logvar = tf.clip_by_value(logvar, -30.0, 20.0)
+    std = tf.exp(0.5 * logvar)
+    sample = tf.random.normal(tf.shape(mean), dtype=mean.dtype)
+    return mean + std * sample
+
+  def save_weights(
+      self, filepath, overwrite=True, save_format=None, options=None
+  ):
+    # Overriding this method will allow us to use the `ModelCheckpoint`
+    # callback directly with this trainer class. In this case, it will
+    # only checkpoint the `diffusion_model` since that's what we're training
+    # during fine-tuning.
+    self.diffusion_model.save_weights(
+        filepath=filepath,
+        overwrite=overwrite,
+        save_format=save_format,
+        options=options,
+    )
+
+
+def main(_) -> None:
+  # _INPUT_CSV_PATH and _OUTPUT_MODEL_DIR should have the format as
+  # gs://<bucket_name>/<object_name>.
+  if _INPUT_CSV_PATH.value:
+    if not _INPUT_CSV_PATH.value.startswith(constants.GCS_URI_PREFIX):
+      raise ValueError('The input csv path should be a gcs path like gs://<>')
+  if _OUTPUT_MODEL_DIR.value:
+    if not _OUTPUT_MODEL_DIR.value.startswith(constants.GCS_URI_PREFIX):
+      raise ValueError('The output model dir should be a gcs path like gs://<>')
+
+  if _USE_MP.value:
+    keras.mixed_precision.set_global_policy('mixed_float16')
+
+  image_encoder = ImageEncoder(RESOLUTION, RESOLUTION)
+  diffusion_ft_trainer = Trainer(
+      diffusion_model=DiffusionModel(RESOLUTION, RESOLUTION, MAX_PROMPT_LENGTH),
+      # Remove the top layer from the encoder, which cuts off the variance and
+      # only returns the mean.
+      vae=tf.keras.Model(
+          image_encoder.input,
+          image_encoder.layers[-2].output,
+      ),
+      noise_scheduler=NoiseScheduler(),
+      use_mixed_precision=_USE_MP.value,
+  )
+
+  optimizer = tf.keras.optimizers.experimental.AdamW(
+      learning_rate=_LEARNING_RATE.value,
+      weight_decay=_WEIGHT_DECAY.value,
+      beta_1=_BETA_1.value,
+      beta_2=_BETA_2.value,
+      epsilon=_EPSILON.value,
+  )
+  diffusion_ft_trainer.compile(optimizer=optimizer, loss='mse')
+
+  training_dataset = prepare_training_dataset(_INPUT_CSV_PATH.value)
+
+  # Note: gcsfuse does not work for Keras. We saves the trained models locally
+  # first, and then copy to gcs storages.
+  if not os.path.exists(constants.LOCAL_MODEL_DIR):
+    os.makedirs(constants.LOCAL_MODEL_DIR)
+  # The default saved model is in HDF5.
+  ckpt_path = os.path.join(constants.LOCAL_MODEL_DIR, 'saved_model.h5')
+  ckpt_callback = tf.keras.callbacks.ModelCheckpoint(
+      ckpt_path,
+      save_weights_only=True,
+      monitor='loss',
+      mode='min',
+  )
+  diffusion_ft_trainer.fit(
+      training_dataset, epochs=_EPOCHS.value, callbacks=[ckpt_callback]
+  )
+
+  # Copies the files in constants.LOCAL_MODEL_DIR to output_model_dir.
+  fileutils.upload_local_dir_to_gcs(
+      constants.LOCAL_MODEL_DIR, _OUTPUT_MODEL_DIR.value
+  )
+
+  return
+
+
+if __name__ == '__main__':
+  app.run(main)

community-content/vertex_vision_model_garden/model_oss/movinet/dockerfile/base.Dockerfile

@@ -0,0 +1,64 @@
+FROM tensorflow/build:2.12-python3.9
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# This is added to fix docker build error related to Nvidia key update.
+RUN rm -f /etc/apt/sources.list.d/cuda.list
+RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
+
+# Install basic libs.
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        cmake \
+        curl \
+        wget \
+        sudo \
+        gnupg \
+        libsm6 \
+        libxext6 \
+        libxrender-dev \
+        lsb-release \
+        ca-certificates \
+        build-essential \
+        git \
+        vim \
+        libtcmalloc-minimal4
+
+
+# Install google cloud CLI.
+RUN wget -q https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-430.0.0-linux-x86.tar.gz
+RUN tar xzf google-cloud-cli-430.0.0-linux-x86.tar.gz
+RUN ./google-cloud-sdk/install.sh -q
+# Make sure gsutil will use the default service account.
+RUN echo '[GoogleCompute]\nservice_account = default' > /etc/boto.cfg
+
+
+# Install required libs.
+RUN pip install --upgrade pip
+RUN pip install cloud-tpu-client==0.10
+RUN pip install pyyaml==6.0
+RUN pip install fsspec==2023.4.0
+RUN pip install gcsfs==2023.4.0
+RUN pip install tf-models-official==2.12.0
+RUN pip install cloudml-hypertune==0.1.0.dev6
+RUN pip install pylint==2.17.3
+
+# Installs Reduction Server NCCL plugin.
+RUN echo "deb https://packages.cloud.google.com/apt google-fast-socket main" | tee /etc/apt/sources.list.d/google-fast-socket.list \
+&&  curl -s -L https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - \
+&&  apt update && apt install -y google-reduction-server
+
+ENV PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp
+
+# Lower the memory fragmentation, and speed up the training.
+# https://github.com/tensorflow/tensorflow/issues/44176#issuecomment-783768033
+ENV LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libtcmalloc_minimal.so.4
+
+# Enable userspace DNS cache
+ENV GCS_RESOLVE_REFRESH_SECS=60
+ENV GCS_REQUEST_CONNECTION_TIMEOUT_SECS=300
+ENV GCS_METADATA_REQUEST_TIMEOUT_SECS=300
+ENV GCS_READ_REQUEST_TIMEOUT_SECS=300
+ENV GCS_WRITE_REQUEST_TIMEOUT_SECS=600
+# Each opened GCS file takes GCS_READ_CACHE_BLOCK_SIZE_MB of RAM, reduce the
+# value from the default 64MB to 8MB to decrease memory footprint.
+ENV GCS_READ_CACHE_BLOCK_SIZE_MB=8

community-content/vertex_vision_model_garden/model_oss/movinet/dockerfile/model_export.Dockerfile

@@ -0,0 +1,13 @@
+FROM gcr.io/automl-migration-test/movinet-base:latest
+
+# Copy license.
+RUN wget https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/LICENSE
+
+RUN wget https://raw.githubusercontent.com/tensorflow/models/954dd73bffd43174bd3ca26a4a34abebe4147570/official/projects/movinet/tools/export_saved_model.py \
+  -O /usr/local/lib/python3.9/dist-packages/official/projects/movinet/tools/export_saved_model.py
+
+WORKDIR /automl_vision
+
+ENV PYTHONPATH "${PYTHONPATH}:/automl_vision/util"
+
+ENTRYPOINT ["python3", "-m", "official.projects.movinet.tools.export_saved_model"]

community-content/vertex_vision_model_garden/model_oss/movinet/dockerfile/serve.Dockerfile

@@ -0,0 +1,18 @@
+FROM gcr.io/automl-migration-test/movinet-base:latest
+
+# Copy license.
+RUN wget https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/LICENSE
+
+RUN pip install flask==2.3.2
+RUN pip install waitress==2.1.2
+
+RUN mkdir -p /automl_vision/movinet/serving
+COPY model_oss/movinet/serving /automl_vision/movinet/serving
+COPY model_oss/util /automl_vision/util
+
+WORKDIR /automl_vision
+
+ENV PYTHONPATH "${PYTHONPATH}:/automl_vision/util"
+
+ENTRYPOINT ["flask", "--app", "movinet.serving.serving_main", "run"]
+CMD ["--host=0.0.0.0", "--port=8501"]

community-content/vertex_vision_model_garden/model_oss/movinet/dockerfile/train.Dockerfile

@@ -0,0 +1,18 @@
+FROM gcr.io/automl-migration-test/movinet-base:latest
+
+# Copy license.
+RUN wget https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/LICENSE
+
+RUN mkdir -p /automl_vision/movinet
+COPY model_oss/movinet/*.py /automl_vision/movinet/
+COPY model_oss/util /automl_vision/util
+
+WORKDIR /automl_vision
+
+ENV PYTHONPATH "${PYTHONPATH}:/automl_vision/util"
+
+# Run pylint to validate code.
+COPY .pylintrc /automl_vision/.pylintrc
+RUN find . -type f -name "*.py" | xargs pylint --rcfile=./.pylintrc --errors-only
+
+ENTRYPOINT ["python3","movinet/train.py"]

community-content/vertex_vision_model_garden/model_oss/movinet/serving/serving_main.py

@@ -0,0 +1,142 @@
+"""Main executable for MoViNet online / batch predictions."""
+
+from collections.abc import Sequence
+import json
+import os
+
+from absl import app
+from absl import logging
+import flask
+import tensorflow as tf
+import waitress
+
+from movinet.serving import video_serving_lib
+from util import constants
+
+
+flask_app = flask.Flask(__name__)
+logging.set_verbosity(logging.INFO)
+
+movinet_model = None
+
+_BATCH_SIZE = int(os.environ.get('BATCH_SIZE', '1'))
+_NUM_FRAMES = int(os.environ.get('NUM_FRAMES', '32'))
+_FPS = float(os.environ.get('FPS', '5'))
+_OVERLAP_FRAMES = int(os.environ.get('OVERLAP_FRAMES', '24'))
+_OBJECTIVE = os.environ.get(
+    'OBJECTIVE', constants.OBJECTIVE_VIDEO_CLASSIFICATION
+).lower()
+
+# VAR parameters.
+_CONFIDENCE_THRESHOLD = float(os.environ.get('CONFIDENCE_THRESHOLD', '0.5'))
+_MIN_GAP_TIME = float(os.environ.get('MIN_GAP_TIME', '1.5'))
+
+
+def load_movinet_model() -> None:
+  model_path = os.environ.get('MODEL_PATH')
+
+  if not model_path:
+    raise app.UsageError('Missing MODEL_PATH environment variable.')
+
+  # We just reload the weights of the fine-tuned diffusion model.
+  logging.info('Initialize finetuned models from: %s', model_path)
+  global movinet_model
+  movinet_model = tf.saved_model.load(model_path)
+
+
+load_movinet_model()
+
+
+def error(message: str) -> str:
+  """Returns a JSON representing an error response."""
+  return json.dumps({
+      'success': False,
+      'error': message,
+  })
+
+
+# The health check route is required for docker deployment in google cloud.
+@flask_app.route('/ping')
+def ping() -> flask.Response:
+  """Health checks."""
+  return flask.Response(status=200)
+
+
+# The return should be `Response` for docker deployment in google cloud.
+@flask_app.route('/predict', methods=['GET', 'POST'])
+def predict_model() -> flask.Response:
+  """Predictions."""
+  if flask.request.method == 'POST':
+    contents = flask.request.get_json(force=True)
+
+    logging.info('The input contents are: %s', contents)
+    instances = contents.get('instances', [])
+
+    try:
+      predictions = []
+      for instance in instances:
+        executor = video_serving_lib.parse_request(instance)
+        prediction = executor.get_prediction(
+            movinet_model,
+            _BATCH_SIZE,
+            _FPS,
+            _NUM_FRAMES,
+            _OVERLAP_FRAMES,
+            _OBJECTIVE,
+        )
+        if _OBJECTIVE == constants.OBJECTIVE_VIDEO_CLASSIFICATION:
+          prediction = video_serving_lib.postprocess_vcn(prediction)
+        elif _OBJECTIVE == constants.OBJECTIVE_VIDEO_ACTION_RECOGNITION:
+          prediction = video_serving_lib.postprocess_var(
+              executor.windows, prediction, _CONFIDENCE_THRESHOLD, _MIN_GAP_TIME
+          )
+        predictions.append(prediction)
+    except ValueError as e:
+      return flask.Response(
+          error(str(e)), status=500, mimetype='application/json'
+      )
+
+    return flask.Response(
+        response=json.dumps({
+            'success': True,
+            'predictions': predictions,
+        }),
+        status=200,
+        mimetype='application/json',
+    )
+  else:
+    return flask.Response(
+        response=json.dumps({
+            'success': True,
+            'isalive': movinet_model is not None,
+        }),
+        status=200,
+        mimetype='application/json',
+    )
+
+
+def main(argv: Sequence[str]) -> None:
+  if len(argv) > 1:
+    raise app.UsageError('Too many command-line arguments.')
+  # This is used when running locally only. When deploying to Google App
+  # Engine, a webserver process such as Gunicorn will serve the app.
+  # # Debug deployment.
+  # flask_app.run(host='0.0.0.0', port=8501, debug=True)
+  # Prod deployment.
+  if _OBJECTIVE not in [
+      constants.OBJECTIVE_VIDEO_CLASSIFICATION,
+      constants.OBJECTIVE_VIDEO_ACTION_RECOGNITION,
+  ]:
+    raise app.UsageError('Objective must be vcn or var.')
+  logging.info(
+      'Env: batch_size: %s, num_frames: %s, fps: %s, overlap_frames: %s',
+      _BATCH_SIZE,
+      _NUM_FRAMES,
+      _FPS,
+      _OVERLAP_FRAMES,
+  )
+  waitress.serve(flask_app, host='0.0.0.0', port=8501)
+
+
+if __name__ == '__main__':
+  app.run(main)

community-content/vertex_vision_model_garden/model_oss/movinet/serving/video_serving_lib.py

@@ -0,0 +1,462 @@
+"""Lib for handling video prediction requests.
+
+The VCN inference algorithm is as follows:
+1. Find all video frames within the given clip according to the sampling FPS.
+2. Create possibly overlapping sliding windows according to the num_frames and
+overlap_frames parameters. The last window might have a larger overlap if it
+doesn't exactly fit.
+3. Run model inference on each sliding window and compute softmax to obtain
+probabilities.
+4. Average the probabilities over all sliding windows.
+
+The VAR inference algorithm is very similar to VCN, with a few differences:
+1. The last sliding window is discarded if it does not exactly fit.
+2. Instead of averaging, the postprocessing consists of temporal nonmaximal
+suppression and removing background and low-confidence labels.
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import os
+from typing import Any, Dict, Optional, Sequence, Union, cast
+
+from absl import logging
+import cv2
+import numpy as np
+import tensorflow as tf
+
+from util import constants
+from util import fileutils
+
+
+_JSON_LABEL_KEY = 'label'
+_JSON_GCS_URI_KEY = 'content'
+_JSON_CONFIDENCE_KEY = 'confidence'
+_JSON_START_TIME_KEY = 'timeSegmentStart'
+_JSON_END_TIME_KEY = 'timeSegmentEnd'
+_BACKGROUND_LABEL = 0
+_JSON_REQUIRED_KEYS = [
+    _JSON_GCS_URI_KEY,
+    _JSON_START_TIME_KEY,
+    _JSON_END_TIME_KEY,
+]
+_IMAGE_WIDTH = int(os.environ.get('IMAGE_WIDTH', '172'))
+_IMAGE_HEIGHT = int(os.environ.get('IMAGE_HEIGHT', '172'))
+
+
+@dataclasses.dataclass
+class DetectionOutput:
+  timestamp: float
+  label: int
+  confidence: float
+
+  def to_json_obj(self) -> Dict[str, Union[int, float]]:
+    """Encodes self as a dict for JSON serialization."""
+    return {
+        _JSON_LABEL_KEY: self.label,
+        _JSON_START_TIME_KEY: self.timestamp,
+        _JSON_END_TIME_KEY: self.timestamp,
+        _JSON_CONFIDENCE_KEY: self.confidence,
+    }
+
+
+def create_detection_output(
+    timestamp: float, predictions: np.ndarray
+) -> DetectionOutput:
+  label = np.argmax(predictions).item()
+  confidence: float = predictions[label].item()
+  return DetectionOutput(timestamp, label, confidence)
+
+
+class SlidingWindow:
+  """Represents a sliding window with start / end timestamps."""
+
+  def __init__(self, fps: float, frames: Sequence[int]):
+    if not frames:
+      raise ValueError('Sliding window cannot be empty.')
+    self.frames = frames
+    self.start_time = frames[0] / fps
+    self.end_time = frames[-1] / fps
+    self.frame_data: list[Optional[np.ndarray]] = []
+    self.clear_frame_data()
+
+  def load_cache_from(self, other: SlidingWindow) -> int:
+    """Loads cache from another sliding window if possible."""
+    cache_count = 0
+    for i, frame in enumerate(self.frames):
+      try:
+        other_idx = other.frames.index(frame)
+        self.frame_data[i] = other.frame_data[other_idx]
+        cache_count += 1
+      except ValueError:
+        # Cache miss.
+        pass
+    return cache_count
+
+  def load_frames(self, video: Any) -> Sequence[np.ndarray]:
+    """Loads frames of this sliding window from a video."""
+    for i, frame in enumerate(self.frames):
+      if self.frame_data[i] is None:
+        video.set(cv2.CAP_PROP_POS_FRAMES, frame)
+        ret, frame = video.read()
+        if not ret:
+          raise IOError(f'Failed to read video at frame {frame}.')
+        self.frame_data[i] = cv2.resize(frame, (_IMAGE_WIDTH, _IMAGE_HEIGHT))
+    return cast(Sequence[np.ndarray], self.frame_data)
+
+  def clear_frame_data(self) -> None:
+    """Clears frame data of this sliding window to reduce memory usage."""
+    self.frame_data: list[Optional[np.ndarray]] = [None] * len(self)
+
+  def __len__(self) -> int:
+    return len(self.frames)
+
+  @property
+  def middle_timestamp(self) -> float:
+    return (self.start_time + self.end_time) / 2
+
+
+def _get_sliding_windows(
+    frames: Sequence[int],
+    original_fps: float,
+    window_size: int,
+    overlap: int,
+    flush_last_window: bool,
+) -> Sequence[SlidingWindow]:
+  """Computes a list of sliding windows from frames.
+
+  Args:
+    frames: A list of frame indices.
+    original_fps: Frames per second of the original video.
+    window_size: Number of frames in a single window.
+    overlap: Number of overlapping frames in adjacent windows.
+    flush_last_window: Where to flush the last window if there are not enough
+      frames left.
+
+  Returns:
+    A list of sliding windows, each has a list of frame indices. The last two
+    windows might have a larger overlap if the last window does not exactly fit
+    and flush_last_window is set to True.
+
+  Raises:
+    ValueError: Arguments are invalid.
+  """
+  if window_size <= overlap:
+    raise ValueError(f'Window size {window_size} <= overlap {overlap}')
+  total_frames = len(frames)
+  windows: list[SlidingWindow] = []
+  for i in range(0, total_frames, window_size - overlap):
+    if i == 0 or i + window_size <= total_frames:
+      windows.append(SlidingWindow(original_fps, frames[i : i + window_size]))
+    elif i + overlap < total_frames and flush_last_window:
+      # Some frames in this window are not covered by the previous window.
+      windows.append(
+          SlidingWindow(
+              original_fps, frames[total_frames - window_size : total_frames]
+          )
+      )
+  return windows
+
+
+def _sample_frame_indices(
+    start_time: float,
+    end_time: float,
+    original_fps: float,
+    sample_fps: float,
+    max_frames: int,
+    padding_left: int = 0,
+    padding_right: int = 0,
+) -> Sequence[int]:
+  """Samples frames from start_time to end_time by sample_fps.
+
+  Args:
+    start_time: Start timestamp in seconds.
+    end_time: End timestamp in seconds.
+    original_fps: Frames per second of the original video.
+    sample_fps: Number of frames to sample per second.
+    max_frames: Total number of frames in the video.
+    padding_left: Padding to add to the start in frames. Padded frames will be
+      duplicates of the first frame.
+    padding_right: Padding to add to the end in frames. Padded frames will be
+      duplicates of the last frame.
+
+  Returns:
+    A list of sampled frame indices.
+  """
+  ret = [
+      min(max_frames - 1, round(t * original_fps))
+      for t in np.arange(start_time, end_time, 1 / sample_fps)
+  ]
+  if ret:
+    ret = [ret[0]] * padding_left + ret + [ret[-1]] * padding_right
+  return ret
+
+
+class VideoPredictionExecutor:
+  """Represents a Video prediction request with a video clip."""
+
+  def __init__(self, gcs_uri: str, start_time: float, end_time: float):
+    self._gcs_uri = gcs_uri
+    self._start_time = start_time
+    self._end_time = end_time
+    self.windows: Sequence[SlidingWindow] = []
+    self._last_window: SlidingWindow = None
+
+  def _read_frames_from_window(
+      self, video: Any, new_window: SlidingWindow
+  ) -> Sequence[np.ndarray]:
+    """Reads video frames from the new window.
+
+    Args:
+      video: Video loaded with cv2.
+      new_window: A list of sorted frame indices in the new window.
+
+    Returns:
+      Frame data from the video as a list of numpy arrays.
+
+    Raises:
+      IOError: Failed to read video.
+    """
+    # Caches frames as much as possible.
+    if self._last_window is not None:
+      cache_count = new_window.load_cache_from(self._last_window)
+      logging.info('Cached %d frames.', cache_count)
+      self._last_window.clear_frame_data()
+    self._last_window = new_window
+    return new_window.load_frames(video)
+
+  def _predict(
+      self, model: Any, video: Any, batched_windows: Sequence[SlidingWindow]
+  ) -> np.ndarray:
+    """Run model inference on specific frames of a video.
+
+    Args:
+      model: MoViNet model.
+      video: Video loaded with cv2.
+      batched_windows: A batch of sliding windows to predict. Each element is an
+        integer frame index. Must have equal number of frames in each window.
+
+    Returns:
+      Prediction results.
+
+    Raises:
+      ValueError: Batched windows are not sorted, or do not have equal number of
+        frames in each window.
+      IOError: Failed to read video.
+    """
+    if any(
+        (
+            len(window) != len(batched_windows[0])
+            for window in batched_windows[1:]
+        )
+    ):
+      raise ValueError(
+          'Batched windows do not have equal number of frames in each window.'
+      )
+    batch = []
+    logging.info('Loading video frames...')
+    for window in batched_windows:
+      logging.info('Predict frames: %s', window.frames)
+      frames = self._read_frames_from_window(video, window)
+      batch.append(frames)
+    input_tensor = tf.convert_to_tensor(batch, dtype=tf.float32) / 255.0
+    logging.info('Predict: Input tensor shape %s', input_tensor.shape)
+    predictions = model({'image': input_tensor})
+    logging.info('Running softmax on predictions...')
+    predictions = tf.nn.softmax(predictions, axis=1)
+    return predictions.numpy()
+
+  def get_prediction(
+      self,
+      model: Any,
+      batch_size: int,
+      fps: float,
+      num_frames: int,
+      overlap_frames: int,
+      objective: str,
+  ) -> Sequence[np.ndarray]:
+    """Predicts the video clip with the model.
+
+    Args:
+      model: The loaded MoViNet model.
+      batch_size: Batch size for prediction.
+      fps: Video sampling FPS.
+      num_frames: Number of frames in a single predictions. If the model is
+        exported with a fixed input shape, this must match its num_frames
+        dimension.
+      overlap_frames: Number of overlapping frames of consecutive sliding
+        windows.
+      objective: A string `vcn` or `var`.
+
+    Returns:
+      A list of floats as the prediction response.
+
+    Raises:
+      IOError: The video fails to load.
+      ValueError: Some arguments are invalid.
+    """
+    if objective not in [
+        constants.OBJECTIVE_VIDEO_CLASSIFICATION,
+        constants.OBJECTIVE_VIDEO_ACTION_RECOGNITION,
+    ]:
+      raise ValueError(f'{objective} objective is not supported.')
+
+    # cv2 expects a local path so we need to download the video from GCS.
+    local_file_path = fileutils.generate_tmp_path(
+        os.path.splitext(self._gcs_uri)[1]
+    )
+    logging.info('Downloading %s to %s...', self._gcs_uri, local_file_path)
+    fileutils.download_gcs_file_to_local(self._gcs_uri, local_file_path)
+    logging.info('Download %s complete.', self._gcs_uri)
+
+    # Loads video.
+    video = cv2.VideoCapture(local_file_path)
+    total_frames = video.get(cv2.CAP_PROP_FRAME_COUNT)
+    original_fps = video.get(cv2.CAP_PROP_FPS)
+    if not original_fps:
+      # 0 or None indicates the video is invalid.
+      raise IOError(f'Failed to load {self._gcs_uri}.')
+    video_length = total_frames / original_fps
+    self._start_time = max(0, self._start_time)
+    self._end_time = min(video_length, self._end_time)
+    padding = (
+        (num_frames // 2)
+        if objective == constants.OBJECTIVE_VIDEO_ACTION_RECOGNITION
+        else 0
+    )
+
+    # Computes sliding windows.
+    frame_indices = _sample_frame_indices(
+        self._start_time,
+        self._end_time,
+        original_fps,
+        fps,
+        total_frames,
+        padding,
+        padding,
+    )
+    logging.info('Frame indices: %s', frame_indices)
+    self.windows = _get_sliding_windows(
+        frame_indices,
+        original_fps,
+        num_frames,
+        overlap_frames,
+        objective != 'var',
+    )
+    if not self.windows:
+      raise ValueError(
+          f'No sliding windows found from {self._start_time} to'
+          f' {self._end_time}.'
+      )
+    self._last_window = None
+
+    # Runs inference.
+    predictions = []
+    for i in range(0, len(self.windows), batch_size):
+      predictions.extend(
+          self._predict(model, video, self.windows[i : i + batch_size])
+      )
+    return predictions
+
+
+def parse_request(req_json: Any) -> VideoPredictionExecutor:
+  """Parses VideoPredictionExecutor from request JSON object.
+
+  Args:
+    req_json: Request JSON object.
+
+  Returns:
+    Parsed VideoPredictionExecutor.
+
+  Raises:
+    ValueError: Request JSON object is invalid.
+  """
+  for key in _JSON_REQUIRED_KEYS:
+    if key not in req_json:
+      raise ValueError(f'{key} not found in {req_json}.')
+  gcs_uri = req_json[_JSON_GCS_URI_KEY]
+  start_time = float(req_json[_JSON_START_TIME_KEY].removesuffix('s'))
+  end_time = float(req_json[_JSON_END_TIME_KEY].removesuffix('s'))
+  return VideoPredictionExecutor(gcs_uri, start_time, end_time)
+
+
+def postprocess_vcn(predictions: Sequence[np.ndarray]) -> Sequence[float]:
+  """Aggregates VCN predictions of sliding windows."""
+  return np.mean(predictions, axis=0).tolist()
+
+
+def temporal_nonmaximal_suppression(
+    detections: Sequence[DetectionOutput], min_gap_time: float
+) -> Sequence[DetectionOutput]:
+  """Nonmaximal suppression for key frame detection.
+
+  For consecutive packets of the same label within a pre-defined duration, we
+  only keep the one with the highest confidence score. Such duration can be
+  determined by performing data analysis on users' dataset.
+
+  Args:
+    detections: A list of DetectionOutputs.
+    min_gap_time: Minimum time between consecutive key frames of the same label
+      in seconds.
+
+  Returns:
+    DetectionOutput after nonmaximal suppression sorted in ascending timestamps.
+  """
+  max_label = max([detection.label for detection in detections])
+  prev_detections: list[Optional[DetectionOutput]] = [None] * (max_label + 1)
+  ret: list[DetectionOutput] = []
+  by_time = lambda x: x.timestamp
+  for detection in sorted(detections, key=by_time):
+    prev_detection = prev_detections[detection.label]
+    prev_detections[detection.label] = detection
+    if not prev_detection:
+      continue
+    if detection.timestamp - prev_detection.timestamp > min_gap_time:
+      ret.append(prev_detection)
+      continue
+    detection.confidence = max(detection.confidence, prev_detection.confidence)
+  ret.extend((d for d in prev_detections if d is not None))
+  return sorted(ret, key=by_time)
+
+
+def postprocess_var(
+    windows: Sequence[SlidingWindow],
+    predictions: Sequence[np.ndarray],
+    confidence_threshold: float,
+    min_gap_time: float,
+) -> Sequence[Dict[str, Any]]:
+  """Generates a list of detected keyframes from sliding window predictions.
+
+  Args:
+    windows: Sliding windows.
+    predictions: A list of predictions of sliding windows.
+    confidence_threshold: Only probabilities greater than this threshold will
+      contribute to the final result.
+    min_gap_time: Minimum time between consecutive key frames of the same label
+      in seconds. Used in temporal nonmaximal suppression.
+
+  Returns:
+    A sequence of dictionaries, each item has the following keys:
+    - label: Integer label of the detection result.
+    - timeSegmentStart: Start timestamp in seconds.
+    - timeSegmentEnd: End timestamp in seconds. Always equals timeSegmentStart.
+  """
+  if len(windows) != len(predictions):
+    raise ValueError('Mismatched # of windows with # of predictions.')
+
+  # Creates detection results from windows, filtering out the background label.
+  detections = [
+      create_detection_output(window.middle_timestamp, predictions[i])
+      for i, window in enumerate(windows)
+  ]
+
+  # Temporal nonmaximal suppression.
+  detections = temporal_nonmaximal_suppression(detections, min_gap_time)
+
+  # Filters out ones with low confidence and the background label.
+  return [
+      x.to_json_obj()
+      for x in detections
+      if x.label != _BACKGROUND_LABEL and x.confidence > confidence_threshold
+  ]

community-content/vertex_vision_model_garden/model_oss/movinet/train.py

@@ -0,0 +1,210 @@
+"""Main executable for MoViNet docker."""
+
+import json
+import os
+from typing import Sequence, Any
+
+from absl import app
+from absl import flags
+from absl import logging
+import gin
+import hypertune
+import tensorflow as tf
+
+from util import constants
+from util import hypertune_utils
+from official.common import distribute_utils
+from official.common import flags as tfm_flags
+from official.core import task_factory
+from official.core import train_lib
+from official.core import train_utils
+from official.modeling import performance
+# Import movinet libraries to register the backbone and model into tf.vision
+# model garden factory.
+# pylint: disable=unused-import
+from official.projects.movinet.modeling import movinet
+from official.projects.movinet.modeling import movinet_model
+from official.vision import registry_imports
+# pylint: enable=unused-import
+
+
+FLAGS = flags.FLAGS
+
+_FILE_TYPE_TFRECORD = 'tfrecord'
+
+_LEARNING_RATE = flags.DEFINE_float(
+    'learning_rate', None, 'The learning rate of this training job.'
+)
+
+_NUM_CLASSES = flags.DEFINE_integer(
+    'num_classes', None, 'The number of classes.'
+)
+
+_INIT_CHECKPOINT = flags.DEFINE_string(
+    'init_checkpoint', None, 'The initial checkpoint of this training job.'
+)
+
+_INPUT_TRAIN_DATA_PATH = flags.DEFINE_string(
+    'input_train_data_path', None, 'Input train data path.'
+)
+
+_INPUT_VALIDATION_DATA_PATH = flags.DEFINE_string(
+    'input_validation_data_path', None, 'Input validation data path.'
+)
+
+_GLOBAL_BATCH_SIZE = flags.DEFINE_integer(
+    'global_batch_size', None, 'Global batch size.'
+)
+
+_PREFETCH_BUFFER_SIZE = flags.DEFINE_integer(
+    'prefetch_buffer_size', None, 'Prefetch buffer size.'
+)
+
+_SHUFFLE_BUFFER_SIZE = flags.DEFINE_integer(
+    'shuffle_buffer_size', None, 'Shuffle buffer size.'
+)
+
+_TRAIN_STEPS = flags.DEFINE_integer('train_steps', None, 'Train steps.')
+_LOG_LEVEL = flags.DEFINE_enum(
+    'log_level',
+    'INFO',
+    ['FATAL', 'ERROR', 'WARNING', 'INFO', 'DEBUG'],
+    'Log level.',
+)
+
+
+def parse_params() -> Any:
+  """Parses parameters."""
+  gin.parse_config_files_and_bindings(FLAGS.gin_file, FLAGS.gin_params)
+  params = train_utils.parse_configuration(FLAGS, lock_return=False)
+  if _INIT_CHECKPOINT.value:
+    params.task.init_checkpoint = _INIT_CHECKPOINT.value
+    params.task.init_checkpoint_modules = 'backbone'
+  if _NUM_CLASSES.value:
+    params.task.model.num_classes = _NUM_CLASSES.value
+    params.task.train_data.num_classes = _NUM_CLASSES.value
+    params.task.validation_data.num_classes = _NUM_CLASSES.value
+  # If users set input train/validation data path, we assume the data are
+  # converted from data converter as tfrecord. Users can use tfds by writing
+  # their own config directly, and no need to override this parameter.
+  if _INPUT_TRAIN_DATA_PATH.value:
+    params.task.train_data.input_path = _INPUT_TRAIN_DATA_PATH.value
+    params.task.train_data.file_type = _FILE_TYPE_TFRECORD
+    params.task.train_data.tfds_name = ''
+  if _INPUT_VALIDATION_DATA_PATH.value:
+    params.task.validation_data.input_path = _INPUT_VALIDATION_DATA_PATH.value
+    params.task.validation_data.file_type = _FILE_TYPE_TFRECORD
+    params.task.validation_data.tfds_name = ''
+  if _GLOBAL_BATCH_SIZE.value:
+    params.task.train_data.global_batch_size = _GLOBAL_BATCH_SIZE.value
+    params.task.validation_data.global_batch_size = _GLOBAL_BATCH_SIZE.value
+  if _PREFETCH_BUFFER_SIZE.value:
+    params.task.train_data.prefetch_buffer_size = _PREFETCH_BUFFER_SIZE.value
+    params.task.validation_data.prefetch_buffer_size = (
+        _PREFETCH_BUFFER_SIZE.value
+    )
+  if _SHUFFLE_BUFFER_SIZE.value:
+    params.task.train_data.shuffle_buffer_size = _SHUFFLE_BUFFER_SIZE.value
+  if _TRAIN_STEPS.value:
+    params.trainer.train_steps = _TRAIN_STEPS.value
+  if _LEARNING_RATE.value:
+    logging.info('Updating learning_rate: %s', _LEARNING_RATE.value)
+    # Use `get` method of train_utils.hyperparams.OneOfConfig to get learning
+    # rate config.
+    learning_rate = params.trainer.optimizer_config.learning_rate.get()
+    if hasattr(learning_rate, 'initial_learning_rate'):
+      learning_rate.initial_learning_rate = _LEARNING_RATE.value
+    else:
+      logging.warning('Cannot set learning rate for %s', learning_rate)
+  # Set default params for best checkpoints.
+  params.trainer.best_checkpoint_export_subdir = constants.BEST_CKPT_DIRNAME
+  params.trainer.best_checkpoint_metric_comp = constants.BEST_CKPT_METRIC_COMP
+  params.trainer.best_checkpoint_eval_metric = (
+      constants.VIDEO_CLASSIFICATION_BEST_EVAL_METRIC
+  )
+  return params
+
+
+def main(argv: Sequence[str]) -> None:
+  logging.set_verbosity(_LOG_LEVEL.value)
+  if len(argv) > 1:
+    raise app.UsageError('Too many command-line arguments.')
+  params = parse_params()
+  logging.info('The actual training parameters are:\n%s', params.as_dict())
+  model_dir: str = os.path.join(
+      FLAGS.model_dir,
+      constants.TRIAL_PREFIX + hypertune_utils.get_trial_id_from_environment(),
+  )
+  logging.info('model_dir: %s', model_dir)
+
+  if 'train' in FLAGS.mode:
+    # Pure eval modes do not output yaml files. Otherwise continuous eval job
+    # may race against the train job for writing the same file.
+    train_utils.serialize_config(params, model_dir)
+
+  # Sets mixed_precision policy. Using 'mixed_float16' or 'mixed_bfloat16'
+  # can have significant impact on model speeds by utilizing float16 in case of
+  # GPUs, and bfloat16 in the case of TPUs. loss_scale takes effect only when
+  # dtype is float16
+  if params.runtime.mixed_precision_dtype:
+    performance.set_mixed_precision_policy(params.runtime.mixed_precision_dtype)
+  distribution_strategy = distribute_utils.get_distribution_strategy(
+      distribution_strategy=params.runtime.distribution_strategy,
+      all_reduce_alg=params.runtime.all_reduce_alg,
+      num_gpus=params.runtime.num_gpus,
+      tpu_address=params.runtime.tpu,
+  )
+
+  # Create task and run experiment.
+  with distribution_strategy.scope():
+    task = task_factory.get_task(params.task, logging_dir=model_dir)
+
+  train_lib.run_experiment(
+      distribution_strategy=distribution_strategy,
+      task=task,
+      mode=FLAGS.mode,
+      params=params,
+      model_dir=model_dir,
+  )
+
+  train_utils.save_gin_config(FLAGS.mode, model_dir)
+
+  eval_metric_name = constants.VIDEO_CLASSIFICATION_BEST_EVAL_METRIC
+
+  eval_filepath = os.path.join(
+      model_dir, constants.BEST_CKPT_DIRNAME, constants.BEST_CKPT_EVAL_FILENAME
+  )
+  logging.info('Load eval metrics from: %s.', eval_filepath)
+
+  with tf.io.gfile.GFile(eval_filepath, 'rb') as f:
+    eval_metric_results = json.load(f)
+  logging.info('eval metrics are: %s.', eval_metric_results)
+  if (
+      eval_metric_name in eval_metric_results
+      and constants.BEST_CKPT_STEP_NAME in eval_metric_results
+  ):
+    hp_metric = eval_metric_results[eval_metric_name]
+    hp_step = int(eval_metric_results[constants.BEST_CKPT_STEP_NAME])
+    hpt = hypertune.HyperTune()
+    hpt.report_hyperparameter_tuning_metric(
+        hyperparameter_metric_tag=constants.HP_METRIC_TAG,
+        metric_value=hp_metric,
+        global_step=hp_step,
+    )
+    logging.info(
+        'Send HP metric: %f and steps %d to hyperparameter tuning.',
+        hp_metric,
+        hp_step,
+    )
+  else:
+    logging.info(
+        'Either %s or %s is not included in the evaluation results: %s.',
+        eval_metric_name,
+        constants.BEST_CKPT_STEP_NAME,
+        eval_metric_results,
+    )
+
+
+if __name__ == '__main__':
+  tfm_flags.define_flags()
+  app.run(main)

community-content/vertex_vision_model_garden/model_oss/open_clip/dockerfile/serve.Dockerfile

@@ -0,0 +1,67 @@
+# Dockerfile for basic serving dockers for OpenCLIP.
+#
+# To build:
+# docker build -f model_oss/open_clip/dockerfile/serve.Dockerfile . -t ${YOUR_IMAGE_TAG}
+#
+# To push to gcr:
+# docker tag ${YOUR_IMAGE_TAG} gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
+# docker push gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
+
+# Switch to this base image for gpu serve.
+FROM pytorch/torchserve:0.7.1-gpu
+
+USER root
+
+# Install tools.
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        curl \
+        wget \
+        vim
+
+# Copy license.
+RUN wget https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/LICENSE
+
+ENV infer_port=7080
+ENV mng_port=7081
+ENV model_name="transformers_serving"
+ENV PATH="/home/model-server/:${PATH}"
+
+# Install libraries.
+RUN python3 -m pip install --upgrade pip
+RUN pip install torch==1.13.1
+RUN pip install open_clip_torch==2.20.0
+RUN pip install pillow==9.5.0
+RUN pip install google-cloud-storage==2.7.0
+
+# Copy model artifacts.
+COPY model_oss/open_clip/handler.py /home/model-server/handler.py
+COPY model_oss/util/ /home/model-server/util/
+ENV PYTHONPATH /home/model-server/
+
+# Create torchserve configuration file.
+RUN echo \
+    "default_response_timeout=1800\n" \
+    "service_envelope=json\n" \
+    "inference_address=http://0.0.0.0:${infer_port}\n"  \
+    "management_address=http://0.0.0.0:${mng_port}" >> /home/model-server/config.properties
+
+# Expose ports.
+EXPOSE ${infer_port}
+EXPOSE ${mng_port}
+
+# Archive model artifacts and dependencies.
+# Do not set --model-file and --serialized-file because model and checkpoint will be dynamically loaded in handler.py.
+RUN torch-model-archiver \
+--model-name=${model_name} \
+--version=1.0 \
+--handler=/home/model-server/handler.py \
+--runtime=python3 \
+--export-path=/home/model-server/model-store \
+--archive-format=default \
+--force
+
+# Run Torchserve HTTP serve to respond to prediction requests.
+CMD ["torchserve", "--start", \
+     "--ts-config", "/home/model-server/config.properties", \
+     "--models", "${model_name}=${model_name}.mar", \
+     "--model-store", "/home/model-server/model-store"]

community-content/vertex_vision_model_garden/model_oss/open_clip/dockerfile/train.Dockerfile

@@ -0,0 +1,53 @@
+# Dockerfile for training dockers with OpenCLIP.
+#
+# To build:
+# docker build -f model_oss/open_clilp/dockerfile/train.Dockerfile . -t ${YOUR_IMAGE_TAG}
+#
+# To push to gcr:
+# docker tag ${YOUR_IMAGE_TAG} gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
+# docker push gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
+
+FROM pytorch/pytorch:2.0.0-cuda11.7-cudnn8-devel
+
+# Install tools.
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update
+RUN apt-get install -y --no-install-recommends apt-utils
+RUN apt-get install -y --no-install-recommends curl
+RUN apt-get install -y --no-install-recommends wget
+RUN apt-get install -y --no-install-recommends git
+RUN apt-get install -y --no-install-recommends jq
+RUN apt-get install -y --no-install-recommends gnupg
+RUN apt-get install -y --no-install-recommends build-essential
+
+ENV PIP_ROOT_USER_ACTION=ignore
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        curl \
+        wget \
+        vim
+
+# Copy license.
+RUN wget https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/LICENSE
+
+# Prepare artifacts.
+WORKDIR /workspace
+RUN git clone --branch main https://github.com/mlfoundations/open_clip.git
+WORKDIR ./open_clip
+RUN git reset --hard 67e5e5ec8741281eb9b30f640c26f91c666308b7
+
+# Install libraries.
+RUN pip install webdataset==0.2.5
+RUN pip install regex==2023.6.3
+RUN pip install ftfy==6.1.1
+RUN pip install pandas==2.0.3
+RUN pip install braceexpand==0.1.7
+RUN pip install huggingface_hub==0.16.4
+RUN pip install transformers==4.31.0
+RUN pip install timm==0.9.2
+RUN pip install fsspec==2023.6.0
+RUN pip install sentencepiece==0.1.99
+RUN pip install protobuf==3.20.3
+RUN pip install tensorboard==2.12.2
+
+# Switch work folder for training.
+WORKDIR ./src

community-content/vertex_vision_model_garden/model_oss/open_clip/handler.py

@@ -0,0 +1,142 @@
+"""Custom handler for OpenCLIP model."""
+
+# pylint:disable=g-importing-member
+import enum
+import logging
+import os
+from typing import Any, Dict, List
+
+import open_clip
+import torch
+from ts.torch_handler.base_handler import BaseHandler
+
+from google3.cloud.ml.applications.vision.model_garden.model_oss.util import constants
+from google3.cloud.ml.applications.vision.model_garden.model_oss.util import fileutils
+from google3.cloud.ml.applications.vision.model_garden.model_oss.util import image_format_converter
+
+
+@enum.unique
+class Precision(enum.Enum):
+  AMP = "amp"
+  AMP_BF16 = "amp_bf16"
+  AMP_BFLOAT16 = "amp_bfloat16"
+  BF16 = "bf16"
+  FP16 = "fp16"
+  PURE_BF16 = "pure_bf16"
+  PURE_FP16 = "pure_fp16"
+  FP32 = "fp32"
+
+
+# Supported checkpoint&model pairs:
+# https://github.com/mlfoundations/open_clip#pretrained-model-interface
+_DEFAULT_CHECKPOINT = "openai"
+_DEFAULT_MODEL = "RN50"
+_DEFAULT_PRECISION = Precision.AMP
+_ZERO_CLASSIFICATION = "zero-shot-image-classification"
+_FEATURE_EMBEDDING = "feature-embedding"
+_VALID_TASKS = frozenset([_ZERO_CLASSIFICATION, _FEATURE_EMBEDDING])
+
+_IMAGE_KEY = "image"
+_TEXT_KEY = "text"
+_IMAGE_FEATURES_KEY = "image_features"
+_TEXT_FEATURES_KEY = "text_features"
+
+
+class OpenclipHandler(BaseHandler):
+  """Custom handler for OpenCLIP."""
+
+  def initialize(self, context: Any):
+    """Custom initialize."""
+
+    properties = context.system_properties
+    self.map_location = (
+        "cuda"
+        if torch.cuda.is_available() and properties.get("gpu_id") is not None
+        else "cpu"
+    )
+    self.device = torch.device(
+        self.map_location + ":" + str(properties.get("gpu_id"))
+        if torch.cuda.is_available() and properties.get("gpu_id") is not None
+        else self.map_location
+    )
+    self.manifest = context.manifest
+
+    model_name = os.environ.get("MODEL", _DEFAULT_MODEL)
+    precision = os.environ.get("PRECISION", _DEFAULT_PRECISION)
+    checkpoint = os.environ.get("CHECKPOINT", _DEFAULT_CHECKPOINT)
+    self.task = os.environ.get("TASK", _FEATURE_EMBEDDING)
+    if self.task not in _VALID_TASKS:
+      raise ValueError(f"Invalid task: {self.task}.")
+    logging.info(
+        "Handler initializing task:%s, model:%s, precision:%s, checkpoint:%s",
+        self.task,
+        model_name,
+        precision,
+        checkpoint,
+    )
+
+    if checkpoint != _DEFAULT_CHECKPOINT:
+      local_fname = os.path.join(constants.LOCAL_MODEL_DIR, "model.pt")
+      fileutils.download_gcs_file_to_local(checkpoint, local_fname)
+      checkpoint = local_fname
+
+    self.model, _, self.preprocessor = open_clip.create_model_and_transforms(
+        model_name, pretrained=checkpoint, precision=precision
+    )
+    self.tokenizer = open_clip.get_tokenizer(model_name)
+
+    self.initialized = True
+
+  def preprocess(self, data: Any) -> List[Dict[str, Any]]:
+    """Preprocess input data."""
+    logging.info("preprocessing: %d instances received.", len(data))
+    processed_list = []
+    for item in data:
+      sample = {}
+      if _IMAGE_KEY in item:
+        sample[_IMAGE_KEY] = self.preprocessor(
+            image_format_converter.base64_to_image(item[_IMAGE_KEY])
+        ).unsqueeze(0)
+      if _TEXT_KEY in item:
+        sample[_TEXT_KEY] = self.tokenizer(item[_TEXT_KEY])
+      processed_list.append(sample)
+    return processed_list
+
+  def inference(
+      self, data: List[Dict[str, Any]], *args, **kwargs
+  ) -> List[Dict[str, Any]]:
+    feature_list = []
+    with torch.no_grad(), torch.cuda.amp.autocast():
+      for item in data:
+        sample = {}
+        if _IMAGE_KEY in item:
+          sample[_IMAGE_FEATURES_KEY] = self.model.encode_image(
+              item[_IMAGE_KEY]
+          )
+        if _TEXT_KEY in item:
+          sample[_TEXT_FEATURES_KEY] = self.model.encode_text(item[_TEXT_KEY])
+        feature_list.append(sample)
+    return feature_list
+
+  def postprocess(self, features: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    """Postprocess the image/text featreus for downstream task."""
+    preds = []
+    if self.task == _FEATURE_EMBEDDING:
+      for item in features:
+        preds.append({k: v.tolist() for k, v in item.items()})
+    elif self.task == _ZERO_CLASSIFICATION:
+      for item in features:
+        image_features = item.get(_IMAGE_FEATURES_KEY, None)
+        text_features = item.get(_TEXT_FEATURES_KEY, None)
+        if image_features is None or text_features is None:
+          raise ValueError(
+              "Missing input for {} task. {} received.".format(
+                  _ZERO_CLASSIFICATION, item.keys()
+              )
+          )
+        image_features /= image_features.norm(dim=-1, keepdim=True)
+        text_features /= text_features.norm(dim=-1, keepdim=True)
+        text_probs = (100.0 * image_features @ text_features.T).softmax(dim=-1)
+        preds.append(text_probs.tolist())
+
+    return preds

community-content/vertex_vision_model_garden/model_oss/pic2word/dockerfile/serve.Dockerfile

@@ -0,0 +1,115 @@
+FROM pytorch/torchserve:0.7.1-gpu
+
+USER root
+
+ENV infer_port=7080
+ENV mng_port=7081
+ENV model_name="pic2word"
+ENV PATH="/home/model-server/:${PATH}"
+
+# Copy license.
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        wget
+
+RUN wget https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/LICENSE
+
+# Install dependencies.
+ENV PIP_ROOT_USER_ACTION=ignore
+RUN python3 -m pip install --upgrade pip
+RUN pip install google-cloud-storage==2.7.0
+RUN pip install open_clip_torch==2.20.0
+RUN pip install numpy==1.22.0
+RUN pip install scikit-image==0.21.0
+RUN pip install scikit-learn==1.0.2
+RUN pip install torch==2.0.0
+RUN pip install torchvision==0.15.2
+RUN pip install tensorboard==2.13.0
+RUN pip install ase==3.21.1
+RUN pip install braceexpand==0.1.7
+RUN pip install cached-property==1.5.2
+RUN pip install configparser==5.0.2
+RUN pip install cycler==0.10.0
+RUN pip install decorator==4.4.2
+RUN pip install docker-pycreds==0.4.0
+RUN pip install gitdb==4.0.7
+RUN pip install gitpython==3.1.30
+RUN pip install googledrivedownloader==0.4
+RUN pip install h5py==3.1.0
+RUN pip install isodate==0.6.0
+RUN pip install jinja2==3.0.1
+RUN pip install kiwisolver==1.3.1
+RUN pip install littleutils==0.2.2
+RUN pip install llvmlite==0.36.0
+RUN pip install markupsafe==2.0.1
+RUN pip install matplotlib==3.3.4
+RUN pip install networkx==2.5.1
+RUN pip install numba==0.53.1
+RUN pip install ogb==1.3.1
+RUN pip install outdated==0.2.1
+RUN pip install pathtools==0.1.2
+RUN pip install promise==2.3
+RUN pip install psutil==5.8.0
+RUN pip install pyarrow==4.0.0
+RUN pip install pyparsing==2.4.7
+RUN pip install python-louvain==0.15
+RUN pip install pyyaml==5.4.1
+RUN pip install rdflib==5.0.0
+RUN pip install sentry-sdk==1.14.0
+RUN pip install shortuuid==1.0.1
+RUN pip install sklearn==0.0
+RUN pip install smmap==4.0.0
+RUN pip install subprocess32==3.5.4
+RUN pip install torch-geometric==1.7.0
+RUN pip install wandb==0.10.30
+RUN pip install wilds==1.1.0
+RUN pip install ftfy==6.1.1
+RUN pip install regex==2023.6.3
+RUN pip install webdataset==0.2.48
+RUN pip install requests==2.31.0
+RUN pip install hydra-core==1.3.2
+RUN pip install omegaconf==2.3.0
+RUN pip install fairseq==0.10.0
+RUN pip install bitarray==2.7.6
+
+# Get 'composed_image_retrieval' repository from github.
+RUN git clone https://github.com/google-research/composed_image_retrieval
+# Set workdir to composed_image_retrieval.
+WORKDIR ./composed_image_retrieval
+# Using git reset command to pin it down to a specific version.
+RUN git reset --hard 8c053297c2fae9cd17ddcded48445a4f47208dbd
+
+# Fix issue introduced by installing composed_image_retrieval
+# https://github.com/huggingface/transformers/issues/8638#issuecomment-790772391
+RUN pip uninstall dataclasses -y
+
+# Copy model artifacts.
+COPY model_oss/pic2word/handler.py /home/model-server/handler.py
+
+# Create torchserve configuration file.
+RUN echo \
+    "default_response_timeout=1800\n" \
+    "service_envelope=json\n" \
+    "inference_address=http://0.0.0.0:${infer_port}\n"  \
+    "management_address=http://0.0.0.0:${mng_port}" >> /home/model-server/config.properties
+
+# Expose ports.
+EXPOSE ${infer_port}
+EXPOSE ${mng_port}
+
+# Archive model artifacts and dependencies.
+# Do not set --model-file and --serialized-file because model and checkpoint
+# will be dynamically loaded in handler.py.
+RUN torch-model-archiver \
+--model-name=${model_name} \
+--version=1.0 \
+--handler=/home/model-server/handler.py \
+--runtime=python3 \
+--export-path=/home/model-server/model-store \
+--archive-format=default \
+--force
+
+# Run Torchserve HTTP serve to respond to prediction requests.
+CMD ["torchserve", "--start", \
+     "--ts-config", "/home/model-server/config.properties", \
+     "--models", "${model_name}=${model_name}.mar", \
+     "--model-store", "/home/model-server/model-store"]

community-content/vertex_vision_model_garden/model_oss/pic2word/handler.py

@@ -0,0 +1,167 @@
+"""Custom handler for Pic2Word."""
+
+from argparse import Namespace  # pylint: disable=g-importing-member
+import os
+from typing import Any
+
+from absl import logging
+from data import CustomFolder
+from eval_utils import visualize_results
+from model.clip import load
+from model.model import convert_weights
+from model.model import IM2TEXT
+from params import get_project_root
+import torch
+from torch.utils.data import DataLoader
+from ts.torch_handler.base_handler import BaseHandler
+
+from util import fileutils
+
+# The COCO dataset is stored in a publicly accessible bucket.
+_COCO_STORAGE_DIR = "gs://pic2word-bucket/data/coco/"
+_COCO_LOCAL_DIR = "/home/model-server/composed_image_retrieval/data/coco/"
+_COCO_VAL2017_PATH = "coco/val2017"
+_COCO_DATASET_NAME = "coco"
+_MODEL_NAME = "ViT-L/14"
+_LOCAL_QUERY_PATH = "./query/"
+_IMAGE_OUTPUT_LOCAL_DIR = "demo_out/images"
+_OUTPUT_LOCAL_DIR = "/demo_out/"
+_DATA_DIR = "data"
+_CHECKPOINT_DIR = "checkpoint/pic2word_model.pt"
+_REQUEST_PROMPTS = "prompts"
+_REQUEST_OUTPUT_STORAGE_DIR = "output_storage_dir"
+_REQUEST_IMAGE_PATH = "image_path"
+_REQUEST_IMAGE_FILE_NAME = "image_file_name"
+_RESPONSE_MSG = "Successfully retrieved images."
+
+
+class ModelHandler(BaseHandler):
+  """A custom model handler implementation."""
+
+  def __init__(self):
+    self.initialized = False
+    self.gpu = 0
+    self.model = None
+    self.dataloader = None
+    self.prompt = None
+    self.output_storage_dir = None
+
+  def initialize(self, context: Any):
+    """Initialize."""
+    logging.info("Initializing pic2word.")
+
+    # Download COCO dataset. The model looks for this folder specifically
+    # during image retrieval to generate a response for each request.
+    # This is a publicly accessible bucket.
+    fileutils.download_gcs_dir_to_local(
+        _COCO_STORAGE_DIR,
+        _COCO_LOCAL_DIR,
+    )
+
+    # Load the model.
+
+    self.initialized = True
+
+    torch.cuda.set_device(self.gpu)
+    model, _, preprocess_val = load(_MODEL_NAME, jit=False)
+
+    img2text = IM2TEXT(
+        embed_dim=model.embed_dim,
+        output_dim=model.token_embedding.weight.shape[1],
+    )
+
+    model.cuda(self.gpu)
+    img2text.cuda(self.gpu)
+
+    convert_weights(model)
+    convert_weights(img2text)
+
+    self.model = model
+    self.img2text = img2text
+
+    # Load the dataset
+    logging.info("Loading dataset.")
+
+    root_project = os.path.join(get_project_root(), _DATA_DIR)
+    dataset = CustomFolder(
+        os.path.join(root_project, _COCO_VAL2017_PATH), transform=preprocess_val
+    )
+
+    # Initialize the dataloader. This is used to create the pickle file from
+    # the dataset.
+    dataloader = DataLoader(
+        dataset,
+        batch_size=64,
+        shuffle=False,
+        num_workers=1,
+        pin_memory=True,
+        drop_last=False,
+    )
+
+    self.dataloader = dataloader
+
+    logging.info("Finished initializing Pic2Word server.")
+
+  def preprocess(self, data: Any) -> str:
+    """Preprocess input data."""
+    logging.info("Preprocessing Pic2Word inference request.")
+    query = data[0]
+
+    self.output_storage_dir = query[_REQUEST_OUTPUT_STORAGE_DIR]
+    prompts = query[_REQUEST_PROMPTS]
+    prompts = prompts.split(",")
+    self.prompt = prompts
+
+    image_path = query[_REQUEST_IMAGE_PATH]
+    # The query image is only supported via GCS bucket upload.
+    fileutils.download_gcs_dir_to_local(image_path, _LOCAL_QUERY_PATH)
+    image_file_name = query[_REQUEST_IMAGE_FILE_NAME]
+
+    query_file = f"./query/{image_file_name}"
+
+    logging.info("Setting model args.")
+
+    args = {
+        "openai-pretrained": True,
+        "resume": _CHECKPOINT_DIR,
+        "retrieval_data": _COCO_DATASET_NAME,
+        "query_file": query_file,
+        "demo_out": _OUTPUT_LOCAL_DIR,
+        "prompts": prompts,
+        "distributed": False,
+        "dp": False,
+        "gpu": 0,
+        "model": _MODEL_NAME,
+        "world_size": 1,
+    }
+    model_input = Namespace(**args)
+
+    logging.info("Finished preprocessing Pic2Word inference request.")
+    return model_input
+
+  def inference(self, model_input: Any):
+    """Runs inference."""
+    logging.info("Running model-inference.")
+    visualize_results(
+        model=self.model,
+        img2text=self.img2text,
+        args=model_input,
+        prompt=self.prompt,
+        dataloader=self.dataloader,
+    )
+
+  def postprocess(self):
+    """Upload the output images to the bucket."""
+    logging.info("Running request postprocess.")
+    fileutils.upload_local_dir_to_gcs(
+        _IMAGE_OUTPUT_LOCAL_DIR, self.output_storage_dir
+    )
+
+  def handle(self, data: Any, context: Any) -> str:  # pylint: disable=unused-argument
+    """Runs preprocess, inference, and post-processing."""
+    logging.info("Received Pic2Word inference request")
+    model_input = self.preprocess(data)
+    self.inference(model_input)
+    self.postprocess()
+    logging.info("Done handling input.")
+    return _RESPONSE_MSG

community-content/vertex_vision_model_garden/model_oss/transformers/dockerfile/serve.Dockerfile

@@ -0,0 +1,79 @@
+# Dockerfile for serving dockers for transformers.
+#
+# To build:
+# docker build -f model_oss/transformers/dockerfile/serve.Dockerfile . -t ${YOUR_IMAGE_TAG}
+#
+# To push to gcr:
+# docker tag ${YOUR_IMAGE_TAG} gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
+# docker push gcr.io/${YOUR_PROJECT}/${YOUR_IMAGE_TAG}
+# Switch to this base image for gpu serve.
+
+FROM pytorch/torchserve:0.7.0-gpu
+
+USER root
+
+ENV infer_port=7080
+ENV mng_port=7081
+ENV model_name="transformers_serving"
+ENV PATH="/home/model-server/:${PATH}"
+
+# Install libraries.
+ENV PIP_ROOT_USER_ACTION=ignore
+RUN python3 -m pip install --upgrade pip
+RUN pip install torch==1.13.1
+RUN pip install torchvision==0.14.1
+RUN pip install transformers==4.27.4
+RUN pip install datasets==2.9.0
+RUN pip install accelerate==0.17.0
+RUN pip install triton==2.0.0.dev20221120
+RUN pip install xformers==0.0.16
+RUN pip install google-cloud-storage==2.7.0
+RUN pip install absl-py==1.4.0
+
+# Install libraries for document-question-answering.
+RUN apt-get update
+RUN apt-get install -y --no-install-recommends tesseract-ocr
+RUN pip install tesseract==0.1.3
+RUN pip install pytesseract==0.3.10
+
+# Install tools.
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        curl \
+        wget \
+        vim
+
+# Copy license.
+RUN wget https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/LICENSE
+
+# Copy model artifacts.
+COPY model_oss/transformers/handler.py /home/model-server/handler.py
+COPY model_oss/util/ /home/model-server/util/
+ENV PYTHONPATH /home/model-server/
+
+# Create torchserve configuration file.
+RUN echo \
+    "default_response_timeout=1800\n" \
+    "service_envelope=json\n" \
+    "inference_address=http://0.0.0.0:${infer_port}\n"  \
+    "management_address=http://0.0.0.0:${mng_port}" >> /home/model-server/config.properties
+
+# Expose ports.
+EXPOSE ${infer_port}
+EXPOSE ${mng_port}
+
+# Archive model artifacts and dependencies.
+# Do not set --model-file and --serialized-file because model and checkpoint will be dynamically loaded in handler.py.
+RUN torch-model-archiver \
+--model-name=${model_name} \
+--version=1.0 \
+--handler=/home/model-server/handler.py \
+--runtime=python3 \
+--export-path=/home/model-server/model-store \
+--archive-format=default \
+--force
+
+# Run Torchserve HTTP serve to respond to prediction requests.
+CMD ["torchserve", "--start", \
+     "--ts-config", "/home/model-server/config.properties", \
+     "--models", "${model_name}=${model_name}.mar", \
+     "--model-store", "/home/model-server/model-store"]

community-content/vertex_vision_model_garden/model_oss/transformers/handler.py

@@ -0,0 +1,233 @@
+"""Custom handler for huggingface/transformers models."""
+
+# pylint: disable=g-multiple-import
+# pylint: disable=g-importing-member
+
+import logging
+import os
+from typing import Any, List, Optional, Tuple
+
+from PIL import Image
+import torch
+from transformers import (
+    AutoProcessor,
+    AutoTokenizer,
+    Blip2ForConditionalGeneration,
+    Blip2Processor,
+    BlipForConditionalGeneration,
+    BlipForQuestionAnswering,
+    BlipProcessor,
+    CLIPModel,
+)
+from transformers import pipeline
+from ts.torch_handler.base_handler import BaseHandler
+
+from util import constants
+from util import fileutils
+from util import image_format_converter
+
+DEFAULT_MODEL_ID = "openai/clip-vit-base-patch32"
+SALESFORCE_BLIP = "Salesforce/blip"
+SALESFORCE_BLIP2 = "Salesforce/blip2"
+FLAN_T5 = "flan-t5"
+BART_LARGE_CNN = "facebook/bart-large-cnn"
+
+ZERO_CLASSIFICATION = "zero-shot-image-classification"
+FEATURE_EMBEDDING = "feature-embedding"
+ZERO_DETECTION = "zero-shot-object-detection"
+IMAGE_CAPTIONING = "image-to-text"
+VQA = "visual-question-answering"
+DQA = "document-question-answering"
+SUMMARIZATION = "summarization"
+SUMMARIZATION_TEMPLATE = (
+    "Summarize the following news article:\n{input}\nSummary:\n"
+)
+
+
+class TransformersHandler(BaseHandler):
+  """Custom handler for huggingface/transformers models."""
+
+  def initialize(self, context: Any):
+    """Custom initialize."""
+
+    properties = context.system_properties
+    self.map_location = (
+        "cuda"
+        if torch.cuda.is_available() and properties.get("gpu_id") is not None
+        else "cpu"
+    )
+    self.device = torch.device(
+        self.map_location + ":" + str(properties.get("gpu_id"))
+        if torch.cuda.is_available() and properties.get("gpu_id") is not None
+        else self.map_location
+    )
+
+    self.manifest = context.manifest
+    # The model id is can be either:
+    # 1) a huggingface model card id, like "Salesforce/blip", or
+    # 2) a GCS path to the model files, like "gs://foo/bar".
+    # If it's a model card id, the model will be loaded from huggingface.
+    self.model_id = (
+        DEFAULT_MODEL_ID
+        if os.environ.get("MODEL_ID") is None
+        else os.environ["MODEL_ID"]
+    )
+    # Else it will be downloaded from GCS to local first.
+    # Since the transformers from_pretrained API can't read from GCS.
+    if self.model_id.startswith(constants.GCS_URI_PREFIX):
+      gcs_path = self.model_id[len(constants.GCS_URI_PREFIX) :]
+      local_model_dir = os.path.join(constants.LOCAL_MODEL_DIR, gcs_path)
+      logging.info("Download %s to %s", self.model_id, local_model_dir)
+      fileutils.download_gcs_dir_to_local(self.model_id, local_model_dir)
+      self.model_id = local_model_dir
+
+    self.task = (
+        ZERO_CLASSIFICATION
+        if os.environ.get("TASK") is None
+        else os.environ["TASK"]
+    )
+    logging.info(
+        "Handler initializing task:%s, model:%s", self.task, self.model_id
+    )
+
+    if SALESFORCE_BLIP in self.model_id:
+      # pipeline() hasn't been ready for Salesforce/blip models.
+      self.salesforce_blip = True
+      self._create_blip_model()
+    else:
+      self.salesforce_blip = False
+      if self.task == FEATURE_EMBEDDING:
+        self.model = CLIPModel.from_pretrained(self.model_id).to(
+            self.map_location
+        )
+        self.tokenizer = AutoTokenizer.from_pretrained(self.model_id)
+        self.processor = AutoProcessor.from_pretrained(self.model_id)
+      elif self.task == SUMMARIZATION and FLAN_T5 in self.model_id:
+        self.pipeline = pipeline(
+            task=self.task,
+            model=self.model_id,
+            torch_dtype=torch.bfloat16,
+            device_map="auto",
+        )
+      else:
+        self.pipeline = pipeline(
+            task=self.task, model=self.model_id, device=self.device
+        )
+
+    self.initialized = True
+    logging.info("Handler initialization done.")
+
+  def _create_blip_model(self):
+    """A helper for creating BLIP and BLIP2 models."""
+    if SALESFORCE_BLIP2 in self.model_id:
+      self.torch_type = torch.float16
+      self.processor = Blip2Processor.from_pretrained(self.model_id)
+      self.model = Blip2ForConditionalGeneration.from_pretrained(
+          self.model_id, torch_dtype=self.torch_type
+      ).to(self.map_location)
+    else:
+      self.torch_type = torch.float32
+      self.processor = BlipProcessor.from_pretrained(self.model_id)
+      if self.task == IMAGE_CAPTIONING:
+        self.model = BlipForConditionalGeneration.from_pretrained(
+            self.model_id
+        ).to(self.map_location)
+      elif self.task == VQA:
+        self.model = BlipForQuestionAnswering.from_pretrained(self.model_id).to(
+            self.map_location
+        )
+
+  def _reformat_detection_result(self, data: List[Any]) -> List[Any]:
+    """Reformat zero-shot-object-detection output."""
+    if not data:
+      return [data]
+    boxes = {}
+    boxes["label"] = data[0]["label"]
+    boxes["boxes"] = []
+    for item in data:
+      box = {}
+      box["score"] = item["score"]
+      box.update(item["box"])
+      boxes["boxes"].append(box)
+    outputs = [boxes]
+    return outputs
+
+  def preprocess(
+      self, data: Any
+  ) -> Tuple[Optional[List[str]], Optional[List[Image.Image]]]:
+    """Preprocess input data."""
+    texts = None
+    images = None
+    if "text" in data[0]:
+      texts = [item["text"] for item in data]
+    if "image" in data[0]:
+      images = [
+          image_format_converter.base64_to_image(item["image"]) for item in data
+      ]
+    return texts, images
+
+  def inference(self, data: Any, *args, **kwargs) -> List[Any]:
+    """Run the inference."""
+    texts, images = data
+    preds = None
+    if self.task == ZERO_CLASSIFICATION:
+      preds = self.pipeline(images=images, candidate_labels=texts)
+    elif self.task == ZERO_DETECTION:
+      # The object detection pipeline doesn't support batch prediction.
+      preds = self.pipeline(image=images[0], candidate_labels=texts[0])
+    elif self.task == IMAGE_CAPTIONING:
+      if self.salesforce_blip:
+        inputs = self.processor(images[0], return_tensors="pt").to(
+            self.map_location, self.torch_type
+        )
+        preds = self.model.generate(**inputs)
+        preds = [
+            self.processor.decode(preds[0], skip_special_tokens=True).strip()
+        ]
+      else:
+        preds = self.pipeline(images=images)
+    elif self.task == VQA:
+      # The VQA pipelines doesn't support batch prediction.
+      if self.salesforce_blip:
+        inputs = self.processor(images[0], texts[0], return_tensors="pt").to(
+            self.map_location, self.torch_type
+        )
+        preds = self.model.generate(**inputs)
+        preds = [
+            self.processor.decode(preds[0], skip_special_tokens=True).strip()
+        ]
+      else:
+        preds = self.pipeline(image=images[0], question=texts[0])
+    elif self.task == DQA:
+      # The DQA pipelines doesn't support batch prediction.
+      preds = self.pipeline(image=images[0], question=texts[0])
+    elif self.task == FEATURE_EMBEDDING:
+      preds = {}
+      if texts:
+        inputs = self.tokenizer(
+            text=texts, padding=True, return_tensors="pt"
+        ).to(self.map_location)
+        text_features = self.model.get_text_features(**inputs)
+        preds["text_features"] = text_features.detach().cpu().numpy().tolist()
+      if images:
+        inputs = self.processor(images=images, return_tensors="pt").to(
+            self.map_location
+        )
+        image_features = self.model.get_image_features(**inputs)
+        preds["image_features"] = image_features.detach().cpu().numpy().tolist()
+      preds = [preds]
+    elif self.task == SUMMARIZATION and FLAN_T5 in self.model_id:
+      texts = [SUMMARIZATION_TEMPLATE.format(input=text) for text in texts]
+      preds = self.pipeline(texts, max_length=130)
+    elif self.task == SUMMARIZATION and self.model_id == BART_LARGE_CNN:
+      preds = self.pipeline(
+          texts[0], max_length=130, min_length=30, do_sample=False
+      )
+    else:
+      raise ValueError(f"Invalid TASK: {self.task}")
+    return preds
+
+  def postprocess(self, data: Any) -> List[Any]:
+    if self.task == ZERO_DETECTION:
+      data = self._reformat_detection_result(data)
+    return data

community-content/vertex_vision_model_garden/model_oss/util/commons.py

@@ -0,0 +1,79 @@
+"""Common utility lib for prediction on images."""
+
+from typing import Any, Dict, List
+
+import numpy as np
+from PIL import Image
+import tensorflow as tf
+import yaml
+
+from util import image_format_converter
+
+
+def get_prediction_instances(image: Image.Image) -> List[Dict[str, Any]]:
+  """Gets prediction instances.
+
+  Args:
+    image: Image instance.
+
+  Returns:
+    List[Dict[str, Any]]: List of prediction instances.
+  """
+  instances = [{
+      "encoded_image": {"b64": image_format_converter.image_to_base64(image)},
+  }]
+  return instances
+
+
+def get_label_map(label_map_yaml_filepath: str) -> Dict[str, Any]:
+  """Gets the label map from a YAML file.
+
+  Args:
+      label_map_yaml_filepath: Filepath to the label map YAML file.
+
+  Returns:
+      dict: Label map.
+  """
+  with tf.io.gfile.GFile(label_map_yaml_filepath, "rb") as input_file:
+    label_map = yaml.safe_load(input_file.read())
+  return label_map
+
+
+def get_object_detection_endpoint_predictions(
+    detection_endpoint: ...,
+    input_image: np.ndarray,
+    detection_thresh: float = 0.2,
+) -> np.ndarray:
+  """Gets endpoint predictions.
+
+  Args:
+      detection_endpoint: image object detection endpoint.
+      input_image: Input image.
+      detection_thresh: Detection threshold.
+
+  Returns:
+      Object detection predictions from endpoints.
+  """
+  height, width, _ = input_image.shape
+  predictions = detection_endpoint.predict(
+      get_prediction_instances(Image.fromarray(input_image))
+  ).predictions
+  detection_scores = np.array(predictions[0]["detection_scores"])
+  detection_classes = np.array(predictions[0]["detection_classes"])
+  detection_boxes = np.array(
+      [
+          [b[1] * width, b[0] * height, b[3] * width, b[2] * height]
+          for b in predictions[0]["detection_boxes"]
+      ]
+  )
+  thresh_indices = [
+      x for x, val in enumerate(detection_scores) if val > detection_thresh
+  ]
+  preds_merge_conf = np.column_stack((
+      detection_boxes[thresh_indices],
+      detection_scores[thresh_indices],
+  ))
+  preds_merge_cls = np.column_stack(
+      (preds_merge_conf, detection_classes[thresh_indices])
+  )
+  return preds_merge_cls

community-content/vertex_vision_model_garden/model_oss/util/constants.py

@@ -0,0 +1,73 @@
+"""Vertex vision model garden util constants."""
+
+# Objectives.
+OBJECTIVE_IMAGE_CLASSIFICATION = 'icn'
+OBJECTIVE_IMAGE_OBJECT_DETECTION = 'iod'
+OBJECTIVE_IMAGE_SEGMENTATION = 'isg'
+OBJECTIVE_VIDEO_CLASSIFICATION = 'vcn'
+OBJECTIVE_VIDEO_ACTION_RECOGNITION = 'var'
+
+# Input file types.
+INPUT_FILE_TYPE_CSV = 'csv'
+INPUT_FILE_TYPE_JSONL = 'jsonl'
+INPUT_FILE_TYPE_COCO_JSON = 'coco_json'
+
+# Output file types.
+OUTPUT_FILE_TYPE_TFRECORD = 'tfrecord'
+OUTPUT_FILE_TYPE_COCO_JSON = 'coco_json'
+
+# Best evaluation metrics.
+IMAGE_CLASSIFICATION_SINGLE_LABEL_BEST_EVAL_METRIC = 'accuracy'
+IMAGE_CLASSIFICATION_MULTI_LABEL_BEST_EVAL_METRIC = 'meanPR-AUC'
+
+IMAGE_OBJECT_DETECTION_BEST_EVAL_METRIC = 'AP50'
+IMAGE_SEGMENTATION_BEST_EVAL_METRIC = 'mean_iou'
+
+VIDEO_CLASSIFICATION_BEST_EVAL_METRIC = 'accuracy'
+
+# Best checkpoints.
+BEST_CKPT_DIRNAME = 'best_ckpt'
+BEST_CKPT_EVAL_FILENAME = 'info.json'
+BEST_CKPT_STEP_NAME = 'best_ckpt_global_step'
+BEST_CKPT_METRIC_COMP = 'higher'
+
+# Reported hyperparameter tuning metric tag.
+HP_METRIC_TAG = 'model_performance'
+
+# HPT trial prefix.
+TRIAL_PREFIX = 'trial_'
+
+# ML uses from user input.
+ML_USE_TRAINING = 'training'
+ML_USE_VALIDATION = 'validation'
+ML_USE_TEST = 'test'
+
+# COCO json keys
+COCO_JSON_ANNOTATIONS = 'annotations'
+COCO_JSON_ANNOTATION_IMAGE_ID = 'image_id'
+COCO_JSON_ANNOTATION_CATEGORY_ID = 'category_id'
+COCO_JSON_CATEGORIES = 'categories'
+COCO_JSON_CATEGORY_ID = 'id'
+COCO_JSON_CATEGORY_NAME = 'name'
+COCO_JSON_FILE_NAME = 'file_name'
+COCO_JSON_IMAGES = 'images'
+COCO_JSON_IMAGE_ID = 'id'
+COCO_JSON_IMAGE_WIDTH = 'width'
+COCO_JSON_IMAGE_HEIGHT = 'height'
+COCO_JSON_IMAGE_COCO_URL = 'coco_url'
+COCO_ANNOTATION_BBOX = 'bbox'
+
+# GCS prefixes
+GCS_URI_PREFIX = 'gs://'
+GCSFUSE_URI_PREFIX = '/gcs/'
+
+LOCAL_EVALUATION_RESULT_DIR = '/tmp/evaluation_result_dir'
+LOCAL_MODEL_DIR = '/tmp/model_dir'
+LOCAL_BASE_MODEL_DIR = '/tmp/base_model_dir'
+LOCAL_DATA_DIR = '/tmp/data'
+
+# PEFT finetuning constants.
+TEXT_TO_IMAGE_LORA = 'text-to-image-lora'
+SEQUENCE_CLASSIFICATION_LORA = 'sequence-classification-lora'
+CAUSAL_LANGUAGE_MODELING_LORA = 'causal-language-modeling-lora'
+INSTRUCT_LORA = 'instruct-lora'

community-content/vertex_vision_model_garden/model_oss/util/fileutils.py

@@ -0,0 +1,246 @@
+"""Fileutil lib to copy files between gcs and local."""
+
+import glob
+import os
+import pathlib
+import shutil
+from typing import Tuple
+import uuid
+
+from absl import logging
+from google.cloud import storage
+
+from util import constants
+
+
+def generate_tmp_path(extension: str = '') -> str:
+  """Generates a temporary file path with UUID.
+
+  Args:
+    extension: File extension, e.g. '.jpg', '.avi'. If not given, no extension
+      will be appended to the filename.
+
+  Returns:
+    Generated file path.
+  """
+  return os.path.join(constants.LOCAL_DATA_DIR, uuid.uuid1().hex) + extension
+
+
+def force_gcs_fuse_path(gcs_uri: str) -> str:
+  """Converts gs:// uris to their /gcs/ equivalents. No-op for other uris."""
+  if is_gcs_path(gcs_uri):
+    return (
+        constants.GCSFUSE_URI_PREFIX + gcs_uri[len(constants.GCS_URI_PREFIX) :]
+    )
+  else:
+    return gcs_uri
+
+
+def download_gcs_file_to_local_dir(gcs_uri: str, local_dir: str):
+  """Download a gcs file to a local dir.
+
+  Args:
+    gcs_uri: A string of file path on GCS.
+    local_dir: A string of local directory.
+  """
+  if not is_gcs_path(gcs_uri):
+    raise ValueError(
+        f'{gcs_uri} is not a GCS path starting with {constants.GCS_URI_PREFIX}.'
+    )
+  filename = os.path.basename(gcs_uri)
+  download_gcs_file_to_local(gcs_uri, os.path.join(local_dir, filename))
+
+
+def download_gcs_file_to_local(gcs_uri: str, local_path: str):
+  """Download a gcs file to a local path.
+
+  Args:
+    gcs_uri: A string of file path on GCS.
+    local_path: A string of local file path.
+  """
+  if not is_gcs_path(gcs_uri):
+    raise ValueError(
+        f'{gcs_uri} is not a GCS path starting with {constants.GCS_URI_PREFIX}.'
+    )
+  client = storage.Client()
+  os.makedirs(os.path.dirname(local_path), exist_ok=True)
+  with open(local_path, 'wb') as f:
+    client.download_blob_to_file(gcs_uri, f)
+
+
+def download_gcs_dir_to_local(gcs_dir: str, local_dir: str):
+  """Downloads files in a GCS directory to a local directory.
+
+  For example:
+    download_gcs_dir_to_local(gs://bucket/foo, /tmp/bar)
+    gs://bucket/foo/a -> /tmp/bar/a
+    gs://bucket/foo/b/c -> /tmp/bar/b/c
+
+  Arguments:
+    gcs_dir: A string of directory path on GCS.
+    local_dir: A string of local directory path.
+  """
+  if not is_gcs_path(gcs_dir):
+    raise ValueError(f'{gcs_dir} is not a GCS path starting with gs://.')
+  bucket_name = gcs_dir.split('/')[2]
+  prefix = gcs_dir[len(constants.GCS_URI_PREFIX + bucket_name) :].strip('/')
+  client = storage.Client()
+  blobs = client.list_blobs(bucket_name, prefix=prefix)
+  for blob in blobs:
+    if blob.name[-1] == '/':
+      continue
+    file_path = blob.name[len(prefix) :].strip('/')
+    local_file_path = os.path.join(local_dir, file_path)
+    os.makedirs(os.path.dirname(local_file_path), exist_ok=True)
+    logging.info('Downloading %s to %s', file_path, local_file_path)
+    blob.download_to_filename(local_file_path)
+
+
+def upload_local_dir_to_gcs(local_dir: str, gcs_dir: str):
+  """Uploads local dir to gcs.
+
+  For example:
+    upload_local_dir_to_gcs(/tmp/bar, gs://bucket/foo)
+    gs://bucket/foo/a -> /tmp/bar/a
+    gs://bucket/foo/b/c -> /tmp/bar/b/c
+
+  Arguments:
+    local_dir: A string of local directory path.
+    gcs_dir: A string of directory path on GCS.
+  """
+  bucket_name = gcs_dir.split('/')[2]
+  blob_dir = '/'.join(gcs_dir.split('/')[3:])
+  client = storage.Client()
+  bucket = client.bucket(bucket_name)
+  for local_file in glob.glob(local_dir + '/**'):
+    if os.path.isfile(local_file):
+      logging.info(
+          'Uploading %s to %s',
+          local_file,
+          os.path.join(constants.GCS_URI_PREFIX, bucket_name, blob_dir),
+      )
+      blob = bucket.blob(os.path.join(blob_dir, os.path.basename(local_file)))
+      blob.upload_from_filename(local_file)
+
+
+def upload_file_to_gcs_path(
+    source_path: str,
+    destination_uri: str,
+):
+  """Uploads local files to GCS uri.
+
+  After upload the destination_uri will contain the same data as the
+  source_path.
+
+  Args:
+      source_path: Required. Path of the local data to copy to GCS.
+      destination_uri: Required. GCS URI where the data should be uploaded.
+
+  Raises:
+      RuntimeError: When source_path does not exist.
+      GoogleCloudError: When the upload process fails.
+  """
+  source_path_obj = pathlib.Path(source_path)
+  if not source_path_obj.exists():
+    raise RuntimeError(f'Source path does not exist: {source_path}')
+
+  storage_client = storage.Client()
+  source_file_path = source_path
+  destination_file_uri = destination_uri
+  logging.info('Uploading "%s" to "%s"', source_file_path, destination_file_uri)
+  destination_blob = storage.Blob.from_string(
+      destination_file_uri, client=storage_client
+  )
+  destination_blob.upload_from_filename(filename=source_file_path)
+
+
+def is_gcs_path(input_path: str) -> bool:
+  """Checks if the input path is a Google Cloud Storage (GCS) path.
+
+  Args:
+      input_path: The input path to be checked.
+
+  Returns:
+      True if the input path is a GCS path, False otherwise.
+  """
+  return input_path.startswith(constants.GCS_URI_PREFIX)
+
+
+def release_text_assets(
+    output_bucket: str, local_text_file_name: str, remote_text_file_name: str
+) -> None:
+  """Releases text assets.
+
+  Args:
+    output_bucket: gcs output bucket.
+    local_text_file_name: Local text file name.
+    remote_text_file_name: Remote text file name.
+
+  Returns:
+    None
+  """
+  remote_file_path = '{}/{}'.format(output_bucket, remote_text_file_name)
+  logging.info('Uploading "%s" to "%s"', local_text_file_name, remote_file_path)
+  upload_file_to_gcs_path(local_text_file_name, remote_file_path)
+  os.remove(local_text_file_name)
+
+
+def upload_video_from_local_to_gcs(
+    output_bucket: str,
+    local_video_file_name: str,
+    remote_video_file_name: str,
+    temp_local_video_file_name: str,
+) -> None:
+  """Uploads video from local to gcs buckent and releases video assets.
+
+  Args:
+    output_bucket: GCS bucket address.
+    local_video_file_name: Local video file name.
+    remote_video_file_name: Remote video file name.
+    temp_local_video_file_name: Temporary local video file name.
+
+  Returns:
+    None
+  """
+  upload_file_to_gcs_path(
+      temp_local_video_file_name,
+      '{}/{}'.format(output_bucket, remote_video_file_name),
+  )
+  shutil.rmtree(local_video_file_name, ignore_errors=True)
+  shutil.rmtree(temp_local_video_file_name, ignore_errors=True)
+
+
+def download_video_from_gcs_to_local(video_file_path: str) -> Tuple[str, str]:
+  """Downloads video from gcs to local folders.
+
+  Args:
+    video_file_path: Path to the video file.
+
+  Returns:
+    Local and remote video file paths.
+  """
+  _, local_video_file_name = os.path.split(video_file_path)
+  file_extension = os.path.splitext(video_file_path)[1]
+  remote_video_file_name = local_video_file_name.replace(
+      file_extension, '_overlay.mp4'
+  )
+  local_file_path = generate_tmp_path(os.path.splitext(video_file_path)[1])
+  logging.info('Downloading %s to %s...', video_file_path, local_file_path)
+  download_gcs_file_to_local(video_file_path, local_file_path)
+  return local_file_path, remote_video_file_name
+
+
+def get_output_video_file(video_output_file_path: str) -> str:
+  """Gets the output video file name for writing video.
+
+  Args:
+    video_output_file_path: Path to the video output file.
+
+  Returns:
+    str: Local video output file path.
+  """
+  file_extension = os.path.splitext(video_output_file_path)[1]
+  out_local_video_file_name = video_output_file_path.replace(
+      file_extension, '_overlay' + file_extension
+  )
+  return out_local_video_file_name

community-content/vertex_vision_model_garden/model_oss/util/hypertune_util.py

@@ -0,0 +1,21 @@
+"""Utility functions for Vertex Hyperparameter Tuning Jobs."""
+
+import os
+
+from absl import logging
+
+_ENVIRONMENT_VARIABLE_FOR_TRIAL_ID = 'CLOUD_ML_TRIAL_ID'
+
+
+def get_trial_id_from_environment() -> str:
+  """Gets the trial id from environment variable.
+
+  Returns:
+    The trial id from environement or '0' if not found.
+  """
+  if _ENVIRONMENT_VARIABLE_FOR_TRIAL_ID not in os.environ:
+    logging.warning(
+        'Environment variable %s not found, return 0 as default trial id.',
+        _ENVIRONMENT_VARIABLE_FOR_TRIAL_ID,
+    )
+  return os.environ.get(_ENVIRONMENT_VARIABLE_FOR_TRIAL_ID, '0')

community-content/vertex_vision_model_garden/model_oss/util/hypertune_utils.py

@@ -0,0 +1,22 @@
+"""Utility functions for Vertex Hyperparameter Tuning Jobs."""
+
+import os
+
+from absl import logging
+
+
+_ENVIRONMENT_VARIABLE_FOR_TRIAL_ID = 'CLOUD_ML_TRIAL_ID'
+
+
+def get_trial_id_from_environment() -> str:
+  """Gets the trial id from environment variable.
+
+  Returns:
+    The trial id from environement or '0' if not found.
+  """
+  if _ENVIRONMENT_VARIABLE_FOR_TRIAL_ID not in os.environ:
+    logging.warning(
+        'Environment variable %s not found, return 0 as default trial id.',
+        _ENVIRONMENT_VARIABLE_FOR_TRIAL_ID,
+    )
+  return os.environ.get(_ENVIRONMENT_VARIABLE_FOR_TRIAL_ID, '0')

community-content/vertex_vision_model_garden/model_oss/util/image_format_converter.py

@@ -0,0 +1,20 @@
+"""Image format converter util lib."""
+
+import base64
+import io
+
+from PIL import Image
+
+
+def image_to_base64(image: Image.Image) -> str:
+  """Convert a PIL image to a base64 string."""
+  buffer = io.BytesIO()
+  image.save(buffer, format="JPEG")
+  image_str = base64.b64encode(buffer.getvalue()).decode("utf-8")
+  return image_str
+
+
+def base64_to_image(image_str: str) -> Image.Image:
+  """Convert a base64 string to a PIL image."""
+  image = Image.open(io.BytesIO(base64.b64decode(image_str)))
+  return image

notebooks/community/CODEOWNERS

@@ -33,6 +33,7 @@
 /notebooks/community/cohere/cohere_embedding_with_matching_engine.ipynb @stewart-co
 /notebooks/community/ml_ops/stage2/get_started_vertex_hpt_r_kernel.ipynb @fhirschmann
 /notebooks/community/ml_ops/stage2/get_started_vertex_training_r_using_r_kernel.ipynb @fhirschmann
+/notebooks/community/ml_ops/stage3/get_started_with_dataflow_flex_template_component.ipynb @wintwoo
 /notebooks/community/vertex-ai-samples/notebooks/community/model_registry/vertex_ai_model_registry_bqml_custom_model_versioning.ipynb @inardini
 /notebooks/community/vertex-ai-samples/notebooks/community/model_registry/vertex_ai_model_registry_automl_model_versioning.ipynb @inardini
 /notebooks/community/vizier/conversions_vertex_vizier_and_open_source_vizier.ipynb @halio-g
@@ -40,4 +41,44 @@
 /notebooks/community/pipelines/google_cloud_pipeline_components_bqml_pipeline_anomaly_detection.ipynb @inardini
 /notebooks/community/pipelines/google_cloud_pipeline_components_cloud_natural_language_pipeline.ipynb @Narwhalprime
 /notebooks/community/pipelines/google_cloud_pipeline_components_ready_to_go_text_classification_pipeline.ipynb @Narwhalprime
+/notebooks/community/feature_store/get_started_vertex_feature_store.ipynb @junkourata
+/notebooks/community/model_garden/model_garden_huggingface_local_inference.ipynb @dstnluong-google
+/notebooks/community/model_garden/model_garden_mediapipe_image_classification.ipynb @schmidt-sebastian
+/notebooks/community/model_garden/model_garden_mediapipe_gesture_recognition.ipynb @schmidt-sebastian
+/notebooks/community/model_garden/model_garden_mediapipe_object_detection.ipynb @schmidt-sebastian
+/notebooks/community/model_garden/model_garden_mediapipe_text_classification.ipynb @schmidt-sebastian
+/notebooks/community/model_garden/model_garden_proprietary_image_classification.ipynb @weigary
+/notebooks/community/model_garden/model_garden_proprietary_image_object_detection.ipynb @weigary
+/notebooks/community/model_garden/model_garden_tfvision_image_classification.ipynb @genquan9
+/notebooks/community/model_garden/model_garden_tfvision_image_object_detection.ipynb @genquan9
+/notebooks/community/model_garden/model_garden_tfvision_image_segmentation.ipynb @genquan9
+/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion.ipynb @xiangxu-google
+/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion_2_1.ipynb @bingatgoogle
+/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion_inpainting.ipynb @xiangxu-google
+/notebooks/community/model_garden/model_garden_pytorch_instructpix2pix.ipynb @xiangxu-google
+/notebooks/community/model_garden/model_garden_pytorch_controlnet.ipynb @xiangxu-google
+/notebooks/community/model_garden/model_garden_pytorch_blip_image_captioning.ipynb @xiangxu-google
+/notebooks/community/model_garden/model_garden_pytorch_blip_vqa.ipynb @xiangxu-google
+/notebooks/community/model_garden/model_garden_pytorch_vilt_vqa.ipynb @xiangxu-google
+/notebooks/community/model_garden/model_garden_pytorch_vit_gpt2_image_captioning.ipynb @xiangxu-google
+/notebooks/community/model_garden/model_garden_pytorch_clip.ipynb @xiangxu-google
+/notebooks/community/model_garden/model_garden_pytorch_owlvit.ipynb @xiangxu-google
+/notebooks/community/model_garden/model_garden_pytorch_layoutml_document_qa.ipynb @xiangxu-google
+/notebooks/community/model_garden/model_garden_pytorch_blip2.ipynb @xiangxu-google
+/notebooks/community/model_garden/model_garden_pytorch_detectron2.ipynb @lavraicse
+/notebooks/community/model_garden/model_garden_pytorch_dolly_v2.ipynb @lavraicse
+/notebooks/community/model_garden/model_garden_pytorch_bart_large_cnn.ipynb @lavraicse
+/notebooks/community/model_garden/model_garden_jax_vision_transformer.ipynb @lavraicse
+/notebooks/community/model_garden/model_garden_pytorch_text_to_video_zero_shot.ipynb @bingatgoogle
+/notebooks/community/model_garden/model_garden_pytorch_text_to_video.ipynb @KCFindstr
+/notebooks/community/generative_ai/text_embedding_api_semantic_search_with_scann.ipynb @henrytansetiawan
+/notebooks/community/bigquery_ml_inference/bq_ml_with_vision_translation_nlp.ipynb @deaconsmith
+/notebooks/community/model_garden/model_garden_keras_stable_diffusion.ipynb @genquan9
+/notebooks/community/model_garden/model_garden_pytorch_sam.ipynb @huguensjean
+/notebooks/community/model_garden/model_garden_pytorch_pic2word.ipynb @jismailyan
+/notebooks/community/model_garden/model_garden_pytorch_peft.ipynb @genquan9
+/notebooks/community/model_garden/model_garden_pytorch_openllama_peft.ipynb @genquan9
+/notebooks/community/model_garden/model_garden_pytorch_falcon_instruct_peft.ipynb @genquan9
+/notebooks/community/model_garden/model_garden_movinet_clip_classification.ipynb @KCFindstr
+/notebooks/community/model_garden/model_garden_pytorch_open_clip.ipynb @lydhr
 

notebooks/community/bigquery_ml/README.md

@@ -0,0 +1,19 @@
+
+[Unstructured data analytics with BigQuery ML and Vertex AI pre-trained models](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/bigquery_ml/bq_ml_with_vision_translation_nlp.ipynb)
+
+```
+Learn how to analyze unstructured data within BigQuery using BigQuery's inference engine. You will use BigQuery ML to connect to three pretrained Vertex AI APIs - Vision API, Translation API and Natural Language Processing API.
+
+The steps performed include:
+
+- Define pre-trained models for Vision AI, Translation AI and NLP AI in BigQuery ML
+- Call the Vision API (`ML.ANNOTATE_IMAGE`) to detect text in images stored in Cloud Storage
+  You will need to create an object table in BigQuery to do this
+- Call the Translation API (`ML.TRANSLATE`) to detect the language of text, and translate non-English movie titles to English
+- Call the Natural Language API (`ML.UNDERSTAND_TEXT`) to run sentiment analysis over movie reviews stored in BigQuery
+
+```
+
+   Check out the [blog for this notebook](https://cloud.google.com/blog/products/data-analytics/how-simplify-unstructured-data-analytics-using-bigquery-ml-and-vertex-ai).
+   Learn more about [BigQuery ML inference engine](https://cloud.google.com/bigquery/docs/reference/standard-sql/inference-overview).
+

notebooks/community/generative_ai/text_embedding_api_semantic_search_with_scann.ipynb

@@ -0,0 +1,373 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "KSP1duKDeaDR",
+      "metadata": {
+        "id": "KSP1duKDeaDR"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2023 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "67b2c5cc-8fc6-4082-9052-69fa0377d770",
+      "metadata": {
+        "id": "67b2c5cc-8fc6-4082-9052-69fa0377d770"
+      },
+      "source": [
+        "# Semantic Search using Embeddings\n",
+        "\n",
+        "Semantic search is a type of search that uses the meaning of words and phrases to find relevant results.\n",
+        "\n",
+        "In this tutorial, we will demonstrate how to do semantic search with embeddings generated from the news text and using [Google ScaNN: Efficient Vector Similarity Search](https://ai.googleblog.com/2020/07/announcing-scann-efficient-vector.html) to retrieve the most relevant news semantically.\n",
+        "\n",
+        "## Pre-requisites:\n",
+        "- Vertex LLM SDK\n",
+        "- ScaNN [github](https://github.com/google-research/google-research/tree/master/scann)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "FyyMdUeAJIVv",
+      "metadata": {
+        "id": "FyyMdUeAJIVv"
+      },
+      "source": [
+        "## Install Vertex LLM SDK\n",
+        "\n",
+        "DISCLAIMER: Text Embedding API is now in Experimental Preview. This release focuses on validating model prototypes and these models are not guaranteed to be released. Use of Text Embedding API is governed by the Google Cloud Terms of Service, the Pre-GA Offerings Terms of the GCP Service Specific Terms. The Acceptance Use Policy, and the Generative AI Prohibited Use Policy. Vertex Text Embedding API’s features may be unstable, change in backward-incompatible ways, and are not guaranteed to be released. There are no SLAs provided and no technical support obligations. GCP’s Cloud Data Processing Addendum does not apply to Pre-GA Offerings and customers should not use Text Embedding API to process personal data or other data subject to legal or regulatory compliance requirements. See description of launch stage for details.\n",
+        "\n",
+        "The information in this documentation is provided to the customer on an “as is” and “with all faults” basis without any warranty of any kind, either express or implied. Google does not warrant or guarantee the correctness, accuracy or reliability of the information in here. In no event will Google or its affiliates or licensors be liable for any damage or harm to customers from customer’s use of these materials."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "h6KaE3XRJdxc",
+      "metadata": {
+        "id": "h6KaE3XRJdxc"
+      },
+      "outputs": [],
+      "source": [
+        "from google.colab import auth as google_auth\n",
+        "\n",
+        "google_auth.authenticate_user()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "snBUuUamoJPz",
+      "metadata": {
+        "id": "snBUuUamoJPz"
+      },
+      "outputs": [],
+      "source": [
+        "!pip3 install google-cloud-aiplatform>=1.25 \"shapely<2.0.0\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "zgVQcE0ewO8W",
+      "metadata": {
+        "id": "zgVQcE0ewO8W"
+      },
+      "outputs": [],
+      "source": [
+        "PROJECT_ID = \"cloud-nl-llm-embedding\"  # @param {type:\"string\"}\n",
+        "LOCATION = \"us-central1\"  # @param {type:\"string\"}\n",
+        "\n",
+        "import vertexai\n",
+        "\n",
+        "vertexai.init(project=PROJECT_ID, location=LOCATION)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "4xFzXmPbY7FC",
+      "metadata": {
+        "id": "4xFzXmPbY7FC"
+      },
+      "source": [
+        "**Attention**: you would need to restart runtime so that the right package is installed."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "xnfG88OPZI18",
+      "metadata": {
+        "id": "xnfG88OPZI18"
+      },
+      "source": [
+        "## Import TextEmbeddingModel"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "1e6e1b98-a632-44a2-afb8-fc212018ef4f",
+      "metadata": {
+        "id": "1e6e1b98-a632-44a2-afb8-fc212018ef4f"
+      },
+      "outputs": [],
+      "source": [
+        "from vertexai.preview.language_models import TextEmbeddingModel\n",
+        "\n",
+        "model = TextEmbeddingModel.from_pretrained(\"textembedding-gecko@001\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "1suA-1HuaGj6",
+      "metadata": {
+        "id": "1suA-1HuaGj6"
+      },
+      "source": [
+        "## Install ScaNN Package"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "770255d3-54dd-48c8-bbdd-fbc0be41f085",
+      "metadata": {
+        "id": "770255d3-54dd-48c8-bbdd-fbc0be41f085"
+      },
+      "outputs": [],
+      "source": [
+        "!pip install scann"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "dae340cb-0583-4e7e-a562-6817ee4d7f6d",
+      "metadata": {
+        "id": "dae340cb-0583-4e7e-a562-6817ee4d7f6d"
+      },
+      "source": [
+        "## Imports packages"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "412d00f1-08db-4880-8ced-52a9583757b8",
+      "metadata": {
+        "id": "412d00f1-08db-4880-8ced-52a9583757b8"
+      },
+      "outputs": [],
+      "source": [
+        "import json\n",
+        "import time\n",
+        "\n",
+        "import numpy as np\n",
+        "import pandas as pd\n",
+        "import scann"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "f50f22f3-ec85-463e-b6fe-5c8e6b80b07b",
+      "metadata": {
+        "id": "f50f22f3-ec85-463e-b6fe-5c8e6b80b07b"
+      },
+      "source": [
+        "## Create Embedding Dataset.\n",
+        "\n",
+        "The dataset is solely to demonstrate the use of the Text Embedding API with a vector database. It is not intended to be used for any other purpose, such as evaluating models. The dataset is small and does not represent a comprehensive sample of all possible text."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "2OUg-Qf8iFta",
+      "metadata": {
+        "id": "2OUg-Qf8iFta"
+      },
+      "outputs": [],
+      "source": [
+        "!gsutil cp gs://cloud-samples-data/vertex-ai/dataset-management/datasets/bert_finetuning/wide_and_deep_trainer_container_tests_input.jsonl ."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "BNPapKXviHlE",
+      "metadata": {
+        "id": "BNPapKXviHlE"
+      },
+      "outputs": [],
+      "source": [
+        "records = []\n",
+        "with open(\"wide_and_deep_trainer_container_tests_input.jsonl\") as f:\n",
+        "    for line in f:\n",
+        "        record = json.loads(line)\n",
+        "        records.append(record)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "Z1Y9Bx2miJba",
+      "metadata": {
+        "id": "Z1Y9Bx2miJba"
+      },
+      "outputs": [],
+      "source": [
+        "# Peek at the data.\n",
+        "df = pd.DataFrame(records)\n",
+        "df.head(50)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "1v7zUwoWiPl-",
+      "metadata": {
+        "id": "1v7zUwoWiPl-"
+      },
+      "outputs": [],
+      "source": [
+        "def get_embedding(text):\n",
+        "    get_embedding.counter += 1\n",
+        "    try:\n",
+        "        if get_embedding.counter % 100 == 0:\n",
+        "            time.sleep(3)\n",
+        "        return model.get_embeddings([text])[0].values\n",
+        "    except:\n",
+        "        return []\n",
+        "\n",
+        "\n",
+        "get_embedding.counter = 0\n",
+        "\n",
+        "# This may take several minutes to complete.\n",
+        "df[\"embedding\"] = df[\"textContent\"].apply(lambda x: get_embedding(x))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "ba4f49b6-65e1-49ea-988b-c4e195219deb",
+      "metadata": {
+        "id": "ba4f49b6-65e1-49ea-988b-c4e195219deb"
+      },
+      "source": [
+        "## Create an Index"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "245bc8cd-038b-484a-acb2-3a705d4cc4cf",
+      "metadata": {
+        "id": "245bc8cd-038b-484a-acb2-3a705d4cc4cf"
+      },
+      "outputs": [],
+      "source": [
+        "record_count = len(records)\n",
+        "dataset = np.empty((record_count, 768))\n",
+        "for i in range(record_count):\n",
+        "    dataset[i] = df.embedding[i]\n",
+        "\n",
+        "normalized_dataset = dataset / np.linalg.norm(dataset, axis=1)[:, np.newaxis]\n",
+        "# configure ScaNN as a tree - asymmetric hash hybrid with reordering\n",
+        "# anisotropic quantization as described in the paper; see README\n",
+        "\n",
+        "# use scann.scann_ops.build() to instead create a TensorFlow-compatible searcher\n",
+        "searcher = (\n",
+        "    scann.scann_ops_pybind.builder(normalized_dataset, 10, \"dot_product\")\n",
+        "    .tree(\n",
+        "        num_leaves=record_count,\n",
+        "        num_leaves_to_search=record_count,\n",
+        "        training_sample_size=record_count,\n",
+        "    )\n",
+        "    .score_ah(2, anisotropic_quantization_threshold=0.2)\n",
+        "    .reorder(100)\n",
+        "    .build()\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "9f1689eb-c27c-4566-afc7-7fbc55552aad",
+      "metadata": {
+        "id": "9f1689eb-c27c-4566-afc7-7fbc55552aad"
+      },
+      "source": [
+        "## Queries the Index"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "494079df-d8f6-4a6a-b26b-0b0477791adc",
+      "metadata": {
+        "id": "494079df-d8f6-4a6a-b26b-0b0477791adc"
+      },
+      "outputs": [],
+      "source": [
+        "def search(query):\n",
+        "    start = time.time()\n",
+        "    query = model.get_embeddings([query])[0].values\n",
+        "    neighbors, distances = searcher.search(query, final_num_neighbors=3)\n",
+        "    end = time.time()\n",
+        "\n",
+        "    for id, dist in zip(neighbors, distances):\n",
+        "        print(f\"[docid:{id}] [{dist}] -- {df.textContent[int(id)][:125]}...\")\n",
+        "    print(\"Latency (ms):\", 1000 * (end - start))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "50497681-6112-4147-a13e-afedb72b54f5",
+      "metadata": {
+        "id": "50497681-6112-4147-a13e-afedb72b54f5"
+      },
+      "outputs": [],
+      "source": [
+        "search(\"tell me about shark or animal\")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "c814f835-3e2a-4366-a334-ccc636f00b83",
+      "metadata": {
+        "id": "c814f835-3e2a-4366-a334-ccc636f00b83"
+      },
+      "outputs": [],
+      "source": [
+        "search(\"tell me about an important moment or event in your life\")"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "text_embedding_api_semantic_search_with_scann.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/community/ml_ops/stage3/get_started_vertex_pipelines_sklearn_with_prediction.ipynb

@@ -560,6 +560,30 @@
         "    print(\"Service Account:\", SERVICE_ACCOUNT)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "set_service_account:pipelines"
+      },
+      "source": [
+        "#### Set service account access for Vertex AI Pipelines\n",
+        "\n",
+        "Run the following commands to grant your service account access to read and write pipeline artifacts in the bucket that you created in the previous step -- you only need to run these once per service account."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "set_service_account:pipelines"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
+        "\n",
+        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {

notebooks/community/ml_ops/stage3/get_started_vertex_pipelines_xgboost_with_experiments.ipynb

@@ -557,6 +557,30 @@
         "    print(\"Service Account:\", SERVICE_ACCOUNT)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "set_service_account:pipelines"
+      },
+      "source": [
+        "#### Set service account access for Vertex AI Pipelines\n",
+        "\n",
+        "Run the following commands to grant your service account access to read and write pipeline artifacts in the bucket that you created in the previous step -- you only need to run these once per service account."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 13,
+      "metadata": {
+        "id": "set_service_account:pipelines"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
+        "\n",
+        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {

notebooks/community/ml_ops/stage3/get_started_vertex_pipelines_xgboost_with_prediction.ipynb

@@ -196,7 +196,7 @@
         "    USER_FLAG = \"--user\"\n",
         "\n",
         "! pip3 install {USER_FLAG} --upgrade --quiet google-cloud-aiplatform \\\n",
-        "                                             google-cloud-pipeline-components \\\n",
+        "                                             google-cloud-pipeline-components==1.0.25 \\\n",
         "                                             kfp "
       ]
     },
@@ -557,6 +557,30 @@
         "    print(\"Service Account:\", SERVICE_ACCOUNT)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "set_service_account:pipelines"
+      },
+      "source": [
+        "#### Set service account access for Vertex AI Pipelines\n",
+        "\n",
+        "Run the following commands to grant your service account access to read and write pipeline artifacts in the bucket that you created in the previous step -- you only need to run these once per service account."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 13,
+      "metadata": {
+        "id": "set_service_account:pipelines"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
+        "\n",
+        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {

notebooks/community/ml_ops/stage3/get_started_with_dataflow_flex_template_component.ipynb

@@ -0,0 +1,832 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "copyright"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2023 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "title:generic,gcp"
+      },
+      "source": [
+        "# E2E ML on GCP: MLOps stage 3 : formalization: get started with the Dataflow Flex Template component\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_dataflow_pipeline_components.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_dataflow_pipeline_components.ipynb\">\n",
+        "<img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> \n",
+        "        Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage3/get_started_with_dataflow_pipeline_components.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "  </td>   \n",
+        "    \n",
+        "</table>\n",
+        "<br/><br/><br/>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "overview:mlops"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "\n",
+        "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 3 : formalization: get started with the Dataflow Flex Template component."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "objective:mlops,stage3,get_started_dataflow_pipeline_components"
+      },
+      "source": [
+        "### Objective\n",
+        "\n",
+        "In this tutorial, you learn how to use prebuilt `Google Cloud Pipeline Components` to execute `Dataflow` [Flex Template](https://cloud.google.com/dataflow/docs/guides/templates/using-flex-templates) jobs.\n",
+        "\n",
+        "This tutorial uses the following Google Cloud ML services:\n",
+        "\n",
+        "- `Vertex AI Pipelines`\n",
+        "- `Google Cloud Pipeline Components`\n",
+        "- `Dataflow`\n",
+        "\n",
+        "The steps performed include:\n",
+        "\n",
+        "- Defining a pipeline step to execute a Dataflow Flex Template job within a Vertex AI pipeline.\n",
+        "- Execute a Vertex AI pipeline."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dataset:gsod,lrg"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the sample data from the [Apache Beam Mobile Gaming Pipeline example](https://beam.apache.org/get-started/mobile-gaming-example/)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0c997d8d92ce"
+      },
+      "source": [
+        "### Costs\n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Cloud Storage\n",
+        "* Dataflow\n",
+        "\n",
+        "Learn about [Vertex AI\n",
+        "pricing](https://cloud.google.com/vertex-ai/pricing), [Cloud Storage\n",
+        "pricing](https://cloud.google.com/storage/pricing), and [Dataflow pricing](https://cloud.google.com/dataflow/pricing)\n",
+        "and use the [Pricing\n",
+        "Calculator](https://cloud.google.com/products/calculator/)\n",
+        "to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "install_mlops"
+      },
+      "source": [
+        "## Installations\n",
+        "\n",
+        "Install the required packages for executing the notebook."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "install_mlops"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
+        "\n",
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "USER_FLAG = \"\"\n",
+        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\"\n",
+        "\n",
+        "! pip3 install --pre --upgrade google-cloud-pipeline-components $USER_FLAG -q"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "restart"
+      },
+      "source": [
+        "### Restart the kernel\n",
+        "\n",
+        "Once you've installed the additional packages, you need to restart the notebook kernel so it can find the packages."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "restart"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "if not os.getenv(\"IS_TESTING\"):\n",
+        "    # Automatically restart kernel after installs\n",
+        "    import IPython\n",
+        "\n",
+        "    app = IPython.Application.instance()\n",
+        "    app.kernel.do_shutdown(True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "BF1j6f9HApxa"
+      },
+      "source": [
+        "## Before you begin\n",
+        "\n",
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI API and Dataflow API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,dataflow.googleapis.com).\n",
+        "\n",
+        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "project_id"
+      },
+      "source": [
+        "#### Set your project ID\n",
+        "\n",
+        "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "set_project_id"
+      },
+      "outputs": [],
+      "source": [
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "autoset_project_id"
+      },
+      "outputs": [],
+      "source": [
+        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
+        "    # Get your GCP project id from gcloud\n",
+        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "    PROJECT_ID = shell_output[0]\n",
+        "    print(\"Project ID:\", PROJECT_ID)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "250cb8c648d5"
+      },
+      "outputs": [],
+      "source": [
+        "! gcloud config set project $PROJECT_ID"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "region"
+      },
+      "source": [
+        "#### Region\n",
+        "\n",
+        "You can also change the `REGION` variable, which is used for operations\n",
+        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
+        "\n",
+        "- Americas: `us-central1`\n",
+        "- Europe: `europe-west4`\n",
+        "- Asia Pacific: `asia-east1`\n",
+        "\n",
+        "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
+        "\n",
+        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "region"
+      },
+      "outputs": [],
+      "source": [
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "timestamp"
+      },
+      "source": [
+        "#### Timestamp\n",
+        "\n",
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "timestamp"
+      },
+      "outputs": [],
+      "source": [
+        "from datetime import datetime\n",
+        "\n",
+        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "927085b84a07"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
+        "\n",
+        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
+        "\n",
+        "**Otherwise**, follow these steps:\n",
+        "\n",
+        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
+        "\n",
+        "**Click Create service account**.\n",
+        "\n",
+        "In the **Service account name** field, enter a name, and click **Create**.\n",
+        "\n",
+        "In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex\" into the filter box, and select **Vertex Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "\n",
+        "Click Create. A JSON file that contains your key downloads to your local environment.\n",
+        "\n",
+        "Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "89788a802687"
+      },
+      "outputs": [],
+      "source": [
+        "# If you are running this notebook in Colab, run this cell and follow the\n",
+        "# instructions to authenticate your GCP account. This provides access to your\n",
+        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
+        "# requests.\n",
+        "\n",
+        "import os\n",
+        "import sys\n",
+        "\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = False\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
+        "    if \"google.colab\" in sys.modules:\n",
+        "        IS_COLAB = True\n",
+        "        from google.colab import auth as google_auth\n",
+        "\n",
+        "        google_auth.authenticate_user()\n",
+        "\n",
+        "    # If you are running this notebook locally, replace the string below with the\n",
+        "    # path to your service account key and run this cell to authenticate your GCP\n",
+        "    # account.\n",
+        "    elif not os.getenv(\"IS_TESTING\"):\n",
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "40ed98f5cc48"
+      },
+      "source": [
+        "#### If you are using Colab Notebooks, set the project using gcloud config."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "fde1a355f1e9"
+      },
+      "outputs": [],
+      "source": [
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
+        "    if \"google.colab\" in sys.modules:\n",
+        "        ! gcloud config set project $PROJECT_ID"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "bucket:mbsdk"
+      },
+      "source": [
+        "### Create a Cloud Storage bucket\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "When you initialize the Vertex SDK for Python, you specify a Cloud Storage staging bucket. The staging bucket is where all the data associated with your dataset and model resources are retained across sessions.\n",
+        "\n",
+        "Set the name of your Cloud Storage bucket below. Bucket names must be globally unique across all Google Cloud projects, including those outside of your organization."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "bucket"
+      },
+      "outputs": [],
+      "source": [
+        "BUCKET_URI = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "autoset_bucket"
+      },
+      "outputs": [],
+      "source": [
+        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "create_bucket"
+      },
+      "source": [
+        "**Only if your bucket doesn't already exist**: Run the following cell to create your Cloud Storage bucket."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "create_bucket"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil mb -l $REGION $BUCKET_URI"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "validate_bucket"
+      },
+      "source": [
+        "Finally, validate access to your Cloud Storage bucket by examining its contents:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "validate_bucket"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil ls -al $BUCKET_URI"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "set_service_account"
+      },
+      "source": [
+        "#### Service Account\n",
+        "\n",
+        "**If you don't know your service account**, try to get your service account using `gcloud` command by executing the second cell below."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "set_service_account"
+      },
+      "outputs": [],
+      "source": [
+        "SERVICE_ACCOUNT = \"[your-service-account]\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "autoset_service_account"
+      },
+      "outputs": [],
+      "source": [
+        "if (\n",
+        "    SERVICE_ACCOUNT == \"\"\n",
+        "    or SERVICE_ACCOUNT is None\n",
+        "    or SERVICE_ACCOUNT == \"[your-service-account]\"\n",
+        "):\n",
+        "    # Get your service account from gcloud\n",
+        "    if not IS_COLAB:\n",
+        "        shell_output = !gcloud auth list 2>/dev/null\n",
+        "        SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
+        "\n",
+        "    if IS_COLAB:\n",
+        "        shell_output = ! gcloud projects describe  $PROJECT_ID\n",
+        "        project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n",
+        "        SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n",
+        "\n",
+        "    print(\"Service Account:\", SERVICE_ACCOUNT)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "set_service_account:pipelines"
+      },
+      "source": [
+        "#### Set service account access for Vertex AI Pipelines\n",
+        "\n",
+        "Run the following commands to grant your service account access to read and write pipeline artifacts in the bucket that you created in the previous step -- you only need to run these once per service account."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "set_service_account:pipelines"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
+        "\n",
+        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "setup_vars"
+      },
+      "source": [
+        "### Set up variables\n",
+        "\n",
+        "Next, set up some variables used throughout the tutorial.\n",
+        "### Import libraries and define constants"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "import_aip:mbsdk"
+      },
+      "outputs": [],
+      "source": [
+        "import google.cloud.aiplatform as aip\n",
+        "from google_cloud_pipeline_components.experimental.dataflow import \\\n",
+        "    DataflowFlexTemplateJobOp\n",
+        "from google_cloud_pipeline_components.v1.wait_gcp_resources import \\\n",
+        "    WaitGcpResourcesOp\n",
+        "from kfp import dsl\n",
+        "from kfp.v2 import compiler"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "init_aip:mbsdk"
+      },
+      "source": [
+        "### Initialize Vertex AI SDK for Python\n",
+        "\n",
+        "Initialize the Vertex AI SDK for Python for your project and corresponding bucket."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "init_aip:mbsdk"
+      },
+      "outputs": [],
+      "source": [
+        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "writefile:wc.py"
+      },
+      "source": [
+        "### Prepare a Flex Template job to convert a CSV file to Parquet\n",
+        "\n",
+        "In this tutorial, you use the Google-provided [File Format Conversion template](https://cloud.google.com/dataflow/docs/guides/templates/provided/file-format-conversion) to convert a CSV file to Parquet format."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "writefile:requirements,wc"
+      },
+      "source": [
+        "#### Create the Avro schema file\n",
+        "\n",
+        "First, create an Avro schema file that describes the example data."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "writefile:requirements,wc"
+      },
+      "outputs": [],
+      "source": [
+        "%%writefile gaming_schema.avsc\n",
+        "\n",
+        "{\n",
+        "   \"type\" : \"record\",\n",
+        "   \"name\" : \"user_score\",\n",
+        "   \"fields\" : [\n",
+        "      { \"name\" : \"user\" , \"type\" : \"string\" },\n",
+        "      { \"name\" : \"team\" , \"type\" : \"string\" },\n",
+        "      { \"name\" : \"score\" , \"type\" : \"int\" },\n",
+        "      { \"name\" : \"ts_epoch\" , \"type\" : \"long\" },\n",
+        "      { \"name\" : \"ts_str\" , \"type\" : \"string\" }\n",
+        "   ]\n",
+        "}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "copy_to_gcs:wc"
+      },
+      "source": [
+        "#### Copy the Avro schema file to Cloud Storage\n",
+        "\n",
+        "Next, you copy the Avro schema file to your Cloud Storage bucket.\n",
+        "\n",
+        "Additional, you set the following:\n",
+        "\n",
+        "- The Cloud Storage location of the Flex Template definition file. In this tutorial, you use the location of the File Conversion template.\n",
+        "- A `Dict` containing the user parameters required by the Flex Template. These parameters include:\n",
+        "\n",
+        "    - `inputFileFormat`: The file format of the input files.\n",
+        "    - `outputFileFormat`: The file format of the output files.\n",
+        "    - `inputFileSpec`: The input filepattern to read from.\n",
+        "    - `outputBucket`: The Cloud Storage path to write the output files.\n",
+        "    - `schema`: The Cloud Storage path top the avro schema file used for the conversion.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "copy_to_gcs:wc"
+      },
+      "outputs": [],
+      "source": [
+        "GCS_AVRO_SCHEMA = BUCKET_URI + \"/gaming_schema.avsc\"\n",
+        "! gsutil cp gaming_schema.avsc $GCS_AVRO_SCHEMA\n",
+        "\n",
+        "GCS_FLEX_TEMPLATE_PATH = \"gs://dataflow-templates/latest/flex/File_Format_Conversion\"\n",
+        "GCS_CONVERT_IN = \"gs://dataflow-samples/game/5000_gaming_data.csv\"\n",
+        "GCS_CONVERT_OUT = BUCKET_URI + \"/parquet_out/\"\n",
+        "\n",
+        "TEMPLATE_PARAMETERS = {\n",
+        "    \"inputFileFormat\": \"csv\",\n",
+        "    \"outputFileFormat\": \"parquet\",\n",
+        "    \"inputFileSpec\": GCS_CONVERT_IN,\n",
+        "    \"outputBucket\": GCS_CONVERT_OUT,\n",
+        "    \"schema\": GCS_AVRO_SCHEMA,\n",
+        "}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "create_dataflow_pipeline:wc"
+      },
+      "source": [
+        "### Create and execute the pipeline job\n",
+        "\n",
+        "In this example, the `DataflowFlexTemplateJobOp` component takes the following parameters:\n",
+        "\n",
+        "- `project_id`: The project ID.\n",
+        "- `location`: The region.\n",
+        "- `container_spec_gcs_path`: The Cloud Storage path to a file that contains the Flex Template definition. This file contains a json serialized `ContainerSpec` as content.\n",
+        "- `temp_location`: The Cloud Storage path to use for temporary files.\n",
+        "- `parameters`: The parameters for the Flex Template.\n",
+        "\n",
+        "Learn more about [Google Cloud Pipeline Components for Dataflow](https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-2.0.0b2/google_cloud_pipeline_components.experimental.dataflow.html)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "create_dataflow_pipeline:wc"
+      },
+      "outputs": [],
+      "source": [
+        "PIPELINE_ROOT = \"{}/pipeline_root/dataflow_file_conversion\".format(BUCKET_URI)\n",
+        "\n",
+        "\n",
+        "@dsl.pipeline(\n",
+        "    name=\"dataflow-file-conversion\", description=\"Dataflow file format conversion\"\n",
+        ")\n",
+        "def pipeline(\n",
+        "    project_id: str = PROJECT_ID,\n",
+        "    location: str = REGION,\n",
+        "    container_spec_gcs_path: str = GCS_FLEX_TEMPLATE_PATH,\n",
+        "    temp_location: str = PIPELINE_ROOT,\n",
+        "    parameters: dict = TEMPLATE_PARAMETERS,\n",
+        "):\n",
+        "    flex_template_op = DataflowFlexTemplateJobOp(\n",
+        "        project=project_id,\n",
+        "        location=location,\n",
+        "        container_spec_gcs_path=container_spec_gcs_path,\n",
+        "        temp_location=temp_location,\n",
+        "        parameters=parameters,\n",
+        "    )\n",
+        "\n",
+        "    _ = WaitGcpResourcesOp(gcp_resources=flex_template_op.outputs[\"gcp_resources\"])\n",
+        "\n",
+        "\n",
+        "compiler.Compiler().compile(\n",
+        "    pipeline_func=pipeline, package_path=\"dataflow_file_conversion.yaml\"\n",
+        ")\n",
+        "\n",
+        "pipeline = aip.PipelineJob(\n",
+        "    display_name=\"dataflow_file_conversion\",\n",
+        "    template_path=\"dataflow_file_conversion.yaml\",\n",
+        "    pipeline_root=PIPELINE_ROOT,\n",
+        "    enable_caching=False,\n",
+        ")\n",
+        "\n",
+        "pipeline.run()\n",
+        "\n",
+        "! gsutil ls $GCS_CONVERT_OUT\n",
+        "\n",
+        "! rm -f dataflow_file_conversion.yaml gaming_schema.avsc"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "delete_pipeline"
+      },
+      "source": [
+        "### Delete a pipeline job\n",
+        "\n",
+        "After a pipeline job is completed, you can delete the pipeline job with the method `delete()`.  Prior to completion, a pipeline job can be canceled with the method `cancel()`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "delete_pipeline"
+      },
+      "outputs": [],
+      "source": [
+        "pipeline.delete()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "cleanup:mbsdk"
+      },
+      "source": [
+        "# Cleaning up\n",
+        "\n",
+        "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
+        "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
+        "\n",
+        "Otherwise, you can delete the individual resources you created in this tutorial:\n",
+        "\n",
+        "- Dataset\n",
+        "- Cloud Storage Bucket"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cleanup:mbsdk"
+      },
+      "outputs": [],
+      "source": [
+        "# Warning: Setting this to true will delete everything in your bucket\n",
+        "delete_bucket = False\n",
+        "\n",
+        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "    ! gsutil rm -r $BUCKET_URI"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "get_started_with_dataflow_flex_template_component.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/community/ml_ops/stage3/get_started_with_hpt_pipeline_components.ipynb

@@ -1333,7 +1333,6 @@
         "Next, you compile the pipeline and then exeute it. The pipeline takes the following parameters, which are passed as the dictionary `parameter_values`:\n",
         "\n",
         "- `display_name`: A human readable name for the pipeline job.\n",
-        "- `import_file`: The Cloud Storage location to the dataset.\n",
         "- `worker_pool_specs`: The the machine and container, and auto-scaling requirements, as well as command line arguments.\n",
         "- `study_spec_metrics`: The metrics to optimize in the study trials.\n",
         "- `study_spec_parameters`: The parameters to tune."

notebooks/community/ml_ops/stage4/get_started_with_vertex_ml_metadata.ipynb

@@ -29,22 +29,22 @@
         "id": "title:generic,gcp"
       },
       "source": [
-        "# E2E ML on GCP: MLOps stage 4 : formalization: get started with Vertex ML Metadata\n",
+        "# Get started with Vertex ML Metadata\n",
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage4/get_started_with_vertex_ml_metadata.ipynb\">\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/ml_metadata/get_started_with_vertex_ml_metadata.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage4/get_started_with_vertex_ml_metadata.ipynb\">\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/ml_metadata/get_started_with_vertex_ml_metadata.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage4/get_started_with_vertex_ml_metadata.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/ml_metadata/get_started_with_vertex_ml_metadata.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
@@ -62,7 +62,9 @@
         "## Overview\n",
         "\n",
         "\n",
-        "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 4 : formalization: get started with Vertex ML Metadata."
+        "This tutorial demonstrates how to use Vertex ML Metadata.\n",
+        "\n",
+        "Learn more about [Vertex ML Metadata](https://cloud.google.com/vertex-ai/docs/ml-metadata)."
       ]
     },
     {
@@ -144,19 +146,32 @@
       "source": [
         "import os\n",
         "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
+        "! pip3 install --upgrade google-cloud-aiplatform[tensorboard] \\\n",
+        "                         google-cloud-pipeline-components  --quiet"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "restart"
+      },
+      "source": [
+        "### Colab only: Uncomment the following cell to restart the kernel"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "D-ZBOjErv5mM"
+      },
+      "outputs": [],
+      "source": [
+        "# Automatically restart kernel after installs so that your environment can access the new packages\n",
+        "# import IPython\n",
         "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
-        "! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG -q\n",
-        "! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG -q"
+        "# app = IPython.Application.instance()\n",
+        "# app.kernel.do_shutdown(True)"
       ]
     },
     {
@@ -170,63 +185,27 @@
         "Once you've installed the additional packages, you need to restart the notebook kernel so it can find the packages."
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "restart"
-      },
-      "outputs": [],
-      "source": [
-        "import os\n",
-        "\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    # Automatically restart kernel after installs\n",
-        "    import IPython\n",
-        "\n",
-        "    app = IPython.Application.instance()\n",
-        "    app.kernel.do_shutdown(True)"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "before_you_begin"
       },
       "source": [
-        "## Before you begin\n",
-        "\n",
-        "### GPU runtime\n",
-        "\n",
-        "*Make sure you're running this notebook in a GPU runtime if you have that option. In Colab, select* **Runtime > Change Runtime Type > GPU**\n",
-        "\n",
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "2. [Make sure that billing is enabled for your project.](https://cloud.google.com/billing/docs/how-to/modify-project)\n",
-        "\n",
-        "3. [Enable the following APIs: Vertex AI APIs, Compute Engine APIs, and Cloud Storage.](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component,storage-component.googleapis.com)\n",
-        "\n",
-        "4. If you are running this notebook locally, you need to install the [Cloud SDK]((https://cloud.google.com/sdk)).\n",
-        "\n",
-        "5. Enter your project ID in the cell below. Then run the  cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$`."
+        "## Before you begin"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "project_id"
+        "id": "before_you_begin:nogpu"
       },
       "source": [
-        "#### Set your project ID\n",
+        "### Set your project ID\n",
         "\n",
-        "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
+        "**If you don't know your project ID**, try the following:\n",
+        "* Run `gcloud config list`.\n",
+        "* Run `gcloud projects list`.\n",
+        "* See the support page: [Locate the project ID](https://support.google.com/googleapi/answer/7014113)"
       ]
     },
     {
@@ -237,33 +216,10 @@
       },
       "outputs": [],
       "source": [
-        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "autoset_project_id"
-      },
-      "outputs": [],
-      "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
-        "    # Get your GCP project id from gcloud\n",
-        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
-        "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID:\", PROJECT_ID)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "set_gcloud_project_id"
-      },
-      "outputs": [],
-      "source": [
-        "! gcloud config set project $PROJECT_ID"
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# Set the project id\n",
+        "! gcloud config set project {PROJECT_ID}"
       ]
     },
     {
@@ -274,16 +230,7 @@
       "source": [
         "#### Region\n",
         "\n",
-        "You can also change the `REGION` variable, which is used for operations\n",
-        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
-        "\n",
-        "- Americas: `us-central1`\n",
-        "- Europe: `europe-west4`\n",
-        "- Asia Pacific: `asia-east1`\n",
-        "\n",
-        "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
-        "\n",
-        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
+        "You can also change the `REGION` variable used by Vertex AI. Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
       ]
     },
     {
@@ -294,34 +241,7 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "timestamp"
-      },
-      "source": [
-        "#### Timestamp\n",
-        "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "timestamp"
-      },
-      "outputs": [],
-      "source": [
-        "from datetime import datetime\n",
-        "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
       ]
     },
     {
@@ -332,57 +252,70 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
-        "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
-        "\n",
-        "**Click Create service account**.\n",
-        "\n",
-        "In the **Service account name** field, enter a name, and click **Create**.\n",
-        "\n",
-        "In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex\" into the filter box, and select **Vertex Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "Click Create. A JSON file that contains your key downloads to your local environment.\n",
-        "\n",
-        "Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
+        "Depending on your Jupyter environment, you may have to manually authenticate. Follow the relevant instructions below."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "FvQeFm3Gv5mR"
+      },
+      "source": [
+        "**1. Vertex AI Workbench**\n",
+        "* Do nothing as you are already authenticated."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ad1138a125ea"
+      },
+      "source": [
+        "**2. Local JupyterLab instance, uncomment and run:**"
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "gcp_authenticate"
+        "id": "ce6043da7b33"
+      },
+      "outputs": [],
+      "source": [
+        "# ! gcloud auth login"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0367eac06a10"
+      },
+      "source": [
+        "**3. Colab, uncomment and run:**"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "21ad4dbb4a61"
       },
       "outputs": [],
       "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
         "IS_COLAB = False\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        IS_COLAB = True\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+        "# from google.colab import auth\n",
+        "# auth.authenticate_user()\n",
+        "# IS_COLAB = True"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "c13224697bfb"
+      },
+      "source": [
+        "**4. Service account or other**\n",
+        "* See how to grant Cloud Storage permissions to your service account at https://cloud.google.com/storage/docs/gsutil/commands/iam#ch-examples."
       ]
     },
     {
@@ -393,11 +326,7 @@
       "source": [
         "### Create a Cloud Storage bucket\n",
         "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "When you initialize the Vertex SDK for Python, you specify a Cloud Storage staging bucket. The staging bucket is where all the data associated with your dataset and model resources are retained across sessions.\n",
-        "\n",
-        "Set the name of your Cloud Storage bucket below. Bucket names must be globally unique across all Google Cloud projects, including those outside of your organization."
+        "Create a storage bucket to store intermediate artifacts such as datasets."
       ]
     },
     {
@@ -408,21 +337,7 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "autoset_bucket"
-      },
-      "outputs": [],
-      "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
-        "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
+        "BUCKET_URI = f\"gs://your-bucket-name-{PROJECT_ID}-unique\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -445,26 +360,6 @@
         "! gsutil mb -l $REGION $BUCKET_URI"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "validate_bucket"
-      },
-      "source": [
-        "Finally, validate access to your Cloud Storage bucket by examining its contents:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "validate_bucket"
-      },
-      "outputs": [],
-      "source": [
-        "! gsutil ls -al $BUCKET_URI"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -545,19 +440,7 @@
       "source": [
         "### Set up variables\n",
         "\n",
-        "Next, set up some variables used throughout the tutorial.\n",
-        "### Import libraries and define constants"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "import_aip:mbsdk"
-      },
-      "outputs": [],
-      "source": [
-        "import google.cloud.aiplatform as aip"
+        "Next, set up some variables used throughout the tutorial."
       ]
     },
     {
@@ -579,7 +462,10 @@
       },
       "outputs": [],
       "source": [
-        "import google.cloud.aiplatform_v1beta1 as aip_beta"
+        "from datetime import datetime\n",
+        "\n",
+        "import google.cloud.aiplatform_v1beta1 as aip_beta\n",
+        "from google.cloud import aiplatform"
       ]
     },
     {
@@ -707,7 +593,7 @@
       "outputs": [],
       "source": [
         "metadata_store = clients[\"metadata\"].create_metadata_store(\n",
-        "    parent=PARENT, metadata_store_id=\"my-metadata-store\"\n",
+        "    parent=PARENT, metadata_store_id=\"my-metadata-store-unique\"\n",
         ")\n",
         "\n",
         "metadata_store_id = str(metadata_store.result())[7:-2]\n",
@@ -1144,7 +1030,7 @@
       "source": [
         "from kfp.v2 import compiler, dsl\n",
         "from kfp.v2.dsl import (Artifact, Dataset, Input, Metrics, Model, Output,\n",
-        "                        OutputPath, component, pipeline)"
+        "                        OutputPath, component)"
       ]
     },
     {
@@ -1196,7 +1082,7 @@
       "outputs": [],
       "source": [
         "@component(\n",
-        "    packages_to_install=[\"google-cloud-bigquery\", \"pandas\", \"pyarrow\"],\n",
+        "    packages_to_install=[\"google-cloud-bigquery\", \"pandas\", \"pyarrow\", \"db-dtypes\"],\n",
         "    base_image=\"python:3.9\",\n",
         "    output_component_file=\"create_dataset.yaml\",\n",
         ")\n",
@@ -1214,7 +1100,7 @@
         "\n",
         "\n",
         "@component(\n",
-        "    packages_to_install=[\"sklearn\", \"pandas\", \"joblib\"],\n",
+        "    packages_to_install=[\"scikit-learn\", \"pandas\", \"joblib\"],\n",
         "    base_image=\"python:3.9\",\n",
         "    output_component_file=\"beans_model_component.yaml\",\n",
         ")\n",
@@ -1298,7 +1184,7 @@
         "    # A name for the pipeline.\n",
         "    name=\"mlmd-pipeline\",\n",
         ")\n",
-        "def pipeline(\n",
+        "def my_pipeline(\n",
         "    bq_table: str = \"\",\n",
         "    output_data_path: str = \"data.csv\",\n",
         "    project: str = PROJECT_ID,\n",
@@ -1332,20 +1218,22 @@
       "source": [
         "NOW = datetime.now().isoformat().replace(\".\", \":\")[:-7]\n",
         "\n",
-        "compiler.Compiler().compile(pipeline_func=pipeline, package_path=\"mlmd_pipeline.json\")\n",
+        "compiler.Compiler().compile(\n",
+        "    pipeline_func=my_pipeline, package_path=\"mlmd_pipeline.json\"\n",
+        ")\n",
         "\n",
-        "run1 = aip.PipelineJob(\n",
+        "run1 = aiplatform.PipelineJob(\n",
         "    display_name=\"mlmd-pipeline\",\n",
         "    template_path=\"mlmd_pipeline.json\",\n",
-        "    job_id=\"mlmd-pipeline-small-{}\".format(TIMESTAMP),\n",
+        "    job_id=\"mlmd-pipeline-small-unique\",\n",
         "    parameter_values={\"bq_table\": \"sara-vertex-demos.beans_demo.small_dataset\"},\n",
         "    enable_caching=True,\n",
         ")\n",
         "\n",
-        "run2 = aip.PipelineJob(\n",
+        "run2 = aiplatform.PipelineJob(\n",
         "    display_name=\"mlmd-pipeline\",\n",
         "    template_path=\"mlmd_pipeline.json\",\n",
-        "    job_id=\"mlmd-pipeline-large-{}\".format(TIMESTAMP),\n",
+        "    job_id=\"mlmd-pipeline-large-unique\",\n",
         "    parameter_values={\"bq_table\": \"sara-vertex-demos.beans_demo.large_dataset\"},\n",
         "    enable_caching=True,\n",
         ")\n",
@@ -1382,7 +1270,7 @@
       },
       "outputs": [],
       "source": [
-        "df = aip.get_pipeline_df(pipeline=\"mlmd-pipeline\")\n",
+        "df = aiplatform.get_pipeline_df(pipeline=\"mlmd-pipeline\")\n",
         "print(df)"
       ]
     },
@@ -1466,6 +1354,10 @@
       },
       "outputs": [],
       "source": [
+        "metadata_store_id = (\n",
+        "    f\"projects/{PROJECT_ID}/locations/{REGION}/metadataStores/my-metadata-store-unique\"\n",
+        ")\n",
+        "\n",
         "clients[\"metadata\"].delete_metadata_store(name=metadata_store_id)"
       ]
     },

notebooks/community/model_garden/model_garden_huggingface_local_inference.ipynb

@@ -0,0 +1,358 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7d9bbf86da5e"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2023 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "4dc4391f6be7"
+      },
+      "source": [
+        "# Vertex AI Model Garden - Hugging Face Local Inference\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_huggingface_local_inference.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_huggingface_local_inference.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/notebooks/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/model_garden/model_garden_huggingface_local_inference.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "    (a Python-3 GPU notebook with preinstalled HuggingFace/transformer libraries is recommended)\n",
+        "  </td>\n",
+        "</table>"
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "4e8a0fdd6f44"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "This notebook demonstrates how to run local inference with various Hugging Face models by using [Colab](https://colab.research.google.com/) and installing the necessary libraries or by deploying a [Vertex AI Workbench Instance](https://cloud.google.com/vertex-ai-workbench) with preinstalled transformer and diffuser libraries.\n",
+        "\n",
+        "### Objective\n",
+        "\n",
+        "* Run local inference with various transformer or diffusion models.\n",
+        "\n",
+        "### Costs\n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "\n",
+        "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing) and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "69453bf7230e"
+      },
+      "source": [
+        "## Before you begin"
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "68990d91bc5f"
+      },
+      "source": [
+        "### Colab only"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3d342b32fb08"
+      },
+      "outputs": [],
+      "source": [
+        "if \"google.colab\" in str(get_ipython()):\n",
+        "    ! pip3 install --upgrade google-cloud-aiplatform\n",
+        "    from google.colab import auth as google_auth\n",
+        "\n",
+        "    google_auth.authenticate_user()\n",
+        "    ! pip3 install --upgrade pip\n",
+        "    ! pip3 install torchvision==0.14.1\n",
+        "    ! pip3 install transformers==4.27.1\n",
+        "    ! pip3 install diffusers==0.15.1\n",
+        "    ! apt-get update\n",
+        "    ! apt-get install -y --no-install-recommends tesseract-ocr\n",
+        "    ! pip3 install tesseract==0.1.3\n",
+        "    ! pip3 install pytesseract==0.3.10\n",
+        "    ! pip3 install datasets==2.9.0\n",
+        "    ! pip3 install accelerate==0.18.0\n",
+        "    ! pip3 install triton==2.0.0.dev20221120\n",
+        "    ! pip3 install xformers==0.0.16\n",
+        "    ! pip3 install modelscope==1.4.2\n",
+        "    ! pip3 install open_clip_torch==2.17.1\n",
+        "    ! pip3 install pytorch-lightning==1.9.5\n",
+        "    ! pip3 install opencv-python-headless==4.7.0.72\n",
+        "    # Install gdown for downloading example training images.\n",
+        "    ! pip3 install gdown\n",
+        "    # Remove wrong cublas version.\n",
+        "    ! pip3 uninstall nvidia_cublas_cu11 --yes\n",
+        "\n",
+        "    # Restart the notebook kernel after installs.\n",
+        "    import IPython\n",
+        "\n",
+        "    app = IPython.Application.instance()\n",
+        "    app.kernel.do_shutdown(True)"
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "05e23144b125"
+      },
+      "source": [
+        "### Workbench only\n",
+        "\n",
+        "1.   Follow [this link](https://console.cloud.google.com/vertex-ai/notebooks/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/model_garden/model_garden_huggingfacE_local_inference.ipynb) to deploy the notebook to a Vertex AI Workbench Instance.\n",
+        "2.   Select `Create a new Notebook`.\n",
+        "3.   Click `Advanced Options`.\n",
+        "4.   In the **Environment** tab, select `Debian 10` for **Operating System** and select `Custom Container` for **Environment**.\n",
+        "5.   Set the **Docker container image** field to `us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/transformers-notebook`.\n",
+        "6.   In the **Machine Type** tab, select a 1 `T4` GPU and select `Install NVIDIA GPU driver automatically for me`.\n",
+        "7.   Click `Create` to create the Vertex AI Workbench instance.\n"
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ad1a690839d5"
+      },
+      "source": [
+        "## Sample code"
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0a4008240483"
+      },
+      "source": [
+        "#### [runwayml/stable-diffusion-v1-5](https://huggingface.co/runwayml/stable-diffusion-v1-5) (Text-to-image)\n",
+        "Generate photo-realistic images given any text input."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "5ec6b474d1be"
+      },
+      "outputs": [],
+      "source": [
+        "import torch\n",
+        "from diffusers import StableDiffusionPipeline\n",
+        "\n",
+        "model_id = \"runwayml/stable-diffusion-v1-5\"\n",
+        "pipe = StableDiffusionPipeline.from_pretrained(model_id, torch_dtype=torch.float16)\n",
+        "pipe = pipe.to(\"cuda\")\n",
+        "\n",
+        "prompt = \"a photo of an astronaut riding a horse on mars\"\n",
+        "image = pipe(prompt).images[0]\n",
+        "\n",
+        "display(image)"
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ae94b9b23a52"
+      },
+      "source": [
+        "#### [runwayml/stable-diffusion-v1-5](https://huggingface.co/runwayml/stable-diffusion-v1-5) (Text guided image-to-image)\n",
+        "Generate an image based on an initial image and a text prompt."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "0acd70f4d08a"
+      },
+      "outputs": [],
+      "source": [
+        "from io import BytesIO\n",
+        "\n",
+        "import requests\n",
+        "import torch\n",
+        "from diffusers import StableDiffusionImg2ImgPipeline\n",
+        "from PIL import Image\n",
+        "\n",
+        "device = \"cuda\"\n",
+        "model_id_or_path = \"runwayml/stable-diffusion-v1-5\"\n",
+        "pipe = StableDiffusionImg2ImgPipeline.from_pretrained(\n",
+        "    model_id_or_path, torch_dtype=torch.float16\n",
+        ")\n",
+        "pipe = pipe.to(device)\n",
+        "\n",
+        "url = \"https://raw.githubusercontent.com/CompVis/stable-diffusion/main/assets/stable-samples/img2img/sketch-mountains-input.jpg\"\n",
+        "\n",
+        "response = requests.get(url)\n",
+        "init_image = Image.open(BytesIO(response.content)).convert(\"RGB\")\n",
+        "init_image = init_image.resize((768, 512))\n",
+        "\n",
+        "prompt = \"A fantasy landscape, trending on artstation\"\n",
+        "\n",
+        "images = pipe(prompt=prompt, image=init_image, strength=0.75, guidance_scale=7.5).images\n",
+        "display(images[0])"
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e76b3fe8d10c"
+      },
+      "source": [
+        "#### [runwayml/stable-diffusion-inpainting](https://huggingface.co/runwayml/stable-diffusion-inpainting) (Image-inpainting)\n",
+        "Generate an image based on an original image and prompt, only editing the areas denoted by a mask image."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "8bc3238be4e7"
+      },
+      "outputs": [],
+      "source": [
+        "from io import BytesIO\n",
+        "\n",
+        "import requests\n",
+        "import torch\n",
+        "from diffusers import StableDiffusionInpaintPipeline\n",
+        "from PIL import Image\n",
+        "\n",
+        "image_url = \"https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo.png\"\n",
+        "image_response = requests.get(image_url)\n",
+        "init_image = Image.open(BytesIO(image_response.content)).convert(\"RGB\")\n",
+        "display(init_image)\n",
+        "\n",
+        "mask_url = \"https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo_mask.png\"\n",
+        "mask_response = requests.get(mask_url)\n",
+        "mask_image = Image.open(BytesIO(mask_response.content)).convert(\"RGB\")\n",
+        "\n",
+        "pipe = StableDiffusionInpaintPipeline.from_pretrained(\n",
+        "    \"runwayml/stable-diffusion-inpainting\",\n",
+        "    revision=\"fp16\",\n",
+        "    torch_dtype=torch.float16,\n",
+        ")\n",
+        "pipe.to(\"cuda\")\n",
+        "\n",
+        "prompt = \"Face of a yellow cat, high resolution, sitting on a park bench\"\n",
+        "images = pipe(prompt=prompt, image=init_image, mask_image=mask_image).images\n",
+        "display(images[0])"
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "1ade95a9b20e"
+      },
+      "source": [
+        "#### [impira/layoutlm-document-qa](https://huggingface.co/impira/layoutlm-document-qa) (Document question answering)\n",
+        "Answer questions about a given document."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "492d9f1de3f2"
+      },
+      "outputs": [],
+      "source": [
+        "from transformers import pipeline\n",
+        "\n",
+        "nlp = pipeline(\n",
+        "    \"document-question-answering\",\n",
+        "    model=\"impira/layoutlm-document-qa\",\n",
+        ")\n",
+        "\n",
+        "print(\n",
+        "    nlp(\n",
+        "        \"https://templates.invoicehome.com/invoice-template-us-neat-750px.png\",\n",
+        "        \"What is the invoice number?\",\n",
+        "    )\n",
+        ")\n",
+        "# [{'score': 0.9943977, 'answer': 'us-001', 'start': 15, 'end': 15}]\n",
+        "\n",
+        "print(\n",
+        "    nlp(\n",
+        "        \"https://miro.medium.com/max/787/1*iECQRIiOGTmEFLdWkVIH2g.jpeg\",\n",
+        "        \"What is the purchase amount?\",\n",
+        "    )\n",
+        ")\n",
+        "# [{'score': 0.9912159, 'answer': '$1,000,000,000', 'start': 97, 'end': 97}]\n",
+        "\n",
+        "print(\n",
+        "    nlp(\n",
+        "        \"https://www.accountingcoach.com/wp-content/uploads/2013/10/income-statement-example@2x.png\",\n",
+        "        \"What are the 2020 net sales?\",\n",
+        "    )\n",
+        ")\n",
+        "# [{'score': 0.978011429309845, 'answer': '$ 3,980', 'start': 15, 'end': 16}]"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "model_garden_huggingface_local_inference.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/community/model_garden/model_garden_jax_vision_transformer.ipynb

@@ -0,0 +1,879 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ur8xi4C7S06n"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2023 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "99c1c3fc2ca5"
+      },
+      "source": [
+        "# Vertex AI Model Garden - JAX Vision Transformer\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_jax_vision_transformer.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_jax_vision_transformer.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/notebooks/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/model_garden/model_garden_jax_vision_transformer.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "  </td>\n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "24743cf4a1e1"
+      },
+      "source": [
+        "**_NOTE_**: This notebook has been tested in the following environment:\n",
+        "\n",
+        "* Python version = 3.9"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tvgnzT1CKxrO"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "This notebook demonstrates finetuning a [JAX ViT-B16 model](https://github.com/google-research/vision_transformer#available-vit-models) for image classification task on GPU and deploying them on Vertex AI for online prediction.\n",
+        "\n",
+        "Learn more about [Generative AI Support in Vertex AI](https://cloud.google.com/blog/products/ai-machine-learning/vertex-ai-model-garden-and-generative-ai-studio)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d975e698c9a4"
+      },
+      "source": [
+        "### Objective\n",
+        "\n",
+        "In this tutorial, you learn how fine-tune, deploy and predict with a Vertex AI pretrained JAX Vision Transformer based model.\n",
+        "\n",
+        "This tutorial uses the following Google Cloud ML services and resources:\n",
+        "\n",
+        "- Vertex AI Model Garden\n",
+        "- Vertex AI Training\n",
+        "- Vertex AI Model Registry\n",
+        "- Vertex AI Online Prediction\n",
+        "\n",
+        "The steps performed are:\n",
+        "\n",
+        "- Finetune a JAX Vision Transformer based model.\n",
+        "- Upload the model to [Model Registry](https://cloud.google.com/vertex-ai/docs/model-registry/introduction).\n",
+        "- Deploy the model on [Endpoint](https://cloud.google.com/vertex-ai/docs/predictions/using-private-endpoints).\n",
+        "- Run online predictions for image classification.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "08d289fa873f"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "This notebook uses the [tf_flowers dataset](https://www.tensorflow.org/datasets/catalog/tf_flowers) and has a section which shows how to download and prepare it. You can follow similar process to use your own custom dataset too."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "aed92deeb4a0"
+      },
+      "source": [
+        "### Costs\n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage pricing](https://cloud.google.com/storage/pricing), and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "i7EUnXsZhAGF"
+      },
+      "source": [
+        "## Installation\n",
+        "\n",
+        "Install the following packages required to execute this notebook."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2b4ef9b72d43"
+      },
+      "outputs": [],
+      "source": [
+        "# Install the packages.\n",
+        "! pip3 install --upgrade google-cloud-aiplatform"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "58707a750154"
+      },
+      "source": [
+        "### Colab only"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "f200f10a1da3"
+      },
+      "outputs": [],
+      "source": [
+        "# Automatically restart kernel after installs so that your environment can access the new packages.\n",
+        "# import IPython\n",
+        "\n",
+        "# app = IPython.Application.instance()\n",
+        "# app.kernel.do_shutdown(True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "BF1j6f9HApxa"
+      },
+      "source": [
+        "## Before you begin\n",
+        "\n",
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI API and Compute Engine API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component).\n",
+        "\n",
+        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WReHDGG5g0XY"
+      },
+      "source": [
+        "#### Set your project ID\n",
+        "\n",
+        "**If you don't know your project ID**, try the following:\n",
+        "* Run `gcloud config list`.\n",
+        "* Run `gcloud projects list`.\n",
+        "* See the support page: [Locate the project ID](https://support.google.com/googleapi/answer/7014113)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "oM1iC_MfAts1"
+      },
+      "outputs": [],
+      "source": [
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# Set the project id\n",
+        "! gcloud config set project {PROJECT_ID}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "region"
+      },
+      "source": [
+        "#### Region\n",
+        "\n",
+        "You can also change the `REGION` variable used by Vertex AI. Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "twgKk-LsLmX3"
+      },
+      "outputs": [],
+      "source": [
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "sBCra4QMA2wR"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "Depending on your Jupyter environment, you may have to manually authenticate. Follow the relevant instructions below."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "74ccc9e52986"
+      },
+      "source": [
+        "**1. Vertex AI Workbench**\n",
+        "* Do nothing as you are already authenticated."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "de775a3773ba"
+      },
+      "source": [
+        "**2. Local JupyterLab instance, uncomment and run:**"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "254614fa0c46"
+      },
+      "outputs": [],
+      "source": [
+        "# ! gcloud auth login"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ef21552ccea8"
+      },
+      "source": [
+        "**3. Colab, uncomment and run:**"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "603adbbf0532"
+      },
+      "outputs": [],
+      "source": [
+        "# from google.colab import auth\n",
+        "# auth.authenticate_user()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "f6b2ccc891ed"
+      },
+      "source": [
+        "**4. Service account or other**\n",
+        "* See how to grant Cloud Storage permissions to your service account at https://cloud.google.com/storage/docs/gsutil/commands/iam#ch-examples."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "zgPO1eR3CYjk"
+      },
+      "source": [
+        "### Create a Cloud Storage bucket\n",
+        "\n",
+        "Create a storage bucket to store intermediate artifacts such as datasets."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "MzGDU7TWdts_"
+      },
+      "outputs": [],
+      "source": [
+        "BUCKET_URI = f\"gs://your-bucket-name-{PROJECT_ID}-unique\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "-EcIXiGsCePi"
+      },
+      "source": [
+        "**Only if your bucket doesn't already exist**: Run the following cell to create your Cloud Storage bucket."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "NIq7R4HZCfIc"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil mb -l {REGION} -p {PROJECT_ID} {BUCKET_URI}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "960505627ddf"
+      },
+      "source": [
+        "### Import libraries"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "PyQmSRbKA8r-"
+      },
+      "outputs": [],
+      "source": [
+        "import base64\n",
+        "import glob\n",
+        "import os\n",
+        "import random\n",
+        "import shutil\n",
+        "from datetime import datetime\n",
+        "from io import BytesIO\n",
+        "\n",
+        "import numpy as np\n",
+        "from google.cloud import aiplatform\n",
+        "from google.protobuf import json_format\n",
+        "from google.protobuf.struct_pb2 import Value\n",
+        "from PIL import Image"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "init_aip:mbsdk,all"
+      },
+      "source": [
+        "### Initialize Vertex AI SDK for Python\n",
+        "\n",
+        "Initialize the Vertex AI SDK for Python for your project."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "vS1hQiGuLmX4"
+      },
+      "outputs": [],
+      "source": [
+        "staging_bucket = os.path.join(BUCKET_URI, \"jax_vit_staging\")\n",
+        "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=staging_bucket)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2cc825514deb"
+      },
+      "source": [
+        "### Define constants"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b42bd4fa2b2d"
+      },
+      "outputs": [],
+      "source": [
+        "# The pre-built training docker image.\n",
+        "TRAIN_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai-restricted/vertex-vision-model-garden-dockers/jax-vit-train-gpu\"\n",
+        "# The pre-built TF SavedModel conversion docker image.\n",
+        "MODEL_CONVERSION_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai-restricted/vertex-vision-model-garden-dockers/jax-vit-model-conversion\"\n",
+        "# The pre-built prediction docker image.\n",
+        "OPTIMIZED_TF_RUNTIME_IMAGE_URI = (\n",
+        "    \"us-docker.pkg.dev/vertex-ai-restricted/prediction/tf_opt-gpu.nightly:latest\"\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0c250872074f"
+      },
+      "source": [
+        "### Define common functions\n",
+        "\n",
+        "This section defines functions for:\n",
+        "\n",
+        "- Splitting the [tf_flowers dataset](https://www.tensorflow.org/datasets/catalog/tf_flowers) images into `train` and `test` folders.\n",
+        "- Converting a Cloud Storage path such as `gs://bucket-name` to GCSFuse path format such as `/gcsfuse/bucket-name`.\n",
+        "- Encoding a local image file to a string for prediction input."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "XcYUGwr-AJGY"
+      },
+      "outputs": [],
+      "source": [
+        "def split(base_dir, test_ratio=0.1):\n",
+        "    \"\"\"Splits images and moves them to train and test folders.\"\"\"\n",
+        "    paths = glob.glob(f\"{base_dir}/*/*.jpg\")\n",
+        "    random.shuffle(paths)\n",
+        "    counts = dict(test=0, train=0)\n",
+        "    for i, path in enumerate(paths):\n",
+        "        split = \"test\" if i < test_ratio * len(paths) else \"train\"\n",
+        "        *_, class_name, basename = path.split(\"/\")\n",
+        "        dst = f\"{base_dir}/{split}/{class_name}/{basename}\"\n",
+        "        if not os.path.isdir(os.path.dirname(dst)):\n",
+        "            os.makedirs(os.path.dirname(dst))\n",
+        "        shutil.move(path, dst)\n",
+        "        counts[split] += 1\n",
+        "    print(f'Moved {counts[\"train\"]:,} train and {counts[\"test\"]:,} test images.')\n",
+        "\n",
+        "\n",
+        "def gcs_fuse_path(path: str) -> str:\n",
+        "    \"\"\"Try to convert path to gcsfuse path if it starts with gs:// else do not modify it.\"\"\"\n",
+        "    path = path.strip()\n",
+        "    if path.startswith(\"gs://\"):\n",
+        "        return \"/gcs/\" + path[5:]\n",
+        "    return path\n",
+        "\n",
+        "\n",
+        "def load_bytes_from_local_image(local_image_path, new_width=-1):\n",
+        "    \"\"\"Returns encoded image string for prediction input.\"\"\"\n",
+        "    image = Image.open(local_image_path)\n",
+        "    if new_width <= 0:\n",
+        "        new_image = image\n",
+        "    else:\n",
+        "        width, height = image.size\n",
+        "        print(\"original input image size: \", width, \" , \", height)\n",
+        "        new_height = int(height * new_width / width)\n",
+        "        print(\"new input image size: \", new_width, \" , \", new_height)\n",
+        "        new_image = image.resize((new_width, new_height))\n",
+        "    buffered = BytesIO()\n",
+        "    new_image.save(buffered, format=\"JPEG\")\n",
+        "    encoded_string = base64.b64encode(buffered.getvalue()).decode(\"utf-8\")\n",
+        "    return encoded_string"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "X0wWrfhDC8ni"
+      },
+      "source": [
+        "### Prepare dataset\n",
+        "\n",
+        "If you are not using [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/overview#all_datasets), then you need to prepare your dataset and store it on Cloud Storage. The following example shows\n",
+        "how to do this for the [tf_flowers dataset](https://www.tensorflow.org/datasets/catalog/tf_flowers). If using TensorFlow Datasets, you pass\n",
+        "the dataset name such as `tf_flowers` to the `--config.dataset` flag and bypass this section."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "LW31Ws1RN9AC"
+      },
+      "outputs": [],
+      "source": [
+        "local_flower_data_directory = \"./flower_photos\"  # @param {type:\"string\"}\n",
+        "FLOWER_DATA_GCS_PATH = os.path.join(BUCKET_URI, \"flower_dataset\")\n",
+        "# The flower dataset has 5 classes.\n",
+        "NUM_CLASSES = 5\n",
+        "# NOTE: For custom dataset, the training code picks the class names\n",
+        "# from the folder structure and then sorts them to create a mapping\n",
+        "# from class-index to class-name. This is why the mapping below\n",
+        "# looks different from default `tf_flowers` documentation.\n",
+        "LABEL_IDX_TO_STR = {\n",
+        "    0: \"daisy\",\n",
+        "    1: \"dandelion\",\n",
+        "    2: \"roses\",\n",
+        "    3: \"sunflowers\",\n",
+        "    4: \"tulips\",\n",
+        "}"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "heMhYO-DD4II"
+      },
+      "outputs": [],
+      "source": [
+        "# Download flower data to a local directory.\n",
+        "! rm -rf $local_flower_data_directory;\n",
+        "! (cd \"./\" && curl https://storage.googleapis.com/download.tensorflow.org/example_images/flower_photos.tgz | tar xz)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "YtWxe2y8Gqzl"
+      },
+      "outputs": [],
+      "source": [
+        "# Since the default file format of above \"tf_flowers\" dataset is\n",
+        "# flower_photos/{class_name}/{filename}.jpg\n",
+        "# we first need to split it into a \"train\" (90%) and a \"test\" (10%) set:\n",
+        "# flower_photos/train/{class_name}/{filename}.jpg\n",
+        "# flower_photos/test/{class_name}/{filename}.jpg\n",
+        "\n",
+        "split(local_flower_data_directory)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "g043ydQ_wlpk"
+      },
+      "outputs": [],
+      "source": [
+        "# Move Flower data from local directory to Cloud Storage.\n",
+        "# This step takes around 2 mins to finish.\n",
+        "! gsutil -m cp -R $local_flower_data_directory/train/* $FLOWER_DATA_GCS_PATH/train/\n",
+        "! gsutil -m cp -R $local_flower_data_directory/test/* $FLOWER_DATA_GCS_PATH/test/"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "aCpLmWPMpJQ8"
+      },
+      "source": [
+        "## Finetune with JAX Vision Transformer\n",
+        "\n",
+        "Create and run the training job with the model-garden JAX vision transformer training docker using the Vertex AI SDK. The training uses one V100 GPU and runs for around 10 mins once the training job begins."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "aec22792ee84"
+      },
+      "outputs": [],
+      "source": [
+        "# Set up training docker arguments.\n",
+        "\n",
+        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d_%H%M%S\")\n",
+        "JOB_NAME = \"jax_vision_transformer\" + TIMESTAMP\n",
+        "\n",
+        "finetuning_workdir = os.path.join(BUCKET_URI, JOB_NAME)\n",
+        "pre_trained_dir = \"gs://vit_models/imagenet21k\"\n",
+        "docker_args_list = [\n",
+        "    \"--config\",\n",
+        "    \"vit_jax/configs/vit.py:b16\",\n",
+        "    \"--config.dataset\",\n",
+        "    f\"{gcs_fuse_path(FLOWER_DATA_GCS_PATH)}\",\n",
+        "    \"--config.pp.train\",\n",
+        "    \"train\",\n",
+        "    \"--config.pp.test\",\n",
+        "    \"test\",\n",
+        "    \"--config.pretrained_dir\",\n",
+        "    f\"{gcs_fuse_path(pre_trained_dir)}\",\n",
+        "    \"--config.batch\",\n",
+        "    \"128\",\n",
+        "    \"--config.batch_eval\",\n",
+        "    \"128\",\n",
+        "    \"--config.base_lr\",\n",
+        "    \"0.01\",\n",
+        "    \"--config.shuffle_buffer\",\n",
+        "    \"1000\",\n",
+        "    \"--config.total_steps\",\n",
+        "    \"100\",\n",
+        "    \"--config.warmup_steps\",\n",
+        "    \"10\",\n",
+        "    \"--config.pp.crop\",\n",
+        "    \"224\",\n",
+        "    \"--workdir\",\n",
+        "    f\"{gcs_fuse_path(finetuning_workdir)}\",\n",
+        "]\n",
+        "print(docker_args_list)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2ELphfgj1f3Q"
+      },
+      "outputs": [],
+      "source": [
+        "# Create and run the training job.\n",
+        "# Click on the generated link in the output under \"View backing custom job:\" to see your run in the Cloud Console.\n",
+        "NUM_GPU = 1\n",
+        "container_uri = TRAIN_DOCKER_URI\n",
+        "job = aiplatform.CustomContainerTrainingJob(\n",
+        "    display_name=JOB_NAME,\n",
+        "    container_uri=container_uri,\n",
+        ")\n",
+        "model = job.run(\n",
+        "    args=docker_args_list,\n",
+        "    base_output_dir=f\"{finetuning_workdir}\",\n",
+        "    replica_count=1,\n",
+        "    machine_type=\"n1-standard-4\",\n",
+        "    accelerator_type=\"NVIDIA_TESLA_V100\",\n",
+        "    accelerator_count=NUM_GPU,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "-2qiROKIONnI"
+      },
+      "source": [
+        "## Convert JAX Vision Transformer model to TF SavedModel\n",
+        "\n",
+        "Convert the previously fine-tuned JAX model to a TF SavedModel for online prediction."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6Y7slAFLOwlV"
+      },
+      "outputs": [],
+      "source": [
+        "# Set up model conversion docker arguments.\n",
+        "# Note: Many of the arguments below are similar to the training job\n",
+        "# such as the model name and train and test data related parameters.\n",
+        "\n",
+        "jax_checkpoint_dir = finetuning_workdir\n",
+        "\n",
+        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d_%H%M%S\")\n",
+        "JOB_NAME = \"jax_model_conversion\" + TIMESTAMP\n",
+        "saved_model_dir = os.path.join(BUCKET_URI, \"jax2tf_\" + TIMESTAMP)\n",
+        "\n",
+        "docker_args_list = [\n",
+        "    \"--config\",\n",
+        "    \"vit_jax/configs/vit.py:b16\",\n",
+        "    \"--num_classes\",\n",
+        "    f\"{NUM_CLASSES}\",\n",
+        "    \"--saved_model_dir\",\n",
+        "    f\"{saved_model_dir}\",\n",
+        "    \"--jax_checkpoint_dir\",\n",
+        "    f\"{jax_checkpoint_dir}\",\n",
+        "    \"--config.pretrained_dir\",\n",
+        "    f\"{pre_trained_dir}\",\n",
+        "    \"--config.dataset\",\n",
+        "    f\"{gcs_fuse_path(FLOWER_DATA_GCS_PATH)}\",\n",
+        "    \"--config.pp.train\",\n",
+        "    \"train\",\n",
+        "    \"--config.pp.test\",\n",
+        "    \"test\",\n",
+        "    \"--config.pp.crop\",\n",
+        "    \"224\",\n",
+        "]\n",
+        "print(docker_args_list)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "0Acfh1VWUsTL"
+      },
+      "outputs": [],
+      "source": [
+        "# Create and run the model conversion job.\n",
+        "# Click on the generated link in the output under \"View backing custom job:\" to see your run in the Cloud Console.\n",
+        "container_uri = MODEL_CONVERSION_DOCKER_URI\n",
+        "job = aiplatform.CustomContainerTrainingJob(\n",
+        "    display_name=JOB_NAME,\n",
+        "    container_uri=container_uri,\n",
+        ")\n",
+        "model_conversion_workdir = os.path.join(BUCKET_URI, JOB_NAME)\n",
+        "model = job.run(\n",
+        "    args=docker_args_list,\n",
+        "    base_output_dir=f\"{model_conversion_workdir}\",\n",
+        "    replica_count=1,\n",
+        "    machine_type=\"n1-standard-4\",\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "iILhhP3TfO8B"
+      },
+      "source": [
+        "## Run online prediction\n",
+        "\n",
+        "Run online prediction with the converted TF SavedModel."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "XswgX6JqRwFK"
+      },
+      "source": [
+        "Upload TF SavedModel and deploy it to an endpoint for prediction. This step takes around 15 minutes to finish."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "74yqis5ufO8B"
+      },
+      "outputs": [],
+      "source": [
+        "jax_vit_model = aiplatform.Model.upload(\n",
+        "    display_name=\"jax_vit\",\n",
+        "    artifact_uri=saved_model_dir,\n",
+        "    serving_container_image_uri=OPTIMIZED_TF_RUNTIME_IMAGE_URI,\n",
+        "    serving_container_args=[],\n",
+        "    location=REGION,\n",
+        ")\n",
+        "\n",
+        "jax_vit_endpoint = jax_vit_model.deploy(\n",
+        "    deployed_model_display_name=\"jax_vit_deployed\",\n",
+        "    traffic_split={\"0\": 100},\n",
+        "    machine_type=\"n1-standard-4\",\n",
+        "    accelerator_type=\"NVIDIA_TESLA_V100\",\n",
+        "    accelerator_count=1,\n",
+        "    min_replica_count=1,\n",
+        "    max_replica_count=1,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "iiozz1aVR7Pe"
+      },
+      "source": [
+        "Load a local test image file, encode it into a string, send it to the endpoint for prediction, and then generate the final class label from the predicted class probabilities."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "qxj4Xv_DhHXj"
+      },
+      "outputs": [],
+      "source": [
+        "test_directory = os.path.join(local_flower_data_directory, \"test/tulips\")\n",
+        "local_test_image_path = os.path.join(test_directory, os.listdir(test_directory)[0])\n",
+        "print(local_test_image_path)\n",
+        "instances_list = [\n",
+        "    {\n",
+        "        \"bytes_inputs\": {\n",
+        "            \"b64\": load_bytes_from_local_image(local_test_image_path, new_width=240)\n",
+        "        }\n",
+        "    }\n",
+        "]\n",
+        "instances = [json_format.ParseDict(s, Value()) for s in instances_list]\n",
+        "results = jax_vit_endpoint.predict(instances=instances)\n",
+        "logits = results.predictions[0]\n",
+        "predicted_label = LABEL_IDX_TO_STR[int(np.argmax(logits))]\n",
+        "print(\"predicted_label: \", predicted_label)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "TpV-iwP9qw9c"
+      },
+      "source": [
+        "## Cleaning up\n",
+        "\n",
+        "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
+        "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
+        "\n",
+        "Otherwise, you can delete the individual resources you created in this tutorial:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "sx_vKniMq9ZX"
+      },
+      "outputs": [],
+      "source": [
+        "# Delete endpoint resource.\n",
+        "jax_vit_endpoint.delete(force=True)\n",
+        "\n",
+        "# Delete model resource.\n",
+        "jax_vit_model.delete()\n",
+        "\n",
+        "# Delete Cloud Storage objects that were created.\n",
+        "delete_bucket = True\n",
+        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "    ! gsutil -m rm -r $BUCKET_URI"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "model_garden_jax_vision_transformer.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/community/model_garden/model_garden_keras_stable_diffusion.ipynb

@@ -0,0 +1,745 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ur8xi4C7S06n"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2023 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "TirJ-SGQseby"
+      },
+      "source": [
+        "# Vertex AI Model Garden Keras Stable Diffusion\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_keras_stable_diffusion.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_keras_stable_diffusion.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/notebooks/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/model_garden/model_garden_keras_stable_diffusion.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "  </td>\n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dwGLvtIeECLK"
+      },
+      "source": [
+        "**_NOTE_**: This notebook has been tested in the following environment:\n",
+        "\n",
+        "* Python version = 3.9\n",
+        "\n",
+        "You can open this notebook directly in colab, or create [google managed](https://cloud.google.com/vertex-ai/docs/workbench/managed/create-instance) or [user managed](https://cloud.google.com/vertex-ai/docs/workbench/user-managed/create-new) workbench instances."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tvgnzT1CKxrO"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "This notebook demonstrates how to use [Keras Stable Diffusion](https://keras.io/api/keras_cv/models/stable_diffusion) in Vertex AI Model Garden."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0z9r_mBmDeYh"
+      },
+      "source": [
+        "### Objective\n",
+        "\n",
+        "* Run local inferences for pretrained or customized models\n",
+        "\n",
+        "* Deploy pretrained or customized models in Google Cloud Vertex AI\n",
+        "\n",
+        "* Finetune models in Google Cloud Vertex AI"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "xxo28lDtDxn-"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "We use the dataset\n",
+        "[Pokémon BLIP captions](https://huggingface.co/datasets/lambdalabs/pokemon-blip-captions) to show how to finetune the stable diffusion models.\n",
+        "However, we'll use a slightly different version which was derived from the original\n",
+        "dataset to fit better with `tf.data`. Refer to\n",
+        "[the documentation](https://huggingface.co/datasets/sayakpaul/pokemon-blip-original-version)\n",
+        "for more details."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "AEnkHABrDijz"
+      },
+      "source": [
+        "### Costs\n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI\n",
+        "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
+        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
+        "Calculator](https://cloud.google.com/products/calculator/)\n",
+        "to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "z__i0w0lCAsW"
+      },
+      "source": [
+        "## Installation\n",
+        "\n",
+        "Install the following packages required to execute this notebook."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Jvqs-ehKlaYh"
+      },
+      "outputs": [],
+      "source": [
+        "if \"google.colab\" in str(get_ipython()):\n",
+        "    # Configs for colab notebooks.\n",
+        "    ! pip3 install --upgrade --quiet google-cloud-aiplatform\n",
+        "\n",
+        "    # Automatically restart kernel after installs\n",
+        "    import IPython\n",
+        "\n",
+        "    app = IPython.Application.instance()\n",
+        "    app.kernel.do_shutdown(True)\n",
+        "\n",
+        "    from google.colab import auth as google_auth\n",
+        "\n",
+        "    google_auth.authenticate_user()\n",
+        "\n",
+        "# Configs for all notebooks.\n",
+        "! pip3 install --quiet keras-cv==0.4.1"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "KEukV6uRk_S3"
+      },
+      "source": [
+        "## Before you begin\n",
+        "\n",
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI API and Compute Engine API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component).\n",
+        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "\n",
+        "1. [Create a service account](https://cloud.google.com/iam/docs/service-accounts-create#iam-service-accounts-create-console) with `Vertex AI User` and `Storage Object Admin` roles for deploying fine tuned model to Vertex AI endpoint.\n",
+        "\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "BF1j6f9HApxa"
+      },
+      "source": [
+        "### Set your project, region and buckets\n",
+        "\n",
+        "**If you don't know your project ID**, try the following:\n",
+        "* Run `gcloud config list`.\n",
+        "* Run `gcloud projects list`.\n",
+        "* See the support page: [Locate the project ID](https://support.google.com/googleapi/answer/7014113)\n",
+        "\n",
+        "You can change the `REGION` variable used by Vertex AI. Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations).\n",
+        "\n",
+        "You can create a storage bucket to store intermediate artifacts such as datasets, trained models etc."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "YjNCFxq0JxlA"
+      },
+      "outputs": [],
+      "source": [
+        "# The project and bucket are for experiments below.\n",
+        "PROJECT_ID = \"\"  # @param {type:\"string\"}\n",
+        "\n",
+        "! gcloud config set project $PROJECT_ID\n",
+        "\n",
+        "# The form for BUCKET_URI is gs://<bucket-name>.\n",
+        "BUCKET_URI = \"\"  # @param {type:\"string\"}\n",
+        "REGION = \"us-central1\"  # @param {type: \"string\"}\n",
+        "\n",
+        "import os\n",
+        "\n",
+        "STAGING_BUCKET = os.path.join(BUCKET_URI, \"temporal\")\n",
+        "EXPERIMENT_BUCKET = os.path.join(BUCKET_URI, \"keras\")\n",
+        "DATA_BUCKET = os.path.join(EXPERIMENT_BUCKET, \"data\")\n",
+        "MODEL_BUCKET = os.path.join(EXPERIMENT_BUCKET, \"model\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "uDjp76aaLZY9"
+      },
+      "source": [
+        "### Initialize Vertex AI SDK for Python\n",
+        "\n",
+        "Initialize the Vertex AI SDK for Python for your project."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "5uv7-iDKLbO0"
+      },
+      "outputs": [],
+      "source": [
+        "from google.cloud import aiplatform\n",
+        "\n",
+        "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=STAGING_BUCKET)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ZZFPe_GezXg8"
+      },
+      "source": [
+        "### Define constants and common functions"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "XcYUGwr-AJGY"
+      },
+      "outputs": [],
+      "source": [
+        "import base64\n",
+        "import os\n",
+        "from datetime import datetime\n",
+        "from io import BytesIO\n",
+        "\n",
+        "import matplotlib.pyplot as plt\n",
+        "from google.cloud import storage\n",
+        "from PIL import Image\n",
+        "\n",
+        "GCS_URI_PREFIX = \"gs://\"\n",
+        "\n",
+        "# Training constants.\n",
+        "TRAINING_JOB_PREFIX = \"train\"\n",
+        "TRAIN_CONTAINER_URI = \"us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/keras-train:latest\"\n",
+        "TRAIN_MACHINE_TYPE = \"a2-highgpu-1g\"\n",
+        "TRAIN_ACCELERATOR_TYPE = \"NVIDIA_TESLA_A100\"\n",
+        "TRAIN_NUM_GPU = 1\n",
+        "RESOLUTION = 512\n",
+        "\n",
+        "# Prediction constants.\n",
+        "PREDICTION_CONTAINER_URI = \"us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/keras-serve:latest\"\n",
+        "PREDICTION_ACCELERATOR_TYPE = \"NVIDIA_TESLA_V100\"\n",
+        "PREDICTION_MACHINE_TYPE = \"n1-standard-8\"\n",
+        "DEPLOY_JOB_PREFIX = \"deploy\"\n",
+        "\n",
+        "\n",
+        "def get_job_name_with_datetime(prefix: str):\n",
+        "    return prefix + datetime.now().strftime(\"_%Y%m%d_%H%M%S\")\n",
+        "\n",
+        "\n",
+        "def download_data_to_gcs(tar_filepath, gcs_bucket):\n",
+        "    filename_with_ext = os.path.basename(tar_filepath)\n",
+        "    filename_without_ext = filename_with_ext.replace(\".tar.gz\", \"\")\n",
+        "    print(\"Download files from: \", tar_filepath)\n",
+        "    ! wget $tar_filepath -O $filename_with_ext\n",
+        "    ! mkdir -p $filename_without_ext\n",
+        "    ! tar -xvf $filename_with_ext -C .\n",
+        "\n",
+        "    ! gsutil -m cp -r $filename_without_ext $gcs_bucket/\n",
+        "    gcs_path = os.path.join(gcs_bucket, filename_without_ext)\n",
+        "    print(\"Upload files to: \", gcs_path)\n",
+        "    return gcs_path\n",
+        "\n",
+        "\n",
+        "def download_gcs_file_to_local(gcs_uri: str, local_path: str):\n",
+        "    \"\"\"Download a gcs file to a local path.\n",
+        "\n",
+        "    Args:\n",
+        "      gcs_uri: A string of file path on GCS.\n",
+        "      local_path: A string of local file path.\n",
+        "    \"\"\"\n",
+        "    if not gcs_uri.startswith(GCS_URI_PREFIX):\n",
+        "        raise ValueError(f\"{gcs_uri} is not a GCS path starting with {GCS_URI_PREFIX}.\")\n",
+        "    client = storage.Client()\n",
+        "    os.makedirs(os.path.dirname(local_path), exist_ok=True)\n",
+        "    with open(local_path, \"wb\") as f:\n",
+        "        client.download_blob_to_file(gcs_uri, f)\n",
+        "\n",
+        "\n",
+        "def deploy_model(model_path, service_account):\n",
+        "\n",
+        "    deploy_model_name = get_job_name_with_datetime(DEPLOY_JOB_PREFIX)\n",
+        "    print(\"The deployed job name is: \", deploy_model_name)\n",
+        "    serving_env = {\n",
+        "        \"MODEL_PATH\": f\"{model_path}\",\n",
+        "        \"IMAGE_WIDTH\": f\"{RESOLUTION}\",\n",
+        "        \"IMAGE_HEIGHT\": f\"{RESOLUTION}\",\n",
+        "    }\n",
+        "\n",
+        "    endpoint = aiplatform.Endpoint.create(display_name=f\"{deploy_model_name}-endpoint\")\n",
+        "    model = aiplatform.Model.upload(\n",
+        "        display_name=deploy_model_name,\n",
+        "        serving_container_image_uri=PREDICTION_CONTAINER_URI,\n",
+        "        serving_container_ports=[8501],\n",
+        "        serving_container_predict_route=\"/predict\",\n",
+        "        serving_container_health_route=\"/ping\",\n",
+        "        serving_container_environment_variables=serving_env,\n",
+        "    )\n",
+        "    model.deploy(\n",
+        "        endpoint=endpoint,\n",
+        "        machine_type=PREDICTION_MACHINE_TYPE,\n",
+        "        accelerator_type=PREDICTION_ACCELERATOR_TYPE,\n",
+        "        accelerator_count=1,\n",
+        "        min_replica_count=1,\n",
+        "        max_replica_count=1,\n",
+        "        deploy_request_timeout=1800,\n",
+        "        service_account=service_account,\n",
+        "    )\n",
+        "    return model, endpoint\n",
+        "\n",
+        "\n",
+        "def base64_to_image(image_str):\n",
+        "    image = Image.open(BytesIO(base64.b64decode(image_str)))\n",
+        "    return image\n",
+        "\n",
+        "\n",
+        "def display_image(image):\n",
+        "    _ = plt.figure(figsize=(20, 15))\n",
+        "    plt.grid(False)\n",
+        "    plt.imshow(image)\n",
+        "\n",
+        "\n",
+        "def display_image_grid(imgs, rows=2, cols=2):\n",
+        "    w, h = imgs[0].size\n",
+        "    grid = Image.new(\"RGB\", size=(cols * w, rows * h))\n",
+        "    for i, img in enumerate(imgs):\n",
+        "        grid.paste(img, box=(i % cols * w, i // cols * h))\n",
+        "    return grid"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "epo-RHXzcBBT"
+      },
+      "source": [
+        "## Run inferences\n",
+        "\n",
+        "This section shows how to run inferences with Keras Stable Diffusion models.\n",
+        "\n",
+        "1. Run inferences locally\n",
+        "2. Run inferences with serving dockers\n",
+        "\n",
+        "You can run inferences with pre-trained models from Keras team, or your own finetuned models.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6zsa9vnBHhvO"
+      },
+      "outputs": [],
+      "source": [
+        "# Sets the model_path to empty to load the pre-trained model from Keras team.\n",
+        "# Sets the model_path to a gcs uri to load the finetuned models.\n",
+        "model_path = \"\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ld39hkcIceE2"
+      },
+      "source": [
+        "### Run inferences locally\n",
+        "Local inferences can finish in seconds with GPUs.\n",
+        "\n",
+        "Load models first."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "G1nCKVSac3Y5"
+      },
+      "outputs": [],
+      "source": [
+        "from keras_cv.models import StableDiffusion\n",
+        "\n",
+        "model = StableDiffusion(img_height=RESOLUTION, img_width=RESOLUTION, jit_compile=True)\n",
+        "if model_path.startswith(GCS_URI_PREFIX):\n",
+        "    local_model_path = \"/tmp/saved_model.h5\"\n",
+        "    download_gcs_file_to_local(model_path, local_model_path)\n",
+        "    model.diffusion_model.load_weights(local_model_path)\n",
+        "elif model_path:\n",
+        "    model.diffusion_model.load_weights(model_path)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ABaCSIWuP-_G"
+      },
+      "source": [
+        "Then run inferences."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "pnyeVsh8RNI5"
+      },
+      "outputs": [],
+      "source": [
+        "batch_size = 1\n",
+        "img = model.text_to_image(\n",
+        "    prompt=\"a squirrel in Picasso style\",\n",
+        "    batch_size=batch_size,  # How many images to generate at once\n",
+        "    num_steps=25,  # Number of iterations (controls image quality)\n",
+        "    seed=123,  # A fixed seed guarantees the same prompt always generates the same image\n",
+        ")\n",
+        "for i in range(batch_size):\n",
+        "    display_image(img[i])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "kY87SU9Adq4o"
+      },
+      "source": [
+        "### Serve models with dockers\n",
+        "When serve models with dockers, we will deploy models in Google Cloud Vertex AI. The default setting will use 1 V100 GPU for deployment.\n",
+        "\n",
+        "Please create a Service Account for serving with dockers if you do not have one yet.\n",
+        "\n",
+        "The model deployment will take ~10 minutes to finish."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "yCB9vu7RenY6"
+      },
+      "outputs": [],
+      "source": [
+        "# The service account looks like:\n",
+        "# '<account_name>@<project>.iam.gserviceaccount.com'\n",
+        "# Please go to https://cloud.google.com/iam/docs/service-accounts-create#iam-service-accounts-create-console\n",
+        "# and create service account with `Vertex AI User` and `Storage Object Admin` roles.\n",
+        "SERVICE_ACCOUNT = \"\"  # @param {type:\"string\"}\n",
+        "\n",
+        "model, endpoint = deploy_model(\n",
+        "    model_path=model_path,\n",
+        "    service_account=SERVICE_ACCOUNT,\n",
+        ")\n",
+        "\n",
+        "endpoint_id = endpoint.name\n",
+        "print(\"endpoint id is: \", endpoint_id)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "72_BW_BgfvYT"
+      },
+      "source": [
+        "Once deployed, you can send a batch of text prompts to the endpoint to generate images.\n",
+        "\n",
+        "Note, the inference time for the first request for a fresh deployment will need more time to process and take ~45 seconds on one V100 GPU. The inferences for further request is ~12 seconds on one V100 GPU per image."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "U_jrNcZ5eVbH"
+      },
+      "outputs": [],
+      "source": [
+        "# # Loads an existing endpoint as below.\n",
+        "# endpoint_id = <An Existing Endpoint ID>\n",
+        "# aip_endpoint_name = (\n",
+        "#     f\"projects/{PROJECT_ID}/locations/{REGION}/endpoints/{endpoint_id}\"\n",
+        "# )\n",
+        "# endpoint = aiplatform.Endpoint(aip_endpoint_name)\n",
+        "\n",
+        "instances = [\n",
+        "    {\"prompt\": \"a squirrel in Picasso style\"},\n",
+        "    {\"prompt\": \"a dog in Picasso style\"},\n",
+        "    {\"prompt\": \"a cat in Picasso style\"},\n",
+        "    {\"prompt\": \"a deer in Picasso style\"},\n",
+        "]\n",
+        "\n",
+        "parameters = {\n",
+        "    \"batch_size\": 1,  # How many images to generate at once\n",
+        "    \"num_steps\": 25,  # Number of iterations (controls image quality)\n",
+        "    \"seed\": 123,  # A fixed seed guarantees the same prompt always generates the same image\n",
+        "}\n",
+        "response = endpoint.predict(instances=instances, parameters=parameters)\n",
+        "# prediction['predicted_image'] will contains the prediction images in a batch.\n",
+        "# The batch size in this example is 1, and the visualization only parses the\n",
+        "# first predicted image.\n",
+        "images = [\n",
+        "    base64_to_image(prediction[\"predicted_image\"][0])\n",
+        "    for prediction in response.predictions\n",
+        "]\n",
+        "display_image_grid(images, rows=2, cols=2)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "LiQF7fm6f842"
+      },
+      "source": [
+        "### Clean up"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "eqJyypt-f9K6"
+      },
+      "outputs": [],
+      "source": [
+        "# Undeploys models and deletes endpoints.\n",
+        "endpoint.delete(force=True)\n",
+        "# Deletes models.\n",
+        "model.delete()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "RB_xY9ipr7ZU"
+      },
+      "source": [
+        "## Finetune models\n",
+        "This section shows how to finetune Keras Stable diffusion models with training dockers.\n",
+        "\n",
+        "If you would like to use finetuned models, please go to the section `Run inferences`."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "OD3TtaWs5b4v"
+      },
+      "source": [
+        "### Download data\n",
+        " We download the data to GCS storage for the experiments with training dockers."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2TVB8MU-5i-q"
+      },
+      "outputs": [],
+      "source": [
+        "# Skips this step if you have already downloaded the dataset.\n",
+        "download_data_to_gcs(\n",
+        "    \"https://huggingface.co/datasets/sayakpaul/pokemon-blip-original-version/resolve/main/pokemon_dataset.tar.gz\",\n",
+        "    DATA_BUCKET,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Ee7Hzq8O5jgF"
+      },
+      "source": [
+        "### Start training jobs\n",
+        "We finetune 512*512 stable diffusion models with 1 epoch and it takes ~15 minutes to finish using 1 A100 GPU with default settings."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "riG_qUokg0XZ"
+      },
+      "outputs": [],
+      "source": [
+        "data_csv = os.path.join(DATA_BUCKET, \"pokemon_dataset/data.csv\")\n",
+        "epochs = 1\n",
+        "\n",
+        "train_job_name = get_job_name_with_datetime(TRAINING_JOB_PREFIX)\n",
+        "model_dir = os.path.join(MODEL_BUCKET, train_job_name)\n",
+        "worker_pool_specs = [\n",
+        "    {\n",
+        "        \"machine_spec\": {\n",
+        "            \"machine_type\": TRAIN_MACHINE_TYPE,\n",
+        "            \"accelerator_type\": TRAIN_ACCELERATOR_TYPE,\n",
+        "            \"accelerator_count\": TRAIN_NUM_GPU,\n",
+        "        },\n",
+        "        \"replica_count\": 1,\n",
+        "        \"disk_spec\": {\n",
+        "            \"boot_disk_type\": \"pd-ssd\",\n",
+        "            \"boot_disk_size_gb\": 500,\n",
+        "        },\n",
+        "        \"container_spec\": {\n",
+        "            \"image_uri\": TRAIN_CONTAINER_URI,\n",
+        "            \"command\": [],\n",
+        "            \"env\": [\n",
+        "                {\n",
+        "                    \"name\": \"RESOLUTION\",\n",
+        "                    \"value\": f\"{RESOLUTION}\",\n",
+        "                },\n",
+        "            ],\n",
+        "            \"args\": [\n",
+        "                f\"--epochs={epochs}\",\n",
+        "                f\"--input_csv_path={data_csv}\",\n",
+        "                f\"--output_model_dir={model_dir}\",\n",
+        "            ],\n",
+        "        },\n",
+        "    }\n",
+        "]\n",
+        "\n",
+        "train_job = aiplatform.CustomJob(\n",
+        "    display_name=train_job_name,\n",
+        "    project=PROJECT_ID,\n",
+        "    worker_pool_specs=worker_pool_specs,\n",
+        "    staging_bucket=STAGING_BUCKET,\n",
+        ")\n",
+        "\n",
+        "train_job.run()\n",
+        "\n",
+        "model_path = os.path.join(model_dir, \"saved_model.h5\")\n",
+        "print(\"The trained model is saved as: \", model_path)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "wBlQ6FQlJhBi"
+      },
+      "source": [
+        "After the training finishes, you can use `model_path` and then go to the `Run inferences` section above to run predictions."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "kkH2nrpdp4sp"
+      },
+      "source": [
+        "### Clean up"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Ax6vQVZhp9pR"
+      },
+      "outputs": [],
+      "source": [
+        "train_job.delete()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "1dijQDiZWegt"
+      },
+      "source": [
+        "## References\n",
+        "\n",
+        "- [Fine-tuning Stable Diffusion](https://keras.io/examples/generative/finetune_stable_diffusion/)\n",
+        "- [StableDiffusion image-generation model](https://keras.io/api/keras_cv/models/stable_diffusion/)\n",
+        "- [High-performance image generation using Stable Diffusion in KerasCV](https://keras.io/guides/keras_cv/generate_images_with_stable_diffusion/)"
+      ]
+    }
+  ],
+  "metadata": {
+    "accelerator": "GPU",
+    "colab": {
+      "name": "model_garden_keras_stable_diffusion.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/community/model_garden/model_garden_mediapipe_gesture_recognition.ipynb

@@ -0,0 +1,641 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ur8xi4C7S06n"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2023 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "TirJ-SGQseby"
+      },
+      "source": [
+        "# Vertex AI Model Garden MediaPipe with gesture recognition\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_mediapipe_gesture_recognition.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_mediapipe_gesture_recognition.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>                                                                                               <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/notebooks/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/model_garden/model_garden_mediapipe_gesture_recognition.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "  </td>\n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dwGLvtIeECLK"
+      },
+      "source": [
+        "**_NOTE_**: This notebook has been tested in the following environment:\n",
+        "\n",
+        "* Python version = 3.9"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tvgnzT1CKxrO"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "This notebook demonstrates how to use [MediaPipe Model Maker](https://developers.google.com/mediapipe/solutions/model_maker) to train an on-device gesture recognition model in Vertex AI Model Garden.\n",
+        "\n",
+        "### Objective\n",
+        "\n",
+        "* Train new models\n",
+        "  * Convert input data to training formats\n",
+        "  * Create [custom jobs](https://cloud.google.com/vertex-ai/docs/training/create-custom-job) to train new models\n",
+        "  * Export models\n",
+        "\n",
+        "* Cleanup resources\n",
+        "\n",
+        "### Costs\n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI\n",
+        "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
+        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
+        "Calculator](https://cloud.google.com/products/calculator/)\n",
+        "to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "KEukV6uRk_S3"
+      },
+      "source": [
+        "## Before you begin"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "z__i0w0lCAsW"
+      },
+      "source": [
+        "### Colab only\n",
+        "Run the following commands to install dependencies and to authenticate with Google Cloud if running on Colab."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Jvqs-ehKlaYh"
+      },
+      "outputs": [],
+      "source": [
+        "! pip3 install --upgrade pip\n",
+        "\n",
+        "import sys\n",
+        "\n",
+        "if \"google.colab\" in sys.modules:\n",
+        "    ! pip3 install --upgrade google-cloud-aiplatform\n",
+        "\n",
+        "    # Automatically restart kernel after installs\n",
+        "    import IPython\n",
+        "\n",
+        "    app = IPython.Application.instance()\n",
+        "    app.kernel.do_shutdown(True)\n",
+        "\n",
+        "    from google.colab import auth as google_auth\n",
+        "\n",
+        "    google_auth.authenticate_user()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WReHDGG5g0XY"
+      },
+      "source": [
+        "#### Set your project ID\n",
+        "\n",
+        "**If you don't know your project ID**, see the support page: [Locate the project ID](https://support.google.com/googleapi/answer/7014113)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "oM1iC_MfAts1"
+      },
+      "outputs": [],
+      "source": [
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# Set the project id\n",
+        "! gcloud config set project {PROJECT_ID}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "region"
+      },
+      "source": [
+        "#### Region\n",
+        "\n",
+        "You can also change the `REGION` variable used by Vertex AI. Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "tTy1gX11kCJY"
+      },
+      "outputs": [],
+      "source": [
+        "REGION = \"us-central1\"  # @param {type: \"string\"}\n",
+        "REGION_PREFIX = REGION.split(\"-\")[0]\n",
+        "assert REGION_PREFIX in (\n",
+        "    \"us\",\n",
+        "    \"europe\",\n",
+        "    \"asia\",\n",
+        "), f'{REGION} is not supported. It must be prefixed by \"us\", \"asia\", or \"europe\".'"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "zgPO1eR3CYjk"
+      },
+      "source": [
+        "### Create a Cloud Storage bucket\n",
+        "\n",
+        "Create a storage bucket to store intermediate artifacts such as datasets."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "MzGDU7TWdts_"
+      },
+      "outputs": [],
+      "source": [
+        "BUCKET_URI = f\"gs://your-bucket-name-{PROJECT_ID}-unique\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "-EcIXiGsCePi"
+      },
+      "source": [
+        "**Only if your bucket doesn't already exist**: Run the following cell to create your Cloud Storage bucket."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "NIq7R4HZCfIc"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil mb -l {REGION} -p {PROJECT_ID} {BUCKET_URI}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "960505627ddf"
+      },
+      "source": [
+        "### Import libraries"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "PyQmSRbKA8r-"
+      },
+      "outputs": [],
+      "source": [
+        "import json\n",
+        "import os\n",
+        "from datetime import datetime\n",
+        "\n",
+        "import tensorflow\n",
+        "from google.cloud import aiplatform"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "init_aip:mbsdk,all"
+      },
+      "source": [
+        "### Initialize Vertex AI SDK for Python\n",
+        "\n",
+        "Initialize the Vertex AI SDK for Python for your project."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9wExiMUxFk91"
+      },
+      "outputs": [],
+      "source": [
+        "now = datetime.now().strftime(\"%Y%m%d-%H%M%S\")\n",
+        "\n",
+        "STAGING_BUCKET = os.path.join(BUCKET_URI, \"temp/%s\" % now)\n",
+        "\n",
+        "EVALUATION_RESULT_OUTPUT_DIRECTORY = os.path.join(STAGING_BUCKET, \"evaluation\")\n",
+        "EVALUATION_RESULT_OUTPUT_FILE = os.path.join(\n",
+        "    EVALUATION_RESULT_OUTPUT_DIRECTORY, \"evaluation.json\"\n",
+        ")\n",
+        "\n",
+        "EXPORTED_MODEL_OUTPUT_DIRECTORY = os.path.join(STAGING_BUCKET, \"model\")\n",
+        "EXPORTED_MODEL_OUTPUT_FILE = os.path.join(\n",
+        "    EXPORTED_MODEL_OUTPUT_DIRECTORY, \"gesture_recognizer.task\"\n",
+        ")\n",
+        "\n",
+        "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=STAGING_BUCKET)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "n6IFz75WGCam"
+      },
+      "source": [
+        "### Define training machine specs"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "riG_qUokg0XZ"
+      },
+      "outputs": [],
+      "source": [
+        "TRAINING_JOB_DISPLAY_NAME = \"mediapipe_gesture_recognizer_%s\" % now\n",
+        "TRAINING_CONTAINER = f\"{REGION_PREFIX}-docker.pkg.dev/vertex-ai-restricted/vertex-vision-model-garden-dockers/mediapipe-train\"\n",
+        "TRAINING_MACHINE_TYPE = \"n1-highmem-16\"\n",
+        "TRAINING_ACCELARATOR_TYPE = \"NVIDIA_TESLA_V100\"\n",
+        "TRAINING_ACCELERATOR_COUNT = 2"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "-rsdAcBV-vlf"
+      },
+      "source": [
+        "## Train your customized models"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "zgPO1eR3CYjk"
+      },
+      "source": [
+        "### Prepare input data for training\n",
+        "\n",
+        "Finetuning a model for gesture recognition requires a dataset with a directory structure following the pattern `<dataset_path>/<label_name>/<img_name>.*` (e.g. `my_custom_dataset/thumbs_up/img12.jpg`). In addition, one of the label names must be none. The none label represents any gesture that isn't classified as one of the other gestures.\n",
+        "\n",
+        "This example uses a rock paper scissors dataset sample which is available on Cloud Storage.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "IndQ_m6ddUEM"
+      },
+      "outputs": [],
+      "source": [
+        "training_data_path = (\n",
+        "    \"gs://mediapipe-tasks/gesture_recognizer/rps_data_sample\"  # @param {type:\"string\"}\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ci4IV6vdXRMD"
+      },
+      "source": [
+        "When Model Maker loads the dataset, it runs the pre-packaged hand detection model from MediaPipe Hands to detect the hand landmarks from the images. Any images without detected hands are ommitted from the dataset. The resulting dataset will contain the extracted hand landmark positions from each image, rather than images themselves.\n",
+        "\n",
+        "You can configure a few options that determine how the dataset is loaded:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "aNHLSyFtXP7I"
+      },
+      "outputs": [],
+      "source": [
+        "# A boolean controlling whether to shuffle the dataset. Defaults to true.\n",
+        "shuffle = True  # @param {type:\"boolean\"}\n",
+        "# A float between 0 and 1 controlling the confidence threshold for hand detection\n",
+        "min_detection_confidence = 0.6  # @param {type:\"number\"}\n",
+        "# Configures how to split the dataset between training, validation and test data. Must sum to up 1.\n",
+        "split_ratio = \"0.8,0.1,0.1\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "aaff6f5be7f6"
+      },
+      "source": [
+        "### Set fine-tuning options\n",
+        "\n",
+        "You can customize the model using the by specifying ModelOptions and HParams. The ModelOptions contain parameters related to the model itself, while the HParams  contains parameters related to training and saving the model.\n",
+        "\n",
+        "The ModelOptions contain these customizable parameter that affects accuracy:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "bDxsEaoGcibW"
+      },
+      "outputs": [],
+      "source": [
+        "# The fraction of the input units to drop. Used in dropout layer.\n",
+        "dropout_rate: float = 0.05  # @param {type:\"number\"}\n",
+        "# A list of hidden layer widths for the gesture model. Each element\n",
+        "# in the list will create a new hidden layer with the specified width.\n",
+        "# The hidden layers are separated with BatchNorm, Dropout, and ReLU.\n",
+        "layer_widths: str = \"\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "fk0TTZbDdJPX"
+      },
+      "source": [
+        "HParams has the following list of customizable parameters which affect model accuracy:\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "um_XKbmpTaHx"
+      },
+      "outputs": [],
+      "source": [
+        "# The learning rate to use for gradient descent training.\n",
+        "learning_rate: float = 0.001  # @param {type:\"number\"}\n",
+        "# Batch size for training.\n",
+        "batch_size: int = 2  # @param {type:\"number\"}\n",
+        "# Number of training iterations over the dataset.\n",
+        "epochs: int = 10  # @param {type:\"slider\", min:0, max:100, step:1}\n",
+        "# An optional integer that indicates the number of training steps per\n",
+        "# epoch. If set to 0, the training pipeline calculates the default\n",
+        "# steps per epoch as the training dataset size divided by batch size.\n",
+        "steps_per_epoch: int = 0  # @param {type:\"number\"}\n",
+        "# Whether to shuffle the dataset before training\n",
+        "shuffle: bool = False  # @param {type:\"boolean\"}\n",
+        "# Learning rate decay to use for gradient descent training.\n",
+        "lr_decay: float = 0.99  # @param {type:\"number\"}\n",
+        "# Gamma parameter for focal loss. Defaults to 2\n",
+        "gamma: float = 2  # @param {type:\"number\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "HwcCjwlBTQIz"
+      },
+      "source": [
+        "### Run fine-tuning\n",
+        "With your training dataset and fine-tuning options prepared, you are ready to start the fine-tuning process. This process is resource intensive and can take a few minutes to complete. On Vertex AI with GPU processing, the example fine-tuning below takes between 1-2 minutes to train on approximately 500 images.\n",
+        "\n",
+        "To begin the fine-tuning process, use the following code:\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "aec22792ee84"
+      },
+      "outputs": [],
+      "source": [
+        "model_export_path = EXPORTED_MODEL_OUTPUT_DIRECTORY\n",
+        "evaluation_result_path = EVALUATION_RESULT_OUTPUT_DIRECTORY\n",
+        "\n",
+        "model_options = {\"dropout_rate\": dropout_rate}\n",
+        "if layer_widths:\n",
+        "    model_options[\"layer_widths\"] = layer_widths\n",
+        "\n",
+        "hparams = {\n",
+        "    \"learning_rate\": learning_rate,\n",
+        "    \"batch_size\": batch_size,\n",
+        "    \"epochs\": epochs,\n",
+        "    \"shuffle\": shuffle,\n",
+        "    \"lr_decay\": lr_decay,\n",
+        "    \"gamma\": gamma,\n",
+        "}\n",
+        "if steps_per_epoch:\n",
+        "    hparams[\"steps_per_epoch\"] = steps_per_epoch\n",
+        "\n",
+        "worker_pool_specs = [\n",
+        "    {\n",
+        "        \"machine_spec\": {\n",
+        "            \"machine_type\": TRAINING_MACHINE_TYPE,\n",
+        "            \"accelerator_type\": TRAINING_ACCELARATOR_TYPE,\n",
+        "            \"accelerator_count\": TRAINING_ACCELERATOR_COUNT,\n",
+        "        },\n",
+        "        \"replica_count\": 1,\n",
+        "        \"container_spec\": {\n",
+        "            \"image_uri\": TRAINING_CONTAINER,\n",
+        "            \"command\": [],\n",
+        "            \"args\": [\n",
+        "                \"--task_name=gesture_recognizer\",\n",
+        "                \"--training_data_path=%s\" % training_data_path,\n",
+        "                \"--model_export_path=%s\" % model_export_path,\n",
+        "                \"--evaluation_result_path=%s\" % evaluation_result_path,\n",
+        "                \"--split_ratio=%s\" % split_ratio,\n",
+        "                \"--model_options=%s\" % json.dumps(model_options),\n",
+        "                \"--hparams=%s\" % json.dumps(hparams),\n",
+        "            ],\n",
+        "        },\n",
+        "    }\n",
+        "]\n",
+        "\n",
+        "training_job = aiplatform.CustomJob(\n",
+        "    display_name=TRAINING_JOB_DISPLAY_NAME,\n",
+        "    project=PROJECT_ID,\n",
+        "    worker_pool_specs=worker_pool_specs,\n",
+        "    staging_bucket=STAGING_BUCKET,\n",
+        ")\n",
+        "\n",
+        "training_job.run()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "rXMF2tnV_WS0"
+      },
+      "source": [
+        "## Evaluate and export model"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "mV-Djz-frBni"
+      },
+      "source": [
+        "### Evaluate performance\n",
+        "\n",
+        "After fine-tuning the model, we evaluate the training result on a test dataset, which is typically a portion of your original dataset not used during training. Accuracy levels between 0.8 and 0.9 are generally considered very good, but your use case requirements may differ. You should also consider how fast the model can produce an inference. Higher accuracy frequently comes at the cost of longer inference times.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "09Rz1AYspK19"
+      },
+      "outputs": [],
+      "source": [
+        "def get_evaluation_result(evaluation_result_path):\n",
+        "    try:\n",
+        "        with tensorflow.io.gfile.GFile(evaluation_result_path, \"r\") as input_file:\n",
+        "            evalutation_result = json.loads(input_file.read())\n",
+        "        return evalutation_result[\"accuracy\"], evalutation_result[\"loss\"]\n",
+        "    except:\n",
+        "        print(\n",
+        "            \"Evaluation result not found. Your test dataset is likely \"\n",
+        "            + \"empty. You can adjust the size of your test dataset or adjust \"\n",
+        "            + \"how you split your dataset.\"\n",
+        "        )\n",
+        "        return None\n",
+        "\n",
+        "\n",
+        "evaluation_result = get_evaluation_result(EVALUATION_RESULT_OUTPUT_FILE)\n",
+        "\n",
+        "if evaluation_result is not None:\n",
+        "    print(\"Accuracy:\", evaluation_result[0])\n",
+        "    print(\"Loss:\", evaluation_result[1])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "g0BGaofgsMsy"
+      },
+      "source": [
+        "### Export model\n",
+        "After finetuning and evaluating the model, you can save the Tensorflow Lite model, try it out in the [Gesture Recognizer](https://mediapipe-studio.webapps.google.com/demo/gesture_recognizer) demo in MediaPipe Studio or integrate it with your on-device application by following the [Gesture recognizer task guide](https://developers.google.com/mediapipe/solutions/vision/gesture_recognizer). The exported model contains the generates required model metadata, as well as a classification label file."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "NYuQowyZEtxK"
+      },
+      "outputs": [],
+      "source": [
+        "import sys\n",
+        "\n",
+        "\n",
+        "def copy_model(model_source, model_dest):\n",
+        "    ! gsutil cp {model_source} {model_dest}\n",
+        "\n",
+        "copy_model(EXPORTED_MODEL_OUTPUT_FILE, \"gesture_recognizer.task\")\n",
+        "\n",
+        "if \"google.colab\" in sys.modules:\n",
+        "    from google.colab import files\n",
+        "\n",
+        "    files.download(\"gesture_recognizer.task\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "kkH2nrpdp4sp"
+      },
+      "source": [
+        "## Clean up"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Ax6vQVZhp9pR"
+      },
+      "outputs": [],
+      "source": [
+        "# Delete training data and jobs.\n",
+        "if training_job.list(filter=f'display_name=\"{TRAINING_JOB_DISPLAY_NAME}\"'):\n",
+        "    training_job.delete()\n",
+        "\n",
+        "!gsutil rm -r {STAGING_BUCKET}"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "model_garden_mediapipe_gesture_recognition.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/community/model_garden/model_garden_mediapipe_image_classification.ipynb

@@ -0,0 +1,619 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ur8xi4C7S06n"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2023 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "TirJ-SGQseby"
+      },
+      "source": [
+        "# Vertex AI Model Garden MediaPipe with image classification\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_mediapipe_image_classification.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_mediapipe_image_classification.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/notebooks/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/model_garden/model_garden_mediapipe_image_classification.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "  </td>\n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dwGLvtIeECLK"
+      },
+      "source": [
+        "**_NOTE_**: This notebook has been tested in the following environment:\n",
+        "\n",
+        "* Python version = 3.9"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tvgnzT1CKxrO"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "This notebook demonstrates how to use [MediaPipe Model Maker](https://developers.google.com/mediapipe/solutions/model_maker) to train an on-device image classification model in Vertex AI Model Garden.\n",
+        "\n",
+        "### Objective\n",
+        "\n",
+        "* Train new models\n",
+        "  * Convert input data to training formats\n",
+        "  * Create [custom jobs](https://cloud.google.com/vertex-ai/docs/training/create-custom-job) to train new models\n",
+        "  * Export models\n",
+        "\n",
+        "* Cleanup resources\n",
+        "\n",
+        "### Costs\n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI\n",
+        "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
+        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
+        "Calculator](https://cloud.google.com/products/calculator/)\n",
+        "to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "KEukV6uRk_S3"
+      },
+      "source": [
+        "## Before you begin"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "z__i0w0lCAsW"
+      },
+      "source": [
+        "### Colab only\n",
+        "Run the following commands to install dependencies and to authenticate with Google Cloud if running on Colab."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Jvqs-ehKlaYh"
+      },
+      "outputs": [],
+      "source": [
+        "! pip3 install --upgrade pip\n",
+        "\n",
+        "import sys\n",
+        "\n",
+        "if \"google.colab\" in sys.modules:\n",
+        "    ! pip3 install --upgrade google-cloud-aiplatform\n",
+        "\n",
+        "    # Automatically restart kernel after installs\n",
+        "    import IPython\n",
+        "\n",
+        "    app = IPython.Application.instance()\n",
+        "    app.kernel.do_shutdown(True)\n",
+        "\n",
+        "    from google.colab import auth as google_auth\n",
+        "\n",
+        "    google_auth.authenticate_user()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WReHDGG5g0XY"
+      },
+      "source": [
+        "#### Set your project ID\n",
+        "\n",
+        "**If you don't know your project ID**, see the support page: [Locate the project ID](https://support.google.com/googleapi/answer/7014113)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "oM1iC_MfAts1"
+      },
+      "outputs": [],
+      "source": [
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# Set the project id\n",
+        "! gcloud config set project {PROJECT_ID}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "region"
+      },
+      "source": [
+        "#### Region\n",
+        "\n",
+        "You can also change the `REGION` variable used by Vertex AI. Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "tTy1gX11kCJY"
+      },
+      "outputs": [],
+      "source": [
+        "REGION = \"us-central1\"  # @param {type: \"string\"}\n",
+        "REGION_PREFIX = REGION.split(\"-\")[0]\n",
+        "assert REGION_PREFIX in (\n",
+        "    \"us\",\n",
+        "    \"europe\",\n",
+        "    \"asia\",\n",
+        "), f'{REGION} is not supported. It must be prefixed by \"us\", \"asia\", or \"europe\".'"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "zgPO1eR3CYjk"
+      },
+      "source": [
+        "### Create a Cloud Storage bucket\n",
+        "\n",
+        "Create a storage bucket to store intermediate artifacts such as datasets."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "MzGDU7TWdts_"
+      },
+      "outputs": [],
+      "source": [
+        "BUCKET_URI = f\"gs://your-bucket-name-{PROJECT_ID}-unique\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "-EcIXiGsCePi"
+      },
+      "source": [
+        "**Only if your bucket doesn't already exist**: Run the following cell to create your Cloud Storage bucket."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "NIq7R4HZCfIc"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil mb -l {REGION} -p {PROJECT_ID} {BUCKET_URI}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "960505627ddf"
+      },
+      "source": [
+        "### Import libraries"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "PyQmSRbKA8r-"
+      },
+      "outputs": [],
+      "source": [
+        "import json\n",
+        "import os\n",
+        "from datetime import datetime\n",
+        "\n",
+        "import tensorflow\n",
+        "from google.cloud import aiplatform"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "init_aip:mbsdk,all"
+      },
+      "source": [
+        "### Initialize Vertex AI SDK for Python\n",
+        "\n",
+        "Initialize the Vertex AI SDK for Python for your project."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9wExiMUxFk91"
+      },
+      "outputs": [],
+      "source": [
+        "now = datetime.now().strftime(\"%Y%m%d-%H%M%S\")\n",
+        "\n",
+        "# The project and bucket are for experiments below.\n",
+        "PROJECT_ID = \"\"  # @param {type:\"string\"}\n",
+        "# The form for BUCKET_URI is gs://<bucket-name>.\\n\",\n",
+        "BUCKET_URI = \"gs://\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# You can choose a region from https://cloud.google.com/about/locations.\n",
+        "# Only regions prefixed by \"us\", \"asia\", or \"europe\" are supported.\n",
+        "REGION = \"us-central1\"  # @param {type:\"string\"}\n",
+        "REGION_PREFIX = REGION.split(\"-\")[0]\n",
+        "assert REGION_PREFIX in (\n",
+        "    \"us\",\n",
+        "    \"europe\",\n",
+        "    \"asia\",\n",
+        "), f'{REGION} is not supported. It must be prefixed by \"us\", \"asia\", or \"europe\".'\n",
+        "\n",
+        "! gcloud config set project $PROJECT_ID\n",
+        "\n",
+        "STAGING_BUCKET = os.path.join(BUCKET_URI, \"temp/%s\" % now)\n",
+        "\n",
+        "EVALUATION_RESULT_OUTPUT_DIRECTORY = os.path.join(STAGING_BUCKET, \"evaluation\")\n",
+        "EVALUATION_RESULT_OUTPUT_FILE = os.path.join(\n",
+        "    EVALUATION_RESULT_OUTPUT_DIRECTORY, \"evaluation.json\"\n",
+        ")\n",
+        "\n",
+        "EXPORTED_MODEL_OUTPUT_DIRECTORY = os.path.join(STAGING_BUCKET, \"model\")\n",
+        "EXPORTED_MODEL_OUTPUT_FILE = os.path.join(\n",
+        "    EXPORTED_MODEL_OUTPUT_DIRECTORY, \"model.tflite\"\n",
+        ")\n",
+        "\n",
+        "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=STAGING_BUCKET)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "n6IFz75WGCam"
+      },
+      "source": [
+        "### Define training machine specs"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "riG_qUokg0XZ"
+      },
+      "outputs": [],
+      "source": [
+        "TRAINING_JOB_DISPLAY_NAME = \"mediapipe_image_classifier_%s\" % now\n",
+        "TRAINING_CONTAINER = f\"{REGION_PREFIX}-docker.pkg.dev/vertex-ai-restricted/vertex-vision-model-garden-dockers/mediapipe-train\"\n",
+        "TRAINING_MACHINE_TYPE = \"n1-highmem-16\"\n",
+        "TRAINING_ACCELARATOR_TYPE = \"NVIDIA_TESLA_V100\"\n",
+        "TRAINING_ACCELERATOR_COUNT = 2"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "-rsdAcBV-vlf"
+      },
+      "source": [
+        "## Train your customized models"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "zgPO1eR3CYjk"
+      },
+      "source": [
+        "### Prepare input data for training\n",
+        "\n",
+        "Finetuning a model for image classification requires a dataset that includes all kinds of items, or classes, that you want the completed model to be able to identify. You can do this by trimming down a public dataset to only the classes that are relevant to your usecase, compiling your own data, or some combination of both. The dataset can be significantly smaller than what would be required to train a new model from scratch. For example, the [ImageNet](https://www.image-net.org/) dataset used to train many reference models contains millions of images with thousands of categories. Transfer learning with Model Maker can finetune an existing model with a smaller dataset and still perform well, depending on your inference accuracy goals.\n",
+        "\n",
+        "You can re-use an existing dataset such as `gs://cloud-samples-data-us-central1/vision/automl_classification/flowers` to finetune the model or you can upload your own dataset to GCS. If you are using your own dataset, ensure that your image directory contains several subdirectories, each corresponding to specific class labels. Your training data should also follow this pattern: <image_path>/<label_name>/<image_names>.*."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "IndQ_m6ddUEM"
+      },
+      "outputs": [],
+      "source": [
+        "training_data_path = \"gs://cloud-samples-data-us-central1/vision/automl_classification/flowers\"  # @param {type:\"string\"}\n",
+        "split_ratio = \"0.8,0.1,0.1\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "aaff6f5be7f6"
+      },
+      "source": [
+        "### Set fine-tuning options\n",
+        "\n",
+        "You can pick between different model architectures to further customize your training:\n",
+        "\n",
+        "*   MobileNet-V2\n",
+        "*   EfficientNet-Lite0\n",
+        "*   EfficientNet-Lite2\n",
+        "*   EfficientNet-Lite4\n",
+        "\n",
+        "To set the model architecture and other training parameters, adjust the following values:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "um_XKbmpTaHx"
+      },
+      "outputs": [],
+      "source": [
+        "model_architecture = \"mobilenet_v2\"  # @param [\"mobilenet_v2\", \"efficientnet_lite0\", \"efficientnet_lite2\", \"efficientnet_lite4\"]\n",
+        "\n",
+        "# The learning rate to use for gradient descent training.\n",
+        "learning_rate: float = 0.01  # @param {type:\"number\"}\n",
+        "# Batch size for training.\n",
+        "batch_size: int = 2  # @param {type:\"number\"}\n",
+        "# Number of training iterations over the dataset.\n",
+        "epochs: int = 10  # @param {type:\"slider\", min:0, max:100, step:1}\n",
+        "# If true, the base module is trained together with the classification layer on\n",
+        "# top.\n",
+        "do_fine_tuning: bool = False  # @param {type:\"boolean\"}\n",
+        "# A regularizer that applies a L1 regularization penalty.\n",
+        "l1_regularizer: float = 0.0  # @param {type:\"number\"}\n",
+        "# A regularizer that applies a L2 regularization penalty.\n",
+        "l2_regularizer: float = 0.0001  # @param {type:\"number\"}\n",
+        "# Amount of label smoothing to apply. See tf.keras.losses for more details.\n",
+        "label_smoothing: float = 0.1  # @param {type:\"number\"}\n",
+        "# A boolean controlling whether the training dataset is augmented by randomly\n",
+        "# distorting input images, including random cropping, flipping, etc. See\n",
+        "# utils.image_preprocessing documentation for details.\n",
+        "do_data_augmentation: bool = True  # @param {type:\"boolean\"}\n",
+        "# Number of training samples used to calculate the decay steps\n",
+        "# and create the training optimizer.\n",
+        "decay_samples: int = 2560000  # @param {type:\"number\"}\n",
+        "# Number of warmup steps for a linear increasing warmup schedule on learning\n",
+        "# rate. Used to set up warmup schedule by model_util.WarmUp.\n",
+        "warmup_epochs: int = 2  # @param {type:\"number\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "HwcCjwlBTQIz"
+      },
+      "source": [
+        "### Run fine-tuning\n",
+        "With your training dataset and fine-tuning options prepared, you are ready to start the fine-tuning process. This process is resource intensive and can take a few minutes to a few hours depending on your available compute resources. On Vertex AI with GPU processing, the example fine-tuning below takes between 4-6 minutes to train on approximately 3700 images.\n",
+        "\n",
+        "To begin the fine-tuning process, use the following code:\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "aec22792ee84"
+      },
+      "outputs": [],
+      "source": [
+        "model_export_path = EXPORTED_MODEL_OUTPUT_DIRECTORY\n",
+        "evaluation_result_path = EVALUATION_RESULT_OUTPUT_DIRECTORY\n",
+        "\n",
+        "worker_pool_specs = [\n",
+        "    {\n",
+        "        \"machine_spec\": {\n",
+        "            \"machine_type\": TRAINING_MACHINE_TYPE,\n",
+        "            \"accelerator_type\": TRAINING_ACCELARATOR_TYPE,\n",
+        "            \"accelerator_count\": TRAINING_ACCELERATOR_COUNT,\n",
+        "        },\n",
+        "        \"replica_count\": 1,\n",
+        "        \"container_spec\": {\n",
+        "            \"image_uri\": TRAINING_CONTAINER,\n",
+        "            \"command\": [],\n",
+        "            \"args\": [\n",
+        "                \"--task_name=image_classifier\",\n",
+        "                \"--training_data_path=%s\" % training_data_path,\n",
+        "                \"--model_export_path=%s\" % model_export_path,\n",
+        "                \"--evaluation_result_path=%s\" % evaluation_result_path,\n",
+        "                \"--split_ratio=%s\" % split_ratio,\n",
+        "                \"--model_architecture=%s\" % model_architecture,\n",
+        "                \"--hparams=%s\"\n",
+        "                % json.dumps(\n",
+        "                    {\n",
+        "                        \"learning_rate\": learning_rate,\n",
+        "                        \"batch_size\": batch_size,\n",
+        "                        \"epochs\": epochs,\n",
+        "                        \"do_fine_tuning\": do_fine_tuning,\n",
+        "                        \"l1_regularizer\": l1_regularizer,\n",
+        "                        \"l2_regularizer\": l2_regularizer,\n",
+        "                        \"label_smoothing\": label_smoothing,\n",
+        "                        \"do_data_augmentation\": do_data_augmentation,\n",
+        "                        \"decay_samples\": decay_samples,\n",
+        "                        \"warmup_epochs\": warmup_epochs,\n",
+        "                    }\n",
+        "                ),\n",
+        "            ],\n",
+        "        },\n",
+        "    }\n",
+        "]\n",
+        "\n",
+        "training_job = aiplatform.CustomJob(\n",
+        "    display_name=TRAINING_JOB_DISPLAY_NAME,\n",
+        "    project=PROJECT_ID,\n",
+        "    worker_pool_specs=worker_pool_specs,\n",
+        "    staging_bucket=STAGING_BUCKET,\n",
+        ")\n",
+        "\n",
+        "training_job.run()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "rXMF2tnV_WS0"
+      },
+      "source": [
+        "## Evaluate and export model"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "mV-Djz-frBni"
+      },
+      "source": [
+        "### Evaluate performance\n",
+        "\n",
+        "After fine-tuning the model, we evaluate the training result on a test dataset, which is typically a portion of your original dataset not used during training. Accuracy levels between 0.8 and 0.9 are generally considered very good, but your use case requirements may differ. You should also consider how fast the model can produce an inference. Higher accuracy frequently comes at the cost of longer inference times.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "09Rz1AYspK19"
+      },
+      "outputs": [],
+      "source": [
+        "def get_evaluation_result(evaluation_result_path):\n",
+        "    try:\n",
+        "        with tensorflow.io.gfile.GFile(evaluation_result_path, \"r\") as input_file:\n",
+        "            evalutation_result = json.loads(input_file.read())\n",
+        "        return evalutation_result[\"accuracy\"], evalutation_result[\"loss\"]\n",
+        "    except:\n",
+        "        print(\n",
+        "            \"Evaluation result not found. Your test dataset is likely \"\n",
+        "            + \"empty. You can adjust the size of your test dataset or adjust \"\n",
+        "            + \"how you split your dataset.\"\n",
+        "        )\n",
+        "        return None\n",
+        "\n",
+        "\n",
+        "evaluation_result = get_evaluation_result(EVALUATION_RESULT_OUTPUT_FILE)\n",
+        "\n",
+        "if evaluation_result is not None:\n",
+        "    print(\"Accuracy:\", evaluation_result[0])\n",
+        "    print(\"Loss:\", evaluation_result[1])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "g0BGaofgsMsy"
+      },
+      "source": [
+        "### Export model\n",
+        "After finetuning and evaluating the model, you can save the Tensorflow Lite model, try it out in the [Image Classification](https://mediapipe-studio.webapps.google.com/demo/image_classifier) demo in MediaPipe Studio or integrate it with your on-device application by following the [Image classification task guide](https://developers.google.com/mediapipe/solutions/vision/image_classifier). The exported model contains the generates required model metadata, as well as a classification label file."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "NYuQowyZEtxK"
+      },
+      "outputs": [],
+      "source": [
+        "import sys\n",
+        "\n",
+        "def copy_model(model_source, model_dest):\n",
+        "    ! gsutil cp {model_source} {model_dest}\n",
+        "\n",
+        "copy_model(EXPORTED_MODEL_OUTPUT_FILE, \"image_classification_model.tflite\")\n",
+        "\n",
+        "if \"google.colab\" in sys.modules:\n",
+        "    from google.colab import files\n",
+        "\n",
+        "    files.download(\"image_classification_model.tflite\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "kkH2nrpdp4sp"
+      },
+      "source": [
+        "## Clean up"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Ax6vQVZhp9pR"
+      },
+      "outputs": [],
+      "source": [
+        "# Delete training data and jobs.\n",
+        "if training_job.list(filter=f'display_name=\"{TRAINING_JOB_DISPLAY_NAME}\"'):\n",
+        "    training_job.delete()\n",
+        "\n",
+        "!gsutil rm -r {STAGING_BUCKET}"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "model_garden_mediapipe_image_classification.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/community/model_garden/model_garden_mediapipe_object_detection.ipynb

@@ -0,0 +1,681 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ur8xi4C7S06n"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2023 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "TirJ-SGQseby"
+      },
+      "source": [
+        "# Vertex AI Model Garden MediaPipe with object detection\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_mediapipe_object_detection.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_mediapipe_object_detection.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/notebooks/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/model_garden/model_garden_mediapipe_object_detection.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "  </td>\n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dwGLvtIeECLK"
+      },
+      "source": [
+        "**_NOTE_**: This notebook has been tested in the following environment:\n",
+        "\n",
+        "* Python version = 3.9"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tvgnzT1CKxrO"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "This notebook demonstrates how to use [MediaPipe Model Maker](https://developers.google.com/mediapipe/solutions/model_maker) in Vertex AI Model Garden.\n",
+        "\n",
+        "### Objective\n",
+        "\n",
+        "* Train new models\n",
+        "  * Convert input data to training formats\n",
+        "  * Create [custom jobs](https://cloud.google.com/vertex-ai/docs/training/create-custom-job) to train new models\n",
+        "  * Export models\n",
+        "\n",
+        "* Cleanup resources\n",
+        "\n",
+        "### Costs\n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI\n",
+        "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
+        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
+        "Calculator](https://cloud.google.com/products/calculator/)\n",
+        "to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "KEukV6uRk_S3"
+      },
+      "source": [
+        "## Before you begin"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "z__i0w0lCAsW"
+      },
+      "source": [
+        "### Colab only\n",
+        "Run the following commands to install dependencies and to authenticate with Google Cloud if running on Colab."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Jvqs-ehKlaYh"
+      },
+      "outputs": [],
+      "source": [
+        "! pip3 install --upgrade pip\n",
+        "\n",
+        "import sys\n",
+        "\n",
+        "if \"google.colab\" in sys.modules:\n",
+        "    ! pip3 install --upgrade google-cloud-aiplatform\n",
+        "\n",
+        "    # Automatically restart kernel after installs\n",
+        "    import IPython\n",
+        "\n",
+        "    app = IPython.Application.instance()\n",
+        "    app.kernel.do_shutdown(True)\n",
+        "\n",
+        "    from google.colab import auth as google_auth\n",
+        "\n",
+        "    google_auth.authenticate_user()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WReHDGG5g0XY"
+      },
+      "source": [
+        "#### Set your project ID\n",
+        "\n",
+        "**If you don't know your project ID**, see the support page: [Locate the project ID](https://support.google.com/googleapi/answer/7014113)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "oM1iC_MfAts1"
+      },
+      "outputs": [],
+      "source": [
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# Set the project id\n",
+        "! gcloud config set project {PROJECT_ID}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "region"
+      },
+      "source": [
+        "#### Region\n",
+        "\n",
+        "You can also change the `REGION` variable used by Vertex AI. Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "tTy1gX11kCJY"
+      },
+      "outputs": [],
+      "source": [
+        "REGION = \"us-central1\"  # @param {type: \"string\"}\n",
+        "REGION_PREFIX = REGION.split(\"-\")[0]\n",
+        "assert REGION_PREFIX in (\n",
+        "    \"us\",\n",
+        "    \"europe\",\n",
+        "    \"asia\",\n",
+        "), f'{REGION} is not supported. It must be prefixed by \"us\", \"asia\", or \"europe\".'"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "zgPO1eR3CYjk"
+      },
+      "source": [
+        "### Create a Cloud Storage bucket\n",
+        "\n",
+        "Create a storage bucket to store intermediate artifacts such as datasets."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "MzGDU7TWdts_"
+      },
+      "outputs": [],
+      "source": [
+        "BUCKET_URI = f\"gs://your-bucket-name-{PROJECT_ID}-unique\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "-EcIXiGsCePi"
+      },
+      "source": [
+        "**Only if your bucket doesn't already exist**: Run the following cell to create your Cloud Storage bucket."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "NIq7R4HZCfIc"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil mb -l {REGION} -p {PROJECT_ID} {BUCKET_URI}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "960505627ddf"
+      },
+      "source": [
+        "### Import libraries"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "PyQmSRbKA8r-"
+      },
+      "outputs": [],
+      "source": [
+        "import json\n",
+        "import os\n",
+        "from datetime import datetime\n",
+        "\n",
+        "import tensorflow\n",
+        "from google.cloud import aiplatform"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "init_aip:mbsdk,all"
+      },
+      "source": [
+        "### Initialize Vertex AI SDK for Python\n",
+        "\n",
+        "Initialize the Vertex AI SDK for Python for your project."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9wExiMUxFk91"
+      },
+      "outputs": [],
+      "source": [
+        "now = datetime.now().strftime(\"%Y%m%d-%H%M%S\")\n",
+        "\n",
+        "STAGING_BUCKET = os.path.join(BUCKET_URI, \"temp/%s\" % now)\n",
+        "\n",
+        "EVALUATION_RESULT_OUTPUT_DIRECTORY = os.path.join(STAGING_BUCKET, \"evaluation\")\n",
+        "EVALUATION_RESULT_OUTPUT_FILE = os.path.join(\n",
+        "    EVALUATION_RESULT_OUTPUT_DIRECTORY, \"evaluation.json\"\n",
+        ")\n",
+        "\n",
+        "EXPORTED_MODEL_OUTPUT_DIRECTORY = os.path.join(STAGING_BUCKET, \"model\")\n",
+        "EXPORTED_MODEL_OUTPUT_FILE = os.path.join(\n",
+        "    EXPORTED_MODEL_OUTPUT_DIRECTORY, \"model.tflite\"\n",
+        ")\n",
+        "\n",
+        "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=STAGING_BUCKET)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "n6IFz75WGCam"
+      },
+      "source": [
+        "### Define training machine specs"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "riG_qUokg0XZ"
+      },
+      "outputs": [],
+      "source": [
+        "TRAINING_JOB_DISPLAY_NAME = \"mediapipe_object_detector_%s\" % now\n",
+        "TRAINING_CONTAINER = f\"{REGION_PREFIX}-docker.pkg.dev/vertex-ai-restricted/vertex-vision-model-garden-dockers/mediapipe-train\"\n",
+        "TRAINING_MACHINE_TYPE = \"n1-highmem-16\"\n",
+        "TRAINING_ACCELARATOR_TYPE = \"NVIDIA_TESLA_V100\"\n",
+        "TRAINING_ACCELERATOR_COUNT = 2"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "XDq9TiRUc7dV"
+      },
+      "source": [
+        "## Train your customized models"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "zgPO1eR3CYjk"
+      },
+      "source": [
+        "### Prepare input data for training\n",
+        "\n",
+        "Fine-tuning a model for object detection requires a dataset that includes the items, or classes, that you want the completed model to be able to identify. You can do this by trimming down a public dataset to only the classes that are relevant to your usecase, compiling your own dataset, or some combination of both, The dataset can be significantly smaller than what would be required to train a new model from scratch. For example, the [COCO](https://cocodataset.org/) dataset used to train many reference models contains hundreds of thousands of images with 91 classes of objects. Transfer learning with Model Maker can finetune an existing model with a smaller dataset and still perform well, depending on your inference accuracy goals. These instructions use a smaller dataset containing 2 types of android figurines, or 2 classes, with 62 total training images.\n",
+        "\n",
+        "You can re-use an existing dataset such as `gs://mediapipe-tasks/object_detector/android_figurine` to finetune the model. The directory contains two subdirectories for the training and validation datasets, located in android_figurine/train and android_figurine/validation respectively. Each of the train and validation datasets follow the COCO Dataset format described below. If you are using your own dataset, ensure that that it adheres to the format specifications before uploading it to Google Cloud Storage.\n",
+        "\n",
+        "\n",
+        "### Supported dataset formats\n",
+        "Model Maker Object Detection API supports reading the following dataset formats:\n",
+        "\n",
+        "#### COCO format\n",
+        "The COCO dataset format has a `data` directory which stores all of the images and a single `labels.json` file which contains the object annotations for all images.\n",
+        "```\n",
+        "<dataset_dir>/\n",
+        "  data/\n",
+        "    <img0>.<jpg/jpeg>\n",
+        "    <img1>.<jpg/jpeg>\n",
+        "    ...\n",
+        "  labels.json\n",
+        "```\n",
+        "where `labels.json` is formatted as:\n",
+        "```\n",
+        "{\n",
+        "  \"categories\":[\n",
+        "    {\"id\":1, \"name\":<cat1_name>},\n",
+        "    ...\n",
+        "  ],\n",
+        "  \"images\":[\n",
+        "    {\"id\":0, \"file_name\":\"<img0>.<jpg/jpeg>\"},\n",
+        "    ...\n",
+        "  ],\n",
+        "  \"annotations\":[\n",
+        "    {\"id\":0, \"image_id\":0, \"category_id\":1, \"bbox\":[x-top left, y-top left, width, height]},\n",
+        "    ...\n",
+        "  ]\n",
+        "}\n",
+        "```\n",
+        "\n",
+        "#### PASCAL VOC format\n",
+        "\n",
+        "The PASCAL VOC dataset format also has a `data` directory which stores all of the images, however the annotations are split up per image into corresponding xml files in the `Annotations` directory.\n",
+        "```\n",
+        "<dataset_dir>/\n",
+        "  data/\n",
+        "    <file0>.<jpg/jpeg>\n",
+        "    ...\n",
+        "  Annotations/\n",
+        "    <file0>.xml\n",
+        "    ...\n",
+        "```\n",
+        "where the xml files are formatted as:\n",
+        "```\n",
+        "<annotation>\n",
+        "  <filename>file0.jpg</filename>\n",
+        "  <object>\n",
+        "    <name>kangaroo</name>\n",
+        "    <bndbox>\n",
+        "      <xmin>233</xmin>\n",
+        "      <ymin>89</ymin>\n",
+        "      <xmax>386</xmax>\n",
+        "      <ymax>262</ymax>\n",
+        "    </bndbox>\n",
+        "  </object>\n",
+        "  <object>\n",
+        "    ...\n",
+        "  </object>\n",
+        "  ...\n",
+        "</annotation>\n",
+        "```"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "O32DU5RRGhdV"
+      },
+      "source": [
+        "### Configure training dataset\n",
+        "\n",
+        "Once you have completed preparing your data, you can begin fine-tuning a model to recognize the new objects, or classes, defined by your training data. The instructions below use the data prepared in the previous section to finetune an object detection model to recognize the two types of android figurines.\n",
+        "\n",
+        "You can leave the path to the test data empty if you do not have a separate test data set."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "IndQ_m6ddUEM"
+      },
+      "outputs": [],
+      "source": [
+        "training_data_path = \"gs://mediapipe-tasks/object_detector/android_figurine/train\"  # @param {type:\"string\"}\n",
+        "validation_data_path = \"gs://mediapipe-tasks/object_detector/android_figurine/validation\"  # @param {type:\"string\"}\n",
+        "test_data_path = \"\"  # @param {type:\"string\"}\n",
+        "data_format = \"coco\"  # @param [\"coco\", \"pascal_voc\"]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "aaff6f5be7f6"
+      },
+      "source": [
+        "### Set fine-tuning options\n",
+        "\n",
+        "You can pick between different model architectures to further customize your training:\n",
+        "\n",
+        "*   MobileNet-V2\n",
+        "*   MobileNet-MultiHW-AVG\n",
+        "\n",
+        "To set the model architecture and other training parameters, adjust the following values:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "um_XKbmpTaHx"
+      },
+      "outputs": [],
+      "source": [
+        "model_architecture = \"mobilenet_v2\"  # @param [\"mobilenet_v2\", \"mobilenet_multihw_avg\"]\n",
+        "\n",
+        "# The learning rate to use for gradient descent training.\n",
+        "learning_rate: float = 0.01  # @param {type:\"number\"}\n",
+        "# Batch size for training.\n",
+        "batch_size: int = 2  # @param {type:\"number\"}\n",
+        "# Number of training iterations over the dataset.\n",
+        "epochs: int = 10  # @param {type:\"slider\", min:0, max:100, step:1}\n",
+        "# If true, the base module is trained together with the classification layer on\n",
+        "# top.\n",
+        "do_fine_tuning: bool = False  # @param {type:\"boolean\"}\n",
+        "# A regularizer that applies a L1 regularization penalty.\n",
+        "l1_regularizer: float = 0.0  # @param {type:\"number\"}\n",
+        "# A regularizer that applies a L2 regularization penalty.\n",
+        "l2_regularizer: float = 0.0001  # @param {type:\"number\"}\n",
+        "# A boolean controlling whether the training dataset is augmented by randomly\n",
+        "# distorting input images, including random cropping, flipping, etc. See\n",
+        "# utils.image_preprocessing documentation for details.\n",
+        "do_data_augmentation: bool = True  # @param {type:\"boolean\"}\n",
+        "# Number of training samples used to calculate the decay steps\n",
+        "# and create the training optimizer.\n",
+        "decay_samples: int = 2560000  # @param {type:\"number\"}\n",
+        "# Number of warmup steps for a linear increasing warmup schedule on learning\n",
+        "# rate. Used to set up warmup schedule by model_util.WarmUp.\n",
+        "warmup_epochs: int = 2  # @param {type:\"number\"}\n",
+        "# The number of epochs for cosine decay learning rate.\n",
+        "cosine_decay_epochs: int = 5  # @param {type:\"number\"}\n",
+        "# The alpha value for cosine decay learning rate.\n",
+        "cosine_decay_alpha: float = 5  # @param {type:\"number\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "HwcCjwlBTQIz"
+      },
+      "source": [
+        "### Run fine-tuning\n",
+        "With your training dataset and fine-tuning options prepared, you are ready to start the fine-tuning process. This process is resource intensive and can take a few minutes to a few hours depending on your available compute resources. This process is resource intensive and can take a few minutes to a few hours depending on your available compute resources. On Vertex AI with GPU processing, the example fine-tuning below takes about 3 to 4 minutes.\n",
+        "\n",
+        "To begin the fine-tuning process, use the following code:\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "aec22792ee84"
+      },
+      "outputs": [],
+      "source": [
+        "model_export_path = EXPORTED_MODEL_OUTPUT_DIRECTORY\n",
+        "evaluation_result_path = EVALUATION_RESULT_OUTPUT_DIRECTORY\n",
+        "\n",
+        "worker_pool_specs = [\n",
+        "    {\n",
+        "        \"machine_spec\": {\n",
+        "            \"machine_type\": TRAINING_MACHINE_TYPE,\n",
+        "            \"accelerator_type\": TRAINING_ACCELARATOR_TYPE,\n",
+        "            \"accelerator_count\": TRAINING_ACCELERATOR_COUNT,\n",
+        "        },\n",
+        "        \"replica_count\": 1,\n",
+        "        \"container_spec\": {\n",
+        "            \"image_uri\": TRAINING_CONTAINER,\n",
+        "            \"command\": [],\n",
+        "            \"args\": [\n",
+        "                \"--task_name=object_detector\",\n",
+        "                \"--training_data_path=%s\" % training_data_path,\n",
+        "                \"--validation_data_path=%s\" % validation_data_path,\n",
+        "                \"--test_data_path=%s\" % test_data_path,\n",
+        "                \"--data_format=%s\" % data_format,\n",
+        "                \"--model_export_path=%s\" % model_export_path,\n",
+        "                \"--evaluation_result_path=%s\" % evaluation_result_path,\n",
+        "                \"--model_architecture=%s\" % model_architecture,\n",
+        "                \"--hparams=%s\"\n",
+        "                % json.dumps(\n",
+        "                    {\n",
+        "                        \"learning_rate\": learning_rate,\n",
+        "                        \"batch_size\": batch_size,\n",
+        "                        \"epochs\": epochs,\n",
+        "                        \"do_fine_tuning\": do_fine_tuning,\n",
+        "                        \"l1_regularizer\": l1_regularizer,\n",
+        "                        \"l2_regularizer\": l2_regularizer,\n",
+        "                        \"do_data_augmentation\": do_data_augmentation,\n",
+        "                        \"decay_samples\": decay_samples,\n",
+        "                        \"warmup_epochs\": warmup_epochs,\n",
+        "                        \"cosine_decay_epochs\": cosine_decay_epochs,\n",
+        "                        \"cosine_decay_alpha\": cosine_decay_alpha,\n",
+        "                    }\n",
+        "                ),\n",
+        "            ],\n",
+        "        },\n",
+        "    }\n",
+        "]\n",
+        "\n",
+        "training_job = aiplatform.CustomJob(\n",
+        "    display_name=TRAINING_JOB_DISPLAY_NAME,\n",
+        "    project=PROJECT_ID,\n",
+        "    worker_pool_specs=worker_pool_specs,\n",
+        "    staging_bucket=STAGING_BUCKET,\n",
+        ")\n",
+        "\n",
+        "training_job.run()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "zcKzIa5QeIIU"
+      },
+      "source": [
+        "## Evaluate and export model"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "mV-Djz-frBni"
+      },
+      "source": [
+        "### Evaluate performance\n",
+        "\n",
+        "If you have specified test data, you can evaluate it on the test dataset and print the loss and coco metrics. The most important metric for evaluating the model performance is typically the \"AP\" coco metric for Average Precision.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "09Rz1AYspK19"
+      },
+      "outputs": [],
+      "source": [
+        "def get_evaluation_result(evaluation_result_path):\n",
+        "    try:\n",
+        "        with tensorflow.io.gfile.GFile(evaluation_result_path, \"r\") as input_file:\n",
+        "            evalutation_result = json.loads(input_file.read())\n",
+        "        return evalutation_result[\"loss\"], evalutation_result[\"coco_metrics\"]\n",
+        "    except:\n",
+        "        print(\"Evaluation result not found. Did you provide a test dataset?\")\n",
+        "        return None\n",
+        "\n",
+        "\n",
+        "evaluation_result = get_evaluation_result(EVALUATION_RESULT_OUTPUT_FILE)\n",
+        "\n",
+        "if evaluation_result is not None:\n",
+        "    print(f\"Validation loss: {evaluation_result[0]}\")\n",
+        "    print(f\"Validation coco metrics: {evaluation_result[1]}\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "g0BGaofgsMsy"
+      },
+      "source": [
+        "### Export model\n",
+        "After fine-tuning and evaluating the model, you can save it as Tensorflow Lite model, try it out in the [Object Detector](https://mediapipe-studio.webapps.google.com/demo/object_detector) demo in MediaPipe Studio or integrate it with your application by following the [Object detection task guide](https://developers.google.com/mediapipe/solutions/vision/object_detector). The exported model also includes metadata and the label map."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "NYuQowyZEtxK"
+      },
+      "outputs": [],
+      "source": [
+        "import sys\n",
+        "\n",
+        "\n",
+        "def copy_model(model_source, model_dest):\n",
+        "    ! gsutil cp {model_source} {model_dest}\n",
+        "\n",
+        "copy_model(EXPORTED_MODEL_OUTPUT_FILE, \"object_detection_model.tflite\")\n",
+        "\n",
+        "if \"google.colab\" in sys.modules:\n",
+        "    from google.colab import files\n",
+        "\n",
+        "    files.download(\"object_detection_model.tflite\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "kkH2nrpdp4sp"
+      },
+      "source": [
+        "## Clean up"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Ax6vQVZhp9pR"
+      },
+      "outputs": [],
+      "source": [
+        "# Delete training data and jobs.\n",
+        "if training_job.list(filter=f'display_name=\"{TRAINING_JOB_DISPLAY_NAME}\"'):\n",
+        "    training_job.delete()\n",
+        "\n",
+        "!gsutil rm -r {STAGING_BUCKET}"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "model_garden_mediapipe_object_detection.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/community/model_garden/model_garden_mediapipe_text_classification.ipynb

@@ -0,0 +1,616 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ur8xi4C7S06n"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2023 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "TirJ-SGQseby"
+      },
+      "source": [
+        "# Vertex AI Model Garden MediaPipe with text classification\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_mediapipe_text_classification.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_mediapipe_text_classification.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/notebooks/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/model_garden/model_garden_mediapipe_image_classification.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "  </td>\n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dwGLvtIeECLK"
+      },
+      "source": [
+        "**_NOTE_**: This notebook has been tested in the following environment:\n",
+        "\n",
+        "* Python version = 3.9"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tvgnzT1CKxrO"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "This notebook demonstrates how to use [MediaPipe Model Maker](https://developers.google.com/mediapipe/solutions/model_maker) to train an on-device text classification model in Vertex AI Model Garden.\n",
+        "\n",
+        "### Objective\n",
+        "\n",
+        "* Train new models\n",
+        "  * Convert input data to training formats\n",
+        "  * Create [custom jobs](https://cloud.google.com/vertex-ai/docs/training/create-custom-job) to train new models\n",
+        "  * Export models\n",
+        "\n",
+        "* Cleanup resources\n",
+        "\n",
+        "### Costs\n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI\n",
+        "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
+        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
+        "Calculator](https://cloud.google.com/products/calculator/)\n",
+        "to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "KEukV6uRk_S3"
+      },
+      "source": [
+        "## Before you begin"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "z__i0w0lCAsW"
+      },
+      "source": [
+        "### Colab only\n",
+        "Run the following commands to install dependencies and to authenticate with Google Cloud if running on Colab."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Jvqs-ehKlaYh"
+      },
+      "outputs": [],
+      "source": [
+        "! pip3 install --upgrade pip\n",
+        "\n",
+        "import sys\n",
+        "\n",
+        "if \"google.colab\" in sys.modules:\n",
+        "    ! pip3 install --upgrade google-cloud-aiplatform\n",
+        "\n",
+        "    # Automatically restart kernel after installs\n",
+        "    import IPython\n",
+        "\n",
+        "    app = IPython.Application.instance()\n",
+        "    app.kernel.do_shutdown(True)\n",
+        "\n",
+        "    from google.colab import auth as google_auth\n",
+        "\n",
+        "    google_auth.authenticate_user()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WReHDGG5g0XY"
+      },
+      "source": [
+        "#### Set your project ID\n",
+        "\n",
+        "**If you don't know your project ID**, see the support page: [Locate the project ID](https://support.google.com/googleapi/answer/7014113)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "oM1iC_MfAts1"
+      },
+      "outputs": [],
+      "source": [
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# Set the project id\n",
+        "! gcloud config set project {PROJECT_ID}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "region"
+      },
+      "source": [
+        "#### Region\n",
+        "\n",
+        "You can also change the `REGION` variable used by Vertex AI. Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "tTy1gX11kCJY"
+      },
+      "outputs": [],
+      "source": [
+        "REGION = \"us-central1\"  # @param {type: \"string\"}\n",
+        "REGION_PREFIX = REGION.split(\"-\")[0]\n",
+        "assert REGION_PREFIX in (\n",
+        "    \"us\",\n",
+        "    \"europe\",\n",
+        "    \"asia\",\n",
+        "), f'{REGION} is not supported. It must be prefixed by \"us\", \"asia\", or \"europe\".'"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "zgPO1eR3CYjk"
+      },
+      "source": [
+        "### Create a Cloud Storage bucket\n",
+        "\n",
+        "Create a storage bucket to store intermediate artifacts such as datasets."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "MzGDU7TWdts_"
+      },
+      "outputs": [],
+      "source": [
+        "BUCKET_URI = f\"gs://your-bucket-name-{PROJECT_ID}-unique\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "-EcIXiGsCePi"
+      },
+      "source": [
+        "**Only if your bucket doesn't already exist**: Run the following cell to create your Cloud Storage bucket."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "NIq7R4HZCfIc"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil mb -l {REGION} -p {PROJECT_ID} {BUCKET_URI}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "960505627ddf"
+      },
+      "source": [
+        "### Import libraries"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "PyQmSRbKA8r-"
+      },
+      "outputs": [],
+      "source": [
+        "import json\n",
+        "import os\n",
+        "from datetime import datetime\n",
+        "\n",
+        "from google.cloud import aiplatform"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "init_aip:mbsdk,all"
+      },
+      "source": [
+        "### Initialize Vertex AI SDK for Python\n",
+        "\n",
+        "Initialize the Vertex AI SDK for Python for your project."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9wExiMUxFk91"
+      },
+      "outputs": [],
+      "source": [
+        "now = datetime.now().strftime(\"%Y%m%d-%H%M%S\")\n",
+        "\n",
+        "STAGING_BUCKET = os.path.join(BUCKET_URI, \"temp/%s\" % now)\n",
+        "\n",
+        "EVALUATION_RESULT_OUTPUT_DIRECTORY = os.path.join(STAGING_BUCKET, \"evaluation\")\n",
+        "EVALUATION_RESULT_OUTPUT_FILE = os.path.join(\n",
+        "    EVALUATION_RESULT_OUTPUT_DIRECTORY, \"evaluation.json\"\n",
+        ")\n",
+        "\n",
+        "EXPORTED_MODEL_OUTPUT_DIRECTORY = os.path.join(STAGING_BUCKET, \"model\")\n",
+        "EXPORTED_MODEL_OUTPUT_FILE = os.path.join(\n",
+        "    EXPORTED_MODEL_OUTPUT_DIRECTORY, \"model.tflite\"\n",
+        ")\n",
+        "\n",
+        "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=STAGING_BUCKET)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "n6IFz75WGCam"
+      },
+      "source": [
+        "### Define training machine specs"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "riG_qUokg0XZ"
+      },
+      "outputs": [],
+      "source": [
+        "TRAINING_JOB_DISPLAY_NAME = \"mediapipe_text_classifier_%s\" % now\n",
+        "TRAINING_CONTAINER = f\"{REGION_PREFIX}-docker.pkg.dev/vertex-ai-restricted/vertex-vision-model-garden-dockers/mediapipe-train\"\n",
+        "TRAINING_MACHINE_TYPE = \"n1-highmem-16\"\n",
+        "TRAINING_ACCELARATOR_TYPE = \"NVIDIA_TESLA_V100\"\n",
+        "TRAINING_ACCELERATOR_COUNT = 2"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "-rsdAcBV-vlf"
+      },
+      "source": [
+        "## Train your customized models"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "zgPO1eR3CYjk"
+      },
+      "source": [
+        "### Get the Dataset\n",
+        "\n",
+        "The following code block uses the [SST-2](https://nlp.stanford.edu/sentiment/index.html) (Stanford Sentiment Treebank) dataset which contains 67,349 movie reviews for training and 872 movie reviews for testing. The dataset has two classes: positive and negative movie reviews. Positive reviews are labeled with 1 and negative reviews with 0.\n",
+        "\n",
+        "The SST-2 dataset is stored as a TSV file. The only difference between the TSV and CSV formats is that TSV uses a tab `\\t` character as its delimiter and CSV uses a comma `,`.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "IndQ_m6ddUEM"
+      },
+      "outputs": [],
+      "source": [
+        "training_data_path = (\n",
+        "    \"gs://mediapipe-tasks/text_classifier/SST-2/train.tsv\"  # @param {type:\"string\"}\n",
+        ")\n",
+        "validation_data_path = (\n",
+        "    \"gs://mediapipe-tasks/text_classifier/SST-2/dev.tsv\"  # @param {type:\"string\"}\n",
+        ")\n",
+        "\n",
+        "# The delimiter used in the dataset.\n",
+        "delimiter = \"\\t\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# Character used to quote fields that contain special characters\n",
+        "# like the `delimiter`.\n",
+        "quotechar = \"\\t\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# Sequence of keys for the CSV columns (represented as a comma\n",
+        "# separated list). If empty, the first row of the CSV file is used\n",
+        "# as the keys\n",
+        "fieldnames = \"\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# Column name for the input text.\n",
+        "text_column = \"sentence\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# Column name for the labels.\n",
+        "label_column = \"label\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "aaff6f5be7f6"
+      },
+      "source": [
+        "### Set fine-tuning options\n",
+        "\n",
+        "You can pick between different model architectures to further customize your training:\n",
+        "\n",
+        "*   Average Word Embedding Model\n",
+        "*   BERT-classifier\n",
+        "\n",
+        "To set the model architecture and other training parameters, adjust the following values:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "um_XKbmpTaHx"
+      },
+      "outputs": [],
+      "source": [
+        "model_architecture = (\n",
+        "    \"average_word_embedding\"  # @param [\"average_word_embedding\", \"mobilebert\"]\n",
+        ")\n",
+        "\n",
+        "# The learning rate to use for gradient descent-based\n",
+        "# optimizers. Defaults to 3e-5 for the BERT-based classifier\n",
+        "# and 0 for the average word-embedding classifier because\n",
+        "# it does not need such an optimizer.\n",
+        "learning_rate: float = 0.0  # @param {type:\"number\"}\n",
+        "\n",
+        "# Batch size for training. Defaults to 32 for the average\n",
+        "# word-embedding classifier and 48 for the BERT-based\n",
+        "# classifier.\n",
+        "batch_size: int = 48  # @param {type:\"number\"}\n",
+        "\n",
+        "# Number of training iterations over the dataset. Defaults\n",
+        "# to 10 for the average word-embedding classifier and 3\n",
+        "# for the BERT-based classifier.\n",
+        "epochs: int = 10  # @param {type:\"slider\", min:0, max:100, step:1}\n",
+        "\n",
+        "# An integer that indicates the number of training steps per\n",
+        "# epoch. If set to 0, the training pipeline calculates the\n",
+        "# default steps per epoch as the training dataset size\n",
+        "# divided by batch size.\n",
+        "steps_per_epoch: int = 0  # @param {type:\"number\"}\n",
+        "\n",
+        "# Controls whether the dataset is shuffled before training.\n",
+        "shuffle: bool = False  # @param {type:\"boolean\"}\n",
+        "\n",
+        "# Length of the sequence to feed into the model.\n",
+        "seq_len: int = 256  # @param {type:\"number\"}\n",
+        "\n",
+        "# Whether to convert all uppercase characters to lowercase\n",
+        "# during preprocessing.\n",
+        "do_lower_case: bool = True  # @param {type:\"boolean\"}\n",
+        "\n",
+        "# The rate for dropout.\n",
+        "dropout_rate: float = 0.2  # @param {type:\"number\"}\n",
+        "\n",
+        "# Dimension of the word embedding. Only used for the Average Word\n",
+        "# Embedding Model.\n",
+        "wordvec_dim: int = 16  # @param {type:\"number\"}\n",
+        "\n",
+        "# Number of words to generate the vocabulary from data.\n",
+        "# Only used for the Average Word Embedding Model.\n",
+        "vocab_size: int = 10000  # @param {type:\"number\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "HwcCjwlBTQIz"
+      },
+      "source": [
+        "### Run fine-tuning\n",
+        "With your training dataset and fine-tuning options prepared, you are ready to start the fine-tuning process. This process is resource intensive and can take a few minutes to a few hours depending on the model archtiecture and your available compute resources. On Vertex AI with GPU processing, the example fine-tuning below takes between 2-3 minutes to train an Average Word Embedding Model on the SST-2 dataset.\n",
+        "\n",
+        "To begin the fine-tuning process, use the following code:\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "aec22792ee84"
+      },
+      "outputs": [],
+      "source": [
+        "model_export_path = EXPORTED_MODEL_OUTPUT_DIRECTORY\n",
+        "evaluation_result_path = EVALUATION_RESULT_OUTPUT_DIRECTORY\n",
+        "\n",
+        "preprocessing_params = {\n",
+        "    \"text_column\": text_column,\n",
+        "    \"label_column\": label_column,\n",
+        "    \"delimiter\": delimiter,\n",
+        "    \"quotechar\": quotechar,\n",
+        "}\n",
+        "if fieldnames:\n",
+        "    preprocessing_params[\"fieldnames\"] = [\n",
+        "        fieldname.strip() for fieldname in fieldnames.split(\",\")\n",
+        "    ]\n",
+        "\n",
+        "hparams = {\n",
+        "    \"learning_rate\": learning_rate,\n",
+        "    \"batch_size\": batch_size,\n",
+        "    \"epochs\": epochs,\n",
+        "    \"shuffle\": shuffle,\n",
+        "}\n",
+        "if steps_per_epoch:\n",
+        "    hparams[\"steps_per_epoch\"] = steps_per_epoch\n",
+        "\n",
+        "model_options = {\n",
+        "    \"dropout_rate\": dropout_rate,\n",
+        "    \"wordvec_dim\": wordvec_dim,\n",
+        "    \"do_lower_case\": do_lower_case,\n",
+        "    \"vocab_size\": vocab_size,\n",
+        "    \"dropout_rate\": dropout_rate,\n",
+        "}\n",
+        "\n",
+        "worker_pool_specs = [\n",
+        "    {\n",
+        "        \"machine_spec\": {\n",
+        "            \"machine_type\": TRAINING_MACHINE_TYPE,\n",
+        "            \"accelerator_type\": TRAINING_ACCELARATOR_TYPE,\n",
+        "            \"accelerator_count\": TRAINING_ACCELERATOR_COUNT,\n",
+        "        },\n",
+        "        \"replica_count\": 1,\n",
+        "        \"container_spec\": {\n",
+        "            \"image_uri\": TRAINING_CONTAINER,\n",
+        "            \"command\": [],\n",
+        "            \"args\": [\n",
+        "                \"--task_name=text_classifier\",\n",
+        "                \"--training_data_path=%s\" % training_data_path,\n",
+        "                \"--validation_data_path=%s\" % validation_data_path,\n",
+        "                \"--evaluation_result_path=%s\" % evaluation_result_path,\n",
+        "                \"--model_export_path=%s\" % model_export_path,\n",
+        "                \"--model_architecture=%s\" % model_architecture,\n",
+        "                \"--preprocessing_params=%s\" % json.dumps(preprocessing_params),\n",
+        "                \"--hparams=%s\" % json.dumps(hparams),\n",
+        "                \"--model_options=%s\" % json.dumps(model_options),\n",
+        "            ],\n",
+        "        },\n",
+        "    }\n",
+        "]\n",
+        "\n",
+        "training_job = aiplatform.CustomJob(\n",
+        "    display_name=TRAINING_JOB_DISPLAY_NAME,\n",
+        "    project=PROJECT_ID,\n",
+        "    worker_pool_specs=worker_pool_specs,\n",
+        "    staging_bucket=STAGING_BUCKET,\n",
+        ")\n",
+        "\n",
+        "training_job.run()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "rXMF2tnV_WS0"
+      },
+      "source": [
+        "## Export model"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "g0BGaofgsMsy"
+      },
+      "source": [
+        "After finetuning, you can save the Tensorflow Lite model, try it out in the [Text Classification](https://mediapipe-studio.webapps.google.com/demo/text_classifier) demo in MediaPipe Studio or integrate it with your on-device application by following the [Text classification task guide](https://developers.google.com/mediapipe/solutions/text/text_classifier). The exported model contains the generates required model metadata, as well as a classification label file."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "NYuQowyZEtxK"
+      },
+      "outputs": [],
+      "source": [
+        "import sys\n",
+        "\n",
+        "\n",
+        "def copy_model(model_source, model_dest):\n",
+        "    ! gsutil cp {model_source} {model_dest}\n",
+        "\n",
+        "copy_model(EXPORTED_MODEL_OUTPUT_FILE, \"text_classification_model.tflite\")\n",
+        "\n",
+        "if \"google.colab\" in sys.modules:\n",
+        "    from google.colab import files\n",
+        "\n",
+        "    files.download(\"text_classification_model.tflite\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "kkH2nrpdp4sp"
+      },
+      "source": [
+        "## Clean up"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Ax6vQVZhp9pR"
+      },
+      "outputs": [],
+      "source": [
+        "# Delete training data and jobs.\n",
+        "if training_job.list(filter=f'display_name=\"{TRAINING_JOB_DISPLAY_NAME}\"'):\n",
+        "    training_job.delete()\n",
+        "\n",
+        "!gsutil rm -r {STAGING_BUCKET}"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "model_garden_mediapipe_text_classification.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/community/model_garden/model_garden_movinet_clip_classification.ipynb

@@ -0,0 +1,873 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ur8xi4C7S06n"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2023 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "TirJ-SGQseby"
+      },
+      "source": [
+        "# Vertex AI Model Garden MoViNet video clip classification\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_movinet_clip_classification.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_movinet_clip_classification.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>                                                                                               <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/notebooks/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/model_garden/model_garden_movinet_clip_classification.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "  </td>\n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dwGLvtIeECLK"
+      },
+      "source": [
+        "**_NOTE_**: This notebook has been tested in the following environment:\n",
+        "\n",
+        "* Python version = 3.9"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tvgnzT1CKxrO"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "This notebook demonstrates how to use [MoViNet](https://github.com/tensorflow/models/tree/master/official/projects/movinet) in Vertex AI Model Garden.\n",
+        "\n",
+        "### Objective\n",
+        "\n",
+        "* Train new models\n",
+        "  * Convert input data to training formats\n",
+        "  * Create [hyperparameter tuning jobs](https://cloud.google.com/vertex-ai/docs/training/hyperparameter-tuning-overview) to train new models\n",
+        "  * Find and export best models\n",
+        "\n",
+        "* Test trained models\n",
+        "  * Upload models to the [Vertex AI Model Registry](https://cloud.google.com/vertex-ai/docs/model-registry/introduction)\n",
+        "  * Run batch predictions\n",
+        "\n",
+        "* Clean up resources\n",
+        "\n",
+        "### Costs\n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI\n",
+        "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
+        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
+        "Calculator](https://cloud.google.com/products/calculator/)\n",
+        "to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "KEukV6uRk_S3"
+      },
+      "source": [
+        "## Before you begin"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "z__i0w0lCAsW"
+      },
+      "source": [
+        "### Colab Only\n",
+        "Run the following commands for Colab or skip this section if you use Workbench."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Jvqs-ehKlaYh"
+      },
+      "outputs": [],
+      "source": [
+        "import sys\n",
+        "\n",
+        "if \"google.colab\" in sys.modules:\n",
+        "    ! pip3 install --upgrade google-cloud-aiplatform\n",
+        "\n",
+        "    # Automatically restart kernel after installs\n",
+        "    import IPython\n",
+        "\n",
+        "    app = IPython.Application.instance()\n",
+        "    app.kernel.do_shutdown(True)\n",
+        "\n",
+        "    from google.colab import auth as google_auth\n",
+        "\n",
+        "    google_auth.authenticate_user()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "BF1j6f9HApxa"
+      },
+      "source": [
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI API and Compute Engine API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component).\n",
+        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9wExiMUxFk91"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "from google.cloud import aiplatform\n",
+        "\n",
+        "# The GCP project ID for experiments.\n",
+        "PROJECT_ID = \"\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# Bucket URI with gs:// prefix.\n",
+        "BUCKET_URI = \"\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# You can choose a region from https://cloud.google.com/about/locations.\n",
+        "# Only regions prefixed by \"us\", \"asia\", or \"europe\" are supported.\n",
+        "REGION = \"us-central1\"  # @param {type:\"string\"}\n",
+        "REGION_PREFIX = REGION.split(\"-\")[0]\n",
+        "assert REGION_PREFIX in (\n",
+        "    \"us\",\n",
+        "    \"europe\",\n",
+        "    \"asia\",\n",
+        "), f'{REGION} is not supported. It must be prefixed by \"us\", \"asia\", or \"europe\".'\n",
+        "\n",
+        "! gcloud config set project $PROJECT_ID\n",
+        "\n",
+        "STAGING_BUCKET = os.path.join(BUCKET_URI, \"temporal\")\n",
+        "CHECKPOINT_BUCKET = os.path.join(BUCKET_URI, \"ckpt\")\n",
+        "\n",
+        "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=STAGING_BUCKET)\n",
+        "\n",
+        "# Download config files.\n",
+        "CONFIG_DIR = os.path.join(BUCKET_URI, \"config\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "n6IFz75WGCam"
+      },
+      "source": [
+        "### Define constants"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "riG_qUokg0XZ"
+      },
+      "outputs": [],
+      "source": [
+        "OBJECTIVE = \"vcn\"\n",
+        "\n",
+        "# Data converter constants.\n",
+        "DATA_CONVERTER_JOB_PREFIX = \"data_converter\"\n",
+        "DATA_CONVERTER_CONTAINER = f\"{REGION_PREFIX}-docker.pkg.dev/vertex-ai-restricted/vertex-vision-model-garden-dockers/data-converter\"\n",
+        "DATA_CONVERTER_MACHINE_TYPE = \"n1-highmem-8\"\n",
+        "\n",
+        "# Training constants.\n",
+        "TRAINING_JOB_PREFIX = \"train\"\n",
+        "TRAIN_CONTAINER_URI = f\"{REGION_PREFIX}-docker.pkg.dev/vertex-ai-restricted/vertex-vision-model-garden-dockers/movinet-train\"\n",
+        "TRAIN_MACHINE_TYPE = \"n1-highmem-16\"\n",
+        "TRAIN_ACCELERATOR_TYPE = \"NVIDIA_TESLA_V100\"\n",
+        "TRAIN_NUM_GPU = 2\n",
+        "\n",
+        "# Evaluation constants.\n",
+        "EVALUATION_METRIC = \"accuracy\"\n",
+        "\n",
+        "# Export constants.\n",
+        "EXPORT_JOB_PREFIX = \"export\"\n",
+        "EXPORT_CONTAINER_URI = f\"{REGION_PREFIX}-docker.pkg.dev/vertex-ai-restricted/vertex-vision-model-garden-dockers/movinet-model-export\"\n",
+        "EXPORT_MACHINE_TYPE = \"n1-highmem-8\"\n",
+        "\n",
+        "# Prediction constants.\n",
+        "# You can adjust accelerator types and machine types to get faster predictions.\n",
+        "UPLOAD_JOB_PREFIX = \"upload\"\n",
+        "PREDICTION_CONTAINER_URI = f\"{REGION_PREFIX}-docker.pkg.dev/vertex-ai-restricted/vertex-vision-model-garden-dockers/movinet-serve\"\n",
+        "PREDICTION_PORT = 8501\n",
+        "PREDICTION_ACCELERATOR_COUNT = 1\n",
+        "PREDICTION_ACCELERATOR_TYPE = \"NVIDIA_TESLA_T4\"\n",
+        "PREDICTION_MACHINE_TYPE = \"n1-standard-4\"\n",
+        "PREDICTION_JOB_PREFIX = \"predict\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ZZFPe_GezXg8"
+      },
+      "source": [
+        "### Define common helper functions"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "XcYUGwr-AJGY"
+      },
+      "outputs": [],
+      "source": [
+        "import json\n",
+        "from datetime import datetime\n",
+        "\n",
+        "import numpy as np\n",
+        "import tensorflow as tf\n",
+        "import yaml\n",
+        "\n",
+        "\n",
+        "def get_job_name_with_datetime(prefix: str):\n",
+        "    \"\"\"Returns a timestamped job name with the given prefix.\"\"\"\n",
+        "    return prefix + datetime.now().strftime(\"_%Y%m%d_%H%M%S\")\n",
+        "\n",
+        "\n",
+        "def print_response_instance(json_str: str, label_map: dict[int, str]):\n",
+        "    \"\"\"Prints summary of a prediction JSON result from the model response.\"\"\"\n",
+        "    json_obj = json.loads(json_str)\n",
+        "    if \"prediction\" not in json_obj:\n",
+        "        print(\"Error:\", json_str)\n",
+        "        return\n",
+        "    instance = json_obj[\"instance\"]\n",
+        "    prediction = json_obj[\"prediction\"]\n",
+        "    gcs_uri = instance[\"content\"]\n",
+        "    time_start = instance.get(\"timeSegmentStart\", \"0.0s\")\n",
+        "    time_end = instance.get(\"timeSegmentEnd\", \"Infinity\")\n",
+        "    max_idx = np.argmax(prediction)\n",
+        "    print(f\"{gcs_uri} {time_start}-{time_end}:\", label_map[max_idx])\n",
+        "\n",
+        "\n",
+        "def get_label_map(label_map_yaml_filepath: str) -> tuple[dict[int, str], int]:\n",
+        "    \"\"\"Reads label map from a YAML file and returns the label map with the number of classes.\"\"\"\n",
+        "    with tf.io.gfile.GFile(label_map_yaml_filepath, \"rb\") as input_file:\n",
+        "        label_map = yaml.safe_load(input_file.read())[\"label_map\"]\n",
+        "    num_classes = max(label_map.keys()) + 1\n",
+        "    return label_map, num_classes\n",
+        "\n",
+        "\n",
+        "def get_best_trial(model_dir, max_trial_count, evaluation_metric):\n",
+        "    \"\"\"Finds the best trial directory and eval results from a hyperparameter tuning job.\"\"\"\n",
+        "    best_trial_dir = \"\"\n",
+        "    best_trial_evaluation_results = {}\n",
+        "    best_performance = -1\n",
+        "\n",
+        "    for i in range(max_trial_count):\n",
+        "        current_trial = i + 1\n",
+        "        current_trial_dir = os.path.join(model_dir, \"trial_\" + str(current_trial))\n",
+        "        current_trial_best_ckpt_dir = os.path.join(current_trial_dir, \"best_ckpt\")\n",
+        "        current_trial_best_ckpt_evaluation_filepath = os.path.join(\n",
+        "            current_trial_best_ckpt_dir, \"info.json\"\n",
+        "        )\n",
+        "        with tf.io.gfile.GFile(current_trial_best_ckpt_evaluation_filepath, \"rb\") as f:\n",
+        "            eval_metric_results = json.load(f)\n",
+        "            current_performance = eval_metric_results[evaluation_metric]\n",
+        "            if current_performance > best_performance:\n",
+        "                best_performance = current_performance\n",
+        "                best_trial_dir = current_trial_dir\n",
+        "                best_trial_evaluation_results = eval_metric_results\n",
+        "    return best_trial_dir, best_trial_evaluation_results\n",
+        "\n",
+        "\n",
+        "def find_checkpoint_in_dir(checkpoint_dir: str):\n",
+        "    \"\"\"Finds a checkpoint path relative to the directory.\"\"\"\n",
+        "    for root, dirs, files in tf.io.gfile.walk(checkpoint_dir):\n",
+        "        for file in files:\n",
+        "            if file.endswith(\".index\"):\n",
+        "                return os.path.join(root, os.path.splitext(file)[0])\n",
+        "\n",
+        "\n",
+        "def upload_checkpoint_to_gcs(checkpoint_url: str):\n",
+        "    \"\"\"Uploads a compressed .tar.gz checkpoint at the given URL to Cloud Storage.\"\"\"\n",
+        "    filename = os.path.basename(checkpoint_url)\n",
+        "    checkpoint_name = filename.replace(\".tar.gz\", \"\")\n",
+        "    print(\"Download checkpoint from\", checkpoint_url, \"and store to\", CHECKPOINT_BUCKET)\n",
+        "    ! wget $checkpoint_url -O $filename\n",
+        "    ! mkdir -p $checkpoint_name\n",
+        "    ! tar -xvzf $filename -C $checkpoint_name\n",
+        "\n",
+        "    checkpoint_path = find_checkpoint_in_dir(checkpoint_name)\n",
+        "    checkpoint_path = os.path.relpath(checkpoint_path, checkpoint_name)\n",
+        "\n",
+        "    ! gsutil cp -r $checkpoint_name $CHECKPOINT_BUCKET/\n",
+        "    checkpoint_uri = os.path.join(CHECKPOINT_BUCKET, checkpoint_name, checkpoint_path)\n",
+        "    print(\"Checkpoint uploaded to\", checkpoint_uri)\n",
+        "    return checkpoint_uri\n",
+        "\n",
+        "\n",
+        "def upload_config_to_gcs(url: str):\n",
+        "    \"\"\"Uploads a config file at the given URL to Cloud Storage.\"\"\"\n",
+        "    filename = os.path.basename(url)\n",
+        "    destination = os.path.join(CONFIG_DIR, filename)\n",
+        "    print(\"Copy\", url, \"to\", destination)\n",
+        "    ! wget \"$url\" -O \"$filename\"\n",
+        "    ! gsutil cp \"$filename\" \"$destination\"\n",
+        "    return destination"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "RB_xY9ipr7ZU"
+      },
+      "source": [
+        "## Train new models\n",
+        "This section shows how to train new models.\n",
+        "1. Convert input data to training formats\n",
+        "2. Create hyperparameter tuning jobs to train new models\n",
+        "3. Find and export best models\n",
+        "\n",
+        "If you already trained models, please go to the section `Test Trained models`.\n",
+        "\n",
+        "Please select a model:\n",
+        "* `model_id`: MoViNet model variant ID, one of `a0`, `a1`, `a2`, `a3`, `a4`, `a5`. The model with a larger number requires more resources to train, and is expected to have a higher accuracy and latency. Here, we use `a0` for demonstration purpose.\n",
+        "* `model_mode`: MoViNet model type, either `base` or `stream`. The base model has a slightly higher accuracy, while the streaming model is optimized for streaming and faster CPU inference. See [official MoViNet docs](https://github.com/tensorflow/models/tree/master/official/projects/movinet) for more information.\n",
+        "\n",
+        "**Note**: The prediction container only supports base model (non-streaming) for now. If you train a streaming model, you need to download the model and refer to the [MoViNet official guide](https://github.com/tensorflow/models/blob/master/official/projects/movinet/movinet_streaming_model_training_and_inference.ipynb) for running predictions locally."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3Ry1mw6AHLTy"
+      },
+      "outputs": [],
+      "source": [
+        "model_id = \"a0\"  # @param [\"a0\", \"a1\", \"a2\", \"a3\", \"a4\", \"a5\"]\n",
+        "model_mode = \"base\"  # @param [\"base\", \"stream\"]\n",
+        "is_stream = model_mode == \"stream\"\n",
+        "model_name = f\"movinet_{model_id}_{model_mode}\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "zgPO1eR3CYjk"
+      },
+      "source": [
+        "### Prepare input data for training\n",
+        "\n",
+        "Prepare data in the format as described [here](https://cloud.google.com/vertex-ai/docs/video-data/classification/prepare-data), and then convert them to the training formats by running the cell below:\n",
+        "\n",
+        "* `input_file_path`: The input file path to the prepared data.\n",
+        "* `input_file_type`: The input file type, such as `csv` or `jsonl`.\n",
+        "* `split_ratio`: Three comma separated floats indicating the proportion of data to split into train/validation/test. They must add up to 1.\n",
+        "* `num_shard`: Three comma separated integers indicating the shards for train/validation/test.\n",
+        "* `output_dir`: The output directory, which will contain converted train/test/validation data.\n",
+        "* `output_fps`: The sampling rate of the video; Frames per second."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "IndQ_m6ddUEM"
+      },
+      "outputs": [],
+      "source": [
+        "# This job will convert input data as training format, with given split ratios\n",
+        "# and number of shards on train/test/validation.\n",
+        "\n",
+        "data_converter_job_name = get_job_name_with_datetime(\n",
+        "    DATA_CONVERTER_JOB_PREFIX + \"_\" + OBJECTIVE\n",
+        ")\n",
+        "\n",
+        "input_file_path = \"\"  # @param {type:\"string\"}\n",
+        "input_file_type = \"csv\"  # @param [\"csv\", \"jsonl\"]\n",
+        "output_fps = 5  # @param {type:\"integer\"}\n",
+        "split_ratio = \"0.8,0.1,0.1\"\n",
+        "num_shard = \"10,10,10\"\n",
+        "data_converter_output_dir = os.path.join(BUCKET_URI, data_converter_job_name)\n",
+        "\n",
+        "\n",
+        "worker_pool_specs = [\n",
+        "    {\n",
+        "        \"machine_spec\": {\n",
+        "            \"machine_type\": DATA_CONVERTER_MACHINE_TYPE,\n",
+        "        },\n",
+        "        \"replica_count\": 1,\n",
+        "        \"container_spec\": {\n",
+        "            \"image_uri\": DATA_CONVERTER_CONTAINER,\n",
+        "            \"command\": [],\n",
+        "            \"args\": [\n",
+        "                \"--input_file_path=%s\" % input_file_path,\n",
+        "                \"--input_file_type=%s\" % input_file_type,\n",
+        "                \"--objective=%s\" % OBJECTIVE,\n",
+        "                \"--num_shard=%s\" % num_shard,\n",
+        "                \"--split_ratio=%s\" % split_ratio,\n",
+        "                \"--output_dir=%s\" % data_converter_output_dir,\n",
+        "                \"--output_fps=%d\" % output_fps,\n",
+        "            ],\n",
+        "        },\n",
+        "    }\n",
+        "]\n",
+        "\n",
+        "data_converter_custom_job = aiplatform.CustomJob(\n",
+        "    display_name=data_converter_job_name,\n",
+        "    project=PROJECT_ID,\n",
+        "    worker_pool_specs=worker_pool_specs,\n",
+        "    staging_bucket=STAGING_BUCKET,\n",
+        ")\n",
+        "\n",
+        "data_converter_custom_job.run()\n",
+        "\n",
+        "input_train_data_path = os.path.join(data_converter_output_dir, \"train.tfrecord*\")\n",
+        "input_validation_data_path = os.path.join(data_converter_output_dir, \"val.tfrecord*\")\n",
+        "label_map_path = os.path.join(data_converter_output_dir, \"label_map.yaml\")\n",
+        "print(\"input_train_data_path for training: \", input_train_data_path)\n",
+        "print(\"input_validation_data_path for training: \", input_validation_data_path)\n",
+        "print(\"label_map_path for prediction: \", label_map_path)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "aaff6f5be7f6"
+      },
+      "source": [
+        "### Create a Vertex AI custom job with hyperparameter tuning\n",
+        "\n",
+        "You use the Vertex AI SDK to create and run the [hyperparameter tuning job](https://cloud.google.com/vertex-ai/docs/training/hyperparameter-tuning-overview) with Vertex AI Model Garden training docker images.\n",
+        "\n",
+        "#### Define the following specifications\n",
+        "\n",
+        "* `worker_pool_specs`: A list of dictionaries specifying the machine type and docker image. This example defines a single node cluster with one `n1-standard-4` machine with 2 `NVIDIA_TESLA_V100` GPUs.\n",
+        "\n",
+        "  **Note**: We recommend using 8 GPUs for MoViNet-A2 and larger. Since loading video data requires a lot of GPU memory, it is recommended to experiment with a small batch size first.\n",
+        "* `parameter_spec`: Dictionary specifying the parameters to optimize. The dictionary key is the string assigned to the command line argument for each hyperparameter in your training application code, and the dictionary value is the parameter specification. The parameter specification includes the type, min/max values, and scale for the hyperparameter.\n",
+        "* `metric_spec`: Dictionary specifying the metric to optimize. The dictionary key is the `hyperparameter_metric_tag` that you set in your training application code, and the value is the optimization goal."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "um_XKbmpTaHx"
+      },
+      "outputs": [],
+      "source": [
+        "from google.cloud.aiplatform import hyperparameter_tuning as hpt\n",
+        "\n",
+        "# Input train and validation datasets can be found from the section above\n",
+        "# `Prepare input data for training`.\n",
+        "# Or, set prepared datasets paths if already exist.\n",
+        "# input_train_data_path = \"\"\n",
+        "# input_validation_data_path = \"\"\n",
+        "# label_map_path = \"\"\n",
+        "\n",
+        "train_job_name = get_job_name_with_datetime(f\"{TRAINING_JOB_PREFIX}_{model_name}\")\n",
+        "model_dir = os.path.join(BUCKET_URI, train_job_name)\n",
+        "label_map, num_classes = get_label_map(label_map_path)\n",
+        "\n",
+        "# Uploads pretained checkpoint to GCS bucket.\n",
+        "init_checkpoint = f\"https://storage.googleapis.com/tf_model_garden/vision/movinet/{model_name}_with_backbone.tar.gz\"\n",
+        "init_checkpoint = upload_checkpoint_to_gcs(init_checkpoint)\n",
+        "\n",
+        "# Uploads config file according to model_id and streaming options.\n",
+        "config_file = f\"{model_id}_stream\" if is_stream else model_id\n",
+        "config_file = f\"https://raw.githubusercontent.com/tensorflow/models/master/official/projects/movinet/configs/yaml/movinet_{config_file}_gpu.yaml\"\n",
+        "config_file = upload_config_to_gcs(config_file)\n",
+        "\n",
+        "# The parameters here are mainly for demonstration purpose. Please update them\n",
+        "# for better performance.\n",
+        "trainer_args = {\n",
+        "    \"experiment\": \"movinet_kinetics600\",\n",
+        "    \"config_file\": config_file,\n",
+        "    \"input_train_data_path\": input_train_data_path,\n",
+        "    \"input_validation_data_path\": input_validation_data_path,\n",
+        "    \"init_checkpoint\": init_checkpoint,\n",
+        "    \"model_dir\": model_dir,\n",
+        "    \"num_classes\": num_classes,\n",
+        "    \"global_batch_size\": 4,\n",
+        "    \"prefetch_buffer_size\": 8,\n",
+        "    \"shuffle_buffer_size\": 32,\n",
+        "    \"train_steps\": 2000,\n",
+        "}\n",
+        "\n",
+        "worker_pool_specs = [\n",
+        "    {\n",
+        "        \"machine_spec\": {\n",
+        "            \"machine_type\": TRAIN_MACHINE_TYPE,\n",
+        "            \"accelerator_type\": TRAIN_ACCELERATOR_TYPE,\n",
+        "            # Each training job uses TRAIN_NUM_GPU GPUs.\n",
+        "            \"accelerator_count\": TRAIN_NUM_GPU,\n",
+        "        },\n",
+        "        \"replica_count\": 1,\n",
+        "        \"container_spec\": {\n",
+        "            \"image_uri\": TRAIN_CONTAINER_URI,\n",
+        "            \"args\": [\n",
+        "                \"--mode=train_and_eval\",\n",
+        "                \"--params_override=runtime.num_gpus=%d\" % TRAIN_NUM_GPU,\n",
+        "            ]\n",
+        "            + [\"--{}={}\".format(k, v) for k, v in trainer_args.items()],\n",
+        "        },\n",
+        "    }\n",
+        "]\n",
+        "\n",
+        "metric_spec = {\"model_performance\": \"maximize\"}\n",
+        "\n",
+        "# These learning rates might not be optimal for your selected model type; To\n",
+        "# tune learning rates, try hpt.DoubleParameterSpec with more trials.\n",
+        "LEARNING_RATES = [1e-3, 3e-3]\n",
+        "MAX_TRIAL_COUNT = len(LEARNING_RATES)\n",
+        "parameter_spec = {\n",
+        "    \"learning_rate\": hpt.DiscreteParameterSpec(values=LEARNING_RATES, scale=\"linear\"),\n",
+        "}\n",
+        "\n",
+        "print(worker_pool_specs, metric_spec, parameter_spec)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "HwcCjwlBTQIz"
+      },
+      "source": [
+        "#### Run the hyperparameter tuning job\n",
+        "* `max_trial_count`: Sets an upper bound on the number of trials the service will run. The recommended practice is to start with a smaller number of trials and get a sense of how impactful your chosen hyperparameters are before scaling up.\n",
+        "\n",
+        "* `parallel_trial_count`:  If you use parallel trials, the service provisions multiple training processing clusters. The worker pool spec that you specify when creating the job is used for each individual training cluster.  Increasing the number of parallel trials reduces the amount of time the hyperparameter tuning job takes to run; however, it can reduce the effectiveness of the job overall. This is because the default tuning strategy uses results of previous trials to inform the assignment of values in subsequent trials.\n",
+        "\n",
+        "* `search_algorithm`: The available search algorithms are grid, random, or default (None). The default option applies Bayesian optimization to search the space of possible hyperparameter values and is the recommended algorithm.\n",
+        "\n",
+        "Click on the generated link in the output to see your run in the Cloud Console."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "aec22792ee84"
+      },
+      "outputs": [],
+      "source": [
+        "train_custom_job = aiplatform.CustomJob(\n",
+        "    display_name=train_job_name,\n",
+        "    project=PROJECT_ID,\n",
+        "    worker_pool_specs=worker_pool_specs,\n",
+        "    staging_bucket=STAGING_BUCKET,\n",
+        ")\n",
+        "\n",
+        "train_hpt_job = aiplatform.HyperparameterTuningJob(\n",
+        "    display_name=train_job_name,\n",
+        "    custom_job=train_custom_job,\n",
+        "    metric_spec=metric_spec,\n",
+        "    parameter_spec=parameter_spec,\n",
+        "    max_trial_count=MAX_TRIAL_COUNT,\n",
+        "    parallel_trial_count=MAX_TRIAL_COUNT,\n",
+        "    project=PROJECT_ID,\n",
+        "    search_algorithm=None,\n",
+        ")\n",
+        "\n",
+        "train_hpt_job.run()\n",
+        "\n",
+        "print(\"model_dir is:\", model_dir)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "vugUfJEC2HrK"
+      },
+      "source": [
+        "### Export model in Tensorflow SavedModel format"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "09Rz1AYspK19"
+      },
+      "outputs": [],
+      "source": [
+        "# This job will export models from TF checkpoints to TF saved model format.\n",
+        "# model_dir is from the section above.\n",
+        "best_trial_dir, best_trial_evaluation_results = get_best_trial(\n",
+        "    model_dir, MAX_TRIAL_COUNT, EVALUATION_METRIC\n",
+        ")\n",
+        "best_checkpoint_path = find_checkpoint_in_dir(f\"{best_trial_dir}/best_ckpt/\")\n",
+        "print(\"best_trial_dir: \", best_trial_dir)\n",
+        "print(\"best_trial_evaluation_results: \", best_trial_evaluation_results)\n",
+        "print(\"best_checkpoint: \", best_checkpoint_path)\n",
+        "\n",
+        "container_args = {\n",
+        "    \"export_path\": f\"{model_dir}/best_model\",\n",
+        "    \"model_id\": model_id,\n",
+        "    \"num_classes\": num_classes,\n",
+        "    \"causal\": is_stream,\n",
+        "    \"checkpoint_path\": best_checkpoint_path,\n",
+        "    \"assert_checkpoint_objects_matched\": False,\n",
+        "}\n",
+        "\n",
+        "if is_stream:\n",
+        "    container_args.update(\n",
+        "        {\n",
+        "            \"conv_type\": \"2plus1d\",\n",
+        "            \"se_type\": \"2plus3d\",\n",
+        "            \"activation\": \"hard_swish\",\n",
+        "            \"gating_activation\": \"hard_sigmoid\",\n",
+        "            \"use_positional_encoding\": model_id in {\"a3\", \"a4\", \"a5\"},\n",
+        "        }\n",
+        "    )\n",
+        "\n",
+        "worker_pool_specs = [\n",
+        "    {\n",
+        "        \"machine_spec\": {\n",
+        "            \"machine_type\": EXPORT_MACHINE_TYPE,\n",
+        "        },\n",
+        "        \"replica_count\": 1,\n",
+        "        \"container_spec\": {\n",
+        "            \"image_uri\": EXPORT_CONTAINER_URI,\n",
+        "            \"args\": [\"--{}={}\".format(k, v) for k, v in container_args.items()],\n",
+        "        },\n",
+        "    }\n",
+        "]\n",
+        "\n",
+        "model_export_job_name = get_job_name_with_datetime(EXPORT_JOB_PREFIX + \"_\" + OBJECTIVE)\n",
+        "model_export_custom_job = aiplatform.CustomJob(\n",
+        "    display_name=model_export_job_name,\n",
+        "    project=PROJECT_ID,\n",
+        "    worker_pool_specs=worker_pool_specs,\n",
+        "    staging_bucket=STAGING_BUCKET,\n",
+        ")\n",
+        "\n",
+        "model_export_custom_job.run()\n",
+        "\n",
+        "print(\"best model is saved to: \", container_args[\"export_path\"])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "g0BGaofgsMsy"
+      },
+      "source": [
+        "## Test trained models\n",
+        "This section shows the way to test with trained models.\n",
+        "1. Upload and deploy models to the [Vertex AI Model Registry](https://cloud.google.com/vertex-ai/docs/model-registry/introduction)\n",
+        "2. Run batch predictions\n",
+        "\n",
+        "**Note:** The prediction container only works with the base model. If you trained a streaming model, download the model from the exported path and refer to the [MoViNet official guide](https://github.com/tensorflow/models/blob/master/official/projects/movinet/movinet_streaming_model_training_and_inference.ipynb) for running predictions locally."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gdlca3BOypXU"
+      },
+      "source": [
+        "### Upload model to Vertex AI Model Registry"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "NYuQowyZEtxK"
+      },
+      "outputs": [],
+      "source": [
+        "upload_job_name = get_job_name_with_datetime(f\"{UPLOAD_JOB_PREFIX}_{model_name}\")\n",
+        "\n",
+        "serving_env = {\n",
+        "    \"MODEL_PATH\": container_args[\"export_path\"],\n",
+        "    \"BATCH_SIZE\": 1,  # Select a larger batch size to accelerate GPU prediction.\n",
+        "    \"NUM_FRAMES\": 32,\n",
+        "    \"FPS\": output_fps,\n",
+        "    \"OVERLAP_FRAMES\": 24,\n",
+        "    \"OBJECTIVE\": OBJECTIVE,\n",
+        "}\n",
+        "\n",
+        "model = aiplatform.Model.upload(\n",
+        "    display_name=model_name,\n",
+        "    serving_container_image_uri=PREDICTION_CONTAINER_URI,\n",
+        "    serving_container_ports=[PREDICTION_PORT],\n",
+        "    serving_container_predict_route=\"/predict\",\n",
+        "    serving_container_health_route=\"/ping\",\n",
+        "    serving_container_environment_variables=serving_env,\n",
+        ")\n",
+        "\n",
+        "model.wait()\n",
+        "\n",
+        "print(\"The uploaded model name is: \", model_name)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9SZsKGeS3x6S"
+      },
+      "source": [
+        "### Run batch predictions\n",
+        "\n",
+        "We will now run batch predictions with the trained MoViNet clip classification model with [Vertex AI Batch Prediction](https://cloud.google.com/vertex-ai/docs/predictions/get-batch-predictions).\n",
+        "\n",
+        "Please prepare an input JSONL file where each line follows [this format](https://cloud.google.com/vertex-ai/docs/video-data/classification/get-predictions?hl=en#input_data_requirements) and store it in a Cloud Storage bucket."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "vbIW9me1F2RY"
+      },
+      "outputs": [],
+      "source": [
+        "# Path to the prediction input JSONL file.\n",
+        "test_jsonl_path = \"\"  # @param {type:\"string\"}\n",
+        "\n",
+        "predict_job_name = get_job_name_with_datetime(f\"{PREDICTION_JOB_PREFIX}_{model_name}\")\n",
+        "predict_destination_prefix = os.path.join(STAGING_BUCKET, predict_job_name)\n",
+        "\n",
+        "batch_prediction_job = model.batch_predict(\n",
+        "    job_display_name=predict_job_name,\n",
+        "    gcs_source=test_jsonl_path,\n",
+        "    gcs_destination_prefix=predict_destination_prefix,\n",
+        "    machine_type=PREDICTION_MACHINE_TYPE,\n",
+        "    accelerator_count=PREDICTION_ACCELERATOR_COUNT,\n",
+        "    accelerator_type=PREDICTION_ACCELERATOR_TYPE,\n",
+        "    max_replica_count=1,\n",
+        ")\n",
+        "\n",
+        "batch_prediction_job.wait()\n",
+        "\n",
+        "print(batch_prediction_job.display_name)\n",
+        "print(batch_prediction_job.resource_name)\n",
+        "print(batch_prediction_job.state)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ik-XPjfx9OCE"
+      },
+      "source": [
+        "You can then read the prediction response JSONL files in the output directory:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "tdkW9e5B9OU1"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "# The label map file was generated from the section above (`Prepare input data for training`).\n",
+        "for file in tf.io.gfile.glob(os.path.join(predict_destination_prefix, \"*/*\")):\n",
+        "    with tf.io.gfile.GFile(file, \"r\") as f:\n",
+        "        for line in f:\n",
+        "            print_response_instance(line, label_map)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "kkH2nrpdp4sp"
+      },
+      "source": [
+        "## Clean up"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Ax6vQVZhp9pR"
+      },
+      "outputs": [],
+      "source": [
+        "# Delete the trained model.\n",
+        "model.delete()\n",
+        "# Delete custom and hpt jobs.\n",
+        "if data_converter_custom_job.list(filter=f'display_name=\"{data_converter_job_name}\"'):\n",
+        "    data_converter_custom_job.delete()\n",
+        "if train_hpt_job.list(filter=f'display_name=\"{train_job_name}\"'):\n",
+        "    train_hpt_job.delete()\n",
+        "if model_export_custom_job.list(filter=f'display_name=\"{model_export_job_name}\"'):\n",
+        "    model_export_custom_job.delete()\n",
+        "if batch_prediction_job.list(filter=f'display_name=\"{predict_job_name}\"'):\n",
+        "    batch_prediction_job.delete()"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "model_garden_movinet_clip_classification.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/community/model_garden/model_garden_pipeline_templates_bert.ipynb

@@ -0,0 +1,881 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "copyright"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2023 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "title:generic,gcp"
+      },
+      "source": [
+        "# Get started with Model Garden Pipeline Templates for BERT models\n",
+        "\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_template_pipelines_bert.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_template_pipelines_bert.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/communitymodel_garden/model_garden_template_pipelines_bert.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "  </td>                                                                                               \n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "overview:mlops"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "\n",
+        "This tutorial demonstrates how to modify, compile and execute a prebuilt Vertex AI Model Garden pipeline template with Vertex AI Pipelines.\n",
+        "\n",
+        "Learn more about [Create a pipeline template](https://cloud.google.com/vertex-ai/docs/pipelines/create-pipeline-template)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "objective:mlops,stage4,get_started_vertex_model_evaluation"
+      },
+      "source": [
+        "### Objective\n",
+        "\n",
+        "In this tutorial, you learn how to use a prebuilt pipeline template with `Vertex AI Pipelines` to fine-tune a BERT text classification model, where the model is accessed from `Vertex AI Model Garden`.\n",
+        "\n",
+        "This tutorial uses the following Google Cloud ML services:\n",
+        "\n",
+        "- `Vertex AI Pipelines`\n",
+        "- `Vertex AI Training`\n",
+        "- `Vertex AI Model Garden`\n",
+        "- `Google Cloud Pipeline Components`\n",
+        "\n",
+        "\n",
+        "The steps performed include:\n",
+        "\n",
+        "- Create a user-defined repository in the `Artifact Registry`.\n",
+        "- Upload the prebuilt pipeline template to the `Artifact Registry`.\n",
+        "- Create a pipeline job with the prebuilt pipeline template to fine-tune a BERT model.\n",
+        "- Execute the pipeline using `Vertex AI Pipelines`.\n",
+        "    - Load BERT model from Vertex AI Model Garden\n",
+        "    - Fine-tune train the model\n",
+        "    - Do batch prediction\n",
+        "    - Evaluate the model from the batch prediction results\n",
+        "- Obtain the Vertex AI Model resource from the pipeline artifacts.\n",
+        "- Deploy the model to a Vertex AI Endpoint\n",
+        "- Make a prediction"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dataset:bank,lbn"
+      },
+      "source": [
+        "### Model\n",
+        "\n",
+        "This tutorial uses a pre-trained BERT text classification model from `Vertex AI Model Garden`, which is then fine-tuned (transfer learning) on a dataset of text phrases which are classified as either FirstClass or SecondClass.\n",
+        "\n",
+        "Learn more about [BERT pretrained encoder model]( https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/3). "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "costs"
+      },
+      "source": [
+        "### Costs\n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Cloud Storage\n",
+        "* Dataflow\n",
+        "\n",
+        "Learn about [Vertex AI\n",
+        "pricing](https://cloud.google.com/vertex-ai/pricing), [Cloud Storage\n",
+        "pricing](https://cloud.google.com/storage/pricing), and [Dataflow pricing](https://cloud.google.com/dataflow/pricing)\n",
+        "and use the [Pricing\n",
+        "Calculator](https://cloud.google.com/products/calculator/)\n",
+        "to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "install_mlops"
+      },
+      "source": [
+        "## Installations\n",
+        "\n",
+        "Install the packages required for executing this notebook.\n",
+        "\n",
+        "*Note:* This tutorial requires KFP 2.x."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "install_mlops"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "! pip3 install --upgrade google-cloud-aiplatform \\\n",
+        "                         google-cloud-pipeline-components \\\n",
+        "                         kfp==2.0.0b15"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "restart"
+      },
+      "source": [
+        "### Colab only: Uncomment the following cell to restart the kernel"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "D-ZBOjErv5mM"
+      },
+      "outputs": [],
+      "source": [
+        "# Automatically restart kernel after installs so that your environment can access the new packages\n",
+        "# import IPython\n",
+        "\n",
+        "# app = IPython.Application.instance()\n",
+        "# app.kernel.do_shutdown(True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "before_you_begin"
+      },
+      "source": [
+        "## Before you begin\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "before_you_begin:nogpu"
+      },
+      "source": [
+        "### Set your project ID\n",
+        "\n",
+        "**If you don't know your project ID**, try the following:\n",
+        "* Run `gcloud config list`.\n",
+        "* Run `gcloud projects list`.\n",
+        "* See the support page: [Locate the project ID](https://support.google.com/googleapi/answer/7014113)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "set_project_id"
+      },
+      "outputs": [],
+      "source": [
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# Set the project id\n",
+        "! gcloud config set project {PROJECT_ID}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "region"
+      },
+      "source": [
+        "#### Region\n",
+        "\n",
+        "You can also change the `REGION` variable used by Vertex AI. Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "region"
+      },
+      "outputs": [],
+      "source": [
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "c4ccf556d4ea"
+      },
+      "source": [
+        "### Enable APIs\n",
+        "\n",
+        "You can enable the required APIs using `gcloud`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "619529337e6d"
+      },
+      "outputs": [],
+      "source": [
+        "! gcloud services enable compute.googleapis.com         \\\n",
+        "                         containerregistry.googleapis.com  \\\n",
+        "                         aiplatform.googleapis.com  \\\n",
+        "                         artifactregistry.googleapis.com"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gcp_authenticate"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "Depending on your Jupyter environment, you may have to manually authenticate. Follow the relevant instructions below."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "FvQeFm3Gv5mR"
+      },
+      "source": [
+        "**1. Vertex AI Workbench**\n",
+        "* Do nothing as you are already authenticated."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ce6043da7b33"
+      },
+      "outputs": [],
+      "source": [
+        "# ! gcloud auth login"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0367eac06a10"
+      },
+      "source": [
+        "**3. Colab, uncomment and run:**"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "21ad4dbb4a61"
+      },
+      "outputs": [],
+      "source": [
+        "# from google.colab import auth\n",
+        "# auth.authenticate_user()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "c13224697bfb"
+      },
+      "source": [
+        "**4. Service account or other**\n",
+        "* See how to grant Cloud Storage permissions to your service account at https://cloud.google.com/storage/docs/gsutil/commands/iam#ch-examples."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "bucket:mbsdk"
+      },
+      "source": [
+        "### Create a Cloud Storage bucket\n",
+        "\n",
+        "Create a storage bucket to store intermediate artifacts such as datasets."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "bucket"
+      },
+      "outputs": [],
+      "source": [
+        "BUCKET_URI = f\"gs://your-bucket-name-{PROJECT_ID}-unique\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "create_bucket"
+      },
+      "source": [
+        "**Only if your bucket doesn't already exist**: Run the following cell to create your Cloud Storage bucket."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "create_bucket"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil mb -l $REGION $BUCKET_URI"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "set_service_account"
+      },
+      "source": [
+        "#### Service Account\n",
+        "\n",
+        "**If you don't know your service account**, try to get your service account using `gcloud` command by executing the second cell below."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "set_service_account"
+      },
+      "outputs": [],
+      "source": [
+        "SERVICE_ACCOUNT = \"[your-service-account]\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "autoset_service_account"
+      },
+      "outputs": [],
+      "source": [
+        "import sys\n",
+        "\n",
+        "IS_COLAB = \"google.colab\" in sys.modules\n",
+        "if (\n",
+        "    SERVICE_ACCOUNT == \"\"\n",
+        "    or SERVICE_ACCOUNT is None\n",
+        "    or SERVICE_ACCOUNT == \"[your-service-account]\"\n",
+        "):\n",
+        "    # Get your service account from gcloud\n",
+        "    if not IS_COLAB:\n",
+        "        shell_output = !gcloud auth list 2>/dev/null\n",
+        "        SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
+        "\n",
+        "    if IS_COLAB:\n",
+        "        shell_output = ! gcloud projects describe  $PROJECT_ID\n",
+        "        project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n",
+        "        SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n",
+        "\n",
+        "    print(\"Service Account:\", SERVICE_ACCOUNT)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "set_service_account:pipelines"
+      },
+      "source": [
+        "#### Set service account access for Vertex AI Pipelines\n",
+        "\n",
+        "Run the following commands to grant your service account access to read and write pipeline artifacts in the bucket that you created in the previous step -- you only need to run these once per service account."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "set_service_account:pipelines"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator  \n",
+        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer  "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "setup_vars"
+      },
+      "source": [
+        "### Set up variables\n",
+        "\n",
+        "Next, set up some variables used throughout the tutorial.\n",
+        "### Import libraries and define constants"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "import_kfp"
+      },
+      "outputs": [],
+      "source": [
+        "import json\n",
+        "import os\n",
+        "\n",
+        "import google.cloud.aiplatform as aiplatform\n",
+        "from kfp.registry import RegistryClient"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "init_aip:mbsdk,all"
+      },
+      "source": [
+        "### Initialize Vertex AI SDK for Python\n",
+        "\n",
+        "Initialize the Vertex AI SDK for Python for your project and corresponding bucket."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "init_aip:mbsdk,all"
+      },
+      "outputs": [],
+      "source": [
+        "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=BUCKET_URI)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2d242773d707"
+      },
+      "source": [
+        "### Enable Artifact Registry API\n",
+        "You must enable the Artifact Registry API service for your project.\n",
+        "\n",
+        "<a href=\"https://cloud.google.com/artifact-registry/docs/enable-service\">Learn more about Enabling service</a>."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "162b5e8883c2"
+      },
+      "outputs": [],
+      "source": [
+        "! gcloud services enable artifactregistry.googleapis.com\n",
+        "\n",
+        "if os.getenv(\"IS_TESTING\"):\n",
+        "    ! sudo apt-get update --yes && sudo apt-get --only-upgrade --yes install google-cloud-sdk-cloud-run-proxy google-cloud-sdk-harbourbridge google-cloud-sdk-cbt google-cloud-sdk-gke-gcloud-auth-plugin google-cloud-sdk-kpt google-cloud-sdk-local-extract google-cloud-sdk-minikube google-cloud-sdk-app-engine-java google-cloud-sdk-app-engine-go google-cloud-sdk-app-engine-python google-cloud-sdk-spanner-emulator google-cloud-sdk-bigtable-emulator google-cloud-sdk-nomos google-cloud-sdk-package-go-module google-cloud-sdk-firestore-emulator kubectl google-cloud-sdk-datastore-emulator google-cloud-sdk-app-engine-python-extras google-cloud-sdk-cloud-build-local google-cloud-sdk-kubectl-oidc google-cloud-sdk-anthos-auth google-cloud-sdk-app-engine-grpc google-cloud-sdk-pubsub-emulator google-cloud-sdk-datalab google-cloud-sdk-skaffold google-cloud-sdk google-cloud-sdk-terraform-tools google-cloud-sdk-config-connector\n",
+        "    ! gcloud components update --quiet"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9b773e8d2bd2"
+      },
+      "source": [
+        "## Create repo in Artifact Registry\n",
+        "\n",
+        "First, you create your own (user-defined) repository in the `Artifact Registry`. You use this repository to upload and retrieve your pipeline templates."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "520de849cee2"
+      },
+      "outputs": [],
+      "source": [
+        "REPO_NAME = \"my-docker-repo-unique\"\n",
+        "\n",
+        "! gcloud artifacts repositories create {REPO_NAME} --location={REGION} --repository-format=KFP"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "1611d3517c0f"
+      },
+      "source": [
+        "### Upload the pipeline template\n",
+        "\n",
+        "Next, you instantiate a client interface to the Artifact Registry. Then with the `upload_pipeline()` method you upload your pipeline template."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "72db37f6d67c"
+      },
+      "outputs": [],
+      "source": [
+        "BERT_YAML = \"gs://cloud-samples-data/vertex-ai/dataset-management/datasets/bert_finetuning/pipeline.yaml\"\n",
+        "\n",
+        "! gsutil cp {BERT_YAML} pipeline.yaml\n",
+        "\n",
+        "client = RegistryClient(\n",
+        "    host=f\"https://{REGION}-kfp.pkg.dev/{PROJECT_ID}/quickstart-kfp-repo\"\n",
+        ")\n",
+        "\n",
+        "templateName, versionName = client.upload_pipeline(\n",
+        "    file_name=\"pipeline.yaml\",\n",
+        "    tags=[\"v1\", \"latest\"],\n",
+        "    extra_headers={\n",
+        "        \"description\": \"This is a pipeline template for fine-tuning a BERT model.\"\n",
+        "    },\n",
+        ")\n",
+        "\n",
+        "! rm pipeline.yaml"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "02f82754fc0d"
+      },
+      "source": [
+        "### View your artifacts in your registry\n",
+        "\n",
+        "Next, using the `gcloud artifacts files` command you view the artifacts, inclusive of the pipeline template, in your artifacts repository."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b2f641eb2056"
+      },
+      "outputs": [],
+      "source": [
+        "! gcloud artifacts files list  --repository={REPO_NAME} --location={REGION}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9d5296831cfb"
+      },
+      "source": [
+        "## Load and execute the pipeline job\n",
+        "\n",
+        "Next, you create a Vertex AI Pipeline job from your BERT pipeline template by instantiating a PipelineJob(), with the following parameters:\n",
+        "\n",
+        "- `display_name`: The human readable name for the pipeline job.\n",
+        "- `template_path`: The path to the pipeline template in the Artifact Registry.\n",
+        "- `enable_caching`: On re-runs, use the results from previous successful and unchanged steps.\n",
+        "- `pipeline_root`: A Cloud storage location for storing pipeline results.\n",
+        "- `parameter_values`: The parameters and values that are input to the template pipeline. In this example, they are:\n",
+        "    - `project`: Your project ID.\n",
+        "    - `class_labels`: A list of valid class labels, in cardinal order.\n",
+        "    - `root_dir`: A Cloud Storage scratch area.\n",
+        "    - `training_data_path`: A Cloud Storage location to the training data.\n",
+        "    - `ground_truth_gcs_source_uris`: A Cloud Storage location to evaluation data."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "a3c502fc7e41"
+      },
+      "outputs": [],
+      "source": [
+        "PIPELINE_ROOT = f\"{BUCKET_URI}/pipeline_root/bert-finetuning\"\n",
+        "\n",
+        "job = aiplatform.PipelineJob(\n",
+        "    display_name=\"bert-finetuning\",\n",
+        "    template_path=f\"https://{REGION}-kfp.pkg.dev/{PROJECT_ID}/quickstart-kfp-repo/{templateName}/{versionName}\",\n",
+        "    pipeline_root=PIPELINE_ROOT,\n",
+        "    enable_caching=False,\n",
+        "    parameter_values={\n",
+        "        \"project\": PROJECT_ID,\n",
+        "        \"class_labels\": [\"FirstClass\", \"SecondClass\", \"[UNK]\"],\n",
+        "        \"root_dir\": BUCKET_URI,\n",
+        "        \"ground_truth_gcs_source_uris\": [\n",
+        "            \"gs://cloud-samples-data/vertex-ai/dataset-management/datasets/bert_finetuning/wide_and_deep_trainer_container_tests_input.jsonl\"\n",
+        "        ],\n",
+        "        \"training_data_path\": \"gs://cloud-samples-data/vertex-ai/dataset-management/datasets/bert_finetuning/wide_and_deep_trainer_container_tests_input.jsonl\",\n",
+        "    },\n",
+        ")\n",
+        "\n",
+        "job.run()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "view_pipleline_results:bqml"
+      },
+      "source": [
+        "### View the pipeline results"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "view_pipleline_results:bqml"
+      },
+      "outputs": [],
+      "source": [
+        "PROJECT_NUMBER = job.gca_resource.name.split(\"/\")[1]\n",
+        "print(PROJECT_NUMBER)\n",
+        "\n",
+        "\n",
+        "def print_pipeline_output(job, output_task_name):\n",
+        "    JOB_ID = job.name\n",
+        "    print(JOB_ID)\n",
+        "    artifact = \"\"\n",
+        "    for _ in range(len(job.gca_resource.job_detail.task_details)):\n",
+        "        TASK_ID = job.gca_resource.job_detail.task_details[_].task_id\n",
+        "        EXECUTE_OUTPUT = (\n",
+        "            PIPELINE_ROOT\n",
+        "            + \"/\"\n",
+        "            + PROJECT_NUMBER\n",
+        "            + \"/\"\n",
+        "            + JOB_ID\n",
+        "            + \"/\"\n",
+        "            + output_task_name\n",
+        "            + \"_\"\n",
+        "            + str(TASK_ID)\n",
+        "            + \"/executor_output.json\"\n",
+        "        )\n",
+        "        GCP_RESOURCES = (\n",
+        "            PIPELINE_ROOT\n",
+        "            + \"/\"\n",
+        "            + PROJECT_NUMBER\n",
+        "            + \"/\"\n",
+        "            + JOB_ID\n",
+        "            + \"/\"\n",
+        "            + output_task_name\n",
+        "            + \"_\"\n",
+        "            + str(TASK_ID)\n",
+        "            + \"/gcp_resources\"\n",
+        "        )\n",
+        "        EVALUATION_METRICS = (\n",
+        "            PIPELINE_ROOT\n",
+        "            + \"/\"\n",
+        "            + PROJECT_NUMBER\n",
+        "            + \"/\"\n",
+        "            + JOB_ID\n",
+        "            + \"/\"\n",
+        "            + output_task_name\n",
+        "            + \"_\"\n",
+        "            + str(TASK_ID)\n",
+        "            + \"/evaluation_metrics\"\n",
+        "        )\n",
+        "        # Check if file exists, 0 is success\n",
+        "        !gsutil -q stat $EXECUTE_OUTPUT\n",
+        "        if _exit_code == 0:\n",
+        "            ! gsutil cat $EXECUTE_OUTPUT\n",
+        "            artifact = EXECUTE_OUTPUT\n",
+        "            break\n",
+        "        !gsutil -q stat $GCP_RESOURCES\n",
+        "        if _exit_code == 0:\n",
+        "            ! gsutil cat $GCP_RESOURCES\n",
+        "            artifact = GCP_RESOURCES\n",
+        "            break\n",
+        "        !gsutil -q stat $EVALUATION_METRICS\n",
+        "        if _exit_code == 0:\n",
+        "            ! gsutil cat $EVALUATION_METRICS\n",
+        "            artifact = EVALUATION_METRICS\n",
+        "            break\n",
+        "\n",
+        "    return artifact\n",
+        "\n",
+        "\n",
+        "print(\"get-vertex-model\")\n",
+        "artifacts = print_pipeline_output(job, \"get-vertex-model\")\n",
+        "output = !gsutil cat $artifacts\n",
+        "print(output)\n",
+        "output = json.loads(output[0])\n",
+        "model_id = output[\"artifacts\"][\"model\"][\"artifacts\"][0][\"metadata\"][\"resourceName\"]\n",
+        "print(\"\\n\\n\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "f431a9e6f025"
+      },
+      "source": [
+        "### Delete the pipeline job\n",
+        "\n",
+        "The method 'delete()' will delete the pipeline job."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "00bf554abbc6"
+      },
+      "outputs": [],
+      "source": [
+        "job.delete()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3d183db57ae2"
+      },
+      "source": [
+        "### Deploy the model\n",
+        "\n",
+        "Next, you deploy the model to an endpoint:\n",
+        "\n",
+        "- Use the `model_id` obtained from the pipeline artifacts to instaniate a Vertex AI Model resource instance.\n",
+        "- Deploy the Vertex AI Model resource to a Vertex AI Endpoint resource.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "591ccc049ce5"
+      },
+      "outputs": [],
+      "source": [
+        "model = aiplatform.Model(model_id)\n",
+        "endpoint = model.deploy(\n",
+        "    accelerator_count=1,\n",
+        "    accelerator_type=aiplatform.gapic.AcceleratorType.NVIDIA_TESLA_T4.name,\n",
+        "    machine_type=\"n1-standard-4\",\n",
+        ")\n",
+        "print(endpoint)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "edb781a92864"
+      },
+      "source": [
+        "### Make a prediction\n",
+        "\n",
+        "Finally, you make a prediction with the deployed model."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "72d94012a987"
+      },
+      "outputs": [],
+      "source": [
+        "endpoint.predict([\"this is a test\"])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "cleanup:mbsdk"
+      },
+      "source": [
+        "# Cleaning up\n",
+        "\n",
+        "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
+        "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
+        "\n",
+        "Otherwise, you can delete the individual resources you created in this tutorial."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cleanup:mbsdk"
+      },
+      "outputs": [],
+      "source": [
+        "delete_bucket = False\n",
+        "\n",
+        "endpoint.undeploy_all()\n",
+        "endpoint.delete()\n",
+        "model.delete()\n",
+        "\n",
+        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "    ! gsutil rm -r $BUCKET_URI\n",
+        "\n",
+        "! rm -rf custom custom.tar.gz\n",
+        "\n",
+        "! gcloud artifacts repositories delete $REPO_NAME --project {PROJECT_ID} --location {REGION} --quiet"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "model_garden_pipeline_templates_bert.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/community/model_garden/model_garden_pipeline_templates_t5x.ipynb

@@ -0,0 +1,879 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "copyright"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2023 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "title:generic,gcp"
+      },
+      "source": [
+        "# Get started with Model Garden Pipeline Templates for T5X models\n",
+        "\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_template_pipelines_t5x.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_template_pipelines_t5x.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/communitymodel_garden/model_garden_template_pipelines_t5x.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "  </td>                                                                                               \n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "overview:mlops"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "\n",
+        "This tutorial demonstrates how to modify, compile and execute a prebuilt Vertex AI Model Garden pipeline template with Vertex AI Pipelines.\n",
+        "\n",
+        "Learn more about [Create a pipeline template](https://cloud.google.com/vertex-ai/docs/pipelines/create-pipeline-template)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "objective:mlops,stage4,get_started_vertex_model_evaluation"
+      },
+      "source": [
+        "### Objective\n",
+        "\n",
+        "In this tutorial, you learn how to use a prebuilt pipeline template with `Vertex AI Pipelines` to fine-tune a T5X text classification model, where the model is accessed from `Vertex AI Model Garden`.\n",
+        "\n",
+        "This tutorial uses the following Google Cloud ML services:\n",
+        "\n",
+        "- `Vertex AI Pipelines`\n",
+        "- `Vertex AI Training`\n",
+        "- `Vertex AI Model Garden`\n",
+        "- `Google Cloud Pipeline Components`\n",
+        "\n",
+        "\n",
+        "The steps performed include:\n",
+        "\n",
+        "- Create a user-defined repository in the `Artifact Registry`.\n",
+        "- Upload the prebuilt pipeline template to the `Artifact Registry`.\n",
+        "- Create a pipeline job with the prebuilt pipeline template to fine-tune a T5X model.\n",
+        "- Execute the pipeline using `Vertex AI Pipelines`.\n",
+        "    - Load T5X model from Vertex AI Model Garden\n",
+        "    - Fine-tune train the model\n",
+        "- Obtain the Vertex AI Model resource from the pipeline artifacts.\n",
+        "- Deploy the model to a Vertex AI Endpoint\n",
+        "- Make a prediction"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dataset:bank,lbn"
+      },
+      "source": [
+        "### Model\n",
+        "\n",
+        "This tutorial uses a pre-trained T5 text classification model from `Vertex AI Model Garden`, which is then fine-tuned (transfer learning) on a dataset of text phrases which are classified as either FirstClass or SecondClass.\n",
+        "\n",
+        "Learn more about [Text-to-text transfer transformer](https://github.com/google-research/text-to-text-transfer-transformer). "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "costs"
+      },
+      "source": [
+        "### Costs\n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Cloud Storage\n",
+        "* Dataflow\n",
+        "\n",
+        "Learn about [Vertex AI\n",
+        "pricing](https://cloud.google.com/vertex-ai/pricing), [Cloud Storage\n",
+        "pricing](https://cloud.google.com/storage/pricing), and [Dataflow pricing](https://cloud.google.com/dataflow/pricing)\n",
+        "and use the [Pricing\n",
+        "Calculator](https://cloud.google.com/products/calculator/)\n",
+        "to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "install_mlops"
+      },
+      "source": [
+        "## Installations\n",
+        "\n",
+        "Install the packages required for executing this notebook.\n",
+        "\n",
+        "*Note:* This tutorial requires KFP 2.x."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "install_mlops"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "! pip3 install --upgrade google-cloud-aiplatform \\\n",
+        "                         google-cloud-pipeline-components \\\n",
+        "                         kfp==2.0.0b15"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "restart"
+      },
+      "source": [
+        "### Colab only: Uncomment the following cell to restart the kernel"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "D-ZBOjErv5mM"
+      },
+      "outputs": [],
+      "source": [
+        "# Automatically restart kernel after installs so that your environment can access the new packages\n",
+        "# import IPython\n",
+        "\n",
+        "# app = IPython.Application.instance()\n",
+        "# app.kernel.do_shutdown(True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "before_you_begin"
+      },
+      "source": [
+        "## Before you begin\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "before_you_begin:nogpu"
+      },
+      "source": [
+        "### Set your project ID\n",
+        "\n",
+        "**If you don't know your project ID**, try the following:\n",
+        "* Run `gcloud config list`.\n",
+        "* Run `gcloud projects list`.\n",
+        "* See the support page: [Locate the project ID](https://support.google.com/googleapi/answer/7014113)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "set_project_id"
+      },
+      "outputs": [],
+      "source": [
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# Set the project id\n",
+        "! gcloud config set project {PROJECT_ID}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "region"
+      },
+      "source": [
+        "#### Region\n",
+        "\n",
+        "You can also change the `REGION` variable used by Vertex AI. Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "region"
+      },
+      "outputs": [],
+      "source": [
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "c4ccf556d4ea"
+      },
+      "source": [
+        "### Enable APIs\n",
+        "\n",
+        "You can enable the required APIs using `gcloud`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "619529337e6d"
+      },
+      "outputs": [],
+      "source": [
+        "! gcloud services enable compute.googleapis.com         \\\n",
+        "                         containerregistry.googleapis.com  \\\n",
+        "                         aiplatform.googleapis.com  \\\n",
+        "                         artifactregistry.googleapis.com"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gcp_authenticate"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "Depending on your Jupyter environment, you may have to manually authenticate. Follow the relevant instructions below."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "FvQeFm3Gv5mR"
+      },
+      "source": [
+        "**1. Vertex AI Workbench**\n",
+        "* Do nothing as you are already authenticated."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ce6043da7b33"
+      },
+      "outputs": [],
+      "source": [
+        "# ! gcloud auth login"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0367eac06a10"
+      },
+      "source": [
+        "**3. Colab, uncomment and run:**"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "21ad4dbb4a61"
+      },
+      "outputs": [],
+      "source": [
+        "# from google.colab import auth\n",
+        "# auth.authenticate_user()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "c13224697bfb"
+      },
+      "source": [
+        "**4. Service account or other**\n",
+        "* See how to grant Cloud Storage permissions to your service account at https://cloud.google.com/storage/docs/gsutil/commands/iam#ch-examples."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "bucket:mbsdk"
+      },
+      "source": [
+        "### Create a Cloud Storage bucket\n",
+        "\n",
+        "Create a storage bucket to store intermediate artifacts such as datasets."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "bucket"
+      },
+      "outputs": [],
+      "source": [
+        "BUCKET_URI = f\"gs://your-bucket-name-{PROJECT_ID}-unique\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "create_bucket"
+      },
+      "source": [
+        "**Only if your bucket doesn't already exist**: Run the following cell to create your Cloud Storage bucket."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "create_bucket"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil mb -l $REGION $BUCKET_URI"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "set_service_account"
+      },
+      "source": [
+        "#### Service Account\n",
+        "\n",
+        "**If you don't know your service account**, try to get your service account using `gcloud` command by executing the second cell below."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "set_service_account"
+      },
+      "outputs": [],
+      "source": [
+        "SERVICE_ACCOUNT = \"[your-service-account]\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "autoset_service_account"
+      },
+      "outputs": [],
+      "source": [
+        "import sys\n",
+        "\n",
+        "IS_COLAB = \"google.colab\" in sys.modules\n",
+        "if (\n",
+        "    SERVICE_ACCOUNT == \"\"\n",
+        "    or SERVICE_ACCOUNT is None\n",
+        "    or SERVICE_ACCOUNT == \"[your-service-account]\"\n",
+        "):\n",
+        "    # Get your service account from gcloud\n",
+        "    if not IS_COLAB:\n",
+        "        shell_output = !gcloud auth list 2>/dev/null\n",
+        "        SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
+        "\n",
+        "    if IS_COLAB:\n",
+        "        shell_output = ! gcloud projects describe  $PROJECT_ID\n",
+        "        project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n",
+        "        SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n",
+        "\n",
+        "    print(\"Service Account:\", SERVICE_ACCOUNT)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "set_service_account:pipelines"
+      },
+      "source": [
+        "#### Set service account access for Vertex AI Pipelines\n",
+        "\n",
+        "Run the following commands to grant your service account access to read and write pipeline artifacts in the bucket that you created in the previous step -- you only need to run these once per service account."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "set_service_account:pipelines"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator  \n",
+        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer  "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "setup_vars"
+      },
+      "source": [
+        "### Set up variables\n",
+        "\n",
+        "Next, set up some variables used throughout the tutorial.\n",
+        "### Import libraries and define constants"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "import_kfp"
+      },
+      "outputs": [],
+      "source": [
+        "import json\n",
+        "import os\n",
+        "\n",
+        "import google.cloud.aiplatform as aiplatform\n",
+        "from kfp.registry import RegistryClient"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "init_aip:mbsdk,all"
+      },
+      "source": [
+        "### Initialize Vertex AI SDK for Python\n",
+        "\n",
+        "Initialize the Vertex AI SDK for Python for your project and corresponding bucket."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "init_aip:mbsdk,all"
+      },
+      "outputs": [],
+      "source": [
+        "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=BUCKET_URI)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2d242773d707"
+      },
+      "source": [
+        "### Enable Artifact Registry API\n",
+        "You must enable the Artifact Registry API service for your project.\n",
+        "\n",
+        "<a href=\"https://cloud.google.com/artifact-registry/docs/enable-service\">Learn more about Enabling service</a>."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "162b5e8883c2"
+      },
+      "outputs": [],
+      "source": [
+        "! gcloud services enable artifactregistry.googleapis.com\n",
+        "\n",
+        "if os.getenv(\"IS_TESTING\"):\n",
+        "    ! sudo apt-get update --yes && sudo apt-get --only-upgrade --yes install google-cloud-sdk-cloud-run-proxy google-cloud-sdk-harbourbridge google-cloud-sdk-cbt google-cloud-sdk-gke-gcloud-auth-plugin google-cloud-sdk-kpt google-cloud-sdk-local-extract google-cloud-sdk-minikube google-cloud-sdk-app-engine-java google-cloud-sdk-app-engine-go google-cloud-sdk-app-engine-python google-cloud-sdk-spanner-emulator google-cloud-sdk-bigtable-emulator google-cloud-sdk-nomos google-cloud-sdk-package-go-module google-cloud-sdk-firestore-emulator kubectl google-cloud-sdk-datastore-emulator google-cloud-sdk-app-engine-python-extras google-cloud-sdk-cloud-build-local google-cloud-sdk-kubectl-oidc google-cloud-sdk-anthos-auth google-cloud-sdk-app-engine-grpc google-cloud-sdk-pubsub-emulator google-cloud-sdk-datalab google-cloud-sdk-skaffold google-cloud-sdk google-cloud-sdk-terraform-tools google-cloud-sdk-config-connector\n",
+        "    ! gcloud components update --quiet"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9b773e8d2bd2"
+      },
+      "source": [
+        "## Create repo in Artifact Registry\n",
+        "\n",
+        "First, you create your own (user-defined) repository in the `Artifact Registry`. You use this repository to upload and retreive your pipeline templates."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "520de849cee2"
+      },
+      "outputs": [],
+      "source": [
+        "REPO_NAME = \"my-docker-repo-unique\"\n",
+        "\n",
+        "! gcloud artifacts repositories create {REPO_NAME} --location={REGION} --repository-format=KFP"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "1611d3517c0f"
+      },
+      "source": [
+        "### Upload the pipeline template\n",
+        "\n",
+        "Next, you instantiate a client interface to the Artifact Registry. Then with the `upload_pipeline()` method you upload your pipeline template."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7f002e57998a"
+      },
+      "outputs": [],
+      "source": [
+        "T5X_YAML = \"gs://cloud-samples-data/vertex-ai/dataset-management/datasets/t5_finetuning/pipeline.yaml\"\n",
+        "\n",
+        "! gsutil cp {T5X_YAML} pipeline.yaml\n",
+        "\n",
+        "client = RegistryClient(\n",
+        "    host=f\"https://{REGION}-kfp.pkg.dev/{PROJECT_ID}/quickstart-kfp-repo\"\n",
+        ")\n",
+        "\n",
+        "templateName, versionName = client.upload_pipeline(\n",
+        "    file_name=\"pipeline.yaml\",\n",
+        "    tags=[\"v1\", \"latest\"],\n",
+        "    extra_headers={\n",
+        "        \"description\": \"This is a pipeline template for fine-tuning a T5 model.\"\n",
+        "    },\n",
+        ")\n",
+        "\n",
+        "! rm pipeline.yaml"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "02f82754fc0d"
+      },
+      "source": [
+        "### View your artifacts in your registry\n",
+        "\n",
+        "Next, using the `gcloud artifacts files` command you view the artifacts, inclusive of the pipeline template, in your artifacts repository."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b2f641eb2056"
+      },
+      "outputs": [],
+      "source": [
+        "! gcloud artifacts files list  --repository={REPO_NAME} --location={REGION}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "968a46a3cb6d"
+      },
+      "source": [
+        "## Load and execute the pipeline job\n",
+        "\n",
+        "Next, you create a Vertex AI Pipeline job from your T5 pipeline template by instantiating a PipelineJob(), with the following parameters:\n",
+        "\n",
+        "- `display_name`: The human readable name for the pipeline job.\n",
+        "- `template_path`: The path to the pipeline template in the Artifact Registry.\n",
+        "- `enable_caching`: On re-runs, use the results from previous successful and unchanged steps.\n",
+        "- `pipeline_root`: A Cloud storage location for storing pipeline results.\n",
+        "- `parameter_values`: The parameters and values that are input to the template pipeline. In this example, they are:\n",
+        "TODO\n",
+        "    - `project`: Your project ID.\n",
+        "    - `class_labels`: A list of valid class labels, in cardinal order.\n",
+        "    - `root_dir`: A Cloud Storage scratch area.\n",
+        "    - `training_data_path`: A Cloud Storage location to the training data.\n",
+        "    - `ground_truth_gcs_source_uris`: A Cloud Storage location to evaluation data."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "a3c502fc7e41"
+      },
+      "outputs": [],
+      "source": [
+        "PIPELINE_ROOT = f\"{BUCKET_URI}/pipeline_root/t5_finetuning\"\n",
+        "\n",
+        "job = aiplatform.PipelineJob(\n",
+        "    display_name=\"t5x-finetuning\",\n",
+        "    template_path=f\"https://{REGION}-kfp.pkg.dev/{PROJECT_ID}/quickstart-kfp-repo/{templateName}/{versionName}\",\n",
+        "    pipeline_root=PIPELINE_ROOT,\n",
+        "    enable_caching=False,\n",
+        "    parameter_values={\n",
+        "        \"project_id\": PROJECT_ID,\n",
+        "        \"accelerator_count\": 32,\n",
+        "        \"feature_keys\": \"question\",\n",
+        "        \"label_key\": \"answer\",\n",
+        "        \"training_data_path\": \"gs://cloud-llm-public/tfds/natural_questions_open/1.0.0_shortened/natural_questions_open-train.tfrecord-00000-of-00001\",\n",
+        "        \"validation_data_path\": \"gs://cloud-llm-public/tfds/natural_questions_open/1.0.0_shortened/natural_questions_open-validation.tfrecord-00000-of-00001\",\n",
+        "    },\n",
+        ")\n",
+        "\n",
+        "job.run()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "view_pipleline_results:bqml"
+      },
+      "source": [
+        "### View the pipeline results"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "view_pipleline_results:bqml"
+      },
+      "outputs": [],
+      "source": [
+        "PROJECT_NUMBER = job.gca_resource.name.split(\"/\")[1]\n",
+        "print(PROJECT_NUMBER)\n",
+        "\n",
+        "\n",
+        "def print_pipeline_output(job, output_task_name):\n",
+        "    JOB_ID = job.name\n",
+        "    print(JOB_ID)\n",
+        "    artifact = \"\"\n",
+        "    for _ in range(len(job.gca_resource.job_detail.task_details)):\n",
+        "        TASK_ID = job.gca_resource.job_detail.task_details[_].task_id\n",
+        "        EXECUTE_OUTPUT = (\n",
+        "            PIPELINE_ROOT\n",
+        "            + \"/\"\n",
+        "            + PROJECT_NUMBER\n",
+        "            + \"/\"\n",
+        "            + JOB_ID\n",
+        "            + \"/\"\n",
+        "            + output_task_name\n",
+        "            + \"_\"\n",
+        "            + str(TASK_ID)\n",
+        "            + \"/executor_output.json\"\n",
+        "        )\n",
+        "        GCP_RESOURCES = (\n",
+        "            PIPELINE_ROOT\n",
+        "            + \"/\"\n",
+        "            + PROJECT_NUMBER\n",
+        "            + \"/\"\n",
+        "            + JOB_ID\n",
+        "            + \"/\"\n",
+        "            + output_task_name\n",
+        "            + \"_\"\n",
+        "            + str(TASK_ID)\n",
+        "            + \"/gcp_resources\"\n",
+        "        )\n",
+        "        EVALUATION_METRICS = (\n",
+        "            PIPELINE_ROOT\n",
+        "            + \"/\"\n",
+        "            + PROJECT_NUMBER\n",
+        "            + \"/\"\n",
+        "            + JOB_ID\n",
+        "            + \"/\"\n",
+        "            + output_task_name\n",
+        "            + \"_\"\n",
+        "            + str(TASK_ID)\n",
+        "            + \"/evaluation_metrics\"\n",
+        "        )\n",
+        "        # Check if file exists, 0 is success\n",
+        "        !gsutil -q stat $EXECUTE_OUTPUT\n",
+        "        if _exit_code == 0:\n",
+        "            ! gsutil cat $EXECUTE_OUTPUT\n",
+        "            artifact = EXECUTE_OUTPUT\n",
+        "            break\n",
+        "        !gsutil -q stat $GCP_RESOURCES\n",
+        "        if _exit_code == 0:\n",
+        "            ! gsutil cat $GCP_RESOURCES\n",
+        "            artifact = GCP_RESOURCES\n",
+        "            break\n",
+        "        !gsutil -q stat $EVALUATION_METRICS\n",
+        "        if _exit_code == 0:\n",
+        "            ! gsutil cat $EVALUATION_METRICS\n",
+        "            artifact = EVALUATION_METRICS\n",
+        "            break\n",
+        "\n",
+        "    return artifact\n",
+        "\n",
+        "\n",
+        "print(\"model-upload\")\n",
+        "artifacts = print_pipeline_output(job, \"model-upload\")\n",
+        "output = !gsutil cat $artifacts\n",
+        "print(output)\n",
+        "output = json.loads(output[0])\n",
+        "model_id = output[\"artifacts\"][\"model\"][\"artifacts\"][0][\"metadata\"][\"resourceName\"]\n",
+        "print(\"\\n\\n\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "f431a9e6f025"
+      },
+      "source": [
+        "### Delete the pipeline job\n",
+        "\n",
+        "The method 'delete()' will delete the pipeline job."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "00bf554abbc6"
+      },
+      "outputs": [],
+      "source": [
+        "job.delete()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3d183db57ae2"
+      },
+      "source": [
+        "### Deploy the model\n",
+        "\n",
+        "Next, you deploy the model to an endpoint:\n",
+        "\n",
+        "- Use the `model_id` obtained from the pipeline artifacts to instantiate a Vertex AI Model resource instance.\n",
+        "- Deploy the Vertex AI Model resource to a Vertex AI Endpoint resource.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "591ccc049ce5"
+      },
+      "outputs": [],
+      "source": [
+        "model = aiplatform.Model(model_id)\n",
+        "endpoint = model.deploy(\n",
+        "    accelerator_count=1,\n",
+        "    accelerator_type=aiplatform.gapic.AcceleratorType.NVIDIA_TESLA_T4.name,\n",
+        "    machine_type=\"n1-standard-4\",\n",
+        ")\n",
+        "print(endpoint)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "edb781a92864"
+      },
+      "source": [
+        "### Make a prediction\n",
+        "\n",
+        "Finally, you make a prediction with the deployed model."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "72d94012a987"
+      },
+      "outputs": [],
+      "source": [
+        "endpoint.predict([\"this is a test\"])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "cleanup:mbsdk"
+      },
+      "source": [
+        "# Cleaning up\n",
+        "\n",
+        "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
+        "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
+        "\n",
+        "Otherwise, you can delete the individual resources you created in this tutorial."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cleanup:mbsdk"
+      },
+      "outputs": [],
+      "source": [
+        "delete_bucket = False\n",
+        "\n",
+        "endpoint.undeploy_all()\n",
+        "endpoint.delete()\n",
+        "model.delete()\n",
+        "\n",
+        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "    ! gsutil rm -r $BUCKET_URI\n",
+        "\n",
+        "! rm -rf custom custom.tar.gz\n",
+        "\n",
+        "! gcloud artifacts repositories delete $REPO_NAME --project {PROJECT_ID} --location {REGION} --quiet"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "model_garden_pipeline_templates_t5x.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/community/model_garden/model_garden_proprietary_image_classification.ipynb

@@ -0,0 +1,614 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ur8xi4C7S06n"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2023 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "TirJ-SGQseby"
+      },
+      "source": [
+        "# Vertex AI Model Garden: Google Proprietary Model Image Classification\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_proprietary_image_classification.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_proprietary_image_classification.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/notebooks/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/model_garden/model_garden_proprietary_image_classification.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "  </td>\n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dwGLvtIeECLK"
+      },
+      "source": [
+        "**_NOTE_**: This notebook has been tested in the following environment:\n",
+        "\n",
+        "* Python version = 3.9"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tvgnzT1CKxrO"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "This notebook demonstrates how to use Google proprietary image classification model training/deployment in [Vertex AI Model Garden](https://cloud.google.com/model-garden).\n",
+        "\n",
+        "### Objective\n",
+        "\n",
+        "* Train new models using Vertex SDK\n",
+        "\n",
+        "* Test trained models\n",
+        "  * View the trained model in [Vertex Model Registry](https://cloud.google.com/vertex-ai/docs/model-registry/introduction)\n",
+        "  * Deploy uploaded models\n",
+        "  * Run predictions\n",
+        "\n",
+        "* Cleanup resources\n",
+        "\n",
+        "### Costs\n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI\n",
+        "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
+        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
+        "Calculator](https://cloud.google.com/products/calculator/)\n",
+        "to generate a cost estimate based on your projected usage.\n",
+        "\n",
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the [Flowers dataset](https://www.tensorflow.org/datasets/catalog/tf_flowers) from [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/overview). The version of the dataset you use in this tutorial is stored in a public Cloud Storage bucket. The trained model predicts the type of flower an image is from a class of five flowers: daisy, dandelion, rose, sunflower, or tulip."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "KEukV6uRk_S3"
+      },
+      "source": [
+        "## Before you begin"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Jvqs-ehKlaYh"
+      },
+      "outputs": [],
+      "source": [
+        "! pip3 install --upgrade google-cloud-aiplatform\n",
+        "\n",
+        "# Automatically restart kernel after installs\n",
+        "import IPython\n",
+        "\n",
+        "app = IPython.Application.instance()\n",
+        "app.kernel.do_shutdown(True)\n",
+        "if \"google.colab\" in str(get_ipython()):\n",
+        "    from google.colab import auth as google_auth\n",
+        "\n",
+        "    google_auth.authenticate_user()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "BF1j6f9HApxa"
+      },
+      "source": [
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI API and Compute Engine API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component).\n",
+        "\n",
+        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9wExiMUxFk91"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "from google.cloud import aiplatform\n",
+        "\n",
+        "# The project and bucket are for experiments below.\n",
+        "PROJECT_ID = \"\"  # @param {type:\"string\"}\n",
+        "BUCKET_URI = \"\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# You can choose a region from https://cloud.google.com/about/locations.\n",
+        "# Only regions prefixed by \"us\", \"europe\", or \"asia\" are supported.\n",
+        "REGION = \"us-central1\"  # @param {type:\"string\"}\n",
+        "REGION_PREFIX = REGION.split(\"-\")[0]\n",
+        "assert REGION_PREFIX in (\n",
+        "    \"us\",\n",
+        "    \"europe\",\n",
+        "    \"asia\",\n",
+        "), f'{REGION} is not supported. It must be prefixed by \"us\", \"europe\", or \"asia\".'\n",
+        "\n",
+        "! gcloud config set project $PROJECT_ID\n",
+        "\n",
+        "STAGING_BUCKET = os.path.join(BUCKET_URI, \"temporal\")\n",
+        "\n",
+        "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=STAGING_BUCKET)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "n6IFz75WGCam"
+      },
+      "source": [
+        "### Define constants"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "riG_qUokg0XZ"
+      },
+      "outputs": [],
+      "source": [
+        "OBJECTIVE = \"icn\"\n",
+        "\n",
+        "# Dataset constants\n",
+        "DATASET_PREFIX = \"dataset-icn\"\n",
+        "\n",
+        "# Training constants.\n",
+        "TRAINING_JOB_PREFIX = \"train\"\n",
+        "\n",
+        "# The image classification flowers dataset used to train the model.\n",
+        "DATASET_FILE = (\n",
+        "    \"gs://cloud-samples-data/vision/automl_classification/flowers/all_data_v2.csv\"\n",
+        ")\n",
+        "\n",
+        "# Evaluation constants.\n",
+        "EVALUATION_METRIC = \"accuracy\"\n",
+        "\n",
+        "DEPLOY_JOB_PREFIX = \"deploy\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ZZFPe_GezXg8"
+      },
+      "source": [
+        "### Define common libraries"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "XcYUGwr-AJGY"
+      },
+      "outputs": [],
+      "source": [
+        "import base64\n",
+        "from datetime import datetime\n",
+        "from io import BytesIO\n",
+        "\n",
+        "import matplotlib.pyplot as plt\n",
+        "import numpy\n",
+        "import tensorflow as tf\n",
+        "from PIL import Image\n",
+        "\n",
+        "\n",
+        "def get_job_name_with_datetime(prefix: str):\n",
+        "    return prefix + datetime.now().strftime(\"_%Y%m%d_%H%M%S\")\n",
+        "\n",
+        "\n",
+        "def load_img(path):\n",
+        "    img = tf.io.read_file(path)\n",
+        "    img = tf.image.decode_jpeg(img, channels=3)\n",
+        "    return Image.fromarray(numpy.uint8(img)).convert(\"RGB\")\n",
+        "\n",
+        "\n",
+        "def display_image(image):\n",
+        "    _ = plt.figure(figsize=(20, 15))\n",
+        "    plt.grid(False)\n",
+        "    plt.imshow(image)\n",
+        "\n",
+        "\n",
+        "def get_prediction_instances(test_filepath, new_width=-1):\n",
+        "    if new_width <= 0:\n",
+        "        with tf.io.gfile.GFile(test_filepath, \"rb\") as input_file:\n",
+        "            encoded_string = base64.b64encode(input_file.read()).decode(\"utf-8\")\n",
+        "    else:\n",
+        "        img = load_img(test_filepath)\n",
+        "        width, height = img.size\n",
+        "        print(\"original input image size: \", width, \" , \", height)\n",
+        "        new_height = int(height * new_width / width)\n",
+        "        new_img = img.resize((new_width, new_height))\n",
+        "        print(\"resized input image size: \", new_width, \" , \", new_height)\n",
+        "        buffered = BytesIO()\n",
+        "        new_img.save(buffered, format=\"JPEG\")\n",
+        "        encoded_string = base64.b64encode(buffered.getvalue()).decode(\"utf-8\")\n",
+        "\n",
+        "    instances = [{\"content\": encoded_string}]\n",
+        "    return instances"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Q149N3V6Uynm"
+      },
+      "source": [
+        "## Create a dataset\n",
+        "\n",
+        "This tutorial uses a version of the Flowers dataset that is stored in a public Cloud Storage bucket, using a CSV index file.\n",
+        "\n",
+        "Start by doing a quick peek at the data. You count the number of examples by counting the number of rows in the CSV index file  (`wc -l`) and then peek at the first few rows."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "yR4x-5XiVIlv"
+      },
+      "outputs": [],
+      "source": [
+        "count = ! gsutil cat $DATASET_FILE | wc -l\n",
+        "print(\"Number of Examples\", int(count[0]))\n",
+        "\n",
+        "print(\"First 10 rows\")\n",
+        "! gsutil cat $DATASET_FILE | head"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8yfBZ1_8VZvq"
+      },
+      "source": [
+        "Next, create the `Dataset` resource using the `create` method for the `ImageDataset` class, which takes the following parameters:\n",
+        "\n",
+        "- `display_name`: The human readable name for the `Dataset` resource.\n",
+        "- `gcs_source`: A list of one or more dataset index files to import the data items into the `Dataset` resource.\n",
+        "- `import_schema_uri`: The data labeling schema for the data items.\n",
+        "\n",
+        "This operation may take several minutes."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "eehB1gwHVfRh"
+      },
+      "outputs": [],
+      "source": [
+        "dataset = aiplatform.ImageDataset.create(\n",
+        "    display_name=DATASET_PREFIX + \"_flowers\",\n",
+        "    gcs_source=[DATASET_FILE],\n",
+        "    import_schema_uri=aiplatform.schema.dataset.ioformat.image.single_label_classification,\n",
+        ")\n",
+        "\n",
+        "print(dataset.resource_name)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "RB_xY9ipr7ZU"
+      },
+      "source": [
+        "## Train new models\n",
+        "\n",
+        "### Create and run training pipeline\n",
+        "\n",
+        "To train an AutoML model, you perform two steps:\n",
+        "1.  Create a training pipeline.\n",
+        "2.  Run the pipeline.\n",
+        "\n",
+        "#### Create training pipeline\n",
+        "\n",
+        "An AutoML training pipeline is created with the `AutoMLImageTrainingJob` class, with the following parameters:\n",
+        "\n",
+        "- `display_name`: The human readable name for the `TrainingJob` resource.\n",
+        "- `prediction_type`: The type task to train the model for.\n",
+        "  - `classification`: An image classification model.\n",
+        "  - `object_detection`: An image object detection model.\n",
+        "- `multi_label`: If a classification task is single (`False`) or multi-labeled (`True`).\n",
+        "- `model_type`: The type of model for deployment.\n",
+        "  - `EFFICIENTNET`: A model that is available in Vertex Model Garden image classification training with customizable hyperparameters. Best tailored to be used within Google Cloud, and cannot be exported externally.\n",
+        "  - `MAXVIT`: A model that is available in Vertex Model Garden image classification training with customizable hyperparameters. Best tailored to be used within Google Cloud, and cannot be exported externally.\n",
+        "  - `VIT`: A model that is available in Vertex Model Garden image classification training with customizable hyperparameters. Best tailored to be used within Google Cloud, and cannot be exported externally.\n",
+        "- `checkpoint_name`: Optional. The field is reserved for Model Garden model training, based on the provided pre-trained model checkpoint.\n",
+        "- `trainer_config`: Optional. The field is usually used together with the Model Garden model training when passing the customized configs for the trainer.\n",
+        "\n",
+        "  Example with all supported parameters:\n",
+        "```\n",
+        "  trainer_config = {\n",
+        "    'global_batch_size': '8',\n",
+        "    'learning_rate': '0.001',\n",
+        "    'optimizer_type': 'sgd',\n",
+        "    'optimizer_momentum': '0.9',\n",
+        "    'train_steps': '10000',\n",
+        "    'accelerator_count': '1',\n",
+        "  }\n",
+        "```\n",
+        "  The global_batch_size should be divisible by accelerator_count.\n",
+        "  Supported values for optimizer_type are 'sgd', 'adam', 'adamw', 'lamb', 'rmsprop', 'lars', 'adagrad', and 'slide'.\n",
+        "  Supported values for accelerator_count are '1', '2', '4', and '8'.\n",
+        "- `metric_spec`: Dictionary representing metrics to optimize. The dictionary key is the `metric_id`, which is reported by your training job, with possible values being ('loss', 'accuracy') and the dictionary value is the optimization goal of the metric ('minimize' or 'maximize').\n",
+        "For example:  `metric_spec = {'loss': 'minimize', 'accuracy': 'maximize'}`\n",
+        "- `parameter_spec`: Dictionary representing parameters to optimize. The dictionary key is the `metric_id`, which is passed into your training job as a command line key word argument, and the dictionary value is the parameter\n",
+        "specification of the metric. Supported parameter specifications can be found in aiplatform.hyperparameter_tuning.\n",
+        "```\n",
+        "  from google.cloud.aiplatform.aiplatform import hpt as hpt\n",
+        "\n",
+        "  parameter_spec = {\n",
+        "    'learning_rate': hpt.DoubleParameterSpec(min=1e-7, max=1, scale='linear'),\n",
+        "  }\n",
+        "```\n",
+        "- `search_algorithm`: The search algorithm specified for the Study. Accepts one of the following:\n",
+        "  - `None`: If you do not specify an algorithm, your job uses the default\n",
+        "  Vertex AI algorithm. The default algorithm applies Bayesian optimization\n",
+        "  to arrive at the optimal solution with a more effective search over the\n",
+        "  parameter space.\n",
+        "  - `grid`: A simple grid search within the feasible space. This option is\n",
+        "  particularly useful if you want to specify a quantity of trials that is greater than the number of points in the feasible space. In such cases, if you do not specify a grid search, the Vertex AI default algorithm may generate duplicate suggestions. To use grid search, all parameter specs must be of type `IntegerParameterSpec`, `CategoricalParameterSpec`, or `DiscreteParameterSpec`.\n",
+        "  - `random`: A simple random search within the feasible space.\n",
+        "- `measurement_selection`: This indicates which measurement to use\n",
+        "if/when the service automatically selects the final measurement from\n",
+        "previously reported intermediate measurements.\n",
+        "  Accepts: `best`, `last` Choose this based on two considerations:\n",
+        "    - A): Do you expect your measurements to monotonically improve? If so,\n",
+        "    choose `last`. On the other hand, if you\\'re in a situation where\n",
+        "    your system can **over-train** and you expect the performance to get\n",
+        "    better for a while but then start declining, choose `best`.\n",
+        "    - B): Are your measurements significantly noisy and/or irreproducible? If\n",
+        "    so, `best` will tend to be over-optimistic, and it may be better\n",
+        "    to choose `last`. If both or neither of (A) and (B) apply, it\n",
+        "    doesn't matter which selection type is chosen.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "un0tyqU7We_A"
+      },
+      "outputs": [],
+      "source": [
+        "from google.cloud.aiplatform import hyperparameter_tuning as hpt\n",
+        "\n",
+        "TRAINER_CONFIG = {\n",
+        "    \"global_batch_size\": \"8\",\n",
+        "    \"learning_rate\": \"0.001\",\n",
+        "    \"train_steps\": \"10000\",\n",
+        "    \"accelerator_count\": \"1\",\n",
+        "}\n",
+        "METRIC_SPEC_KEY = \"accuracy\"\n",
+        "METRIC_SPEC_VALUE = \"maximize\"\n",
+        "SEARCH_ALGORITHM = \"random\"\n",
+        "MEASUREMENT_SELECTION = \"best\"\n",
+        "MODEL_TYPE = \"MAXVIT\"  # @param {type:\"string\"} one of the values [\"MAXVIT\", \"EFFICIENTNET\", \"VIT\"]\n",
+        "\n",
+        "job = aiplatform.AutoMLImageTrainingJob(\n",
+        "    display_name=get_job_name_with_datetime(TRAINING_JOB_PREFIX),\n",
+        "    prediction_type=\"classification\",\n",
+        "    multi_label=False,\n",
+        "    model_type=MODEL_TYPE,\n",
+        "    base_model=None,\n",
+        "    trainer_config=TRAINER_CONFIG,\n",
+        "    metric_spec={METRIC_SPEC_KEY: METRIC_SPEC_VALUE},\n",
+        "    parameter_spec={\n",
+        "        \"learning_rate\": hpt.DoubleParameterSpec(min=0.001, max=0.1, scale=\"log\"),\n",
+        "    },\n",
+        "    search_algorithm=SEARCH_ALGORITHM,\n",
+        "    measurement_selection=MEASUREMENT_SELECTION,\n",
+        ")\n",
+        "\n",
+        "print(job)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "HwcCjwlBTQIz"
+      },
+      "source": [
+        "#### Run the training pipeline\n",
+        "\n",
+        "Next, run the DAG to start the training job by invoking the method `run`, with the following parameters:\n",
+        "\n",
+        "- `dataset`: The `Dataset` resource to train the model.\n",
+        "- `model_display_name`: The human readable name for the trained model.\n",
+        "- `training_fraction_split`: The percentage of the dataset to use for training.\n",
+        "- `validation_fraction_split`: The percentage of the dataset to use for validation.\n",
+        "- `test_fraction_split`: The percentage of the dataset to use for test (holdout data).\n",
+        "- `budget_milli_node_hours`: (optional) Maximum training time specified in unit of millihours (1000 = hour).\n",
+        "- `disable_early_stopping`: If `True`, training may be completed before using the entire budget if the service believes it cannot further improve on the model objective measurements.\n",
+        "\n",
+        "The `run` method, when completed, returns the `Model` resource.\n",
+        "\n",
+        "The execution of the training pipeline will take up to 60 minutes."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "aec22792ee84"
+      },
+      "outputs": [],
+      "source": [
+        "model = job.run(\n",
+        "    dataset=dataset,\n",
+        "    model_display_name=get_job_name_with_datetime(\"flowers\"),\n",
+        "    training_fraction_split=0.8,\n",
+        "    validation_fraction_split=0.1,\n",
+        "    test_fraction_split=0.1,\n",
+        "    budget_milli_node_hours=8000,\n",
+        "    disable_early_stopping=False,\n",
+        ")\n",
+        "\n",
+        "print(\"Model is: \", model)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "g0BGaofgsMsy"
+      },
+      "source": [
+        "## Test trained models\n",
+        "This section shows how to test the trained models.\n",
+        "1. Deploy model from Model Registry\n",
+        "2. Run online predictions"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "NYuQowyZEtxK"
+      },
+      "outputs": [],
+      "source": [
+        "# @title Deploy model from Model Registry\n",
+        "# Model does not support dedicated deployment resources.\n",
+        "# An n1-standard-4 machine with 1 P100 GPU will be used.\n",
+        "\n",
+        "deploy_model_name = get_job_name_with_datetime(DEPLOY_JOB_PREFIX + \"_\" + OBJECTIVE)\n",
+        "print(\"The deployed job name is: \", deploy_model_name)\n",
+        "\n",
+        "endpoint = model.deploy(\n",
+        "    deployed_model_display_name=deploy_model_name,\n",
+        "    traffic_split={\"0\": 100},\n",
+        "    min_replica_count=1,\n",
+        "    max_replica_count=1,\n",
+        ")\n",
+        "\n",
+        "endpoint_id = endpoint.name\n",
+        "print(\"endpoint id is: \", endpoint_id)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "vbIW9me1F2RY"
+      },
+      "outputs": [],
+      "source": [
+        "# @title Run online predictions\n",
+        "\n",
+        "# test image file path from a GCS bucket\n",
+        "test_filepath = \"\"  # @param {type:\"string\"}\n",
+        "\n",
+        "with tf.io.gfile.GFile(test_filepath, \"rb\") as f:\n",
+        "    content = f.read()\n",
+        "\n",
+        "# The format of each instance should conform to the deployed model's prediction input schema.\n",
+        "instances = [{\"content\": base64.b64encode(content).decode(\"utf-8\")}]\n",
+        "\n",
+        "prediction = endpoint.predict(instances=instances)\n",
+        "\n",
+        "img = load_img(test_filepath)\n",
+        "display_image(img)\n",
+        "print(prediction)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "kkH2nrpdp4sp"
+      },
+      "source": [
+        "## Clean up"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Ax6vQVZhp9pR"
+      },
+      "outputs": [],
+      "source": [
+        "# Undeploy model and delete endpoint.\n",
+        "endpoint.undeploy_all()\n",
+        "endpoint.delete(force=True)\n",
+        "\n",
+        "# Delete models.\n",
+        "model.delete()"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "model_garden_proprietary_image_classification.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/community/model_garden/model_garden_proprietary_image_object_detection.ipynb

@@ -0,0 +1,687 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ur8xi4C7S06n"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2023 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "TirJ-SGQseby"
+      },
+      "source": [
+        "# Vertex AI Model Garden: Google Proprietary Model Image Object Detection\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_proprietary_image_object_detection.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_proprietary_image_object_detection.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/notebooks/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/model_garden/model_garden_proprietary_image_object_detection.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "  </td>\n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dwGLvtIeECLK"
+      },
+      "source": [
+        "**_NOTE_**: This notebook has been tested in the following environment:\n",
+        "\n",
+        "* Python version = 3.9"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tvgnzT1CKxrO"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "This notebook demonstrates how to use Google proprietary image object detection model training/deployment in [Vertex AI Model Garden](https://cloud.google.com/model-garden).\n",
+        "\n",
+        "### Objective\n",
+        "\n",
+        "* Train new models using Vertex SDK\n",
+        "\n",
+        "* Test trained models\n",
+        "  * View the trained model in [Vertex AI Model Registry](https://cloud.google.com/vertex-ai/docs/model-registry/introduction)\n",
+        "  * Deploy uploaded models\n",
+        "  * Run predictions\n",
+        "\n",
+        "* Cleanup resources\n",
+        "\n",
+        "### Costs\n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI\n",
+        "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
+        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
+        "Calculator](https://cloud.google.com/products/calculator/)\n",
+        "to generate a cost estimate based on your projected usage.\n",
+        "\n",
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the Salads category of the [OpenImages dataset](https://www.tensorflow.org/datasets/catalog/open_images_v4) from [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/overview). This dataset does not require any feature engineering. The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket. The trained model predicts the bounding box locations and corresponding type of salad items in an image from a class of five items: salad, seafood, tomato, baked goods, or cheese."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "KEukV6uRk_S3"
+      },
+      "source": [
+        "## Before you begin"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Jvqs-ehKlaYh"
+      },
+      "outputs": [],
+      "source": [
+        "! pip3 install --upgrade google-cloud-aiplatform\n",
+        "\n",
+        "# Automatically restart kernel after installs\n",
+        "import IPython\n",
+        "\n",
+        "app = IPython.Application.instance()\n",
+        "app.kernel.do_shutdown(True)\n",
+        "if \"google.colab\" in str(get_ipython()):\n",
+        "    from google.colab import auth as google_auth\n",
+        "\n",
+        "    google_auth.authenticate_user()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "BF1j6f9HApxa"
+      },
+      "source": [
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI API and Compute Engine API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component).\n",
+        "\n",
+        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9wExiMUxFk91"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "from google.cloud import aiplatform\n",
+        "\n",
+        "# The project and bucket are for experiments below.\n",
+        "PROJECT_ID = \"\"  # @param {type:\"string\"}\n",
+        "BUCKET_URI = \"\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# You can choose a region from https://cloud.google.com/about/locations.\n",
+        "# Only regions prefixed by \"us\", \"europe\", or \"asia\" are supported.\n",
+        "REGION = \"us-central1\"  # @param {type:\"string\"}\n",
+        "REGION_PREFIX = REGION.split(\"-\")[0]\n",
+        "assert REGION_PREFIX in (\n",
+        "    \"us\",\n",
+        "    \"europe\",\n",
+        "    \"asia\",\n",
+        "), f'{REGION} is not supported. It must be prefixed by \"us\", \"europe\", or \"asia\".'\n",
+        "\n",
+        "! gcloud config set project $PROJECT_ID\n",
+        "\n",
+        "STAGING_BUCKET = os.path.join(BUCKET_URI, \"temporal\")\n",
+        "\n",
+        "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=STAGING_BUCKET)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "n6IFz75WGCam"
+      },
+      "source": [
+        "### Define constants"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "riG_qUokg0XZ"
+      },
+      "outputs": [],
+      "source": [
+        "OBJECTIVE = \"iod\"\n",
+        "\n",
+        "# Dataset constants.\n",
+        "DATASET_PREFIX = \"dataset-iod\"\n",
+        "\n",
+        "# Training constants.\n",
+        "TRAINING_JOB_PREFIX = \"train\"\n",
+        "# The image object detection salad dataset used to train the model\n",
+        "DATASET_FILE = \"gs://cloud-samples-data/vision/salads.csv\"\n",
+        "\n",
+        "# Evaluation constants.\n",
+        "EVALUATION_METRIC = \"AP50\"\n",
+        "\n",
+        "DEPLOY_JOB_PREFIX = \"deploy\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ZZFPe_GezXg8"
+      },
+      "source": [
+        "### Define common libraries"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "XcYUGwr-AJGY"
+      },
+      "outputs": [],
+      "source": [
+        "import base64\n",
+        "import os\n",
+        "from datetime import datetime\n",
+        "\n",
+        "import matplotlib.pyplot as plt\n",
+        "import numpy as np\n",
+        "import tensorflow as tf\n",
+        "from google.cloud import aiplatform\n",
+        "from PIL import Image, ImageColor, ImageDraw, ImageFont\n",
+        "\n",
+        "\n",
+        "def get_job_name_with_datetime(prefix: str):\n",
+        "    return prefix + datetime.now().strftime(\"_%Y%m%d_%H%M%S\")\n",
+        "\n",
+        "\n",
+        "def load_img(path):\n",
+        "    img = tf.io.read_file(path)\n",
+        "    img = tf.image.decode_jpeg(img, channels=3)\n",
+        "    return Image.fromarray(np.uint8(img)).convert(\"RGB\")\n",
+        "\n",
+        "\n",
+        "def display_image(image):\n",
+        "    _ = plt.figure(figsize=(20, 15))\n",
+        "    plt.grid(False)\n",
+        "    plt.imshow(image)\n",
+        "\n",
+        "\n",
+        "def draw_bounding_box_on_image(\n",
+        "    image, ymin, xmin, ymax, xmax, color, font, thickness=4, display_str_list=()\n",
+        "):\n",
+        "    \"\"\"Adds a bounding box to an image.\"\"\"\n",
+        "    draw = ImageDraw.Draw(image)\n",
+        "    im_width, im_height = image.size\n",
+        "    (left, right, top, bottom) = (\n",
+        "        xmin * im_width,\n",
+        "        xmax * im_width,\n",
+        "        ymin * im_height,\n",
+        "        ymax * im_height,\n",
+        "    )\n",
+        "    draw.line(\n",
+        "        [(left, top), (left, bottom), (right, bottom), (right, top), (left, top)],\n",
+        "        width=thickness,\n",
+        "        fill=color,\n",
+        "    )\n",
+        "\n",
+        "    # If the total height of the display strings added to the top of the bounding\n",
+        "    # box exceeds the top of the image, stack the strings below the bounding box\n",
+        "    # instead of above.\n",
+        "    display_str_heights = [font.getsize(ds)[1] for ds in display_str_list]\n",
+        "    # Each display_str has a top and bottom margin of 0.05x.\n",
+        "    total_display_str_height = (1 + 2 * 0.05) * sum(display_str_heights)\n",
+        "\n",
+        "    if top > total_display_str_height:\n",
+        "        text_bottom = top\n",
+        "    else:\n",
+        "        text_bottom = top + total_display_str_height\n",
+        "    # Reverse list and print from bottom to top.\n",
+        "    for display_str in display_str_list[::-1]:\n",
+        "        text_width, text_height = font.getsize(display_str)\n",
+        "        margin = np.ceil(0.05 * text_height)\n",
+        "        draw.rectangle(\n",
+        "            [\n",
+        "                (left, text_bottom - text_height - 2 * margin),\n",
+        "                (left + text_width, text_bottom),\n",
+        "            ],\n",
+        "            fill=color,\n",
+        "        )\n",
+        "        draw.text(\n",
+        "            (left + margin, text_bottom - text_height - margin),\n",
+        "            display_str,\n",
+        "            fill=\"black\",\n",
+        "            font=font,\n",
+        "        )\n",
+        "        text_bottom -= text_height - 2 * margin\n",
+        "\n",
+        "\n",
+        "def draw_boxes(image, boxes, class_names, scores, max_boxes=40, min_score=0.05):\n",
+        "    \"\"\"Overlay labeled boxes on an image with formatted scores and label names.\"\"\"\n",
+        "    colors = list(ImageColor.colormap.values())\n",
+        "    try:\n",
+        "        font = ImageFont.truetype(\n",
+        "            \"/usr/share/fonts/truetype/liberation/LiberationSansNarrow-Regular.ttf\", 25\n",
+        "        )\n",
+        "    except OSError:\n",
+        "        print(\"Font not found, using default font.\")\n",
+        "        font = ImageFont.load_default()\n",
+        "\n",
+        "    for i in range(min(len(boxes), max_boxes)):\n",
+        "        if scores[i] >= min_score:\n",
+        "            ymin, xmin, ymax, xmax = boxes[i]\n",
+        "            display_str = \"{}: {}%\".format(class_names[i], int(100 * scores[i]))\n",
+        "            color = colors[hash(class_names[i]) % len(colors)]\n",
+        "            draw_bounding_box_on_image(\n",
+        "                image,\n",
+        "                ymin,\n",
+        "                xmin,\n",
+        "                ymax,\n",
+        "                xmax,\n",
+        "                color,\n",
+        "                font,\n",
+        "                display_str_list=[display_str],\n",
+        "            )\n",
+        "    return image"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "nZLVI9TtUuif"
+      },
+      "source": [
+        "## Create a dataset\n",
+        "\n",
+        "This tutorial uses a version of the Salads dataset that is stored in a public Cloud Storage bucket, using a CSV index file.\n",
+        "\n",
+        "Start by doing a quick peek at the data. You count the number of examples by counting the number of rows in the CSV index file  (`wc -l`) and then peek at the first few rows."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Pr60Y1fpUuO9"
+      },
+      "outputs": [],
+      "source": [
+        "count = ! gsutil cat $DATASET_FILE | wc -l\n",
+        "print(\"Number of Examples\", int(count[0]))\n",
+        "\n",
+        "print(\"First 10 rows\")\n",
+        "! gsutil cat $DATASET_FILE | head"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "SZEdBfNZUxQn"
+      },
+      "source": [
+        "Next, create the `Dataset` resource using the `create` method for the `ImageDataset` class, which takes the following parameters:\n",
+        "\n",
+        "- `display_name`: The human readable name for the `Dataset` resource.\n",
+        "- `gcs_source`: A list of one or more dataset index files to import the data items into the `Dataset` resource.\n",
+        "- `import_schema_uri`: The data labeling schema for the data items.\n",
+        "\n",
+        "This operation may take several minutes."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "FlDqM5APU0As"
+      },
+      "outputs": [],
+      "source": [
+        "dataset = aiplatform.ImageDataset.create(\n",
+        "    display_name=DATASET_PREFIX + \"_salads\",\n",
+        "    gcs_source=[DATASET_FILE],\n",
+        "    import_schema_uri=aiplatform.schema.dataset.ioformat.image.bounding_box,\n",
+        ")\n",
+        "\n",
+        "print(dataset.resource_name)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "RB_xY9ipr7ZU"
+      },
+      "source": [
+        "## Train new models\n",
+        "\n",
+        "### Create and run training pipeline\n",
+        "\n",
+        "To train an AutoML model, you perform two steps:\n",
+        "1.  Create a training pipeline.\n",
+        "2.  Run the pipeline.\n",
+        "\n",
+        "#### Create training pipeline\n",
+        "\n",
+        "An AutoML training pipeline is created with the `AutoMLImageTrainingJob` class, with the following parameters:\n",
+        "\n",
+        "- `display_name`: The human readable name for the `TrainingJob` resource.\n",
+        "- `prediction_type`: The type task to train the model for.\n",
+        "  - `classification`: An image classification model.\n",
+        "  - `object_detection`: An image object detection model.\n",
+        "- `model_type`: The type of model for deployment. For image object detection, we current support the following:\n",
+        "  - `SPINENET`: A model that is available in Vertex Model Garden image object detection training with customizable hyperparameters. Best tailored to be used within Google Cloud, and cannot be exported externally.\n",
+        "  - `YOLO`: A model that is available in Vertex Model Garden image object detection training with customizable hyperparameters. Best tailored to be used within Google Cloud, and cannot be exported externally.\n",
+        "- `checkpoint_name`: Optional. The field is reserved for Model Garden model training, based on the provided pre-trained model checkpoint.\n",
+        "- `trainer_config`: Optional. The field is usually used together with the Model Garden model training, when passing the customized configs for the trainer. `anchor_size` cannot be used with `YOLO`.\n",
+        "\n",
+        "  Example with all supported parameters:\n",
+        "```\n",
+        "  trainer_config = {\n",
+        "    'global_batch_size': '8',\n",
+        "    'learning_rate': '0.001',\n",
+        "    'optimizer_type': 'sgd',\n",
+        "    'optimizer_momentum': '0.9',\n",
+        "    'train_steps': '10000',\n",
+        "    'accelerator_count': '2',\n",
+        "    'anchor_size': '8',\n",
+        "  }\n",
+        "```\n",
+        "  The global_batch_size should be divisible by accelerator_count.\n",
+        "  Supported values for optimizer_type are 'sgd', 'adam', 'adamw', 'lamb', 'rmsprop', 'lars', 'adagrad', and 'slide'.\n",
+        "  Currently, only '2' is supported for accelerator_count.\n",
+        "- `metric_spec`: Dictionary representing metrics to optimize. The dictionary key is the metric_id, which is reported by your training job, with possible values being ('loss', 'AP50') and the dictionary value is the optimization goal of the metric('minimize' or 'maximize').\n",
+        "For example:  `metric_spec = {'loss': 'minimize', 'AP50': 'maximize'}`\n",
+        "- `parameter_spec`:Dictionary representing parameters to optimize. The dictionary key is the `metric_id`, which is passed into your training job as a command line key word argument, and the dictionary value is the parameter\n",
+        "specification of the metric. Supported parameter specifications can be found in aiplatform.hyperparameter_tuning.\n",
+        "```\n",
+        "  from google.cloud.aiplatform.aiplatform import hpt as hpt\n",
+        "\n",
+        "  parameter_spec = {\n",
+        "    'learning_rate': hpt.DoubleParameterSpec(min=1e-7, max=1, scale='linear'), \\\n",
+        "  }\n",
+        "```\n",
+        "- `search_algorithm`: The search algorithm specified for the Study. Accepts one of the following:\n",
+        "  - `None`: If you do not specify an algorithm, your job uses the default\n",
+        "  Vertex AI algorithm. The default algorithm applies Bayesian optimization\n",
+        "  to arrive at the optimal solution with a more effective search over the\n",
+        "  parameter space.\n",
+        "  - `grid`: A simple grid search within the feasible space. This option is\n",
+        "  particularly useful if you want to specify a quantity of trials that is greater than the number of points in the feasible space. In such cases, if you do not specify a grid search, the Vertex AI default algorithm may generate duplicate suggestions. To use grid search, all parameter specs must be of type `IntegerParameterSpec`, `CategoricalParameterSpec`, or `DiscreteParameterSpec`.\n",
+        "  - `random`: A simple random search within the feasible space.\n",
+        "- `measurement_selection`: This indicates which measurement to use\n",
+        "if/when the service automatically selects the final measurement from\n",
+        "previously reported intermediate measurements.\n",
+        "  Accepts: `best`, `last` Choose this based on two considerations:\n",
+        "    - A): Do you expect your measurements to monotonically improve? If so,\n",
+        "    choose `last`. On the other hand, if you\\'re in a situation where\n",
+        "    your system can **over-train** and you expect the performance to get\n",
+        "    better for a while but then start declining, choose `best`.\n",
+        "    - B): Are your measurements significantly noisy and/or irreproducible? If\n",
+        "    so, `best` will tend to be over-optimistic, and it may be better\n",
+        "    to choose `last`. If both or neither of (A) and (B) apply, it\n",
+        "    doesn't matter which selection type is chosen.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "um_XKbmpTaHx"
+      },
+      "outputs": [],
+      "source": [
+        "from google.cloud.aiplatform import hyperparameter_tuning as hpt\n",
+        "\n",
+        "TRAINER_CONFIG = {\n",
+        "    \"global_batch_size\": \"8\",\n",
+        "    \"learning_rate\": \"0.001\",\n",
+        "    \"train_steps\": \"10000\",\n",
+        "    \"accelerator_count\": \"2\",\n",
+        "}\n",
+        "METRIC_SPEC_KEY = \"AP50\"\n",
+        "METRIC_SPEC_VALUE = \"maximize\"\n",
+        "SEARCH_ALGORITHM = \"random\"\n",
+        "MEASUREMENT_SELECTION = \"best\"\n",
+        "MODEL_TYPE = \"SPINENET\"  # @param {type:\"string\"} one of the values [\"SPINENET\", \"YOLO\"]\n",
+        "\n",
+        "PARAMETER_SPEC = {}\n",
+        "if MODEL_TYPE == \"YOLO\":\n",
+        "    PARAMETER_SPEC = {\n",
+        "        \"learning_rate\": hpt.DiscreteParameterSpec(\n",
+        "            values=[0.001, 0.1],\n",
+        "            scale=\"linear\",\n",
+        "        ),\n",
+        "        \"weight_decay\": hpt.DiscreteParameterSpec(\n",
+        "            values=[0.0001, 0.001],\n",
+        "            scale=\"linear\",\n",
+        "        ),\n",
+        "    }\n",
+        "else:\n",
+        "    PARAMETER_SPEC = {\n",
+        "        \"learning_rate\": hpt.DiscreteParameterSpec(\n",
+        "            values=[0.001, 0.01], scale=\"linear\"\n",
+        "        ),\n",
+        "        \"anchor_size\": hpt.DiscreteParameterSpec(values=[2, 4], scale=\"reverse_log\"),\n",
+        "    }\n",
+        "\n",
+        "job = aiplatform.AutoMLImageTrainingJob(\n",
+        "    display_name=get_job_name_with_datetime(TRAINING_JOB_PREFIX),\n",
+        "    prediction_type=\"object_detection\",\n",
+        "    model_type=MODEL_TYPE,\n",
+        "    base_model=None,\n",
+        "    trainer_config=TRAINER_CONFIG,\n",
+        "    metric_spec={METRIC_SPEC_KEY: METRIC_SPEC_VALUE},\n",
+        "    parameter_spec=PARAMETER_SPEC,\n",
+        "    search_algorithm=SEARCH_ALGORITHM,\n",
+        "    measurement_selection=MEASUREMENT_SELECTION,\n",
+        ")\n",
+        "\n",
+        "print(job)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "HwcCjwlBTQIz"
+      },
+      "source": [
+        "#### Run the training pipeline\n",
+        "\n",
+        "Next, run the DAG to start the training job by invoking the method `run`, with the following parameters:\n",
+        "\n",
+        "- `dataset`: The `Dataset` resource to train the model.\n",
+        "- `model_display_name`: The human readable name for the trained model.\n",
+        "- `training_fraction_split`: The percentage of the dataset to use for training.\n",
+        "- `test_fraction_split`: The percentage of the dataset to use for test (holdout data).\n",
+        "- `validation_fraction_split`: The percentage of the dataset to use for validation.\n",
+        "- `budget_milli_node_hours`: (optional) Maximum training time specified in unit of millihours (1000 = hour).\n",
+        "- `disable_early_stopping`: If `True`, training may be completed before using the entire budget if the service believes it cannot further improve on the model objective measurements.\n",
+        "\n",
+        "The `run` method when completed returns the `Model` resource.\n",
+        "\n",
+        "The execution of the training pipeline will take up to 60 minutes."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "aec22792ee84"
+      },
+      "outputs": [],
+      "source": [
+        "model = job.run(\n",
+        "    dataset=dataset,\n",
+        "    model_display_name=get_job_name_with_datetime(\"salads\"),\n",
+        "    training_fraction_split=0.8,\n",
+        "    validation_fraction_split=0.1,\n",
+        "    test_fraction_split=0.1,\n",
+        "    budget_milli_node_hours=20000,\n",
+        "    disable_early_stopping=False,\n",
+        ")\n",
+        "\n",
+        "print(\"Model is: \", model)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "g0BGaofgsMsy"
+      },
+      "source": [
+        "## Test trained models\n",
+        "This section shows how to test the trained models.\n",
+        "1. Deploy models from Model Registry\n",
+        "2. Run online predictions"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "mj723H4NXu4v"
+      },
+      "outputs": [],
+      "source": [
+        "# @title Deploy model from Model Registry\n",
+        "# Model does not support dedicated deployment resources.\n",
+        "# An n1-standard-4 machine with 1 P100 GPU will be used.\n",
+        "\n",
+        "deploy_model_name = get_job_name_with_datetime(DEPLOY_JOB_PREFIX + \"_\" + OBJECTIVE)\n",
+        "print(\"The deployed job name is: \", deploy_model_name)\n",
+        "\n",
+        "endpoint = model.deploy(\n",
+        "    deployed_model_display_name=deploy_model_name,\n",
+        "    traffic_split={\"0\": 100},\n",
+        "    min_replica_count=1,\n",
+        "    max_replica_count=1,\n",
+        ")\n",
+        "\n",
+        "endpoint_id = endpoint.name\n",
+        "print(\"endpoint id is: \", endpoint_id)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "NTYvgFv6XyEe"
+      },
+      "outputs": [],
+      "source": [
+        "# @title Run online predictions\n",
+        "\n",
+        "# test image file path from a GCS bucket\n",
+        "test_filepath = \"\"  # @param {type:\"string\"}\n",
+        "\n",
+        "with tf.io.gfile.GFile(test_filepath, \"rb\") as f:\n",
+        "    content = f.read()\n",
+        "\n",
+        "# The format of each instance should conform to the deployed model's prediction input schema.\n",
+        "instances = [{\"content\": base64.b64encode(content).decode(\"utf-8\")}]\n",
+        "\n",
+        "prediction = endpoint.predict(instances=instances)\n",
+        "\n",
+        "img = load_img(test_filepath)\n",
+        "display_image(img)\n",
+        "print(prediction)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "frcGP5HFX1XN"
+      },
+      "source": [
+        "## Clean up"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "H2m8-u0IX4dX"
+      },
+      "outputs": [],
+      "source": [
+        "# Undeploy model and delete endpoint.\n",
+        "endpoint.undeploy_all()\n",
+        "endpoint.delete(force=True)\n",
+        "\n",
+        "# Delete models.\n",
+        "model.delete()"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "model_garden_proprietary_image_object_detection.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/community/model_garden/model_garden_pytorch_bart_large_cnn.ipynb

@@ -0,0 +1,371 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7d9bbf86da5e"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2023 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2bd716bf3e39"
+      },
+      "source": [
+        "# Vertex AI Model Garden - Bart Large CNN\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_bart_large_cnn.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_bart_large_cnn.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/notebooks/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/model_garden/model_garden_pytorch_bart_large_cnn.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "    (a Python-3 CPU notebook is recommended)\n",
+        "  </td>\n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d8cd12648da4"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "This notebook demonstrates deploying the pre-trained [Bart Large CNN](https://huggingface.co/facebook/bart-large-cnn) model on Vertex AI for online prediction.\n",
+        "\n",
+        "### Objective\n",
+        "\n",
+        "- Upload the model to [Model Registry](https://cloud.google.com/vertex-ai/docs/model-registry/introduction).\n",
+        "- Deploy the model on [Endpoint](https://cloud.google.com/vertex-ai/docs/predictions/using-private-endpoints).\n",
+        "- Run online predictions for image captioning.\n",
+        "\n",
+        "### Costs\n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage pricing](https://cloud.google.com/storage/pricing), and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "264c07757582"
+      },
+      "source": [
+        "## Setup environment\n",
+        "\n",
+        "**NOTE**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d73ffa0c0b83"
+      },
+      "source": [
+        "### Colab only"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2707b02ef5df"
+      },
+      "outputs": [],
+      "source": [
+        "!pip3 install --upgrade google-cloud-aiplatform"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b60a4d7100bf"
+      },
+      "outputs": [],
+      "source": [
+        "from google.colab import auth as google_auth\n",
+        "\n",
+        "google_auth.authenticate_user()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0f826ff482a2"
+      },
+      "source": [
+        "### Setup Google Cloud project\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI API and Compute Engine API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component).\n",
+        "\n",
+        "1. [Create a Cloud Storage bucket](https://cloud.google.com/storage/docs/creating-buckets) for storing experiment outputs."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8958ebc71868"
+      },
+      "source": [
+        "Fill following variables for experiments environment:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9db30f827a65"
+      },
+      "outputs": [],
+      "source": [
+        "# Cloud project id.\n",
+        "PROJECT_ID = \"\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# The region you want to launch jobs in.\n",
+        "REGION = \"us-central1\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# The Cloud Storage bucket for storing experiments output. Fill it without the 'gs://' prefix.\n",
+        "GCS_BUCKET = \"\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "92f16e22c20b"
+      },
+      "source": [
+        "Initialize Vertex AI API:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "1680c257acfb"
+      },
+      "outputs": [],
+      "source": [
+        "from google.cloud import aiplatform\n",
+        "\n",
+        "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=GCS_BUCKET)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6ca48b699d17"
+      },
+      "source": [
+        "### Define constants"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "de9882ea89ea"
+      },
+      "outputs": [],
+      "source": [
+        "# The pre-built serving docker image.\n",
+        "# The model artifacts are embedded within the container, except for model weights which will be downloaded during deployment.\n",
+        "SERVE_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-transformers-serve\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "10188266a5cd"
+      },
+      "source": [
+        "### Define common functions"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cac4478ae098"
+      },
+      "outputs": [],
+      "source": [
+        "def deploy_model(model_id, task):\n",
+        "    \"\"\"Uploads and deploys the model to Vertex AI endpoint for prediction.\"\"\"\n",
+        "    model_name = \"bart_large_cnn\"\n",
+        "    endpoint = aiplatform.Endpoint.create(display_name=f\"{model_name}-endpoint\")\n",
+        "    serving_env = {\n",
+        "        \"MODEL_ID\": model_id,\n",
+        "        \"TASK\": task,\n",
+        "    }\n",
+        "    # If the model_id is a GCS path, use artifact_uri to pass it to serving docker.\n",
+        "    artifact_uri = model_id if model_id.startswith(\"gs://\") else None\n",
+        "    model = aiplatform.Model.upload(\n",
+        "        display_name=model_name,\n",
+        "        serving_container_image_uri=SERVE_DOCKER_URI,\n",
+        "        serving_container_ports=[7080],\n",
+        "        serving_container_predict_route=\"/predictions/transformers_serving\",\n",
+        "        serving_container_health_route=\"/ping\",\n",
+        "        serving_container_environment_variables=serving_env,\n",
+        "        artifact_uri=artifact_uri,\n",
+        "    )\n",
+        "    model.deploy(\n",
+        "        endpoint=endpoint,\n",
+        "        machine_type=\"n1-standard-8\",\n",
+        "        accelerator_type=\"NVIDIA_TESLA_T4\",\n",
+        "        accelerator_count=1,\n",
+        "        deploy_request_timeout=1800,\n",
+        "    )\n",
+        "    return model, endpoint"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d2d72ecdb8c9"
+      },
+      "source": [
+        "## Upload and deploy models"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9448c5f545fa"
+      },
+      "source": [
+        "This section uploads the pre-trained model to Model Registry and deploys it on the Endpoint with 1 T4 GPU.\n",
+        "\n",
+        "The model deployment step will take ~15 minutes to complete."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b4b46c28d8b1"
+      },
+      "outputs": [],
+      "source": [
+        "model, endpoint = deploy_model(model_id=\"facebook/bart-large-cnn\", task=\"summarization\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "nTs3JBT3W_lJ"
+      },
+      "source": [
+        "## Send prediction request"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6be655247cb1"
+      },
+      "outputs": [],
+      "source": [
+        "ARTICLE = \"\"\" New York (CNN)When Liana Barrientos was 23 years old, she got married in Westchester County, New York.\n",
+        "A year later, she got married again in Westchester County, but to a different man and without divorcing her first husband.\n",
+        "Only 18 days after that marriage, she got hitched yet again. Then, Barrientos declared \"I do\" five more times, sometimes only within two weeks of each other.\n",
+        "In 2010, she married once more, this time in the Bronx. In an application for a marriage license, she stated it was her \"first and only\" marriage.\n",
+        "Barrientos, now 39, is facing two criminal counts of \"offering a false instrument for filing in the first degree,\" referring to her false statements on the\n",
+        "2010 marriage license application, according to court documents.\n",
+        "Prosecutors said the marriages were part of an immigration scam.\n",
+        "On Friday, she pleaded not guilty at State Supreme Court in the Bronx, according to her attorney, Christopher Wright, who declined to comment further.\n",
+        "After leaving court, Barrientos was arrested and charged with theft of service and criminal trespass for allegedly sneaking into the New York subway through an emergency exit, said Detective\n",
+        "Annette Markowski, a police spokeswoman. In total, Barrientos has been married 10 times, with nine of her marriages occurring between 1999 and 2002.\n",
+        "All occurred either in Westchester County, Long Island, New Jersey or the Bronx. She is believed to still be married to four men, and at one time, she was married to eight men at once, prosecutors say.\n",
+        "Prosecutors said the immigration scam involved some of her husbands, who filed for permanent residence status shortly after the marriages.\n",
+        "Any divorces happened only after such filings were approved. It was unclear whether any of the men will be prosecuted.\n",
+        "The case was referred to the Bronx District Attorney\\'s Office by Immigration and Customs Enforcement and the Department of Homeland Security\\'s\n",
+        "Investigation Division. Seven of the men are from so-called \"red-flagged\" countries, including Egypt, Turkey, Georgia, Pakistan and Mali.\n",
+        "Her eighth husband, Rashid Rajput, was deported in 2006 to his native Pakistan after an investigation by the Joint Terrorism Task Force.\n",
+        "If convicted, Barrientos faces up to four years in prison.  Her next court appearance is scheduled for May 18.\n",
+        "\"\"\"\n",
+        "\n",
+        "instances = [\n",
+        "    {\"text\": ARTICLE},\n",
+        "]\n",
+        "preds = endpoint.predict(instances=instances).predictions\n",
+        "print(preds)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "xKj4wBQNWuHT"
+      },
+      "source": [
+        "## Clean up"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2ccf3714dbe9"
+      },
+      "outputs": [],
+      "source": [
+        "# Undeploy model and delete endpoint.\n",
+        "endpoint.delete(force=True)\n",
+        "\n",
+        "# Delete models.\n",
+        "model.delete()"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "model_garden_pytorch_bart_large_cnn.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/community/model_garden/model_garden_pytorch_blip2.ipynb

@@ -0,0 +1,465 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7d9bbf86da5e"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2023 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2bd716bf3e39"
+      },
+      "source": [
+        "# Vertex AI Model Garden - BLIP2\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_blip2.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_blip2.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/notebooks/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/model_garden/model_garden_pytorch_blip2.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "    (a Python-3 CPU notebook is recommended)\n",
+        "  </td>\n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d8cd12648da4"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "This notebook demonstrates deploying the pre-trained [BLIP2](https://huggingface.co/Salesforce/blip2-opt-2.7b) model on Vertex AI for online prediction.\n",
+        "\n",
+        "### Objective\n",
+        "\n",
+        "- Upload the model to [Model Registry](https://cloud.google.com/vertex-ai/docs/model-registry/introduction).\n",
+        "- Deploy the model on [Endpoint](https://cloud.google.com/vertex-ai/docs/predictions/using-private-endpoints).\n",
+        "- Run online predictions for image captioning.\n",
+        "\n",
+        "### Costs\n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage pricing](https://cloud.google.com/storage/pricing), and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "264c07757582"
+      },
+      "source": [
+        "## Setup environment\n",
+        "\n",
+        "**NOTE**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d73ffa0c0b83"
+      },
+      "source": [
+        "### Colab only"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2707b02ef5df"
+      },
+      "outputs": [],
+      "source": [
+        "if \"google.colab\" in str(get_ipython()):\n",
+        "    ! pip3 install --upgrade google-cloud-aiplatform\n",
+        "    from google.colab import auth as google_auth\n",
+        "\n",
+        "    google_auth.authenticate_user()\n",
+        "\n",
+        "    # Restart the notebook kernel after installs.\n",
+        "    import IPython\n",
+        "\n",
+        "    app = IPython.Application.instance()\n",
+        "    app.kernel.do_shutdown(True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0f826ff482a2"
+      },
+      "source": [
+        "### Setup Google Cloud project\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI API and Compute Engine API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component).\n",
+        "\n",
+        "1. [Create a Cloud Storage bucket](https://cloud.google.com/storage/docs/creating-buckets) for storing experiment outputs."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8958ebc71868"
+      },
+      "source": [
+        "Fill following variables for experiments environment:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9db30f827a65"
+      },
+      "outputs": [],
+      "source": [
+        "# Cloud project id.\n",
+        "PROJECT_ID = \"\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# The region you want to launch jobs in.\n",
+        "REGION = \"us-central1\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# The Cloud Storage bucket for storing experiments output. Fill it without the 'gs://' prefix.\n",
+        "GCS_BUCKET = \"\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "92f16e22c20b"
+      },
+      "source": [
+        "Initialize Vertex AI API:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "1680c257acfb"
+      },
+      "outputs": [],
+      "source": [
+        "from google.cloud import aiplatform\n",
+        "\n",
+        "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=GCS_BUCKET)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6ca48b699d17"
+      },
+      "source": [
+        "### Define constants"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "de9882ea89ea"
+      },
+      "outputs": [],
+      "source": [
+        "# The pre-built serving docker image.\n",
+        "# The model artifacts are embedded within the container, except for model weights which will be downloaded during deployment.\n",
+        "SERVE_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-transformers-serve\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "10188266a5cd"
+      },
+      "source": [
+        "### Define common functions"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cac4478ae098"
+      },
+      "outputs": [],
+      "source": [
+        "import base64\n",
+        "import os\n",
+        "from datetime import datetime\n",
+        "from io import BytesIO\n",
+        "\n",
+        "import requests\n",
+        "from google.cloud import aiplatform\n",
+        "from PIL import Image\n",
+        "\n",
+        "\n",
+        "def create_job_name(prefix):\n",
+        "    user = os.environ.get(\"USER\")\n",
+        "    now = datetime.now().strftime(\"%Y%m%d_%H%M%S\")\n",
+        "    job_name = f\"{prefix}-{user}-{now}\"\n",
+        "    return job_name\n",
+        "\n",
+        "\n",
+        "def download_image(url):\n",
+        "    response = requests.get(url)\n",
+        "    return Image.open(BytesIO(response.content))\n",
+        "\n",
+        "\n",
+        "def image_to_base64(image, format=\"JPEG\"):\n",
+        "    buffer = BytesIO()\n",
+        "    image.save(buffer, format=format)\n",
+        "    image_str = base64.b64encode(buffer.getvalue()).decode(\"utf-8\")\n",
+        "    return image_str\n",
+        "\n",
+        "\n",
+        "def base64_to_image(image_str):\n",
+        "    image = Image.open(BytesIO(base64.b64decode(image_str)))\n",
+        "    return image\n",
+        "\n",
+        "\n",
+        "def image_grid(imgs, rows=2, cols=2):\n",
+        "    w, h = imgs[0].size\n",
+        "    grid = Image.new(\"RGB\", size=(cols * w, rows * h))\n",
+        "    for i, img in enumerate(imgs):\n",
+        "        grid.paste(img, box=(i % cols * w, i // cols * h))\n",
+        "    return grid\n",
+        "\n",
+        "\n",
+        "def deploy_model(model_id, task):\n",
+        "    model_name = \"blip2\"\n",
+        "    endpoint = aiplatform.Endpoint.create(display_name=f\"{model_name}-endpoint\")\n",
+        "    serving_env = {\n",
+        "        \"MODEL_ID\": model_id,\n",
+        "        \"TASK\": task,\n",
+        "    }\n",
+        "    # If the model_id is a GCS path, use artifact_uri to pass it to serving docker.\n",
+        "    artifact_uri = model_id if model_id.startswith(\"gs://\") else None\n",
+        "    model = aiplatform.Model.upload(\n",
+        "        display_name=model_name,\n",
+        "        serving_container_image_uri=SERVE_DOCKER_URI,\n",
+        "        serving_container_ports=[7080],\n",
+        "        serving_container_predict_route=\"/predictions/transformers_serving\",\n",
+        "        serving_container_health_route=\"/ping\",\n",
+        "        serving_container_environment_variables=serving_env,\n",
+        "        artifact_uri=artifact_uri,\n",
+        "    )\n",
+        "    model.deploy(\n",
+        "        endpoint=endpoint,\n",
+        "        machine_type=\"n1-standard-8\",\n",
+        "        accelerator_type=\"NVIDIA_TESLA_T4\",\n",
+        "        accelerator_count=1,\n",
+        "        deploy_request_timeout=1800,\n",
+        "    )\n",
+        "    return model, endpoint"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d2d72ecdb8c9"
+      },
+      "source": [
+        "## Upload and deploy models"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9448c5f545fa"
+      },
+      "source": [
+        "This section uploads the pre-trained model to Model Registry and deploys it on the Endpoint with 1 T4 GPU.\n",
+        "\n",
+        "The model deployment step will take ~15 minutes to complete.\n",
+        "\n",
+        "Once deployed, you can send images to get descriptions."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "19e8aeec969c"
+      },
+      "source": [
+        "### Image captioning"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b4b46c28d8b1"
+      },
+      "outputs": [],
+      "source": [
+        "model, endpoint = deploy_model(\n",
+        "    model_id=\"Salesforce/blip2-opt-2.7b\", task=\"image-to-text\"\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "80b3fd2ace09"
+      },
+      "source": [
+        "NOTE: The model weights will be downloaded after the deployment succeeds. Thus additional 10 minutes of waiting time is needed **after** the above model deployment step succeeds and before you run the next step below. Otherwise you might see a `ServiceUnavailable: 503 502:Bad Gateway` error when you send requests to the endpoint."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6be655247cb1"
+      },
+      "outputs": [],
+      "source": [
+        "image = download_image(\"http://images.cocodataset.org/val2017/000000039769.jpg\")\n",
+        "display(image)\n",
+        "\n",
+        "instances = [\n",
+        "    {\"image\": image_to_base64(image)},\n",
+        "]\n",
+        "preds = endpoint.predict(instances=instances).predictions\n",
+        "print(preds)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2ccf3714dbe9"
+      },
+      "outputs": [],
+      "source": [
+        "# Undeploy model and delete endpoint.\n",
+        "endpoint.delete(force=True)\n",
+        "\n",
+        "# Delete models.\n",
+        "model.delete()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "01e11d85d504"
+      },
+      "source": [
+        "### VQA (Visual-Question-Answering)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "26018d961cf9"
+      },
+      "outputs": [],
+      "source": [
+        "model, endpoint = deploy_model(\n",
+        "    model_id=\"Salesforce/blip2-opt-2.7b\", task=\"visual-question-answering\"\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "80b3fd2ace09"
+      },
+      "source": [
+        "NOTE: The model weights will be downloaded after the deployment succeeds. Thus additional 10 minutes of waiting time is needed **after** the above model deployment step succeeds and before you run the next step below. Otherwise you might see a `ServiceUnavailable: 503 502:Bad Gateway` error when you send requests to the endpoint."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "f19c342829fd"
+      },
+      "outputs": [],
+      "source": [
+        "image = download_image(\n",
+        "    \"https://media.newyorker.com/cartoons/63dc6847be24a6a76d90eb99/master/w_1160,c_limit/230213_a26611_838.jpg\"\n",
+        ")\n",
+        "display(image)\n",
+        "\n",
+        "question = \"Question: What are they doing? Answer:\"\n",
+        "instances = [\n",
+        "    {\"image\": image_to_base64(image), \"text\": question},\n",
+        "]\n",
+        "preds = endpoint.predict(instances=instances).predictions\n",
+        "print(question)\n",
+        "print(preds)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "712eb9d0b336"
+      },
+      "outputs": [],
+      "source": [
+        "# Undeploy model and delete endpoint.\n",
+        "endpoint.delete(force=True)\n",
+        "\n",
+        "# Delete models.\n",
+        "model.delete()"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "model_garden_pytorch_blip2.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/community/model_garden/model_garden_pytorch_blip_image_captioning.ipynb

@@ -0,0 +1,396 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7d9bbf86da5e"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2023 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2bd716bf3e39"
+      },
+      "source": [
+        "# Vertex AI Model Garden - BLIP Image Captioning\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_blip_image_captioning.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_blip_image_captioning.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/notebooks/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/model_garden/model_garden_pytorch_blip_image_captioning.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "    (a Python-3 CPU notebook is recommended)\n",
+        "  </td>\n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d8cd12648da4"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "This notebook demonstrates deploying the pre-trained [BLIP Image Captioning](https://huggingface.co/Salesforce/blip-image-captioning-base) model on Vertex AI for online prediction.\n",
+        "\n",
+        "### Objective\n",
+        "\n",
+        "- Upload the model to [Model Registry](https://cloud.google.com/vertex-ai/docs/model-registry/introduction).\n",
+        "- Deploy the model on [Endpoint](https://cloud.google.com/vertex-ai/docs/predictions/using-private-endpoints).\n",
+        "- Run online predictions for image captioning.\n",
+        "\n",
+        "### Costs\n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage pricing](https://cloud.google.com/storage/pricing), and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "264c07757582"
+      },
+      "source": [
+        "## Setup environment\n",
+        "\n",
+        "**NOTE**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d73ffa0c0b83"
+      },
+      "source": [
+        "### Colab only"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2707b02ef5df"
+      },
+      "outputs": [],
+      "source": [
+        "if \"google.colab\" in str(get_ipython()):\n",
+        "    ! pip3 install --upgrade google-cloud-aiplatform\n",
+        "    from google.colab import auth as google_auth\n",
+        "\n",
+        "    google_auth.authenticate_user()\n",
+        "\n",
+        "    # Restart the notebook kernel after installs.\n",
+        "    import IPython\n",
+        "\n",
+        "    app = IPython.Application.instance()\n",
+        "    app.kernel.do_shutdown(True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0f826ff482a2"
+      },
+      "source": [
+        "### Setup Google Cloud project\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI API and Compute Engine API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component).\n",
+        "\n",
+        "1. [Create a Cloud Storage bucket](https://cloud.google.com/storage/docs/creating-buckets) for storing experiment outputs."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8958ebc71868"
+      },
+      "source": [
+        "Fill following variables for experiments environment:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9db30f827a65"
+      },
+      "outputs": [],
+      "source": [
+        "# Cloud project id.\n",
+        "PROJECT_ID = \"\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# The region you want to launch jobs in.\n",
+        "REGION = \"us-central1\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# The Cloud Storage bucket for storing experiments output. Fill it without the 'gs://' prefix.\n",
+        "GCS_BUCKET = \"\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "92f16e22c20b"
+      },
+      "source": [
+        "Initialize Vertex AI API:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "1680c257acfb"
+      },
+      "outputs": [],
+      "source": [
+        "from google.cloud import aiplatform\n",
+        "\n",
+        "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=GCS_BUCKET)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6ca48b699d17"
+      },
+      "source": [
+        "### Define constants"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "de9882ea89ea"
+      },
+      "outputs": [],
+      "source": [
+        "# The pre-built serving docker image. It contains serving scripts and models.\n",
+        "SERVE_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-transformers-serve\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "10188266a5cd"
+      },
+      "source": [
+        "### Define common functions"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cac4478ae098"
+      },
+      "outputs": [],
+      "source": [
+        "import base64\n",
+        "import os\n",
+        "from datetime import datetime\n",
+        "from io import BytesIO\n",
+        "\n",
+        "import requests\n",
+        "from google.cloud import aiplatform\n",
+        "from PIL import Image\n",
+        "\n",
+        "\n",
+        "def create_job_name(prefix):\n",
+        "    user = os.environ.get(\"USER\")\n",
+        "    now = datetime.now().strftime(\"%Y%m%d_%H%M%S\")\n",
+        "    job_name = f\"{prefix}-{user}-{now}\"\n",
+        "    return job_name\n",
+        "\n",
+        "\n",
+        "def download_image(url):\n",
+        "    response = requests.get(url)\n",
+        "    return Image.open(BytesIO(response.content))\n",
+        "\n",
+        "\n",
+        "def image_to_base64(image, format=\"JPEG\"):\n",
+        "    buffer = BytesIO()\n",
+        "    image.save(buffer, format=format)\n",
+        "    image_str = base64.b64encode(buffer.getvalue()).decode(\"utf-8\")\n",
+        "    return image_str\n",
+        "\n",
+        "\n",
+        "def base64_to_image(image_str):\n",
+        "    image = Image.open(BytesIO(base64.b64decode(image_str)))\n",
+        "    return image\n",
+        "\n",
+        "\n",
+        "def image_grid(imgs, rows=2, cols=2):\n",
+        "    w, h = imgs[0].size\n",
+        "    grid = Image.new(\"RGB\", size=(cols * w, rows * h))\n",
+        "    for i, img in enumerate(imgs):\n",
+        "        grid.paste(img, box=(i % cols * w, i // cols * h))\n",
+        "    return grid\n",
+        "\n",
+        "\n",
+        "def deploy_model(model_id, task):\n",
+        "    model_name = \"blip-image-captioning\"\n",
+        "    endpoint = aiplatform.Endpoint.create(display_name=f\"{model_name}-endpoint\")\n",
+        "    serving_env = {\n",
+        "        \"MODEL_ID\": model_id,\n",
+        "        \"TASK\": task,\n",
+        "    }\n",
+        "    # If the model_id is a GCS path, use artifact_uri to pass it to serving docker.\n",
+        "    artifact_uri = model_id if model_id.startswith(\"gs://\") else None\n",
+        "    model = aiplatform.Model.upload(\n",
+        "        display_name=model_name,\n",
+        "        serving_container_image_uri=SERVE_DOCKER_URI,\n",
+        "        serving_container_ports=[7080],\n",
+        "        serving_container_predict_route=\"/predictions/transformers_serving\",\n",
+        "        serving_container_health_route=\"/ping\",\n",
+        "        serving_container_environment_variables=serving_env,\n",
+        "        artifact_uri=artifact_uri,\n",
+        "    )\n",
+        "    model.deploy(\n",
+        "        endpoint=endpoint,\n",
+        "        machine_type=\"n1-standard-8\",\n",
+        "        accelerator_type=\"NVIDIA_TESLA_T4\",\n",
+        "        accelerator_count=1,\n",
+        "        deploy_request_timeout=1800,\n",
+        "    )\n",
+        "    return model, endpoint"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d2d72ecdb8c9"
+      },
+      "source": [
+        "## Upload and deploy models"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9448c5f545fa"
+      },
+      "source": [
+        "This section uploads the pre-trained model to Model Registry and deploys it on the Endpoint with 1 T4 GPU.\n",
+        "\n",
+        "The model deployment step will take ~15 minutes to complete.\n",
+        "\n",
+        "Once deployed, you can send images to get descriptions."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b4b46c28d8b1"
+      },
+      "outputs": [],
+      "source": [
+        "model, endpoint = deploy_model(\n",
+        "    model_id=\"Salesforce/blip-image-captioning-base\", task=\"image-to-text\"\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "80b3fd2ace09"
+      },
+      "source": [
+        "NOTE: The model weights will be downloaded after the deployment succeeds. Thus additional 5 minutes of waiting time is needed **after** the above model deployment step succeeds and before you run the next step below. Otherwise you might see a `ServiceUnavailable: 503 502:Bad Gateway` error when you send requests to the endpoint."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6be655247cb1"
+      },
+      "outputs": [],
+      "source": [
+        "image = download_image(\"http://images.cocodataset.org/val2017/000000039769.jpg\")\n",
+        "display(image)\n",
+        "\n",
+        "instances = [\n",
+        "    {\"image\": image_to_base64(image)},\n",
+        "]\n",
+        "preds = endpoint.predict(instances=instances).predictions\n",
+        "print(preds)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "db7ffebdb4be"
+      },
+      "source": [
+        "### Clean up resources"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2ccf3714dbe9"
+      },
+      "outputs": [],
+      "source": [
+        "# Undeploy model and delete endpoint.\n",
+        "endpoint.delete(force=True)\n",
+        "\n",
+        "# Delete models.\n",
+        "model.delete()"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "model_garden_pytorch_blip_image_captioning.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/community/model_garden/model_garden_pytorch_blip_vqa.ipynb

@@ -0,0 +1,398 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7d9bbf86da5e"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2023 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2bd716bf3e39"
+      },
+      "source": [
+        "# Vertex AI Model Garden - BLIP VQA\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_blip_vqa.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_blip_vqa.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/notebooks/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/model_garden/model_garden_pytorch_blip_vqa.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "    (a Python-3 CPU notebook is recommended)\n",
+        "  </td>\n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d8cd12648da4"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "This notebook demonstrates deploying the pre-trained [BLIP VQA](https://huggingface.co/Salesforce/blip-vqa-base) model on Vertex AI for online prediction.\n",
+        "\n",
+        "### Objective\n",
+        "\n",
+        "- Upload the model to [Model Registry](https://cloud.google.com/vertex-ai/docs/model-registry/introduction).\n",
+        "- Deploy the model on [Endpoint](https://cloud.google.com/vertex-ai/docs/predictions/using-private-endpoints).\n",
+        "- Run online predictions for image captioning.\n",
+        "\n",
+        "### Costs\n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage pricing](https://cloud.google.com/storage/pricing), and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "264c07757582"
+      },
+      "source": [
+        "## Setup environment\n",
+        "\n",
+        "**NOTE**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d73ffa0c0b83"
+      },
+      "source": [
+        "### Colab only"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2707b02ef5df"
+      },
+      "outputs": [],
+      "source": [
+        "if \"google.colab\" in str(get_ipython()):\n",
+        "    ! pip3 install --upgrade google-cloud-aiplatform\n",
+        "    from google.colab import auth as google_auth\n",
+        "\n",
+        "    google_auth.authenticate_user()\n",
+        "\n",
+        "    # Restart the notebook kernel after installs.\n",
+        "    import IPython\n",
+        "\n",
+        "    app = IPython.Application.instance()\n",
+        "    app.kernel.do_shutdown(True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0f826ff482a2"
+      },
+      "source": [
+        "### Setup Google Cloud project\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI API and Compute Engine API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component).\n",
+        "\n",
+        "1. [Create a Cloud Storage bucket](https://cloud.google.com/storage/docs/creating-buckets) for storing experiment outputs."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8958ebc71868"
+      },
+      "source": [
+        "Fill following variables for experiments environment:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9db30f827a65"
+      },
+      "outputs": [],
+      "source": [
+        "# Cloud project id.\n",
+        "PROJECT_ID = \"\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# The region you want to launch jobs in.\n",
+        "REGION = \"us-central1\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# The Cloud Storage bucket for storing experiments output. Fill it without the 'gs://' prefix.\n",
+        "GCS_BUCKET = \"\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "92f16e22c20b"
+      },
+      "source": [
+        "Initialize Vertex AI API:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "1680c257acfb"
+      },
+      "outputs": [],
+      "source": [
+        "from google.cloud import aiplatform\n",
+        "\n",
+        "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=GCS_BUCKET)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6ca48b699d17"
+      },
+      "source": [
+        "### Define constants"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "de9882ea89ea"
+      },
+      "outputs": [],
+      "source": [
+        "# The pre-built serving docker image. It contains serving scripts and models.\n",
+        "SERVE_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-transformers-serve\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "10188266a5cd"
+      },
+      "source": [
+        "### Define common functions"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cac4478ae098"
+      },
+      "outputs": [],
+      "source": [
+        "import base64\n",
+        "import os\n",
+        "from datetime import datetime\n",
+        "from io import BytesIO\n",
+        "\n",
+        "import requests\n",
+        "from google.cloud import aiplatform\n",
+        "from PIL import Image\n",
+        "\n",
+        "\n",
+        "def create_job_name(prefix):\n",
+        "    user = os.environ.get(\"USER\")\n",
+        "    now = datetime.now().strftime(\"%Y%m%d_%H%M%S\")\n",
+        "    job_name = f\"{prefix}-{user}-{now}\"\n",
+        "    return job_name\n",
+        "\n",
+        "\n",
+        "def download_image(url):\n",
+        "    response = requests.get(url)\n",
+        "    return Image.open(BytesIO(response.content))\n",
+        "\n",
+        "\n",
+        "def image_to_base64(image, format=\"JPEG\"):\n",
+        "    buffer = BytesIO()\n",
+        "    image.save(buffer, format=format)\n",
+        "    image_str = base64.b64encode(buffer.getvalue()).decode(\"utf-8\")\n",
+        "    return image_str\n",
+        "\n",
+        "\n",
+        "def base64_to_image(image_str):\n",
+        "    image = Image.open(BytesIO(base64.b64decode(image_str)))\n",
+        "    return image\n",
+        "\n",
+        "\n",
+        "def image_grid(imgs, rows=2, cols=2):\n",
+        "    w, h = imgs[0].size\n",
+        "    grid = Image.new(\"RGB\", size=(cols * w, rows * h))\n",
+        "    for i, img in enumerate(imgs):\n",
+        "        grid.paste(img, box=(i % cols * w, i // cols * h))\n",
+        "    return grid\n",
+        "\n",
+        "\n",
+        "def deploy_model(model_id, task):\n",
+        "    model_name = \"blip-vqa\"\n",
+        "    endpoint = aiplatform.Endpoint.create(display_name=f\"{model_name}-endpoint\")\n",
+        "    serving_env = {\n",
+        "        \"MODEL_ID\": model_id,\n",
+        "        \"TASK\": task,\n",
+        "    }\n",
+        "    # If the model_id is a GCS path, use artifact_uri to pass it to serving docker.\n",
+        "    artifact_uri = model_id if model_id.startswith(\"gs://\") else None\n",
+        "    model = aiplatform.Model.upload(\n",
+        "        display_name=model_name,\n",
+        "        serving_container_image_uri=SERVE_DOCKER_URI,\n",
+        "        serving_container_ports=[7080],\n",
+        "        serving_container_predict_route=\"/predictions/transformers_serving\",\n",
+        "        serving_container_health_route=\"/ping\",\n",
+        "        serving_container_environment_variables=serving_env,\n",
+        "        artifact_uri=artifact_uri,\n",
+        "    )\n",
+        "    model.deploy(\n",
+        "        endpoint=endpoint,\n",
+        "        machine_type=\"n1-standard-8\",\n",
+        "        accelerator_type=\"NVIDIA_TESLA_T4\",\n",
+        "        accelerator_count=1,\n",
+        "        deploy_request_timeout=1800,\n",
+        "    )\n",
+        "    return model, endpoint"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d2d72ecdb8c9"
+      },
+      "source": [
+        "## Upload and deploy models"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9448c5f545fa"
+      },
+      "source": [
+        "This section uploads the pre-trained model to Model Registry and deploys it on the Endpoint with 1 T4 GPU.\n",
+        "\n",
+        "The model deployment step will take ~15 minutes to complete.\n",
+        "\n",
+        "Once deployed, you can send images and questions to get answers."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b4b46c28d8b1"
+      },
+      "outputs": [],
+      "source": [
+        "model, endpoint = deploy_model(\n",
+        "    model_id=\"Salesforce/blip-vqa-base\", task=\"visual-question-answering\"\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "80b3fd2ace09"
+      },
+      "source": [
+        "NOTE: The model weights will be downloaded after the deployment succeeds. Thus additional 5 minutes of waiting time is needed **after** the above model deployment step succeeds and before you run the next step below. Otherwise you might see a `ServiceUnavailable: 503 502:Bad Gateway` error when you send requests to the endpoint."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6be655247cb1"
+      },
+      "outputs": [],
+      "source": [
+        "image = download_image(\"http://images.cocodataset.org/val2017/000000039769.jpg\")\n",
+        "display(image)\n",
+        "\n",
+        "question = \"Which cat is bigger?\"\n",
+        "instances = [\n",
+        "    {\"image\": image_to_base64(image), \"text\": question},\n",
+        "]\n",
+        "preds = endpoint.predict(instances=instances).predictions\n",
+        "print(question)\n",
+        "print(preds)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "db7ffebdb4be"
+      },
+      "source": [
+        "### Clean up resources"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2ccf3714dbe9"
+      },
+      "outputs": [],
+      "source": [
+        "# Undeploy model and delete endpoint.\n",
+        "endpoint.delete(force=True)\n",
+        "\n",
+        "# Delete models.\n",
+        "model.delete()"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "model_garden_pytorch_blip_vqa.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/community/model_garden/model_garden_pytorch_clip.ipynb

@@ -0,0 +1,473 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7d9bbf86da5e"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2023 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2bd716bf3e39"
+      },
+      "source": [
+        "# Vertex AI Model Garden - CLIP\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_clip.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_clip.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/notebooks/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/model_garden/model_garden_pytorch_clip.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "    (a Python-3 CPU notebook is recommended)\n",
+        "  </td>\n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d8cd12648da4"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "This notebook demonstrates deploying the pre-trained [CLIP](https://huggingface.co/openai/clip-vit-base-patch32) model on Vertex AI for online prediction.\n",
+        "\n",
+        "### Objective\n",
+        "\n",
+        "- Upload the model to [Model Registry](https://cloud.google.com/vertex-ai/docs/model-registry/introduction).\n",
+        "- Deploy the model on [Endpoint](https://cloud.google.com/vertex-ai/docs/predictions/using-private-endpoints).\n",
+        "- Run online predictions for image captioning.\n",
+        "\n",
+        "### Costs\n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage pricing](https://cloud.google.com/storage/pricing), and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "264c07757582"
+      },
+      "source": [
+        "## Setup environment\n",
+        "\n",
+        "**NOTE**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d73ffa0c0b83"
+      },
+      "source": [
+        "### Colab only"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2707b02ef5df"
+      },
+      "outputs": [],
+      "source": [
+        "if \"google.colab\" in str(get_ipython()):\n",
+        "    ! pip3 install --upgrade google-cloud-aiplatform\n",
+        "    from google.colab import auth as google_auth\n",
+        "\n",
+        "    google_auth.authenticate_user()\n",
+        "\n",
+        "    # Restart the notebook kernel after installs.\n",
+        "    import IPython\n",
+        "\n",
+        "    app = IPython.Application.instance()\n",
+        "    app.kernel.do_shutdown(True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0f826ff482a2"
+      },
+      "source": [
+        "### Setup Google Cloud project\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI API and Compute Engine API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component).\n",
+        "\n",
+        "1. [Create a Cloud Storage bucket](https://cloud.google.com/storage/docs/creating-buckets) for storing experiment outputs."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8958ebc71868"
+      },
+      "source": [
+        "Fill following variables for experiments environment:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9db30f827a65"
+      },
+      "outputs": [],
+      "source": [
+        "# Cloud project id.\n",
+        "PROJECT_ID = \"\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# The region you want to launch jobs in.\n",
+        "REGION = \"us-central1\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# The Cloud Storage bucket for storing experiments output. Fill it without the 'gs://' prefix.\n",
+        "GCS_BUCKET = \"\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "92f16e22c20b"
+      },
+      "source": [
+        "Initialize Vertex AI API:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "1680c257acfb"
+      },
+      "outputs": [],
+      "source": [
+        "from google.cloud import aiplatform\n",
+        "\n",
+        "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=GCS_BUCKET)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6ca48b699d17"
+      },
+      "source": [
+        "### Define constants"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "de9882ea89ea"
+      },
+      "outputs": [],
+      "source": [
+        "# The pre-built serving docker image. It contains serving scripts and models.\n",
+        "SERVE_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-transformers-serve\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "10188266a5cd"
+      },
+      "source": [
+        "### Define common functions"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cac4478ae098"
+      },
+      "outputs": [],
+      "source": [
+        "import base64\n",
+        "import os\n",
+        "from datetime import datetime\n",
+        "from io import BytesIO\n",
+        "\n",
+        "import requests\n",
+        "from google.cloud import aiplatform\n",
+        "from PIL import Image\n",
+        "\n",
+        "\n",
+        "def create_job_name(prefix):\n",
+        "    user = os.environ.get(\"USER\")\n",
+        "    now = datetime.now().strftime(\"%Y%m%d_%H%M%S\")\n",
+        "    job_name = f\"{prefix}-{user}-{now}\"\n",
+        "    return job_name\n",
+        "\n",
+        "\n",
+        "def download_image(url):\n",
+        "    response = requests.get(url)\n",
+        "    return Image.open(BytesIO(response.content))\n",
+        "\n",
+        "\n",
+        "def image_to_base64(image, format=\"JPEG\"):\n",
+        "    buffer = BytesIO()\n",
+        "    image.save(buffer, format=format)\n",
+        "    image_str = base64.b64encode(buffer.getvalue()).decode(\"utf-8\")\n",
+        "    return image_str\n",
+        "\n",
+        "\n",
+        "def base64_to_image(image_str):\n",
+        "    image = Image.open(BytesIO(base64.b64decode(image_str)))\n",
+        "    return image\n",
+        "\n",
+        "\n",
+        "def image_grid(imgs, rows=2, cols=2):\n",
+        "    w, h = imgs[0].size\n",
+        "    grid = Image.new(\"RGB\", size=(cols * w, rows * h))\n",
+        "    for i, img in enumerate(imgs):\n",
+        "        grid.paste(img, box=(i % cols * w, i // cols * h))\n",
+        "    return grid\n",
+        "\n",
+        "\n",
+        "def deploy_model(model_id, task):\n",
+        "    model_name = \"clip\"\n",
+        "    endpoint = aiplatform.Endpoint.create(display_name=f\"{model_name}-endpoint\")\n",
+        "    serving_env = {\n",
+        "        \"MODEL_ID\": model_id,\n",
+        "        \"TASK\": task,\n",
+        "    }\n",
+        "    # If the model_id is a GCS path, use artifact_uri to pass it to serving docker.\n",
+        "    artifact_uri = model_id if model_id.startswith(\"gs://\") else None\n",
+        "    model = aiplatform.Model.upload(\n",
+        "        display_name=model_name,\n",
+        "        serving_container_image_uri=SERVE_DOCKER_URI,\n",
+        "        serving_container_ports=[7080],\n",
+        "        serving_container_predict_route=\"/predictions/transformers_serving\",\n",
+        "        serving_container_health_route=\"/ping\",\n",
+        "        serving_container_environment_variables=serving_env,\n",
+        "        artifact_uri=artifact_uri,\n",
+        "    )\n",
+        "    model.deploy(\n",
+        "        endpoint=endpoint,\n",
+        "        machine_type=\"n1-standard-8\",\n",
+        "        accelerator_type=\"NVIDIA_TESLA_T4\",\n",
+        "        accelerator_count=1,\n",
+        "        deploy_request_timeout=1800,\n",
+        "    )\n",
+        "    return model, endpoint"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d2d72ecdb8c9"
+      },
+      "source": [
+        "## Upload and deploy models"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9448c5f545fa"
+      },
+      "source": [
+        "This section uploads the pre-trained model to Model Registry and deploys it on the Endpoint with 1 T4 GPU.\n",
+        "\n",
+        "The model deployment step will take ~15 minutes to complete."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "99f3c6b404b5"
+      },
+      "source": [
+        "### Zero-shot image classification"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b4b46c28d8b1"
+      },
+      "outputs": [],
+      "source": [
+        "model, endpoint = deploy_model(\n",
+        "    model_id=\"openai/clip-vit-base-patch32\", task=\"zero-shot-image-classification\"\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "80b3fd2ace09"
+      },
+      "source": [
+        "NOTE: The model weights will be downloaded after the deployment succeeds. Thus additional 5 minutes of waiting time is needed **after** the above model deployment step succeeds and before you run the next step below. Otherwise you might see a `ServiceUnavailable: 503 502:Bad Gateway` error when you send requests to the endpoint."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6be655247cb1"
+      },
+      "outputs": [],
+      "source": [
+        "image1 = download_image(\"http://images.cocodataset.org/val2017/000000039769.jpg\")\n",
+        "image2 = download_image(\"http://images.cocodataset.org/val2017/000000000285.jpg\")\n",
+        "grid = image_grid([image1, image2], 1, 2)\n",
+        "display(grid)\n",
+        "\n",
+        "instances = [\n",
+        "    {\"image\": image_to_base64(image1), \"text\": \"two cats\"},\n",
+        "    {\"image\": image_to_base64(image2), \"text\": \"a bear\"},\n",
+        "]\n",
+        "preds = endpoint.predict(instances=instances).predictions\n",
+        "print(preds)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2ccf3714dbe9"
+      },
+      "outputs": [],
+      "source": [
+        "# Undeploy model and delete endpoint.\n",
+        "endpoint.delete(force=True)\n",
+        "\n",
+        "# Delete models.\n",
+        "model.delete()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ddf9e9ec7b58"
+      },
+      "source": [
+        "### Image/text feature embedding"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "8897bbac6887"
+      },
+      "outputs": [],
+      "source": [
+        "model, endpoint = deploy_model(\n",
+        "    model_id=\"openai/clip-vit-base-patch32\", task=\"feature-embedding\"\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9f854c45f8f3"
+      },
+      "source": [
+        "NOTE: The model weights will be downloaded after the deployment succeeds. Thus additional 5 minutes of waiting time is needed after the above model deployment step succeeds and before you run the next step below. Otherwise you might see a ServiceUnavailable: 503 502:Bad Gateway error when you send requests to the endpoint."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2abd54335f36"
+      },
+      "outputs": [],
+      "source": [
+        "import numpy as np\n",
+        "\n",
+        "# Extract feature embedding of images.\n",
+        "image = download_image(\"http://images.cocodataset.org/val2017/000000039769.jpg\")\n",
+        "display(image)\n",
+        "instances = [\n",
+        "    {\"image\": image_to_base64(image)},\n",
+        "]\n",
+        "preds = endpoint.predict(instances=instances).predictions\n",
+        "image_features = np.array(preds[0][\"image_features\"])\n",
+        "print(image_features.shape)\n",
+        "\n",
+        "# Extract feature embedding of texts.\n",
+        "instances = [\n",
+        "    {\"text\": \"two cats\"},\n",
+        "    {\"text\": \"hello world\"},\n",
+        "]\n",
+        "preds = endpoint.predict(instances=instances).predictions\n",
+        "text_features = np.array(preds[0][\"text_features\"])\n",
+        "print(text_features.shape)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "712eb9d0b336"
+      },
+      "outputs": [],
+      "source": [
+        "# Undeploy model and delete endpoint.\n",
+        "endpoint.delete(force=True)\n",
+        "\n",
+        "# Delete models.\n",
+        "model.delete()"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "model_garden_pytorch_clip.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/community/model_garden/model_garden_pytorch_controlnet.ipynb

@@ -0,0 +1,660 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7d9bbf86da5e"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2023 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "99c1c3fc2ca5"
+      },
+      "source": [
+        "# Vertex AI Model Garden - ControlNet\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_controlnet.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_controlnet.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/notebooks/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/model_garden/model_garden_pytorch_controlnet.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "    (a Python-3 CPU notebook is recommended)\n",
+        "  </td>\n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3de7470326a2"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "This notebook demonstrates finetuning the [ControlNet](https://huggingface.co/lllyasviel/ControlNet) with the [fusing/fill50k](https://huggingface.co/datasets/fusing/fill50k) dataset and deploying the model on Vertex AI for online prediction.\n",
+        "\n",
+        "### Objective\n",
+        "\n",
+        "- Finetune the ControlNet model.\n",
+        "- Upload the model to [Model Registry](https://cloud.google.com/vertex-ai/docs/model-registry/introduction).\n",
+        "- Deploy the model on [Endpoint](https://cloud.google.com/vertex-ai/docs/predictions/using-private-endpoints).\n",
+        "- Run online predictions for text-guided-image-to-image.\n",
+        "\n",
+        "### Costs\n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage pricing](https://cloud.google.com/storage/pricing), and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "264c07757582"
+      },
+      "source": [
+        "## Setup environment\n",
+        "\n",
+        "**NOTE**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d73ffa0c0b83"
+      },
+      "source": [
+        "### Colab only"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2707b02ef5df"
+      },
+      "outputs": [],
+      "source": [
+        "!pip3 install --upgrade google-cloud-aiplatform"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b60a4d7100bf"
+      },
+      "outputs": [],
+      "source": [
+        "from google.colab import auth as google_auth\n",
+        "\n",
+        "google_auth.authenticate_user()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "fb671e75ca7b"
+      },
+      "source": [
+        "### Install dependencies"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "dc8ee367fb42"
+      },
+      "outputs": [],
+      "source": [
+        "# Install gdown for downloading example training images.\n",
+        "!pip install gdown\n",
+        "# Install libs for generating conditioning images for ControlNet.\n",
+        "!pip install opencv-python"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5244aac3d929"
+      },
+      "source": [
+        "Restart the notebook kernel after installs."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "567212ff53a6"
+      },
+      "outputs": [],
+      "source": [
+        "import IPython\n",
+        "\n",
+        "app = IPython.Application.instance()\n",
+        "app.kernel.do_shutdown(True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "bb7adab99e41"
+      },
+      "source": [
+        "### Setup Google Cloud project\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI API and Compute Engine API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component).\n",
+        "\n",
+        "1. [Create a Cloud Storage bucket](https://cloud.google.com/storage/docs/creating-buckets) for storing experiment outputs.\n",
+        "\n",
+        "1. [Create a service account](https://cloud.google.com/iam/docs/service-accounts-create#iam-service-accounts-create-console) with `Vertex AI User` and `Storage Object Admin` roles for deploying fine tuned model to Vertex AI endpoint."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6c460088b873"
+      },
+      "source": [
+        "Fill following variables for experiments environment:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "855d6b96f291"
+      },
+      "outputs": [],
+      "source": [
+        "# Cloud project id.\n",
+        "PROJECT_ID = \"\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# The region you want to launch jobs in.\n",
+        "REGION = \"us-central1\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# The Cloud Storage bucket for storing experiments output. Fill it without the 'gs://' prefix.\n",
+        "GCS_BUCKET = \"\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# The service account you created in step-5 above, it's like \"<account_name>@<project>.iam.gserviceaccount.com\"\n",
+        "SERVICE_ACCOUNT = \"\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e828eb320337"
+      },
+      "source": [
+        "Initialize Vertex AI API:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "12cd25839741"
+      },
+      "outputs": [],
+      "source": [
+        "from google.cloud import aiplatform\n",
+        "\n",
+        "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=GCS_BUCKET)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2cc825514deb"
+      },
+      "source": [
+        "### Define constants"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b42bd4fa2b2d"
+      },
+      "outputs": [],
+      "source": [
+        "# The pre-built training docker image. It contains training scripts and models.\n",
+        "TRAIN_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-diffusers-train:latest\"\n",
+        "\n",
+        "# The pre-built serving docker image. It contains serving scripts and models.\n",
+        "SERVE_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-diffusers-serve\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0c250872074f"
+      },
+      "source": [
+        "### Define common functions"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "354da31189dc"
+      },
+      "outputs": [],
+      "source": [
+        "import base64\n",
+        "import os\n",
+        "from datetime import datetime\n",
+        "from io import BytesIO\n",
+        "\n",
+        "import cv2\n",
+        "import numpy as np\n",
+        "import requests\n",
+        "from PIL import Image\n",
+        "\n",
+        "\n",
+        "def create_job_name(prefix):\n",
+        "    user = os.environ.get(\"USER\")\n",
+        "    now = datetime.now().strftime(\"%Y%m%d_%H%M%S\")\n",
+        "    job_name = f\"{prefix}-{user}-{now}\"\n",
+        "    return job_name\n",
+        "\n",
+        "\n",
+        "def download_image(url):\n",
+        "    response = requests.get(url)\n",
+        "    return Image.open(BytesIO(response.content))\n",
+        "\n",
+        "\n",
+        "def image_to_base64(image, format=\"JPEG\"):\n",
+        "    buffer = BytesIO()\n",
+        "    image.save(buffer, format=format)\n",
+        "    image_str = base64.b64encode(buffer.getvalue()).decode(\"utf-8\")\n",
+        "    return image_str\n",
+        "\n",
+        "\n",
+        "def base64_to_image(image_str):\n",
+        "    image = Image.open(BytesIO(base64.b64decode(image_str)))\n",
+        "    return image\n",
+        "\n",
+        "\n",
+        "def image_grid(imgs, rows=2, cols=2):\n",
+        "    w, h = imgs[0].size\n",
+        "    grid = Image.new(\"RGB\", size=(cols * w, rows * h))\n",
+        "    for i, img in enumerate(imgs):\n",
+        "        grid.paste(img, box=(i % cols * w, i // cols * h))\n",
+        "    return grid\n",
+        "\n",
+        "\n",
+        "def canny(image):\n",
+        "    image = np.array(image)\n",
+        "    image = cv2.Canny(image, 100, 200)\n",
+        "    image = image[:, :, None]\n",
+        "    image = np.concatenate([image, image, image], axis=2)\n",
+        "    image = Image.fromarray(image)\n",
+        "    return image\n",
+        "\n",
+        "\n",
+        "def deploy_model(model_id, task):\n",
+        "    model_name = \"controlnet\"\n",
+        "    endpoint = aiplatform.Endpoint.create(display_name=f\"{model_name}-endpoint\")\n",
+        "    serving_env = {\n",
+        "        \"MODEL_ID\": model_id,\n",
+        "        \"TASK\": task,\n",
+        "    }\n",
+        "    model = aiplatform.Model.upload(\n",
+        "        display_name=model_name,\n",
+        "        serving_container_image_uri=SERVE_DOCKER_URI,\n",
+        "        serving_container_ports=[7080],\n",
+        "        serving_container_predict_route=\"/predictions/diffusers_serving\",\n",
+        "        serving_container_health_route=\"/ping\",\n",
+        "        serving_container_environment_variables=serving_env,\n",
+        "    )\n",
+        "    model.deploy(\n",
+        "        endpoint=endpoint,\n",
+        "        machine_type=\"n1-standard-8\",\n",
+        "        accelerator_type=\"NVIDIA_TESLA_V100\",\n",
+        "        accelerator_count=1,\n",
+        "        deploy_request_timeout=1800,\n",
+        "        service_account=SERVICE_ACCOUNT,\n",
+        "    )\n",
+        "    return model, endpoint"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e70e3519ff8b"
+      },
+      "source": [
+        "## Finetune with fill50k dataset"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0dc65d8f0689"
+      },
+      "source": [
+        "This section uses the [fusing/fill50k](https://huggingface.co/datasets/fusing/fill50k) dataset to finetune the ControlNet model.\n",
+        "\n",
+        "The job will run on 1 A100 GPU and take ~7 hours to finish 1 epoch of training.\n",
+        "\n",
+        "The ControlNet model will be saved after the finetuning job finishs and it can be loaded to run inference later."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "65467b361315"
+      },
+      "outputs": [],
+      "source": [
+        "# The pre-trained stable diffusion model to be loaded.\n",
+        "stable_diffusion_model_id = \"runwayml/stable-diffusion-v1-5\"\n",
+        "# The datase id to be loaded.\n",
+        "dataset_id = \"fusing/fill50k\"\n",
+        "# The output path.\n",
+        "output_dir = f\"/gcs/{GCS_BUCKET}/controlnet/output\"\n",
+        "# The training steps. If provided, it overrides num_train_epochs. Set it to None to use num_train_epochs only.\n",
+        "max_train_steps = 10  # @param {type:\"integer\"}\n",
+        "# The training epochs. Set it to a bigger number, like 10, to make training converge better.\n",
+        "num_train_epochs = 1  # @param {type:\"integer\"}\n",
+        "\n",
+        "# Worker pool spec.\n",
+        "machine_type = \"a2-highgpu-1g\"\n",
+        "num_nodes = 1\n",
+        "gpu_type = \"NVIDIA_TESLA_A100\"\n",
+        "num_gpus = 1\n",
+        "\n",
+        "# Setup training job.\n",
+        "job_name = create_job_name(\"controlnet\")\n",
+        "job = aiplatform.CustomContainerTrainingJob(\n",
+        "    display_name=job_name,\n",
+        "    container_uri=TRAIN_DOCKER_URI,\n",
+        ")\n",
+        "\n",
+        "# Pass training arguments and launch job.\n",
+        "# See https://github.com/huggingface/diffusers/blob/main/examples/controlnet/train_controlnet.py\n",
+        "# for a full list of training arguments.\n",
+        "model = job.run(\n",
+        "    args=[\n",
+        "        f\"--num_machines={num_nodes}\",\n",
+        "        f\"--num_processes={num_gpus}\",\n",
+        "        \"--machine_rank=0\",\n",
+        "        \"--mixed_precision=no\",\n",
+        "        \"--gpu_ids=all\",\n",
+        "        \"--same_network\",\n",
+        "        \"--dynamo_backend=no\",\n",
+        "        \"controlnet/train_controlnet.py\",\n",
+        "        \"--tracker_project_name=train_controlnet\",\n",
+        "        f\"--pretrained_model_name_or_path={stable_diffusion_model_id}\",\n",
+        "        f\"--output_dir={output_dir}\",\n",
+        "        f\"--dataset_name={dataset_id}\",\n",
+        "        f\"--max_train_steps={max_train_steps}\",\n",
+        "        f\"--num_train_epochs={num_train_epochs}\",\n",
+        "        \"--resolution=512\",\n",
+        "        \"--learning_rate=1e-5\",\n",
+        "        \"--train_batch_size=2\",\n",
+        "        \"--checkpointing_steps=50000\",\n",
+        "        \"--checkpoints_total_limit=1\",\n",
+        "    ],\n",
+        "    replica_count=num_nodes,\n",
+        "    machine_type=machine_type,\n",
+        "    accelerator_type=gpu_type,\n",
+        "    accelerator_count=num_gpus,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "bf7f82732e61"
+      },
+      "source": [
+        "## Upload and Deploy models"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "1cc26e68d7b0"
+      },
+      "source": [
+        "This section uploads the model to Model Registry and deploys it on the Endpoint.\n",
+        "\n",
+        "The model deployment step will take ~15 minutes to complete."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "cd7b56421392"
+      },
+      "source": [
+        "### Pre-trained canny model"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6d331b1ea337"
+      },
+      "source": [
+        "Deploy the pre-trained [lllyasviel/sd-controlnet-canny](https://huggingface.co/lllyasviel/sd-controlnet-canny) model for the text-guided image-to-image task. When deployed on one V100 GPU, the average inference time of a request is ~15 seconds."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "bf55e38815dc"
+      },
+      "outputs": [],
+      "source": [
+        "model, endpoint = deploy_model(\n",
+        "    model_id=\"lllyasviel/sd-controlnet-canny\", task=\"controlnet\"\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "80b3fd2ace09"
+      },
+      "source": [
+        "NOTE: The model weights will be downloaded after the deployment succeeds. Thus additional 5 minutes of waiting time is needed **after** the above model deployment step succeeds and before you run the next step below. Otherwise you might see a `ServiceUnavailable: 503 502:Bad Gateway` error when you send requests to the endpoint."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "4ab04da3ec9a"
+      },
+      "outputs": [],
+      "source": [
+        "init_image = download_image(\n",
+        "    \"https://huggingface.co/takuma104/controlnet_dev/resolve/main/gen_compare/output_images/diffusers/output_bird_canny_1.png\"\n",
+        ")\n",
+        "display(init_image)\n",
+        "image = canny(init_image)\n",
+        "display(image)\n",
+        "\n",
+        "instances = [\n",
+        "    {\n",
+        "        \"prompt\": \"bird\",\n",
+        "        \"image\": image_to_base64(image),\n",
+        "    },\n",
+        "]\n",
+        "response = endpoint.predict(instances=instances)\n",
+        "images = [base64_to_image(image) for image in response.predictions]\n",
+        "display(images[0])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "af21a3cff1e0"
+      },
+      "source": [
+        "Clean up resources:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "911406c1561e"
+      },
+      "outputs": [],
+      "source": [
+        "# Undeploy model and delete endpoint.\n",
+        "endpoint.delete(force=True)\n",
+        "\n",
+        "# Delete models.\n",
+        "model.delete()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "c1e51f764a60"
+      },
+      "source": [
+        "### Custom finetuned fill50k model"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "fa686a54047c"
+      },
+      "source": [
+        "Deploy the finetuned fill50k model above for the text-guided image-to-image task. When deployed on one V100 GPU, the averaged inference time of a request is ~15 seconds."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "65e32356fbd1"
+      },
+      "outputs": [],
+      "source": [
+        "model, endpoint = deploy_model(\n",
+        "    model_id=f\"gs://{GCS_BUCKET}/controlnet/output\", task=\"controlnet\"\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "80b3fd2ace09"
+      },
+      "source": [
+        "NOTE: The model weights will be downloaded after the deployment succeeds. Thus additional 5 minutes of waiting time is needed **after** the above model deployment step succeeds and before you run the next step below. Otherwise you might see a `ServiceUnavailable: 503 502:Bad Gateway` error when you send requests to the endpoint."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "83a50fd4a1ed"
+      },
+      "outputs": [],
+      "source": [
+        "init_image = download_image(\n",
+        "    \"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/controlnet_training/conditioning_image_1.png\"\n",
+        ")\n",
+        "display(init_image)\n",
+        "\n",
+        "instances = [\n",
+        "    {\n",
+        "        \"prompt\": \"red circle with green background\",\n",
+        "        \"image\": image_to_base64(init_image, format=\"PNG\"),\n",
+        "    },\n",
+        "]\n",
+        "response = endpoint.predict(instances=instances)\n",
+        "images = [base64_to_image(image) for image in response.predictions]\n",
+        "display(images[0])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ed3795d474b9"
+      },
+      "source": [
+        "Clean up resources:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b53b883257b4"
+      },
+      "outputs": [],
+      "source": [
+        "# Undeploy model and delete endpoint.\n",
+        "endpoint.delete(force=True)\n",
+        "\n",
+        "# Delete models.\n",
+        "model.delete()"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "model_garden_pytorch_controlnet.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/community/model_garden/model_garden_pytorch_dolly_v2.ipynb

@@ -0,0 +1,362 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7d9bbf86da5e"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2023 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2bd716bf3e39"
+      },
+      "source": [
+        "# Vertex AI Model Garden - Dolly V2\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_dolly_v2.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_dolly_v2.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/notebooks/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/model_garden/model_garden_pytorch_dolly_v2.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "    (a Python-3 CPU notebook is recommended)\n",
+        "  </td>\n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d8cd12648da4"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "This notebook demonstrates deploying the pre-trained [Dolly-V2-7b](https://huggingface.co/databricks/dolly-v2-7b) model on Vertex AI for online prediction.\n",
+        "\n",
+        "### Objective\n",
+        "\n",
+        "- Upload the model to [Model Registry](https://cloud.google.com/vertex-ai/docs/model-registry/introduction).\n",
+        "- Deploy the model on [Endpoint](https://cloud.google.com/vertex-ai/docs/predictions/using-private-endpoints).\n",
+        "- Run online predictions for image captioning.\n",
+        "\n",
+        "### Costs\n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage pricing](https://cloud.google.com/storage/pricing), and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "264c07757582"
+      },
+      "source": [
+        "## Setup environment\n",
+        "\n",
+        "**NOTE**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d73ffa0c0b83"
+      },
+      "source": [
+        "### Colab only"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2707b02ef5df"
+      },
+      "outputs": [],
+      "source": [
+        "!pip3 install --upgrade google-cloud-aiplatform"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b60a4d7100bf"
+      },
+      "outputs": [],
+      "source": [
+        "from google.colab import auth as google_auth\n",
+        "\n",
+        "google_auth.authenticate_user()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0f826ff482a2"
+      },
+      "source": [
+        "### Setup Google Cloud project\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI API and Compute Engine API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component).\n",
+        "\n",
+        "1. [Create a Cloud Storage bucket](https://cloud.google.com/storage/docs/creating-buckets) for storing experiment outputs."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8958ebc71868"
+      },
+      "source": [
+        "Fill following variables for experiments environment:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9db30f827a65"
+      },
+      "outputs": [],
+      "source": [
+        "# Cloud project id.\n",
+        "PROJECT_ID = \"\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# The region you want to launch jobs in.\n",
+        "REGION = \"us-central1\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# The Cloud Storage bucket for storing experiments output. Fill it without the 'gs://' prefix.\n",
+        "GCS_BUCKET = \"\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "92f16e22c20b"
+      },
+      "source": [
+        "Initialize Vertex AI API:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "1680c257acfb"
+      },
+      "outputs": [],
+      "source": [
+        "from google.cloud import aiplatform\n",
+        "\n",
+        "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=GCS_BUCKET)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6ca48b699d17"
+      },
+      "source": [
+        "### Define constants"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "de9882ea89ea"
+      },
+      "outputs": [],
+      "source": [
+        "# The pre-built serving docker image.\n",
+        "# The model artifacts are embedded within the container, except for model weights which will be downloaded during deployment.\n",
+        "SERVE_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai-restricted/vertex-vision-model-garden-dockers/pytorch-dolly-v2-serve\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "10188266a5cd"
+      },
+      "source": [
+        "### Define common functions"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cac4478ae098"
+      },
+      "outputs": [],
+      "source": [
+        "def deploy_model(model_id):\n",
+        "    \"\"\"Uploads and deploys the model to Vertex AI endpoint for prediction.\"\"\"\n",
+        "    model_name = \"dolly_v2_7b\"\n",
+        "    endpoint = aiplatform.Endpoint.create(display_name=f\"{model_name}-endpoint\")\n",
+        "    serving_env = {\n",
+        "        \"MODEL_ID\": model_id,\n",
+        "    }\n",
+        "    # If the model_id is a GCS path, use artifact_uri to pass it to serving docker.\n",
+        "    artifact_uri = model_id if model_id.startswith(\"gs://\") else None\n",
+        "    model = aiplatform.Model.upload(\n",
+        "        display_name=model_name,\n",
+        "        serving_container_image_uri=SERVE_DOCKER_URI,\n",
+        "        serving_container_ports=[7080],\n",
+        "        serving_container_predict_route=\"/predictions/transformers_serving\",\n",
+        "        serving_container_health_route=\"/ping\",\n",
+        "        serving_container_environment_variables=serving_env,\n",
+        "        artifact_uri=artifact_uri,\n",
+        "    )\n",
+        "    model.deploy(\n",
+        "        endpoint=endpoint,\n",
+        "        machine_type=\"a2-highgpu-1g\",\n",
+        "        accelerator_type=\"NVIDIA_TESLA_A100\",\n",
+        "        accelerator_count=1,\n",
+        "        deploy_request_timeout=1800,\n",
+        "    )\n",
+        "    return model, endpoint"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d2d72ecdb8c9"
+      },
+      "source": [
+        "## Upload and deploy models"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9448c5f545fa"
+      },
+      "source": [
+        "This section uploads the pre-trained model to Model Registry and deploys it on the Endpoint with 1 A100 GPU.\n",
+        "\n",
+        "The model deployment step will take ~15 minutes to complete."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b4b46c28d8b1"
+      },
+      "outputs": [],
+      "source": [
+        "model, endpoint = deploy_model(model_id=\"databricks/dolly-v2-7b\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "80b3fd2ace09"
+      },
+      "source": [
+        "NOTE: The model weights will be downloaded after the deployment succeeds. Thus additional 10 minutes of waiting time is needed **after** the above model deployment step succeeds and before you run the next step below. Otherwise you might see a `ServiceUnavailable: 503 502:Bad Gateway` error when you send requests to the endpoint."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ohPQGAawXtAf"
+      },
+      "source": [
+        "## Send prediction request"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6be655247cb1"
+      },
+      "outputs": [],
+      "source": [
+        "input_text = \"Explain to me the difference between nuclear fission and fusion.\"\n",
+        "\n",
+        "instances = [\n",
+        "    {\"text\": input_text},\n",
+        "]\n",
+        "preds = endpoint.predict(instances=instances).predictions\n",
+        "print(preds)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "uXLWt43CXMgj"
+      },
+      "source": [
+        "## Clean up"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2ccf3714dbe9"
+      },
+      "outputs": [],
+      "source": [
+        "# Undeploy model and delete endpoint.\n",
+        "endpoint.delete(force=True)\n",
+        "\n",
+        "# Delete models.\n",
+        "model.delete()"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "model_garden_pytorch_dolly_v2.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/community/model_garden/model_garden_pytorch_falcon_instruct_peft.ipynb

@@ -0,0 +1,655 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7d9bbf86da5e"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2023 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "99c1c3fc2ca5"
+      },
+      "source": [
+        "# Vertex AI Model Garden - Falcon Instruct (PEFT)\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_falcon_instruct_peft.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_falcon_instruct_peft.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/notebooks/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/model_garden/model_garden_pytorch_falcon_instruct_peft.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "Open in Vertex AI Workbench\n",
+        "    </a> (A Python-3 CPU notebook is recommended)\n",
+        "  </td>\n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3de7470326a2"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "This notebook demonstrates deploying prebuilt Falcon Instruct models, and also finetuning and deploying Falcon Instruct models with performance efficient finetuning libraries ([PEFT](https://github.com/huggingface/peft)) in Vertex AI.\n",
+        "\n",
+        "### Objective\n",
+        "\n",
+        "- Deploy prebuilt Falcon Instruct models\n",
+        "- Finetune and deploy Falcon Instruct models with PEFT, supporting\n",
+        "\n",
+        "| Models | LoRA |\n",
+        "| :- | :- |\n",
+        "| [tiiuae/falcon-7b-instruct](https://huggingface.co/tiiuae/falcon-7b-instruct) | Y |\n",
+        "| [tiiuae/falcon-40b-instruct](https://huggingface.co/tiiuae/falcon-40b-instruct) | Y |\n",
+        "\n",
+        "### Costs\n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage pricing](https://cloud.google.com/storage/pricing), and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "264c07757582"
+      },
+      "source": [
+        "## Before you begin\n",
+        "\n",
+        "**NOTE**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ioensNKM8ned"
+      },
+      "source": [
+        "### Colab only\n",
+        "Run the following commands for Colab and skip this section if you are using Workbench."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2707b02ef5df"
+      },
+      "outputs": [],
+      "source": [
+        "import sys\n",
+        "\n",
+        "if \"google.colab\" in sys.modules:\n",
+        "    ! pip3 install --upgrade google-cloud-aiplatform\n",
+        "    from google.colab import auth as google_auth\n",
+        "\n",
+        "    google_auth.authenticate_user()\n",
+        "    # Install gdown for downloading example training images.\n",
+        "    ! pip3 install gdown\n",
+        "\n",
+        "    # Restart the notebook kernel after installs.\n",
+        "    import IPython\n",
+        "\n",
+        "    app = IPython.Application.instance()\n",
+        "    app.kernel.do_shutdown(True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "bb7adab99e41"
+      },
+      "source": [
+        "### Setup Google Cloud project\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI API and Compute Engine API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component).\n",
+        "\n",
+        "1. [Create a Cloud Storage bucket](https://cloud.google.com/storage/docs/creating-buckets) for storing experiment outputs.\n",
+        "\n",
+        "1. [Create a service account](https://cloud.google.com/iam/docs/service-accounts-create#iam-service-accounts-create-console) with `Vertex AI User` and `Storage Object Admin` roles for deploying fine tuned model to Vertex AI endpoint."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6c460088b873"
+      },
+      "source": [
+        "Fill following variables for experiments environment:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "855d6b96f291"
+      },
+      "outputs": [],
+      "source": [
+        "# Cloud project id.\n",
+        "PROJECT_ID = \"\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# The region you want to launch jobs in.\n",
+        "REGION = \"\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# The Cloud Storage bucket for storing experiments output.\n",
+        "BUCKET_URI = \"\"  # @param {type:\"string\"}\n",
+        "\n",
+        "! gcloud config set project $PROJECT_ID\n",
+        "\n",
+        "import os\n",
+        "\n",
+        "STAGING_BUCKET = os.path.join(BUCKET_URI, \"temporal\")\n",
+        "EXPERIMENT_BUCKET = os.path.join(BUCKET_URI, \"peft\")\n",
+        "DATA_BUCKET = os.path.join(EXPERIMENT_BUCKET, \"data\")\n",
+        "MODEL_BUCKET = os.path.join(EXPERIMENT_BUCKET, \"model\")\n",
+        "\n",
+        "# The service account looks like:\n",
+        "# '@.iam.gserviceaccount.com'\n",
+        "# Please go to https://cloud.google.com/iam/docs/service-accounts-create#iam-service-accounts-create-console\n",
+        "# and create service account with `Vertex AI User` and `Storage Object Admin` roles.\n",
+        "# The service account for deploying fine tuned model.\n",
+        "SERVICE_ACCOUNT = \"\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e828eb320337"
+      },
+      "source": [
+        "### Initialize Vertex AI API"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "12cd25839741"
+      },
+      "outputs": [],
+      "source": [
+        "from google.cloud import aiplatform\n",
+        "\n",
+        "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=STAGING_BUCKET)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2cc825514deb"
+      },
+      "source": [
+        "### Define constants"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b42bd4fa2b2d"
+      },
+      "outputs": [],
+      "source": [
+        "# The pre-built training and serving docker images.\n",
+        "TRAIN_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai-restricted/vertex-vision-model-garden-dockers/pytorch-peft-train\"\n",
+        "PREDICTION_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai-restricted/vertex-vision-model-garden-dockers/pytorch-peft-serve\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0c250872074f"
+      },
+      "source": [
+        "### Define common functions"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "354da31189dc"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "from datetime import datetime\n",
+        "\n",
+        "from google.cloud import aiplatform\n",
+        "\n",
+        "\n",
+        "def get_job_name_with_datetime(prefix: str):\n",
+        "    \"\"\"Gets the job name with date time when triggering training or deployment\n",
+        "    jobs in Vertex AI.\n",
+        "    \"\"\"\n",
+        "    return prefix + datetime.now().strftime(\"_%Y%m%d_%H%M%S\")\n",
+        "\n",
+        "\n",
+        "def deploy_model(\n",
+        "    model_name,\n",
+        "    base_model_id,\n",
+        "    finetuned_lora_model_path,\n",
+        "    service_account,\n",
+        "    task,\n",
+        "    machine_type=\"n1-standard-8\",\n",
+        "    accelerator_type=\"NVIDIA_TESLA_V100\",\n",
+        "):\n",
+        "    \"\"\"Deploys trained models into Vertex AI.\"\"\"\n",
+        "    endpoint = aiplatform.Endpoint.create(display_name=f\"{model_name}-endpoint\")\n",
+        "    serving_env = {\n",
+        "        \"BASE_MODEL_ID\": base_model_id,\n",
+        "        \"TASK\": task,\n",
+        "    }\n",
+        "    if finetuned_lora_model_path:\n",
+        "        serving_env[\"FINETUNED_LORA_MODEL_PATH\"] = finetuned_lora_model_path\n",
+        "    model = aiplatform.Model.upload(\n",
+        "        display_name=model_name,\n",
+        "        serving_container_image_uri=PREDICTION_DOCKER_URI,\n",
+        "        serving_container_ports=[7080],\n",
+        "        serving_container_predict_route=\"/predictions/peft_serving\",\n",
+        "        serving_container_health_route=\"/ping\",\n",
+        "        serving_container_environment_variables=serving_env,\n",
+        "    )\n",
+        "    model.deploy(\n",
+        "        endpoint=endpoint,\n",
+        "        machine_type=machine_type,\n",
+        "        accelerator_type=accelerator_type,\n",
+        "        accelerator_count=1,\n",
+        "        deploy_request_timeout=1800,\n",
+        "        service_account=service_account,\n",
+        "    )\n",
+        "    return model, endpoint"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8neJc8CnDDpu"
+      },
+      "source": [
+        "## Deploy prebuilt Falcon Instruct models\n",
+        "\n",
+        "This section deploys prebuilt Falcon Instruct models on the Endpoint. The model deployment step will take ~15 minutes to complete.\n",
+        "\n",
+        "The peak GPU memory usages for [tiiuae/falcon-7b-instruct](https://huggingface.co/tiiuae/falcon-7b-instruct), and [tiiuae/falcon-40b-instruct](https://huggingface.co/tiiuae/falcon-40b-instruct) are ~15.5G and ~38.2G separately with the default settings. We use V100 in deployments as an example. Please use A100 (40G) or A100 (80G) to get better inferences."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2MjaORIIFDVu"
+      },
+      "source": [
+        "Set the prebuilt model id."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "E8OiHHNNE_wj"
+      },
+      "outputs": [],
+      "source": [
+        "prebuilt_model_id = \"tiiuae/falcon-7b-instruct\"  # @param [\"tiiuae/falcon-7b-instruct\", \"tiiuae/falcon-40b-instruct\"]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dHFW7yvjaVFV"
+      },
+      "source": [
+        "We use the PEFT serving images to deploy prebuilt Falcon Instruct models, by setting finetuning LoRA model paths as empty."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Uak1pyEeExYM"
+      },
+      "outputs": [],
+      "source": [
+        "# Sets V100 to deploy tiiuae/falcon-7b-instruct as an example.\n",
+        "machine_type = \"n1-standard-8\"\n",
+        "accelerator_type = \"NVIDIA_TESLA_V100\"\n",
+        "\n",
+        "# Sets A100 (40G) to deploy tiiuae/falcon-7b-instruct or tiiuae/falcon-40b-instruct.\n",
+        "# machine_type = \"a2-highgpu-1g\"\n",
+        "# accelerator_type = \"NVIDIA_TESLA_A100\"\n",
+        "\n",
+        "# Sets A100 (80G) to deploy falcon-40b-instruct models for faster inferences.\n",
+        "# machine_type = \"a2-ultragpu-1g\"\n",
+        "# accelerator_type = \"NVIDIA_TESLA_A100\"\n",
+        "\n",
+        "model_without_peft, endpoint_without_peft = deploy_model(\n",
+        "    model_name=get_job_name_with_datetime(prefix=\"falcon-instruct-serve\"),\n",
+        "    base_model_id=prebuilt_model_id,\n",
+        "    finetuned_lora_model_path=\"\",  # This will avoid override finetuning models.\n",
+        "    service_account=SERVICE_ACCOUNT,\n",
+        "    task=\"instruct-lora\",\n",
+        "    machine_type=machine_type,\n",
+        "    accelerator_type=accelerator_type,\n",
+        ")\n",
+        "print(\"endpoint_name:\", endpoint_without_peft.name)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "sGKIjgmDFRW2"
+      },
+      "source": [
+        "NOTE: The prebuilt model weights will be downloaded on the fly from the orginal location after the deployment succeeds. Thus additional 5 minutes of waiting time is needed **after** the above model deployment step succeeds and before you run the next step below. Otherwise you might see a `ServiceUnavailable: 503 502:Bad Gateway` error when you send requests to the endpoint.\n",
+        "\n",
+        "Once deployment succeeds, you can send requests to the endpoint with text prompts.\n",
+        "\n",
+        "Example:\n",
+        "\n",
+        "```\n",
+        "Human: What is a car?\n",
+        "Assistant:  A car, or a motor car, is a road-connected human-transportation system used to move people or goods from one place to another. The term also encompasses a wide range of vehicles, including motorboats, trains, and aircrafts. Cars typically have four wheels, a cabin for passengers, and an engine or motor. They have been around since the early 19th century and are now one of the most popular forms of transportation, used for daily commuting, shopping, and other purposes.\n",
+        "```"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "rDHsCOqvFYBi"
+      },
+      "outputs": [],
+      "source": [
+        "# # Loads an existing endpoint as below.\n",
+        "# endpoint_name = endpoint_without_peft.name\n",
+        "# aip_endpoint_name = (\n",
+        "#     f\"projects/{PROJECT_ID}/locations/{REGION}/endpoints/{endpoint_name}\"\n",
+        "# )\n",
+        "# endpoint_without_peft = aiplatform.Endpoint(aip_endpoint_name)\n",
+        "instances = [\n",
+        "    {\"prompt\": \"What is a car?\"},\n",
+        "]\n",
+        "response = endpoint_without_peft.predict(instances=instances)\n",
+        "\n",
+        "for prediction in response.predictions[0]:\n",
+        "    print(prediction[\"generated_text\"])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e70e3519ff8b"
+      },
+      "source": [
+        "## Finetune and deploy Falcon Instruct models with PEFT\n",
+        "\n",
+        "This section demonstrates how to finetune and dpeloy Falcon Instruct models with PEFT LoRA."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5qCrm_kJH5cz"
+      },
+      "source": [
+        "Set the base model id."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "N3UBLiYrM3sU"
+      },
+      "outputs": [],
+      "source": [
+        "base_model_id = \"tiiuae/falcon-7b-instruct\"  # @param [\"tiiuae/falcon-7b-instruct\", \"tiiuae/falcon-40b-instruct\"]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "iWGwJHqI7LMs"
+      },
+      "source": [
+        "### Finetune"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "KKEYoRfiHDVv"
+      },
+      "source": [
+        "Use the Vertex AI SDK to create and run the custom training jobs with Vertex AI Model Garden training images.\n",
+        "\n",
+        "This example uses the dataset [timdettmers/openassistant-guanaco](https://huggingface.co/datasets/timdettmers/openassistant-guanaco).\n",
+        "\n",
+        "The peak GPU memory usages are ~11G and ~34G for finetuning LoRA models for [tiiuae/falcon-7b-instruct](https://huggingface.co/tiiuae/falcon-7b-instruct), and [tiiuae/falcon-40b-instruct](https://huggingface.co/tiiuae/falcon-40b-instruct) separately with default training parameters and the example dataset. Falcon-7b-instruct can be finetuned on 1 P100/V100, and falcon-40b-instruct can be finetuned on 1 A100 (40G)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "65467b361315"
+      },
+      "outputs": [],
+      "source": [
+        "dataset_name = \"timdettmers/openassistant-guanaco\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# Worker pool spec.\n",
+        "# Uses V100 to finetune falcon-70b-instruct.\n",
+        "machine_type = \"n1-standard-8\"\n",
+        "accelerator_type = \"NVIDIA_TESLA_V100\"\n",
+        "# Uses A100 to finetune falcon-40b-instruct.\n",
+        "# machine_type = \"a2-highgpu-1g\"\n",
+        "# accelerator_type = \"NVIDIA_TESLA_A100\"\n",
+        "replica_count = 1\n",
+        "accelerator_count = 1\n",
+        "\n",
+        "# Setup training job.\n",
+        "job_name = get_job_name_with_datetime(\"falcon-instruct-lora-train\")\n",
+        "train_job = aiplatform.CustomContainerTrainingJob(\n",
+        "    display_name=job_name,\n",
+        "    container_uri=TRAIN_DOCKER_URI,\n",
+        ")\n",
+        "output_dir = os.path.join(MODEL_BUCKET, job_name)\n",
+        "output_dir_gcsfuse = output_dir.replace(\"gs://\", \"/gcs/\")\n",
+        "\n",
+        "# Pass training arguments and launch job.\n",
+        "max_steps = 10\n",
+        "train_job.run(\n",
+        "    args=[\n",
+        "        \"--task=instruct-lora\",\n",
+        "        f\"--pretrained_model_id={base_model_id}\",\n",
+        "        f\"--dataset_name={dataset_name}\",\n",
+        "        f\"--output_dir={output_dir_gcsfuse}\",\n",
+        "        \"--lora_rank=64\",\n",
+        "        \"--lora_alpha=16\",\n",
+        "        \"--lora_dropout=0.1\",\n",
+        "        \"--warmup_ratio=0.03\",\n",
+        "        f\"--max_steps={max_steps}\",\n",
+        "        \"--max_seq_length=512\",\n",
+        "        \"--learning_rate=2e-4\",\n",
+        "    ],\n",
+        "    replica_count=replica_count,\n",
+        "    machine_type=machine_type,\n",
+        "    accelerator_type=accelerator_type,\n",
+        "    accelerator_count=accelerator_count,\n",
+        "    boot_disk_size_gb=500,\n",
+        ")\n",
+        "\n",
+        "print(\"Trained models were saved in: \", output_dir)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "jqmCtkGnhDmp"
+      },
+      "source": [
+        "### Deploy\n",
+        "This section uploads the model to Model Registry and deploys it on the Endpoint.\n",
+        "\n",
+        "The model deployment step will take ~15 minutes to complete.\n",
+        "\n",
+        "The peak GPU memory usages for [tiiuae/falcon-7b-instruct](https://huggingface.co/tiiuae/falcon-7b-instruct), and [tiiuae/falcon-40b-instruct](https://huggingface.co/tiiuae/falcon-40b-instruct) with LoRA weights are ~15.5G and ~38.2G separately with the default settings. We use V100 in deployments as an example. Please use A100 (40G) or A100 (80G) to get better inferences."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "bf55e38815dc"
+      },
+      "outputs": [],
+      "source": [
+        "# Sets V100 to deploy tiiuae/falcon-7b-instruct as an example.\n",
+        "machine_type = \"n1-standard-8\"\n",
+        "accelerator_type = \"NVIDIA_TESLA_V100\"\n",
+        "\n",
+        "# Sets A100 (40G) to deploy tiiuae/falcon-7b-instruct or tiiuae/falcon-40b-instruct.\n",
+        "# machine_type = \"a2-highgpu-1g\"\n",
+        "# accelerator_type = \"NVIDIA_TESLA_A100\"\n",
+        "\n",
+        "# Sets A100 (80G) to deploy falcon-40b-instruct models for faster inferences.\n",
+        "# machine_type = \"a2-ultragpu-1g\"\n",
+        "# accelerator_type = \"NVIDIA_TESLA_A100\"\n",
+        "\n",
+        "model_with_peft, endpoint_with_peft = deploy_model(\n",
+        "    model_name=get_job_name_with_datetime(prefix=\"falcon-instruct-peft-serve\"),\n",
+        "    base_model_id=base_model_id,\n",
+        "    finetuned_lora_model_path=os.path.join(output_dir, \"checkpoint-\" + str(max_steps)),\n",
+        "    service_account=SERVICE_ACCOUNT,\n",
+        "    task=\"instruct-lora\",\n",
+        "    machine_type=machine_type,\n",
+        "    accelerator_type=accelerator_type,\n",
+        ")\n",
+        "print(\"endpoint_name:\", endpoint_with_peft.name)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "80b3fd2ace09"
+      },
+      "source": [
+        "NOTE: After the deployment succeeds, the base model weights will be downloaded one the fly from the original location and LoRA model weights will be downloaded from the GCS bucket used in training above. Thus additional 5 minutes of waiting time is needed **after** the above model deployment step succeeds and before you run the next step below. Otherwise you might see a `ServiceUnavailable: 503 502:Bad Gateway` error when you send requests to the endpoint.\n",
+        "\n",
+        "Once deployment succeeds, you can send requests to the endpoint with text prompts.\n",
+        "\n",
+        "Example:\n",
+        "\n",
+        "```\n",
+        "Human: What is a car?\n",
+        "Assistant:  A car, or a motor car, is a road-connected human-transportation system used to move people or goods from one place to another. The term also encompasses a wide range of vehicles, including motorboats, trains, and aircrafts. Cars typically have four wheels, a cabin for passengers, and an engine or motor. They have been around since the early 19th century and are now one of the most popular forms of transportation, used for daily commuting, shopping, and other purposes.\n",
+        "```"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "4ab04da3ec9a"
+      },
+      "outputs": [],
+      "source": [
+        "# # Loads an existing endpoint as below.\n",
+        "# endpoint_name = endpoint_with_peft.name\n",
+        "# aip_endpoint_name = (\n",
+        "#     f\"projects/{PROJECT_ID}/locations/{REGION}/endpoints/{endpoint_name}\"\n",
+        "# )\n",
+        "# endpoint_with_peft = aiplatform.Endpoint(aip_endpoint_name)\n",
+        "instances = [\n",
+        "    {\"prompt\": \"What is a car?\"},\n",
+        "]\n",
+        "response = endpoint_with_peft.predict(instances=instances)\n",
+        "\n",
+        "for prediction in response.predictions[0]:\n",
+        "    print(prediction[\"generated_text\"])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "af21a3cff1e0"
+      },
+      "source": [
+        "## Clean up resources"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "911406c1561e"
+      },
+      "outputs": [],
+      "source": [
+        "# Delete custom train jobs.\n",
+        "train_job.delete()\n",
+        "\n",
+        "# Undeploy model and delete endpoint.\n",
+        "endpoint_without_peft.delete(force=True)\n",
+        "endpoint_with_peft.delete(force=True)\n",
+        "\n",
+        "# Delete models.\n",
+        "model_without_peft.delete()\n",
+        "model_with_peft.delete()"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "model_garden_pytorch_falcon_instruct_peft.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/community/model_garden/model_garden_pytorch_instructpix2pix.ipynb

@@ -0,0 +1,421 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7d9bbf86da5e"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2023 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2bd716bf3e39"
+      },
+      "source": [
+        "# Vertex AI Model Garden - InstructPix2Pix\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_instructpix2pix.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_instructpix2pix.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/notebooks/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/model_garden/model_garden_pytorch_instructpix2pix.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "    (a Python-3 CPU notebook is recommended)\n",
+        "  </td>\n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d8cd12648da4"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "This notebook demonstrates deploying the pre-trained [InstructPix2Pix](https://huggingface.co/timbrooks/instruct-pix2pix) model on Vertex AI for online prediction.\n",
+        "\n",
+        "### Objective\n",
+        "\n",
+        "- Upload the model to [Model Registry](https://cloud.google.com/vertex-ai/docs/model-registry/introduction).\n",
+        "- Deploy the model on [Endpoint](https://cloud.google.com/vertex-ai/docs/predictions/using-private-endpoints).\n",
+        "- Run online predictions for text-guided image-to-image.\n",
+        "\n",
+        "### Costs\n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage pricing](https://cloud.google.com/storage/pricing), and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "264c07757582"
+      },
+      "source": [
+        "## Setup environment\n",
+        "\n",
+        "**NOTE**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d73ffa0c0b83"
+      },
+      "source": [
+        "### Colab only"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2707b02ef5df"
+      },
+      "outputs": [],
+      "source": [
+        "if \"google.colab\" in str(get_ipython()):\n",
+        "    ! pip3 install --upgrade google-cloud-aiplatform\n",
+        "    from google.colab import auth as google_auth\n",
+        "\n",
+        "    google_auth.authenticate_user()\n",
+        "\n",
+        "    # Restart the notebook kernel after installs.\n",
+        "    import IPython\n",
+        "\n",
+        "    app = IPython.Application.instance()\n",
+        "    app.kernel.do_shutdown(True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0f826ff482a2"
+      },
+      "source": [
+        "### Setup Google Cloud project\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI API and Compute Engine API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component).\n",
+        "\n",
+        "1. [Create a Cloud Storage bucket](https://cloud.google.com/storage/docs/creating-buckets) for storing experiment outputs."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8958ebc71868"
+      },
+      "source": [
+        "Fill following variables for experiments environment:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9db30f827a65"
+      },
+      "outputs": [],
+      "source": [
+        "# Cloud project id.\n",
+        "PROJECT_ID = \"\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# The region you want to launch jobs in.\n",
+        "REGION = \"us-central1\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# The Cloud Storage bucket for storing experiments output. Fill it without the 'gs://' prefix.\n",
+        "GCS_BUCKET = \"\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "92f16e22c20b"
+      },
+      "source": [
+        "Initialize Vertex AI API:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "1680c257acfb"
+      },
+      "outputs": [],
+      "source": [
+        "from google.cloud import aiplatform\n",
+        "\n",
+        "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=GCS_BUCKET)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6ca48b699d17"
+      },
+      "source": [
+        "### Define constants"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "de9882ea89ea"
+      },
+      "outputs": [],
+      "source": [
+        "# The pre-built serving docker image. It contains serving scripts and models.\n",
+        "SERVE_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-diffusers-serve\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "10188266a5cd"
+      },
+      "source": [
+        "### Define common functions"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cac4478ae098"
+      },
+      "outputs": [],
+      "source": [
+        "import base64\n",
+        "import os\n",
+        "from datetime import datetime\n",
+        "from io import BytesIO\n",
+        "\n",
+        "import requests\n",
+        "from google.cloud import aiplatform\n",
+        "from PIL import Image\n",
+        "\n",
+        "\n",
+        "def create_job_name(prefix):\n",
+        "    user = os.environ.get(\"USER\")\n",
+        "    now = datetime.now().strftime(\"%Y%m%d_%H%M%S\")\n",
+        "    job_name = f\"{prefix}-{user}-{now}\"\n",
+        "    return job_name\n",
+        "\n",
+        "\n",
+        "def download_image(url):\n",
+        "    response = requests.get(url)\n",
+        "    return Image.open(BytesIO(response.content))\n",
+        "\n",
+        "\n",
+        "def image_to_base64(image, format=\"JPEG\"):\n",
+        "    buffer = BytesIO()\n",
+        "    image.save(buffer, format=format)\n",
+        "    image_str = base64.b64encode(buffer.getvalue()).decode(\"utf-8\")\n",
+        "    return image_str\n",
+        "\n",
+        "\n",
+        "def base64_to_image(image_str):\n",
+        "    image = Image.open(BytesIO(base64.b64decode(image_str)))\n",
+        "    return image\n",
+        "\n",
+        "\n",
+        "def image_grid(imgs, rows=2, cols=2):\n",
+        "    w, h = imgs[0].size\n",
+        "    grid = Image.new(\"RGB\", size=(cols * w, rows * h))\n",
+        "    for i, img in enumerate(imgs):\n",
+        "        grid.paste(img, box=(i % cols * w, i // cols * h))\n",
+        "    return grid\n",
+        "\n",
+        "\n",
+        "def deploy_model(model_id, task):\n",
+        "    model_name = \"instruct-pix2pix\"\n",
+        "    endpoint = aiplatform.Endpoint.create(display_name=f\"{model_name}-endpoint\")\n",
+        "    serving_env = {\n",
+        "        \"MODEL_ID\": model_id,\n",
+        "        \"TASK\": task,\n",
+        "    }\n",
+        "    # If the model_id is a GCS path, use artifact_uri to pass it to serving docker.\n",
+        "    artifact_uri = model_id if model_id.startswith(\"gs://\") else None\n",
+        "    model = aiplatform.Model.upload(\n",
+        "        display_name=model_name,\n",
+        "        serving_container_image_uri=SERVE_DOCKER_URI,\n",
+        "        serving_container_ports=[7080],\n",
+        "        serving_container_predict_route=\"/predictions/diffusers_serving\",\n",
+        "        serving_container_health_route=\"/ping\",\n",
+        "        serving_container_environment_variables=serving_env,\n",
+        "        artifact_uri=artifact_uri,\n",
+        "    )\n",
+        "    model.deploy(\n",
+        "        endpoint=endpoint,\n",
+        "        machine_type=\"n1-standard-8\",\n",
+        "        accelerator_type=\"NVIDIA_TESLA_V100\",\n",
+        "        accelerator_count=1,\n",
+        "        deploy_request_timeout=1800,\n",
+        "    )\n",
+        "    return model, endpoint"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d2d72ecdb8c9"
+      },
+      "source": [
+        "## Upload and deploy models"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9448c5f545fa"
+      },
+      "source": [
+        "This section uploads the pre-trained model to Model Registry and deploys it on the Endpoint.\n",
+        "\n",
+        "The model deployment step will take ~15 minutes to complete."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "c277da31bde6"
+      },
+      "source": [
+        "### Text-guided image-to-image"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "a5a86996222c"
+      },
+      "source": [
+        "Deploy the InstructPix2Pix model for the text-guided image-to-image task.\n",
+        "\n",
+        "Once deployed, you can send prompts to the endpoint to generated images.\n",
+        "\n",
+        "When deployed on one V100 GPU, the averaged inference time of a request is ~15 seconds."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b4b46c28d8b1"
+      },
+      "outputs": [],
+      "source": [
+        "model, endpoint = deploy_model(\n",
+        "    model_id=\"timbrooks/instruct-pix2pix\", task=\"instruct-pix2pix\"\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "80b3fd2ace09"
+      },
+      "source": [
+        "NOTE: The model weights will be downloaded after the deployment succeeds. Thus additional 5 minutes of waiting time is needed **after** the above model deployment step succeeds and before you run the next step below. Otherwise you might see a `ServiceUnavailable: 503 502:Bad Gateway` error when you send requests to the endpoint."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6be655247cb1"
+      },
+      "outputs": [],
+      "source": [
+        "init_image = download_image(\n",
+        "    \"https://huggingface.co/datasets/diffusers/diffusers-images-docs/resolve/main/mountain.png\"\n",
+        ")\n",
+        "display(init_image)\n",
+        "instances = [\n",
+        "    {\n",
+        "        \"prompt\": \"Add fire to the mountain\",\n",
+        "        \"image\": image_to_base64(init_image),\n",
+        "    },\n",
+        "]\n",
+        "response = endpoint.predict(instances=instances)\n",
+        "images = [base64_to_image(image) for image in response.predictions]\n",
+        "display(images[0])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "db7ffebdb4be"
+      },
+      "source": [
+        "### Clean up resources"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2ccf3714dbe9"
+      },
+      "outputs": [],
+      "source": [
+        "# Undeploy model and delete endpoint.\n",
+        "endpoint.delete(force=True)\n",
+        "\n",
+        "# Delete models.\n",
+        "model.delete()"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "model_garden_pytorch_instructpix2pix.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/community/model_garden/model_garden_pytorch_layoutml_document_qa.ipynb

@@ -0,0 +1,400 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7d9bbf86da5e"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2023 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2bd716bf3e39"
+      },
+      "source": [
+        "# Vertex AI Model Garden - LayoutML Document QA\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_layoutml_document_qa.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_layoutml_document_qa.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/notebooks/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/model_garden/model_garden_pytorch_layoutml_document_qa.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "    (a Python-3 CPU notebook is recommended)\n",
+        "  </td>\n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d8cd12648da4"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "This notebook demonstrates deploying the pre-trained [LayoutML](https://huggingface.co/impira/layoutlm-document-qa) model on Vertex AI for online prediction.\n",
+        "\n",
+        "### Objective\n",
+        "\n",
+        "- Upload the model to [Model Registry](https://cloud.google.com/vertex-ai/docs/model-registry/introduction).\n",
+        "- Deploy the model on [Endpoint](https://cloud.google.com/vertex-ai/docs/predictions/using-private-endpoints).\n",
+        "- Run online predictions for image captioning.\n",
+        "\n",
+        "### Costs\n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage pricing](https://cloud.google.com/storage/pricing), and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "264c07757582"
+      },
+      "source": [
+        "## Setup environment\n",
+        "\n",
+        "**NOTE**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d73ffa0c0b83"
+      },
+      "source": [
+        "### Colab only"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2707b02ef5df"
+      },
+      "outputs": [],
+      "source": [
+        "if \"google.colab\" in str(get_ipython()):\n",
+        "    ! pip3 install --upgrade google-cloud-aiplatform\n",
+        "    from google.colab import auth as google_auth\n",
+        "\n",
+        "    google_auth.authenticate_user()\n",
+        "\n",
+        "    # Restart the notebook kernel after installs.\n",
+        "    import IPython\n",
+        "\n",
+        "    app = IPython.Application.instance()\n",
+        "    app.kernel.do_shutdown(True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0f826ff482a2"
+      },
+      "source": [
+        "### Setup Google Cloud project\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI API and Compute Engine API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component).\n",
+        "\n",
+        "1. [Create a Cloud Storage bucket](https://cloud.google.com/storage/docs/creating-buckets) for storing experiment outputs."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8958ebc71868"
+      },
+      "source": [
+        "Fill following variables for experiments environment:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9db30f827a65"
+      },
+      "outputs": [],
+      "source": [
+        "# Cloud project id.\n",
+        "PROJECT_ID = \"\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# The region you want to launch jobs in.\n",
+        "REGION = \"us-central1\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# The Cloud Storage bucket for storing experiments output. Fill it without the 'gs://' prefix.\n",
+        "GCS_BUCKET = \"\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "92f16e22c20b"
+      },
+      "source": [
+        "Initialize Vertex AI API:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "1680c257acfb"
+      },
+      "outputs": [],
+      "source": [
+        "from google.cloud import aiplatform\n",
+        "\n",
+        "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=GCS_BUCKET)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6ca48b699d17"
+      },
+      "source": [
+        "### Define constants"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "de9882ea89ea"
+      },
+      "outputs": [],
+      "source": [
+        "# The pre-built serving docker image. It contains serving scripts and models.\n",
+        "SERVE_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-transformers-serve\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "10188266a5cd"
+      },
+      "source": [
+        "### Define common functions"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cac4478ae098"
+      },
+      "outputs": [],
+      "source": [
+        "import base64\n",
+        "import os\n",
+        "from datetime import datetime\n",
+        "from io import BytesIO\n",
+        "\n",
+        "import requests\n",
+        "from google.cloud import aiplatform\n",
+        "from PIL import Image\n",
+        "\n",
+        "\n",
+        "def create_job_name(prefix):\n",
+        "    user = os.environ.get(\"USER\")\n",
+        "    now = datetime.now().strftime(\"%Y%m%d_%H%M%S\")\n",
+        "    job_name = f\"{prefix}-{user}-{now}\"\n",
+        "    return job_name\n",
+        "\n",
+        "\n",
+        "def download_image(url):\n",
+        "    response = requests.get(url)\n",
+        "    return Image.open(BytesIO(response.content)).convert(\"RGB\")\n",
+        "\n",
+        "\n",
+        "def image_to_base64(image, format=\"JPEG\"):\n",
+        "    buffer = BytesIO()\n",
+        "    image.save(buffer, format=format)\n",
+        "    image_str = base64.b64encode(buffer.getvalue()).decode(\"utf-8\")\n",
+        "    return image_str\n",
+        "\n",
+        "\n",
+        "def base64_to_image(image_str):\n",
+        "    image = Image.open(BytesIO(base64.b64decode(image_str)))\n",
+        "    return image\n",
+        "\n",
+        "\n",
+        "def image_grid(imgs, rows=2, cols=2):\n",
+        "    w, h = imgs[0].size\n",
+        "    grid = Image.new(\"RGB\", size=(cols * w, rows * h))\n",
+        "    for i, img in enumerate(imgs):\n",
+        "        grid.paste(img, box=(i % cols * w, i // cols * h))\n",
+        "    return grid\n",
+        "\n",
+        "\n",
+        "def deploy_model(model_id, task):\n",
+        "    model_name = \"layoutml\"\n",
+        "    endpoint = aiplatform.Endpoint.create(display_name=f\"{model_name}-endpoint\")\n",
+        "    serving_env = {\n",
+        "        \"MODEL_ID\": model_id,\n",
+        "        \"TASK\": task,\n",
+        "    }\n",
+        "    # If the model_id is a GCS path, use artifact_uri to pass it to serving docker.\n",
+        "    artifact_uri = model_id if model_id.startswith(\"gs://\") else None\n",
+        "    model = aiplatform.Model.upload(\n",
+        "        display_name=model_name,\n",
+        "        serving_container_image_uri=SERVE_DOCKER_URI,\n",
+        "        serving_container_ports=[7080],\n",
+        "        serving_container_predict_route=\"/predictions/transformers_serving\",\n",
+        "        serving_container_health_route=\"/ping\",\n",
+        "        serving_container_environment_variables=serving_env,\n",
+        "        artifact_uri=artifact_uri,\n",
+        "    )\n",
+        "    model.deploy(\n",
+        "        endpoint=endpoint,\n",
+        "        machine_type=\"n1-standard-8\",\n",
+        "        accelerator_type=\"NVIDIA_TESLA_T4\",\n",
+        "        accelerator_count=1,\n",
+        "        deploy_request_timeout=1800,\n",
+        "    )\n",
+        "    return model, endpoint"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d2d72ecdb8c9"
+      },
+      "source": [
+        "## Upload and deploy models"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9448c5f545fa"
+      },
+      "source": [
+        "This section uploads the pre-trained model to Model Registry and deploys it on the Endpoint with 1 T4 GPU.\n",
+        "\n",
+        "The model deployment step will take ~15 minutes to complete.\n",
+        "\n",
+        "Once deployed, you can send document images and questions to get answers."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b4b46c28d8b1"
+      },
+      "outputs": [],
+      "source": [
+        "model, endpoint = deploy_model(\n",
+        "    model_id=\"impira/layoutlm-document-qa\", task=\"document-question-answering\"\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "80b3fd2ace09"
+      },
+      "source": [
+        "NOTE: The model weights will be downloaded after the deployment succeeds. Thus additional 5 minutes of waiting time is needed **after** the above model deployment step succeeds and before you run the next step below. Otherwise you might see a `ServiceUnavailable: 503 502:Bad Gateway` error when you send requests to the endpoint."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6be655247cb1"
+      },
+      "outputs": [],
+      "source": [
+        "image = download_image(\n",
+        "    \"https://huggingface.co/spaces/impira/docquery/resolve/2359223c1837a7587402bda0f2643382a6eefeab/invoice.png\"\n",
+        ")\n",
+        "display(image)\n",
+        "\n",
+        "question = \"What is the name of the signer?\"\n",
+        "instances = [\n",
+        "    {\"image\": image_to_base64(image), \"text\": \"\"},\n",
+        "]\n",
+        "preds = endpoint.predict(instances=instances).predictions\n",
+        "print(question)\n",
+        "print(preds)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "db7ffebdb4be"
+      },
+      "source": [
+        "### Clean up resources"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2ccf3714dbe9"
+      },
+      "outputs": [],
+      "source": [
+        "# Undeploy model and delete endpoint.\n",
+        "endpoint.delete(force=True)\n",
+        "\n",
+        "# Delete models.\n",
+        "model.delete()"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "model_garden_pytorch_layoutml_document_qa.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/community/model_garden/model_garden_pytorch_open_clip.ipynb

@@ -0,0 +1,677 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7d9bbf86da5e"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2023 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "99c1c3fc2ca5"
+      },
+      "source": [
+        "# Vertex AI Model Garden - OpenCLIP\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_open_clip.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_open_clip.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/notebooks/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/model_garden/model_garden_pytorch_open_clip.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "  </td>\n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3de7470326a2"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "This notebook demonstrates finetuning [OpenCLIP](https://github.com/mlfoundations/open_clip) with [CC3M](https://ai.google.com/research/ConceptualCaptions/download) dataset and deploying it on Vertex AI for online prediction.\n",
+        "\n",
+        "### Objective\n",
+        "\n",
+        "- Finetune the OpenCLIP model with [Vertex AI custom training](https://cloud.google.com/vertex-ai/docs/training/overview).\n",
+        "- Upload the model to [Vertex AI Model Registry](https://cloud.google.com/vertex-ai/docs/model-registry/introduction).\n",
+        "- Deploy the model to a [Vertex AI Endpoint resource](https://cloud.google.com/vertex-ai/docs/predictions/using-private-endpoints).\n",
+        "- Run online predictions for zero-shot image classification.\n",
+        "\n",
+        "### Costs\n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage pricing](https://cloud.google.com/storage/pricing), and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "264c07757582"
+      },
+      "source": [
+        "## Setup environment\n",
+        "\n",
+        "**NOTE**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d73ffa0c0b83"
+      },
+      "source": [
+        "### Colab only"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2707b02ef5df"
+      },
+      "outputs": [],
+      "source": [
+        "if \"google.colab\" in str(get_ipython()):\n",
+        "    ! pip3 install --upgrade google-cloud-aiplatform\n",
+        "    from google.colab import auth as google_auth\n",
+        "\n",
+        "    google_auth.authenticate_user()\n",
+        "\n",
+        "    # Restart the notebook kernel after installs.\n",
+        "    import IPython\n",
+        "\n",
+        "    app = IPython.Application.instance()\n",
+        "    app.kernel.do_shutdown(True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "bb7adab99e41"
+      },
+      "source": [
+        "### Setup Google Cloud project\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI API and Compute Engine API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component).\n",
+        "\n",
+        "1. [Create a Cloud Storage bucket](https://cloud.google.com/storage/docs/creating-buckets) for storing experiment outputs.\n",
+        "\n",
+        "1. [Create a service account](https://cloud.google.com/iam/docs/service-accounts-create#iam-service-accounts-create-console) with `Vertex AI User`,  `Storage Object Admin`, and `GCS Storage Bucket Owner roles` roles for deploying fine tuned model to Vertex AI endpoint."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6c460088b873"
+      },
+      "source": [
+        "Fill the following variables for experiments environment:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "a12c23679315"
+      },
+      "outputs": [],
+      "source": [
+        "# Cloud project id.\n",
+        "PROJECT_ID = \"\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# The region you want to launch jobs in.\n",
+        "REGION = \"\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# The GCS bucket for storing experiments output. Fill it without the 'gs://' prefix.\n",
+        "GCS_BUCKET = \"\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# The service account for deploying fine tuned model.\n",
+        "# The service account looks like:\n",
+        "# '<account_name>@<project>.iam.gserviceaccount.com'\n",
+        "# Follow step 5 above to create this account.\n",
+        "SERVICE_ACCOUNT = \"\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "12c249e14a5d"
+      },
+      "source": [
+        "### Download data to Google Cloud Storage"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "fb0259bfd059"
+      },
+      "outputs": [],
+      "source": [
+        "# Install the library for downloading training data.\n",
+        "!pip install img2dataset==1.41.0"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "1d6b99305e7e"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "!gcloud storage cp gs://gcc-data/Validation/GCC-1.1.0-Validation.tsv ./data.tsv  # Download list of URLs.\n",
+        "!sed -i '1s/^/caption\\turl\\n/' data.tsv  # Add column name."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "f36f82f74dde"
+      },
+      "outputs": [],
+      "source": [
+        "# Download images from URLs. It takes around 2min.\n",
+        "output_folder = \"data\"\n",
+        "!img2dataset --url_list data.tsv --input_format \"tsv\"\\\n",
+        "    --output_folder {output_folder}\\\n",
+        "    --url_col \"url\" --caption_col \"caption\" --output_format webdataset\\\n",
+        "    --processes_count {os.cpu_count()} --thread_count {os.cpu_count()*4}\\\n",
+        "    --image_size 256"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "514ee0208d47"
+      },
+      "outputs": [],
+      "source": [
+        "GCS_data_dir = f\"{GCS_BUCKET}/CC3M-val-wds\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b90bbd80476a"
+      },
+      "outputs": [],
+      "source": [
+        "# Upload data to GCS.\n",
+        "!gcloud storage cp -r data gs://{GCS_data_dir}"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7ffc8c1aed83"
+      },
+      "outputs": [],
+      "source": [
+        "# Get total number of samples, which is required by OpenCLIP training.\n",
+        "import json\n",
+        "\n",
+        "n_samples = 0\n",
+        "for filename in [f for f in os.listdir(output_folder) if f.endswith(\"_stats.json\")]:\n",
+        "    with open(os.path.join(output_folder, filename), \"r\") as f:\n",
+        "        n_samples += json.load(f)[\"successes\"]\n",
+        "print(n_samples)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e828eb320337"
+      },
+      "source": [
+        "### Initialize Vertex AI SDK for Python\n",
+        "\n",
+        "Initialize the Vertex AI SDK for Python for your project and corresponding bucket."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "12cd25839741"
+      },
+      "outputs": [],
+      "source": [
+        "from google.cloud import aiplatform\n",
+        "\n",
+        "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=GCS_BUCKET)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2cc825514deb"
+      },
+      "source": [
+        "### Define constants"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b42bd4fa2b2d"
+      },
+      "outputs": [],
+      "source": [
+        "# The pre-built training docker image. It contains training scripts and models.\n",
+        "TRAIN_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-open-clip-train\"\n",
+        "\n",
+        "# The pre-built serving docker image. It contains serving scripts and models.\n",
+        "SERVE_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-open-clip-serve\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0c250872074f"
+      },
+      "source": [
+        "### Define common functions for job operations and test data preparations"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "354da31189dc"
+      },
+      "outputs": [],
+      "source": [
+        "import base64\n",
+        "from datetime import datetime\n",
+        "from io import BytesIO\n",
+        "\n",
+        "import matplotlib.pyplot as plt\n",
+        "import requests\n",
+        "from PIL import Image\n",
+        "\n",
+        "\n",
+        "def create_job_name(prefix):\n",
+        "    \"\"\"Create a job name string with a prefix.\"\"\"\n",
+        "    user = os.environ.get(\"USER\")\n",
+        "    now = datetime.now().strftime(\"%Y%m%d_%H%M%S\")\n",
+        "    job_name = f\"{prefix}-{user}-{now}\"\n",
+        "    return job_name\n",
+        "\n",
+        "\n",
+        "def deploy_model(checkpoint, model, precision, task):\n",
+        "    \"\"\"Deploy a model to Vertex AI endpoint.\"\"\"\n",
+        "    model_name = \"openclip\"\n",
+        "    endpoint = aiplatform.Endpoint.create(display_name=f\"{model_name}-{task}-endpoint\")\n",
+        "    serving_env = {\n",
+        "        \"CHECKPOINT\": checkpoint,\n",
+        "        \"MODEL\": model,\n",
+        "        \"PRECISION\": precision,\n",
+        "        \"TASK\": task,\n",
+        "    }\n",
+        "    model = aiplatform.Model.upload(\n",
+        "        display_name=model_name,\n",
+        "        serving_container_image_uri=SERVE_DOCKER_URI,\n",
+        "        serving_container_ports=[7080],\n",
+        "        serving_container_predict_route=\"/predictions/transformers_serving\",\n",
+        "        serving_container_health_route=\"/ping\",\n",
+        "        serving_container_environment_variables=serving_env,\n",
+        "    )\n",
+        "    model.deploy(\n",
+        "        endpoint=endpoint,\n",
+        "        machine_type=\"n1-standard-4\",\n",
+        "        accelerator_type=\"NVIDIA_TESLA_V100\",\n",
+        "        accelerator_count=1,\n",
+        "        deploy_request_timeout=1800,\n",
+        "        service_account=SERVICE_ACCOUNT,\n",
+        "    )\n",
+        "    return model, endpoint\n",
+        "\n",
+        "\n",
+        "def download_image(url):\n",
+        "    response = requests.get(url)\n",
+        "    return Image.open(BytesIO(response.content)).convert(\"RGB\")\n",
+        "\n",
+        "\n",
+        "def image_to_base64(image):\n",
+        "    buffer = BytesIO()\n",
+        "    image.save(buffer, format=\"PNG\")\n",
+        "    image_str = base64.b64encode(buffer.getvalue()).decode(\"utf-8\")\n",
+        "    return image_str\n",
+        "\n",
+        "\n",
+        "def plot_images(images, rows=1, cols=None):\n",
+        "    fig, axes = plt.subplots(rows, cols if cols is not None else len(images))\n",
+        "    for ax, img in zip(axes, images):\n",
+        "        ax.imshow(img)\n",
+        "        ax.axis(\"off\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e70e3519ff8b"
+      },
+      "source": [
+        "## Fine tune the model\n",
+        "\n",
+        "This section fine tunes the OpenCLIP model with [CC3M](https://ai.google.com/research/ConceptualCaptions/download) dataset validation split for the image-text pre-training. It loads the pretrained checkpoint by OpenAI. You use a small model of ***RN50*** here; check [this list](https://github.com/mlfoundations/open_clip#pretrained-model-interface) for other options supported by OpenAI checkpoints, such as ***RN50, ViT-B-32, etc***.\n",
+        "\n",
+        "One `n1-standard-4` machine with 1 `NVIDIA_TESLA_V100` is required to run the fine-tuning job. The fine-tuning job takes about 3min to complete training for 2 epochs.\n",
+        "\n",
+        "The fine-tuned model will be saved after the job finishs and it can then be loaded for inference."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "55dabb1b02e3"
+      },
+      "outputs": [],
+      "source": [
+        "machine_type = \"n1-standard-4\"\n",
+        "gpu_type = \"NVIDIA_TESLA_V100\"\n",
+        "num_gpus = 1\n",
+        "\n",
+        "job_name = create_job_name(\"openclip\")\n",
+        "\n",
+        "model_name = \"RN50\"\n",
+        "precision = \"amp\"\n",
+        "\n",
+        "job = aiplatform.CustomContainerTrainingJob(\n",
+        "    display_name=job_name, container_uri=TRAIN_DOCKER_URI, command=[\"torchrun\"]\n",
+        ")\n",
+        "\n",
+        "job.run(\n",
+        "    args=[\n",
+        "        f\"--nproc_per_node={num_gpus}\",\n",
+        "        \"-m\",\n",
+        "        \"training.main\",\n",
+        "        f\"--name={job_name}\",\n",
+        "        f\"--logs=/gcs/{GCS_BUCKET}\",\n",
+        "        f\"--train-data=/gcs/{GCS_data_dir}/{{00000..00001}}.tar\",\n",
+        "        f\"--train-num-samples={n_samples}\",\n",
+        "        \"--dataset-type=webdataset\",\n",
+        "        \"--batch-size=32\",\n",
+        "        \"--precision=amp\",\n",
+        "        \"--workers=8\",\n",
+        "        \"--dataset-resampled\",\n",
+        "        \"--save-frequency=2\",\n",
+        "        \"--epochs=2\",\n",
+        "        f\"--model={model_name}\",\n",
+        "        \"--pretrained=openai\",\n",
+        "        \"--save-most-recent\",\n",
+        "    ],\n",
+        "    boot_disk_size_gb=600,\n",
+        "    replica_count=1,\n",
+        "    machine_type=machine_type,\n",
+        "    accelerator_type=gpu_type,\n",
+        "    accelerator_count=num_gpus,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "bf7f82732e61"
+      },
+      "source": [
+        "## Upload and Deploy models\n",
+        "\n",
+        "This section uploads the fine-tuned model to Model Registry and deploys it on the Endpoint.\n",
+        "\n",
+        "One `n1-standard-4` machine with 1 `NVIDIA_TESLA_V100` is required to deploy OpenCLIP model.\n",
+        "\n",
+        "The model deployment step will take ~20 minutes to complete."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6a2cf6e84b10"
+      },
+      "outputs": [],
+      "source": [
+        "# Prepare image samples\n",
+        "img_diagram = download_image(\n",
+        "    \"https://raw.githubusercontent.com/mlfoundations/open_clip/main/docs/CLIP.png\"\n",
+        ")\n",
+        "img_cat = download_image(\n",
+        "    \"https://upload.wikimedia.org/wikipedia/commons/thumb/9/9d/Calicocats2222.jpg/220px-Calicocats2222.jpg\"\n",
+        ")\n",
+        "plot_images([img_diagram, img_cat])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5d63ffdd5f09"
+      },
+      "source": [
+        "#### Zero-shot image classification"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "bf55e38815dc"
+      },
+      "outputs": [],
+      "source": [
+        "model, endpoint = deploy_model(\n",
+        "    checkpoint=f\"gs://{GCS_BUCKET}/{job_name}/checkpoints/epoch_latest.pt\",\n",
+        "    model=model_name,\n",
+        "    precision=precision,\n",
+        "    task=\"zero-shot-image-classification\",\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "80b3fd2ace09"
+      },
+      "source": [
+        "NOTE: The model weights are downloaded after the deployment succeeds. An additional 5 minutes of waiting time is needed **after** the above model deployment step succeeds and before you run the next step below. Otherwise you might see a `ServiceUnavailable: 503 502:Bad Gateway` error when you send requests to the endpoint."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "002f438ecfea"
+      },
+      "outputs": [],
+      "source": [
+        "instances = [\n",
+        "    {\n",
+        "        \"text\": [\"a diagram\", \"a dog\", \"a cat\"],\n",
+        "        \"image\": image_to_base64(img_diagram),\n",
+        "    },\n",
+        "    {\n",
+        "        \"text\": [\"a diagram\", \"a dog\", \"two cats\", \"calico cat\"],\n",
+        "        \"image\": image_to_base64(img_cat),\n",
+        "    },\n",
+        "]\n",
+        "response = endpoint.predict(instances=instances).predictions\n",
+        "response"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "af21a3cff1e0"
+      },
+      "source": [
+        "Clean up resources:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "911406c1561e"
+      },
+      "outputs": [],
+      "source": [
+        "# Undeploy model and delete endpoint.\n",
+        "endpoint.delete(force=True)\n",
+        "\n",
+        "# Delete models.\n",
+        "model.delete()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "4fb87c6aeecf"
+      },
+      "source": [
+        "#### Image/text feature embedding"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "bf55e38815dc"
+      },
+      "outputs": [],
+      "source": [
+        "model, endpoint = deploy_model(\n",
+        "    checkpoint=f\"gs://{GCS_BUCKET}/{job_name}/checkpoints/epoch_latest.pt\",\n",
+        "    model=model_name,\n",
+        "    precision=precision,\n",
+        "    task=\"feature-embedding\",\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "80b3fd2ace09"
+      },
+      "source": [
+        "NOTE: The model weights are downloaded after the deployment succeeds. An additional 5 minutes of waiting time is needed **after** the above model deployment step succeeds and before you run the next step below. Otherwise you might see a `ServiceUnavailable: 503 502:Bad Gateway` error when you send requests to the endpoint."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "56d5c001575d"
+      },
+      "outputs": [],
+      "source": [
+        "import numpy as np\n",
+        "\n",
+        "instances = [\n",
+        "    {\n",
+        "        \"text\": [\"a diagram\", \"a dog\", \"a cat\"],\n",
+        "        \"image\": image_to_base64(img_diagram),\n",
+        "    },\n",
+        "    {\n",
+        "        \"image\": image_to_base64(img_cat),\n",
+        "    },\n",
+        "    {\n",
+        "        \"text\": [\"a diagram\", \"a dog\", \"two cats\", \"calico cat\"],\n",
+        "    },\n",
+        "    {\n",
+        "        \"text\": \"a single value\",\n",
+        "    },\n",
+        "]\n",
+        "response = endpoint.predict(instances=instances).predictions\n",
+        "for pred in response:\n",
+        "    for k, v in pred.items():\n",
+        "        print(k, np.array(v).shape)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "af21a3cff1e0"
+      },
+      "source": [
+        "Clean up resources:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "911406c1561e"
+      },
+      "outputs": [],
+      "source": [
+        "# Undeploy model and delete endpoint.\n",
+        "endpoint.delete(force=True)\n",
+        "\n",
+        "# Delete models.\n",
+        "model.delete()"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "model_garden_pytorch_open_clip.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/community/model_garden/model_garden_pytorch_openllama_peft.ipynb

@@ -0,0 +1,610 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7d9bbf86da5e"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2023 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "99c1c3fc2ca5"
+      },
+      "source": [
+        "# Vertex AI Model Garden - OpenLLaMA (PEFT)\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_openllama_peft.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_openllama_peft.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/notebooks/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/model_garden/model_garden_pytorch_openllama_peft.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "Open in Vertex AI Workbench\n",
+        "    </a> (A Python-3 CPU notebook is recommended)\n",
+        "  </td>\n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3de7470326a2"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "This notebook demonstrates deploying prebuilt OpenLLaMA, and also finetuning and deploying OpenLLaMA with performance efficient finetuning libraries ([PEFT](https://github.com/huggingface/peft)) in Vertex AI.\n",
+        "\n",
+        "### Objective\n",
+        "\n",
+        "- Deploy prebuilt OpenLLaMA\n",
+        "- Finetune and deploy OpenLLaMA with PEFT, supporting\n",
+        "\n",
+        "| Models | LoRA |\n",
+        "| :- | :- |\n",
+        "| [openlm-research/open_llama_3b](https://huggingface.co/openlm-research/open_llama_3b) | Y |\n",
+        "| [openlm-research/open_llama_7b](https://huggingface.co/openlm-research/open_llama_7b) | Y |\n",
+        "| [openlm-research/open_llama_13b](https://huggingface.co/openlm-research/open_llama_13b) | Y |\n",
+        "\n",
+        "### Costs\n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage pricing](https://cloud.google.com/storage/pricing), and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "264c07757582"
+      },
+      "source": [
+        "## Before you begin\n",
+        "\n",
+        "**NOTE**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ioensNKM8ned"
+      },
+      "source": [
+        "### Colab only\n",
+        "Run the following commands for Colab and skip this section if you are using Workbench."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2707b02ef5df"
+      },
+      "outputs": [],
+      "source": [
+        "import sys\n",
+        "\n",
+        "if \"google.colab\" in sys.modules:\n",
+        "    ! pip3 install --upgrade google-cloud-aiplatform\n",
+        "    from google.colab import auth as google_auth\n",
+        "\n",
+        "    google_auth.authenticate_user()\n",
+        "    # Install gdown for downloading example training images.\n",
+        "    ! pip3 install gdown\n",
+        "\n",
+        "    # Restart the notebook kernel after installs.\n",
+        "    import IPython\n",
+        "\n",
+        "    app = IPython.Application.instance()\n",
+        "    app.kernel.do_shutdown(True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "bb7adab99e41"
+      },
+      "source": [
+        "### Setup Google Cloud project\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI API and Compute Engine API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component).\n",
+        "\n",
+        "1. [Create a Cloud Storage bucket](https://cloud.google.com/storage/docs/creating-buckets) for storing experiment outputs.\n",
+        "\n",
+        "1. [Create a service account](https://cloud.google.com/iam/docs/service-accounts-create#iam-service-accounts-create-console) with `Vertex AI User` and `Storage Object Admin` roles for deploying fine tuned model to Vertex AI endpoint."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6c460088b873"
+      },
+      "source": [
+        "Fill following variables for experiments environment:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "855d6b96f291"
+      },
+      "outputs": [],
+      "source": [
+        "# Cloud project id.\n",
+        "PROJECT_ID = \"\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# The region you want to launch jobs in.\n",
+        "REGION = \"\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# The Cloud Storage bucket for storing experiments output.\n",
+        "BUCKET_URI = \"\"  # @param {type:\"string\"}\n",
+        "\n",
+        "! gcloud config set project $PROJECT_ID\n",
+        "\n",
+        "import os\n",
+        "\n",
+        "STAGING_BUCKET = os.path.join(BUCKET_URI, \"temporal\")\n",
+        "EXPERIMENT_BUCKET = os.path.join(BUCKET_URI, \"peft\")\n",
+        "DATA_BUCKET = os.path.join(EXPERIMENT_BUCKET, \"data\")\n",
+        "MODEL_BUCKET = os.path.join(EXPERIMENT_BUCKET, \"model\")\n",
+        "\n",
+        "# The service account looks like:\n",
+        "# '@.iam.gserviceaccount.com'\n",
+        "# Please go to https://cloud.google.com/iam/docs/service-accounts-create#iam-service-accounts-create-console\n",
+        "# and create service account with `Vertex AI User` and `Storage Object Admin` roles.\n",
+        "# The service account for deploying fine tuned model.\n",
+        "SERVICE_ACCOUNT = \"\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e828eb320337"
+      },
+      "source": [
+        "### Initialize Vertex AI API"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "12cd25839741"
+      },
+      "outputs": [],
+      "source": [
+        "from google.cloud import aiplatform\n",
+        "\n",
+        "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=STAGING_BUCKET)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2cc825514deb"
+      },
+      "source": [
+        "### Define constants"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b42bd4fa2b2d"
+      },
+      "outputs": [],
+      "source": [
+        "# The pre-built training and serving docker images.\n",
+        "TRAIN_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai-restricted/vertex-vision-model-garden-dockers/pytorch-peft-train\"\n",
+        "PREDICTION_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai-restricted/vertex-vision-model-garden-dockers/pytorch-peft-serve\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0c250872074f"
+      },
+      "source": [
+        "### Define common functions"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "354da31189dc"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "from datetime import datetime\n",
+        "\n",
+        "from google.cloud import aiplatform\n",
+        "\n",
+        "\n",
+        "def get_job_name_with_datetime(prefix: str):\n",
+        "    \"\"\"Gets the job name with date time when triggering training or deployment\n",
+        "    jobs in Vertex AI.\n",
+        "    \"\"\"\n",
+        "    return prefix + datetime.now().strftime(\"_%Y%m%d_%H%M%S\")\n",
+        "\n",
+        "\n",
+        "def deploy_model(\n",
+        "    model_name,\n",
+        "    base_model_id,\n",
+        "    finetuned_lora_model_path,\n",
+        "    service_account,\n",
+        "    task,\n",
+        "    machine_type=\"n1-standard-8\",\n",
+        "    accelerator_type=\"NVIDIA_TESLA_V100\",\n",
+        "):\n",
+        "    \"\"\"Deploys trained models into Vertex AI.\"\"\"\n",
+        "    endpoint = aiplatform.Endpoint.create(display_name=f\"{model_name}-endpoint\")\n",
+        "    serving_env = {\n",
+        "        \"BASE_MODEL_ID\": base_model_id,\n",
+        "        \"TASK\": task,\n",
+        "    }\n",
+        "    if finetuned_lora_model_path:\n",
+        "        serving_env[\"FINETUNED_LORA_MODEL_PATH\"] = finetuned_lora_model_path\n",
+        "    model = aiplatform.Model.upload(\n",
+        "        display_name=model_name,\n",
+        "        serving_container_image_uri=PREDICTION_DOCKER_URI,\n",
+        "        serving_container_ports=[7080],\n",
+        "        serving_container_predict_route=\"/predictions/peft_serving\",\n",
+        "        serving_container_health_route=\"/ping\",\n",
+        "        serving_container_environment_variables=serving_env,\n",
+        "    )\n",
+        "    model.deploy(\n",
+        "        endpoint=endpoint,\n",
+        "        machine_type=machine_type,\n",
+        "        accelerator_type=accelerator_type,\n",
+        "        accelerator_count=1,\n",
+        "        deploy_request_timeout=1800,\n",
+        "        service_account=service_account,\n",
+        "    )\n",
+        "    return model, endpoint"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8neJc8CnDDpu"
+      },
+      "source": [
+        "## Deploy Prebuilt OpenLLaMA\n",
+        "\n",
+        "This section deploys prebuilt OpenLLaMA models on the Endpoint. The model deployment step will take ~15 minutes to complete.\n",
+        "\n",
+        "The peak GPU memory usages for [openlm-research/open_llama_3b](https://huggingface.co/openlm-research/open_llama_3b), [openlm-research/open_llama_7b](https://huggingface.co/openlm-research/open_llama_7b), and [openlm-research/open_llama_13b](https://huggingface.co/openlm-research/open_llama_13b) are ~5.3G, ~8.7G and ~15.2G separately with the default settings. We use V100 in deployments for simplicity."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2MjaORIIFDVu"
+      },
+      "source": [
+        "Set the prebuilt model id."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "E8OiHHNNE_wj"
+      },
+      "outputs": [],
+      "source": [
+        "prebuilt_model_id = \"openlm-research/open_llama_3b\"  # @param [\"openlm-research/open_llama_3b\", \"openlm-research/open_llama_7b\", \"openlm-research/open_llama_13b\"]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dHFW7yvjaVFV"
+      },
+      "source": [
+        "We use the PEFT serving images to deploy prebuilt OpenLLaMA models, by setting finetuning LoRA model paths as empty."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Uak1pyEeExYM"
+      },
+      "outputs": [],
+      "source": [
+        "model_without_peft, endpoint_without_peft = deploy_model(\n",
+        "    model_name=get_job_name_with_datetime(prefix=\"openllama-serve\"),\n",
+        "    base_model_id=prebuilt_model_id,\n",
+        "    finetuned_lora_model_path=\"\",  # This will avoid override finetuning models.\n",
+        "    service_account=SERVICE_ACCOUNT,\n",
+        "    task=\"causal-language-modeling-lora\",\n",
+        ")\n",
+        "print(\"endpoint_name:\", endpoint_without_peft.name)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "sGKIjgmDFRW2"
+      },
+      "source": [
+        "NOTE: The prebuilt model weights will be downloaded on the fly from the orginal location after the deployment succeeds. Thus additional 5 minutes of waiting time is needed **after** the above model deployment step succeeds and before you run the next step below. Otherwise you might see a `ServiceUnavailable: 503 502:Bad Gateway` error when you send requests to the endpoint.\n",
+        "\n",
+        "Once deployment succeeds, you can send requests to the endpoint with text prompts."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "rDHsCOqvFYBi"
+      },
+      "outputs": [],
+      "source": [
+        "# # Loads an existing endpoint as below.\n",
+        "# endpoint_name = endpoint_without_peft.name\n",
+        "# aip_endpoint_name = (\n",
+        "#     f\"projects/{PROJECT_ID}/locations/{REGION}/endpoints/{endpoint_name}\"\n",
+        "# )\n",
+        "# endpoint_without_peft = aiplatform.Endpoint(aip_endpoint_name)\n",
+        "instances = [\n",
+        "    {\"prompt\": \"Hi, Google.\"},\n",
+        "]\n",
+        "response = endpoint_without_peft.predict(instances=instances)\n",
+        "\n",
+        "for prediction in response.predictions[0]:\n",
+        "    print(prediction[\"generated_text\"])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e70e3519ff8b"
+      },
+      "source": [
+        "## Finetune and deploy OpenLLaMA with PEFT\n",
+        "\n",
+        "This section demonstrates how to finetune and dpeloy OpenLLaMA with PEFT LoRA."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5qCrm_kJH5cz"
+      },
+      "source": [
+        "Set the base model id."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "N3UBLiYrM3sU"
+      },
+      "outputs": [],
+      "source": [
+        "base_model_id = \"openlm-research/open_llama_3b\"  # @param [\"openlm-research/open_llama_3b\", \"openlm-research/open_llama_7b\", \"openlm-research/open_llama_13b\"]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "iWGwJHqI7LMs"
+      },
+      "source": [
+        "### Finetune"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "KKEYoRfiHDVv"
+      },
+      "source": [
+        "Use the Vertex AI SDK to create and run the custom training jobs with Vertex AI Model Garden training images.\n",
+        "\n",
+        "This example uses the dataset [Abirate/english_quotes](https://huggingface.co/datasets/Abirate/english_quotes).\n",
+        "\n",
+        "In order to make the finetuning efficiently, we enabled quantization (8bits) when loading pretrained models for finetuning LoRA models. The peak GPU memory usages are ~7G, ~10G and ~16G for finetuning LoRA models for [openlm-research/open_llama_3b](https://huggingface.co/openlm-research/open_llama_3b), [openlm-research/open_llama_7b](https://huggingface.co/openlm-research/open_llama_7b), and [openlm-research/open_llama_13b](https://huggingface.co/openlm-research/open_llama_13b) separately with default training parameters and the example dataset. In theory, open_llama_3b and open_llama_7b can be finetuned on 1 V100, and open_llama_13b can be finetuned on 1 A100 (40G). We choose to use 1 A100 (40G) by default to support all these models in this notebook for simplicity."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "65467b361315"
+      },
+      "outputs": [],
+      "source": [
+        "dataset_name = \"Abirate/english_quotes\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# Worker pool spec.\n",
+        "# machine_type = \"n1-standard-8\"\n",
+        "# accelerator_type = \"NVIDIA_TESLA_V100\"\n",
+        "machine_type = \"a2-highgpu-1g\"\n",
+        "accelerator_type = \"NVIDIA_TESLA_A100\"\n",
+        "replica_count = 1\n",
+        "accelerator_count = 1\n",
+        "\n",
+        "# Setup training job.\n",
+        "job_name = get_job_name_with_datetime(\"openllama-lora-train\")\n",
+        "train_job = aiplatform.CustomContainerTrainingJob(\n",
+        "    display_name=job_name,\n",
+        "    container_uri=TRAIN_DOCKER_URI,\n",
+        ")\n",
+        "output_dir = os.path.join(MODEL_BUCKET, job_name)\n",
+        "output_dir_gcsfuse = output_dir.replace(\"gs://\", \"/gcs/\")\n",
+        "\n",
+        "# Pass training arguments and launch job.\n",
+        "train_job.run(\n",
+        "    args=[\n",
+        "        \"--task=causal-language-modeling-lora\",\n",
+        "        f\"--pretrained_model_id={base_model_id}\",\n",
+        "        f\"--dataset_name={dataset_name}\",\n",
+        "        f\"--output_dir={output_dir_gcsfuse}\",\n",
+        "        \"--lora_rank=16\",\n",
+        "        \"--lora_alpha=32\",\n",
+        "        \"--lora_dropout=0.05\",\n",
+        "        \"--warmup_steps=10\",\n",
+        "        \"--max_steps=10\",\n",
+        "        \"--learning_rate=2e-4\",\n",
+        "    ],\n",
+        "    replica_count=replica_count,\n",
+        "    machine_type=machine_type,\n",
+        "    accelerator_type=accelerator_type,\n",
+        "    accelerator_count=accelerator_count,\n",
+        "    boot_disk_size_gb=500,\n",
+        ")\n",
+        "\n",
+        "print(\"Trained models were saved in: \", output_dir)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "jqmCtkGnhDmp"
+      },
+      "source": [
+        "### Deploy\n",
+        "This section uploads the model to Model Registry and deploys it on the Endpoint.\n",
+        "\n",
+        "The model deployment step will take ~15 minutes to complete.\n",
+        "\n",
+        "The peak GPU memory usages for [openlm-research/open_llama_3b](https://huggingface.co/openlm-research/open_llama_3b), [openlm-research/open_llama_7b](https://huggingface.co/openlm-research/open_llama_7b), and [openlm-research/open_llama_13b](https://huggingface.co/openlm-research/open_llama_13b) with LoRA weights are ~5.3G, ~8.7G and ~15.2G separately with the default settings. We use V100 in deployments for simplicity."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "bf55e38815dc"
+      },
+      "outputs": [],
+      "source": [
+        "model_with_peft, endpoint_with_peft = deploy_model(\n",
+        "    model_name=get_job_name_with_datetime(prefix=\"openllama-peft-serve\"),\n",
+        "    base_model_id=base_model_id,\n",
+        "    finetuned_lora_model_path=output_dir,\n",
+        "    service_account=SERVICE_ACCOUNT,\n",
+        "    task=\"causal-language-modeling-lora\",\n",
+        ")\n",
+        "print(\"endpoint_name:\", endpoint_with_peft.name)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "80b3fd2ace09"
+      },
+      "source": [
+        "NOTE: After the deployment succeeds, the base model weights will be downloaded one the fly from the original location and LoRA model weights will be downloaded from the GCS bucket used in training above. Thus additional 5 minutes of waiting time is needed **after** the above model deployment step succeeds and before you run the next step below. Otherwise you might see a `ServiceUnavailable: 503 502:Bad Gateway` error when you send requests to the endpoint.\n",
+        "\n",
+        "Once deployment succeeds, you can send requests to the endpoint with text prompts."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "4ab04da3ec9a"
+      },
+      "outputs": [],
+      "source": [
+        "# # Loads an existing endpoint as below.\n",
+        "# endpoint_name = endpoint_with_peft.name\n",
+        "# aip_endpoint_name = (\n",
+        "#     f\"projects/{PROJECT_ID}/locations/{REGION}/endpoints/{endpoint_name}\"\n",
+        "# )\n",
+        "# endpoint_with_peft = aiplatform.Endpoint(aip_endpoint_name)\n",
+        "instances = [\n",
+        "    {\"prompt\": \"Hi, Google.\"},\n",
+        "]\n",
+        "response = endpoint_with_peft.predict(instances=instances)\n",
+        "\n",
+        "for prediction in response.predictions[0]:\n",
+        "    print(prediction[\"generated_text\"])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "af21a3cff1e0"
+      },
+      "source": [
+        "## Clean up resources"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "911406c1561e"
+      },
+      "outputs": [],
+      "source": [
+        "# Delete custom train jobs.\n",
+        "train_job.delete()\n",
+        "\n",
+        "# Undeploy model and delete endpoint.\n",
+        "endpoint_without_peft.delete(force=True)\n",
+        "endpoint_with_peft.delete(force=True)\n",
+        "\n",
+        "# Delete models.\n",
+        "model_without_peft.delete()\n",
+        "model_with_peft.delete()"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "model_garden_pytorch_openllama_peft.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/community/model_garden/model_garden_pytorch_owlvit.ipynb

@@ -0,0 +1,415 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7d9bbf86da5e"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2023 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2bd716bf3e39"
+      },
+      "source": [
+        "# Vertex AI Model Garden - OWL-ViT\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_owlvit.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_owlvit.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/notebooks/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/model_garden/model_garden_pytorch_owlvit.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "    (a Python-3 CPU notebook is recommended)\n",
+        "  </td>\n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d8cd12648da4"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "This notebook demonstrates deploying the pre-trained [OWL-ViT](https://huggingface.co/google/owlvit-base-patch32) model on Vertex AI for online prediction.\n",
+        "\n",
+        "### Objective\n",
+        "\n",
+        "- Upload the model to [Model Registry](https://cloud.google.com/vertex-ai/docs/model-registry/introduction).\n",
+        "- Deploy the model on [Endpoint](https://cloud.google.com/vertex-ai/docs/predictions/using-private-endpoints).\n",
+        "- Run online predictions for image captioning.\n",
+        "\n",
+        "### Costs\n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage pricing](https://cloud.google.com/storage/pricing), and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "264c07757582"
+      },
+      "source": [
+        "## Setup environment\n",
+        "\n",
+        "**NOTE**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d73ffa0c0b83"
+      },
+      "source": [
+        "### Colab only"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2707b02ef5df"
+      },
+      "outputs": [],
+      "source": [
+        "if \"google.colab\" in str(get_ipython()):\n",
+        "    ! pip3 install --upgrade google-cloud-aiplatform\n",
+        "    from google.colab import auth as google_auth\n",
+        "\n",
+        "    google_auth.authenticate_user()\n",
+        "\n",
+        "    # Restart the notebook kernel after installs.\n",
+        "    import IPython\n",
+        "\n",
+        "    app = IPython.Application.instance()\n",
+        "    app.kernel.do_shutdown(True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0f826ff482a2"
+      },
+      "source": [
+        "### Setup Google Cloud project\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI API and Compute Engine API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component).\n",
+        "\n",
+        "1. [Create a Cloud Storage bucket](https://cloud.google.com/storage/docs/creating-buckets) for storing experiment outputs."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8958ebc71868"
+      },
+      "source": [
+        "Fill following variables for experiments environment:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9db30f827a65"
+      },
+      "outputs": [],
+      "source": [
+        "# Cloud project id.\n",
+        "PROJECT_ID = \"\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# The region you want to launch jobs in.\n",
+        "REGION = \"us-central1\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# The Cloud Storage bucket for storing experiments output. Fill it without the 'gs://' prefix.\n",
+        "GCS_BUCKET = \"\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "92f16e22c20b"
+      },
+      "source": [
+        "Initialize Vertex AI API:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "1680c257acfb"
+      },
+      "outputs": [],
+      "source": [
+        "from google.cloud import aiplatform\n",
+        "\n",
+        "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=GCS_BUCKET)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6ca48b699d17"
+      },
+      "source": [
+        "### Define constants"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "de9882ea89ea"
+      },
+      "outputs": [],
+      "source": [
+        "# The pre-built serving docker image. It contains serving scripts and models.\n",
+        "SERVE_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-transformers-serve\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "10188266a5cd"
+      },
+      "source": [
+        "### Define common functions"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cac4478ae098"
+      },
+      "outputs": [],
+      "source": [
+        "import base64\n",
+        "import os\n",
+        "from datetime import datetime\n",
+        "from io import BytesIO\n",
+        "\n",
+        "import matplotlib.patches as patches\n",
+        "import matplotlib.pyplot as plt\n",
+        "import requests\n",
+        "from google.cloud import aiplatform\n",
+        "from PIL import Image\n",
+        "\n",
+        "\n",
+        "def create_job_name(prefix):\n",
+        "    user = os.environ.get(\"USER\")\n",
+        "    now = datetime.now().strftime(\"%Y%m%d_%H%M%S\")\n",
+        "    job_name = f\"{prefix}-{user}-{now}\"\n",
+        "    return job_name\n",
+        "\n",
+        "\n",
+        "def download_image(url):\n",
+        "    response = requests.get(url)\n",
+        "    return Image.open(BytesIO(response.content))\n",
+        "\n",
+        "\n",
+        "def image_to_base64(image, format=\"JPEG\"):\n",
+        "    buffer = BytesIO()\n",
+        "    image.save(buffer, format=format)\n",
+        "    image_str = base64.b64encode(buffer.getvalue()).decode(\"utf-8\")\n",
+        "    return image_str\n",
+        "\n",
+        "\n",
+        "def base64_to_image(image_str):\n",
+        "    image = Image.open(BytesIO(base64.b64decode(image_str)))\n",
+        "    return image\n",
+        "\n",
+        "\n",
+        "def image_grid(imgs, rows=2, cols=2):\n",
+        "    w, h = imgs[0].size\n",
+        "    grid = Image.new(\"RGB\", size=(cols * w, rows * h))\n",
+        "    for i, img in enumerate(imgs):\n",
+        "        grid.paste(img, box=(i % cols * w, i // cols * h))\n",
+        "    return grid\n",
+        "\n",
+        "\n",
+        "def draw_image_with_boxes(image, boxes):\n",
+        "    fig, ax = plt.subplots()\n",
+        "    plt.axis(\"off\")\n",
+        "    ax.imshow(image)\n",
+        "    if len(boxes) == 0:\n",
+        "        return\n",
+        "    boxes = boxes[\"boxes\"]\n",
+        "    for box in boxes:\n",
+        "        x, y = box[\"xmin\"], box[\"ymin\"]\n",
+        "        width, height = box[\"xmax\"] - x, box[\"ymax\"] - y\n",
+        "        rect = patches.Rectangle(\n",
+        "            (x, y), width, height, linewidth=2, edgecolor=\"yellow\", facecolor=\"none\"\n",
+        "        )\n",
+        "        ax.add_patch(rect)\n",
+        "    plt.show()\n",
+        "\n",
+        "\n",
+        "def deploy_model(model_id, task):\n",
+        "    model_name = \"owl-vit\"\n",
+        "    endpoint = aiplatform.Endpoint.create(display_name=f\"{model_name}-endpoint\")\n",
+        "    serving_env = {\n",
+        "        \"MODEL_ID\": model_id,\n",
+        "        \"TASK\": task,\n",
+        "    }\n",
+        "    # If the model_id is a GCS path, use artifact_uri to pass it to serving docker.\n",
+        "    artifact_uri = model_id if model_id.startswith(\"gs://\") else None\n",
+        "    model = aiplatform.Model.upload(\n",
+        "        display_name=model_name,\n",
+        "        serving_container_image_uri=SERVE_DOCKER_URI,\n",
+        "        serving_container_ports=[7080],\n",
+        "        serving_container_predict_route=\"/predictions/transformers_serving\",\n",
+        "        serving_container_health_route=\"/ping\",\n",
+        "        serving_container_environment_variables=serving_env,\n",
+        "        artifact_uri=artifact_uri,\n",
+        "    )\n",
+        "    model.deploy(\n",
+        "        endpoint=endpoint,\n",
+        "        machine_type=\"n1-standard-8\",\n",
+        "        accelerator_type=\"NVIDIA_TESLA_T4\",\n",
+        "        accelerator_count=1,\n",
+        "        deploy_request_timeout=1800,\n",
+        "    )\n",
+        "    return model, endpoint"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d2d72ecdb8c9"
+      },
+      "source": [
+        "## Upload and deploy models"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9448c5f545fa"
+      },
+      "source": [
+        "This section uploads the pre-trained model to Model Registry and deploys it on the Endpoint with 1 T4 GPU.\n",
+        "\n",
+        "The model deployment step will take ~15 minutes to complete.\n",
+        "\n",
+        "Once deployed, you can send images and object texts to get bounding boxes."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b4b46c28d8b1"
+      },
+      "outputs": [],
+      "source": [
+        "model, endpoint = deploy_model(\n",
+        "    model_id=\"google/owlvit-base-patch32\", task=\"zero-shot-object-detection\"\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "80b3fd2ace09"
+      },
+      "source": [
+        "NOTE: The model weights will be downloaded after the deployment succeeds. Thus additional 5 minutes of waiting time is needed **after** the above model deployment step succeeds and before you run the next step below. Otherwise you might see a `ServiceUnavailable: 503 502:Bad Gateway` error when you send requests to the endpoint."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6be655247cb1"
+      },
+      "outputs": [],
+      "source": [
+        "image = download_image(\"http://images.cocodataset.org/val2017/000000039769.jpg\")\n",
+        "\n",
+        "instances = [\n",
+        "    {\"image\": image_to_base64(image), \"text\": \"cat\"},\n",
+        "]\n",
+        "preds = endpoint.predict(instances=instances).predictions\n",
+        "draw_image_with_boxes(image, preds[0])\n",
+        "print(preds)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "db7ffebdb4be"
+      },
+      "source": [
+        "### Clean up resources"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2ccf3714dbe9"
+      },
+      "outputs": [],
+      "source": [
+        "# Undeploy model and delete endpoint.\n",
+        "endpoint.delete(force=True)\n",
+        "\n",
+        "# Delete models.\n",
+        "model.delete()"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "model_garden_pytorch_owlvit.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/community/model_garden/model_garden_pytorch_peft.ipynb

@@ -0,0 +1,953 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7d9bbf86da5e"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2023 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "99c1c3fc2ca5"
+      },
+      "source": [
+        "# Vertex AI Model Garden - PEFT\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_peft.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_peft.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/notebooks/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/model_garden/model_garden_pytorch_peft.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "Open in Vertex AI Workbench\n",
+        "    </a> (A Python-3 CPU notebook is recommended)\n",
+        "  </td>\n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3de7470326a2"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "This notebook demonstrates finetuning models with performance efficient finetuning libaries ([PEFT](https://github.com/huggingface/peft)), and running inferences with containers in Vertex AI. There are various models supported in [PEFT](https://github.com/huggingface/peft). This notebook shows examples with some models, such as OpenLLaMA, Falcon-instruct, BERT, RoBERTa-large, and XLM-RoBERTa-large, etc.\n",
+        "\n",
+        "### Objective\n",
+        "\n",
+        "- Finetune causal language models with PEFT and run inferences with Vertex AI, supporting\n",
+        "\n",
+        "| Models | LoRA |\n",
+        "| :- | :- |\n",
+        "| [openlm-research/open_llama_3b](https://huggingface.co/openlm-research/open_llama_3b) | Y |\n",
+        "| [openlm-research/open_llama_7b](https://huggingface.co/openlm-research/open_llama_7b) | Y |\n",
+        "| [openlm-research/open_llama_13b](https://huggingface.co/openlm-research/open_llama_13b) | Y |\n",
+        "\n",
+        "- Finetune instruct models with PEFT and run inferences with Vertex AI, supporting\n",
+        "\n",
+        "| Models | LoRA |\n",
+        "| :- | :- |\n",
+        "| [tiiuae/falcon-7b-instruct](https://huggingface.co/tiiuae/falcon-7b-instruct) | Y |\n",
+        "| [tiiuae/falcon-40b-instruct](https://huggingface.co/tiiuae/falcon-40b-instruct) | Y |\n",
+        "\n",
+        "\n",
+        "- Finetune sequence classification models with PEFT and run inferences with Vertex AI, supporting\n",
+        "\n",
+        "| Models | LoRA |\n",
+        "| :- | :- |\n",
+        "| [bert-base-uncase](https://huggingface.co/bert-base-uncased) | Y |\n",
+        "| [RoBERTa-large](https://huggingface.co/roberta-large) | Y |\n",
+        "| [XLM-RoBERTa-large](https://huggingface.co/xlm-roberta-large) | Y |\n",
+        "\n",
+        "### Costs\n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage pricing](https://cloud.google.com/storage/pricing), and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "264c07757582"
+      },
+      "source": [
+        "## Before you begin\n",
+        "\n",
+        "**NOTE**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ioensNKM8ned"
+      },
+      "source": [
+        "### Colab only\n",
+        "Run the following commands for Colab and skip this section if you are using Workbench."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2707b02ef5df"
+      },
+      "outputs": [],
+      "source": [
+        "import sys\n",
+        "\n",
+        "if \"google.colab\" in sys.modules:\n",
+        "    ! pip3 install --upgrade google-cloud-aiplatform\n",
+        "    from google.colab import auth as google_auth\n",
+        "\n",
+        "    google_auth.authenticate_user()\n",
+        "    # Install gdown for downloading example training images.\n",
+        "    ! pip3 install gdown\n",
+        "\n",
+        "    # Restart the notebook kernel after installs.\n",
+        "    import IPython\n",
+        "\n",
+        "    app = IPython.Application.instance()\n",
+        "    app.kernel.do_shutdown(True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "bb7adab99e41"
+      },
+      "source": [
+        "### Setup Google Cloud project\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI API and Compute Engine API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component).\n",
+        "\n",
+        "1. [Create a Cloud Storage bucket](https://cloud.google.com/storage/docs/creating-buckets) for storing experiment outputs.\n",
+        "\n",
+        "1. [Create a service account](https://cloud.google.com/iam/docs/service-accounts-create#iam-service-accounts-create-console) with `Vertex AI User` and `Storage Object Admin` roles for deploying fine tuned model to Vertex AI endpoint."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6c460088b873"
+      },
+      "source": [
+        "Fill following variables for experiments environment:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "855d6b96f291"
+      },
+      "outputs": [],
+      "source": [
+        "# Cloud project id.\n",
+        "PROJECT_ID = \"\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# The region you want to launch jobs in.\n",
+        "REGION = \"\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# The Cloud Storage bucket for storing experiments output.\n",
+        "BUCKET_URI = \"\"  # @param {type:\"string\"}\n",
+        "\n",
+        "! gcloud config set project $PROJECT_ID\n",
+        "\n",
+        "import os\n",
+        "\n",
+        "STAGING_BUCKET = os.path.join(BUCKET_URI, \"temporal\")\n",
+        "EXPERIMENT_BUCKET = os.path.join(BUCKET_URI, \"peft\")\n",
+        "DATA_BUCKET = os.path.join(EXPERIMENT_BUCKET, \"data\")\n",
+        "MODEL_BUCKET = os.path.join(EXPERIMENT_BUCKET, \"model\")\n",
+        "\n",
+        "# The service account looks like:\n",
+        "# '@.iam.gserviceaccount.com'\n",
+        "# Please go to https://cloud.google.com/iam/docs/service-accounts-create#iam-service-accounts-create-console\n",
+        "# and create service account with `Vertex AI User` and `Storage Object Admin` roles.\n",
+        "# The service account for deploying fine tuned model.\n",
+        "SERVICE_ACCOUNT = \"\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e828eb320337"
+      },
+      "source": [
+        "### Initialize Vertex AI API"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "12cd25839741"
+      },
+      "outputs": [],
+      "source": [
+        "from google.cloud import aiplatform\n",
+        "\n",
+        "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=STAGING_BUCKET)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2cc825514deb"
+      },
+      "source": [
+        "### Define constants"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b42bd4fa2b2d"
+      },
+      "outputs": [],
+      "source": [
+        "# The pre-built training and serving docker images.\n",
+        "TRAIN_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai-restricted/vertex-vision-model-garden-dockers/pytorch-peft-train\"\n",
+        "PREDICTION_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai-restricted/vertex-vision-model-garden-dockers/pytorch-peft-serve\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0c250872074f"
+      },
+      "source": [
+        "### Define common functions"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "354da31189dc"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "from datetime import datetime\n",
+        "\n",
+        "from google.cloud import aiplatform\n",
+        "\n",
+        "\n",
+        "def get_job_name_with_datetime(prefix: str):\n",
+        "    \"\"\"Gets the job name with date time when triggering training or deployment\n",
+        "    jobs in Vertex AI.\n",
+        "    \"\"\"\n",
+        "    return prefix + datetime.now().strftime(\"_%Y%m%d_%H%M%S\")\n",
+        "\n",
+        "\n",
+        "def deploy_model(\n",
+        "    model_name,\n",
+        "    base_model_id,\n",
+        "    finetuned_lora_model_path,\n",
+        "    service_account,\n",
+        "    task,\n",
+        "    machine_type=\"n1-standard-8\",\n",
+        "    accelerator_type=\"NVIDIA_TESLA_V100\",\n",
+        "):\n",
+        "    \"\"\"Deploys trained models into Vertex AI.\"\"\"\n",
+        "    endpoint = aiplatform.Endpoint.create(display_name=f\"{model_name}-endpoint\")\n",
+        "    serving_env = {\n",
+        "        \"BASE_MODEL_ID\": base_model_id,\n",
+        "        \"FINETUNED_LORA_MODEL_PATH\": finetuned_lora_model_path,\n",
+        "        \"TASK\": task,\n",
+        "    }\n",
+        "    model = aiplatform.Model.upload(\n",
+        "        display_name=model_name,\n",
+        "        serving_container_image_uri=PREDICTION_DOCKER_URI,\n",
+        "        serving_container_ports=[7080],\n",
+        "        serving_container_predict_route=\"/predictions/peft_serving\",\n",
+        "        serving_container_health_route=\"/ping\",\n",
+        "        serving_container_environment_variables=serving_env,\n",
+        "    )\n",
+        "    model.deploy(\n",
+        "        endpoint=endpoint,\n",
+        "        machine_type=machine_type,\n",
+        "        accelerator_type=accelerator_type,\n",
+        "        accelerator_count=1,\n",
+        "        deploy_request_timeout=1800,\n",
+        "        service_account=service_account,\n",
+        "    )\n",
+        "    return model, endpoint"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e70e3519ff8b"
+      },
+      "source": [
+        "## Casual Language Modeling + PEFT\n",
+        "\n",
+        "This section demonstrates how to finetune OpenLLaMA with PEFT LoRA, including [openlm-research/open_llama_3b](https://huggingface.co/openlm-research/open_llama_3b), [openlm-research/open_llama_7b](https://huggingface.co/openlm-research/open_llama_7b), and [openlm-research/open_llama_13b](https://huggingface.co/openlm-research/open_llama_13b)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "iWGwJHqI7LMs"
+      },
+      "source": [
+        "### Finetune with LoRA"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5qCrm_kJH5cz"
+      },
+      "source": [
+        "Set the base model id."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "N3UBLiYrM3sU"
+      },
+      "outputs": [],
+      "source": [
+        "base_model_id = \"openlm-research/open_llama_3b\"  # @param [\"openlm-research/open_llama_3b\", \"openlm-research/open_llama_7b\", \"openlm-research/open_llama_13b\"]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "KKEYoRfiHDVv"
+      },
+      "source": [
+        "Use the Vertex AI SDK to create and run the custom training jobs with Vertex AI Model Garden training images.\n",
+        "\n",
+        "This example uses the dataset [Abirate/english_quotes](https://huggingface.co/datasets/Abirate/english_quotes).\n",
+        "\n",
+        "In order to make the finetunig efficiently, we enabled quantization (8bits) when loading pretrained models for finetuning LoRA models. The peak GPU memory usages are ~7G, ~10G and ~16G for finetuning LoRA models for [openlm-research/open_llama_3b](https://huggingface.co/openlm-research/open_llama_3b), [openlm-research/open_llama_7b](https://huggingface.co/openlm-research/open_llama_7b), and [openlm-research/open_llama_13b](https://huggingface.co/openlm-research/open_llama_13b) separately with default training parameters and the example dataset. In theory, open_llama_3b and open_llama_7b can be finetuned on 1 V100, and open_llama_13b can be finetuned on 1 A100 (40G). We choose to use 1 A100 (40G) by default to support all these models in this notebook for simplicity."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "65467b361315"
+      },
+      "outputs": [],
+      "source": [
+        "dataset_name = \"Abirate/english_quotes\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# Worker pool spec.\n",
+        "# machine_type = \"n1-standard-8\"\n",
+        "# accelerator_type = \"NVIDIA_TESLA_V100\"\n",
+        "machine_type = \"a2-highgpu-1g\"\n",
+        "accelerator_type = \"NVIDIA_TESLA_A100\"\n",
+        "replica_count = 1\n",
+        "accelerator_count = 1\n",
+        "\n",
+        "# Setup training job.\n",
+        "job_name = get_job_name_with_datetime(\"openllama-lora-train\")\n",
+        "train_job = aiplatform.CustomContainerTrainingJob(\n",
+        "    display_name=job_name,\n",
+        "    container_uri=TRAIN_DOCKER_URI,\n",
+        ")\n",
+        "output_dir = os.path.join(MODEL_BUCKET, job_name)\n",
+        "output_dir_gcsfuse = output_dir.replace(\"gs://\", \"/gcs/\")\n",
+        "\n",
+        "# Pass training arguments and launch job.\n",
+        "train_job.run(\n",
+        "    args=[\n",
+        "        \"--task=causal-language-modeling-lora\",\n",
+        "        f\"--pretrained_model_id={base_model_id}\",\n",
+        "        f\"--dataset_name={dataset_name}\",\n",
+        "        f\"--output_dir={output_dir_gcsfuse}\",\n",
+        "        \"--lora_rank=16\",\n",
+        "        \"--lora_alpha=32\",\n",
+        "        \"--lora_dropout=0.05\",\n",
+        "        \"--warmup_steps=10\",\n",
+        "        \"--max_steps=10\",\n",
+        "        \"--learning_rate=2e-4\",\n",
+        "    ],\n",
+        "    replica_count=replica_count,\n",
+        "    machine_type=machine_type,\n",
+        "    accelerator_type=accelerator_type,\n",
+        "    accelerator_count=accelerator_count,\n",
+        "    boot_disk_size_gb=500,\n",
+        ")\n",
+        "\n",
+        "print(\"Trained models were saved in: \", output_dir)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "jqmCtkGnhDmp"
+      },
+      "source": [
+        "### Run inferences with serving images\n",
+        "This section uploads the model to Model Registry and deploys it on the Endpoint.\n",
+        "\n",
+        "The model deployment step will take ~15 minutes to complete.\n",
+        "\n",
+        "The peak GPU memory usages for [openlm-research/open_llama_3b](https://huggingface.co/openlm-research/open_llama_3b), [openlm-research/open_llama_7b](https://huggingface.co/openlm-research/open_llama_7b), and [openlm-research/open_llama_13b](https://huggingface.co/openlm-research/open_llama_13b) with LoRA weights are ~5.3G, ~8.7G and ~15.2G separately with the default settings. We use V100 in deployments for simplicity."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "bf55e38815dc"
+      },
+      "outputs": [],
+      "source": [
+        "model, endpoint = deploy_model(\n",
+        "    model_name=get_job_name_with_datetime(prefix=\"openllama-peft-serve\"),\n",
+        "    base_model_id=base_model_id,\n",
+        "    finetuned_lora_model_path=output_dir,\n",
+        "    service_account=SERVICE_ACCOUNT,\n",
+        "    task=\"causal-language-modeling-lora\",\n",
+        ")\n",
+        "print(\"endpoint_name:\", endpoint.name)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "80b3fd2ace09"
+      },
+      "source": [
+        "NOTE: The model weights will be downloaded after the deployment succeeds. Thus additional 5 minutes of waiting time is needed **after** the above model deployment step succeeds and before you run the next step below. Otherwise you might see a `ServiceUnavailable: 503 502:Bad Gateway` error when you send requests to the endpoint.\n",
+        "\n",
+        "Once deployment succeeds, you can send requests to the endpoint with text prompts."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "4ab04da3ec9a"
+      },
+      "outputs": [],
+      "source": [
+        "# # Loads an existing endpoint as below.\n",
+        "# endpoint_name = endpoint.name\n",
+        "# aip_endpoint_name = (\n",
+        "#     f\"projects/{PROJECT_ID}/locations/{REGION}/endpoints/{endpoint_name}\"\n",
+        "# )\n",
+        "# endpoint = aiplatform.Endpoint(aip_endpoint_name)\n",
+        "instances = [\n",
+        "    {\"prompt\": \"Hi, Google.\"},\n",
+        "]\n",
+        "response = endpoint.predict(instances=instances)\n",
+        "\n",
+        "for prediction in response.predictions[0]:\n",
+        "    print(prediction[\"generated_text\"])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "af21a3cff1e0"
+      },
+      "source": [
+        "### Clean up resources"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "911406c1561e"
+      },
+      "outputs": [],
+      "source": [
+        "# Delete custom train jobs.\n",
+        "train_job.delete()\n",
+        "\n",
+        "# Undeploy model and delete endpoint.\n",
+        "endpoint.delete(force=True)\n",
+        "\n",
+        "# Delete models.\n",
+        "model.delete()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "W2USaHtQbE-l"
+      },
+      "source": [
+        "## Instruct + PEFT\n",
+        "\n",
+        "This section demonstrates how to finetune Falcon-instruct with PEFT LoRA, including [tiiuae/falcon-7b-instruct](https://huggingface.co/tiiuae/falcon-7b-instruct), and [tiiuae/falcon-40b-instruct](https://huggingface.co/tiiuae/falcon-40b-instruct)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "PwluMqvYbLu9"
+      },
+      "source": [
+        "### Finetune with LoRA"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ap1pQd-ckAAb"
+      },
+      "source": [
+        "Set the base model id."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "vVTfG2bpbR90"
+      },
+      "outputs": [],
+      "source": [
+        "base_model_id = \"tiiuae/falcon-7b-instruct\"  # @param [\"tiiuae/falcon-7b-instruct\", \"tiiuae/falcon-40b-instruct\"]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "iJe_us8mkZEV"
+      },
+      "source": [
+        "Use the Vertex AI SDK to create and run the custom training jobs with Vertex AI Model Garden training images.\n",
+        "\n",
+        "This example uses the dataset [timdettmers/openassistant-guanaco](https://huggingface.co/datasets/timdettmers/openassistant-guanaco).\n",
+        "\n",
+        "The peak GPU memory usages are ~11G and ~34G for finetuning LoRA models for [tiiuae/falcon-7b-instruct](https://huggingface.co/tiiuae/falcon-7b-instruct), and [tiiuae/falcon-40b-instruct](https://huggingface.co/tiiuae/falcon-40b-instruct) separately with default training parameters and the example dataset. In theory, falcon-7b-instruct can be finetuned on 1 P100/V100, and falcon-40b-instruct can be finetuned on 1 A100 (40G). We choose to use 1 A100 (40G) by default to support all these models in this notebook for simplicity."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "-i4M7mWPbV8s"
+      },
+      "outputs": [],
+      "source": [
+        "dataset_name = \"timdettmers/openassistant-guanaco\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# Worker pool spec.\n",
+        "machine_type = \"a2-highgpu-1g\"\n",
+        "accelerator_type = \"NVIDIA_TESLA_A100\"\n",
+        "replica_count = 1\n",
+        "accelerator_count = 1\n",
+        "\n",
+        "# Setup training job.\n",
+        "job_name = get_job_name_with_datetime(\"falcon-instruct-lora-train\")\n",
+        "train_job = aiplatform.CustomContainerTrainingJob(\n",
+        "    display_name=job_name,\n",
+        "    container_uri=TRAIN_DOCKER_URI,\n",
+        ")\n",
+        "output_dir = os.path.join(MODEL_BUCKET, job_name)\n",
+        "output_dir_gcsfuse = output_dir.replace(\"gs://\", \"/gcs/\")\n",
+        "\n",
+        "# Pass training arguments and launch job.\n",
+        "max_steps = 10\n",
+        "train_job.run(\n",
+        "    args=[\n",
+        "        \"--task=instruct-lora\",\n",
+        "        f\"--pretrained_model_id={base_model_id}\",\n",
+        "        f\"--dataset_name={dataset_name}\",\n",
+        "        f\"--output_dir={output_dir_gcsfuse}\",\n",
+        "        \"--lora_rank=64\",\n",
+        "        \"--lora_alpha=16\",\n",
+        "        \"--lora_dropout=0.1\",\n",
+        "        \"--warmup_ratio=0.03\",\n",
+        "        f\"--max_steps={max_steps}\",\n",
+        "        \"--max_seq_length=512\",\n",
+        "        \"--learning_rate=2e-4\",\n",
+        "    ],\n",
+        "    replica_count=replica_count,\n",
+        "    machine_type=machine_type,\n",
+        "    accelerator_type=accelerator_type,\n",
+        "    accelerator_count=accelerator_count,\n",
+        "    boot_disk_size_gb=500,\n",
+        ")\n",
+        "\n",
+        "print(\"Trained models were saved in: \", output_dir)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "oCgivsdllZ1x"
+      },
+      "source": [
+        "### Run inferences with serving images"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "_lNIFvielcyV"
+      },
+      "source": [
+        "This section uploads the model to Model Registry and deploys it on the Endpoint.\n",
+        "\n",
+        "The model deployment step will take ~15 minutes to complete.\n",
+        "\n",
+        "The peak GPU memory usages for [tiiuae/falcon-7b-instruct](https://huggingface.co/tiiuae/falcon-7b-instruct), and [tiiuae/falcon-40b-instruct](https://huggingface.co/tiiuae/falcon-40b-instruct) with LoRA weights are ~15.5G and ~38.2G separately with the default settings. We use V100 in deployments for simplicity."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "inrBXzoxk53t"
+      },
+      "outputs": [],
+      "source": [
+        "# # If deploy finetuned falcon-40b-instruct models, please set\n",
+        "# machine_type = \"a2-highgpu-1g\",\n",
+        "# accelerator_type = \"NVIDIA_TESLA_A100\"\n",
+        "machine_type = \"n1-standard-8\"\n",
+        "accelerator_type = \"NVIDIA_TESLA_V100\"\n",
+        "\n",
+        "model, endpoint = deploy_model(\n",
+        "    model_name=get_job_name_with_datetime(prefix=\"falcon-peft-serve\"),\n",
+        "    base_model_id=base_model_id,\n",
+        "    finetuned_lora_model_path=os.path.join(output_dir, \"checkpoint-\" + str(max_steps)),\n",
+        "    service_account=SERVICE_ACCOUNT,\n",
+        "    task=\"instruct-lora\",\n",
+        "    machine_type=machine_type,\n",
+        "    accelerator_type=accelerator_type,\n",
+        ")\n",
+        "print(\"endpoint_name: \", endpoint.name)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "_dmO3XooliAs"
+      },
+      "source": [
+        "NOTE: The model weights will be downloaded after the deployment succeeds. Thus additional 5 minutes of waiting time is needed after the above model deployment step succeeds and before you run the next step below. Otherwise you might see a ServiceUnavailable: 503 502:Bad Gateway error when you send requests to the endpoint.\n",
+        "\n",
+        "Once deployment succeeds, you can send requests to the endpoint with text prompts.\n",
+        "\n",
+        "Example:\n",
+        "\n",
+        "```\n",
+        "Human: What is a car?\n",
+        "Assistant:  A car, or a motor car, is a road-connected human-transportation system used to move people or goods from one place to another. The term also encompasses a wide range of vehicles, including motorboats, trains, and aircrafts. Cars typically have four wheels, a cabin for passengers, and an engine or motor. They have been around since the early 19th century and are now one of the most popular forms of transportation, used for daily commuting, shopping, and other purposes.\n",
+        "```"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "JrKsrffGl14T"
+      },
+      "outputs": [],
+      "source": [
+        "# # Loads an existing endpoint as below.\n",
+        "# endpoint_name = endpoint.name\n",
+        "# aip_endpoint_name = (\n",
+        "#     f\"projects/{PROJECT_ID}/locations/{REGION}/endpoints/{endpoint_name}\"\n",
+        "# )\n",
+        "# endpoint = aiplatform.Endpoint(aip_endpoint_name)\n",
+        "instances = [\n",
+        "    {\"prompt\": \"What is a car?\"},\n",
+        "]\n",
+        "response = endpoint.predict(instances=instances)\n",
+        "\n",
+        "for prediction in response.predictions[0]:\n",
+        "    print(prediction[\"generated_text\"])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "uk-t69nFl9rr"
+      },
+      "source": [
+        "### Clean up resources"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "CcIX2WAJmAI7"
+      },
+      "outputs": [],
+      "source": [
+        "# Delete custom train jobs.\n",
+        "train_job.delete()\n",
+        "\n",
+        "# Undeploy model and delete endpoint.\n",
+        "endpoint.delete(force=True)\n",
+        "\n",
+        "# Delete models.\n",
+        "model.delete()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "og_s64QVmJDb"
+      },
+      "source": [
+        "## Sequence Classification + PEFT\n",
+        "\n",
+        "This section demonstrates how to finetune sequence classification models with PEFT LoRA, including [bert-base-uncased](https://huggingface.co/bert-base-uncased), [RoBERTa-large](https://huggingface.co/roberta-large), and [XLM-RoBERTa-large](https://huggingface.co/xlm-roberta-large)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "QqJcvCb2mn8l"
+      },
+      "source": [
+        "### Finetune with PEFT"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "c0E7dfqloeBB"
+      },
+      "source": [
+        "Set the base model id."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "eru79ot6mtNG"
+      },
+      "outputs": [],
+      "source": [
+        "base_model_id = \"xlm-roberta-large\"  # @param [\"bert-base-uncased\", \"roberta-large\", \"xlm-roberta-large\"]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "y9n-qheaotS9"
+      },
+      "source": [
+        "Use the Vertex AI SDK to create and run the custom training jobs with Vertex AI Model Garden training images.\n",
+        "\n",
+        "This example uses the dataset [glue](https://huggingface.co/datasets/glue).\n",
+        "\n",
+        "The peak GPU memory usages are ~8.8G and ~15G for finetuning LoRA models for [RoBERTa-large](https://huggingface.co/roberta-large), and [XLM-RoBERTa-large](https://huggingface.co/xlm-roberta-large) separately with default training parameters and the example dataset. In theory, RoBERTa-large can be finetuned on 1 P100/V100, and XLM-RoBERTa-large can be finetuned on 1 A100 (40G). We choose to use 1 A100 (40G) by default to support all these models in this notebook for simplicity."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "OHeSYEKinfP3"
+      },
+      "outputs": [],
+      "source": [
+        "dataset_name = \"glue\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# Worker pool spec.\n",
+        "# # Please switch to A100 or other powerful machines if you run out of memory.\n",
+        "# machine_type = \"a2-highgpu-1g\"\n",
+        "# accelerator_type = \"NVIDIA_TESLA_A100\"\n",
+        "machine_type = \"n1-standard-8\"\n",
+        "accelerator_type = \"NVIDIA_TESLA_V100\"\n",
+        "replica_count = 1\n",
+        "accelerator_count = 1\n",
+        "\n",
+        "# Setup training job.\n",
+        "job_name = get_job_name_with_datetime(os.path.basename(base_model_id) + \"-lora-train\")\n",
+        "train_job = aiplatform.CustomContainerTrainingJob(\n",
+        "    display_name=job_name,\n",
+        "    container_uri=TRAIN_DOCKER_URI,\n",
+        ")\n",
+        "output_dir = os.path.join(MODEL_BUCKET, job_name)\n",
+        "output_dir_gcsfuse = output_dir.replace(\"gs://\", \"/gcs/\")\n",
+        "\n",
+        "# Pass training arguments and launch job.\n",
+        "train_job.run(\n",
+        "    args=[\n",
+        "        \"--task=sequence-classification-lora\",\n",
+        "        f\"--pretrained_model_id={base_model_id}\",\n",
+        "        f\"--dataset_name={dataset_name}\",\n",
+        "        f\"--output_dir={output_dir_gcsfuse}\",\n",
+        "        \"--lora_rank=8\",\n",
+        "        \"--lora_alpha=16\",\n",
+        "        \"--lora_dropout=0.1\",\n",
+        "        \"--num_epochs=20\",\n",
+        "        \"--batch_size=32\",\n",
+        "        \"--learning_rate=3e-4\",\n",
+        "    ],\n",
+        "    replica_count=replica_count,\n",
+        "    machine_type=machine_type,\n",
+        "    accelerator_type=accelerator_type,\n",
+        "    accelerator_count=accelerator_count,\n",
+        "    boot_disk_size_gb=500,\n",
+        ")\n",
+        "\n",
+        "print(\"Trained models were saved in: \", output_dir)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "kJuHaBCqoacg"
+      },
+      "source": [
+        "### Run inferences with serving images"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "jw1GFXydoft0"
+      },
+      "source": [
+        "This section uploads the model to Model Registry and deploys it on the Endpoint.\n",
+        "\n",
+        "The model deployment step will take ~15 minutes to complete.\n",
+        "\n",
+        "The peak GPU memory usages for [RoBERTa-large](https://huggingface.co/roberta-large), and [XLM-RoBERTa-large](https://huggingface.co/xlm-roberta-large) with LoRA weights are ~2G and ~2.4G separately with the default settings. We use V100 in deployments for simplicity."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "A5j0a3AZoj3h"
+      },
+      "outputs": [],
+      "source": [
+        "model, endpoint = deploy_model(\n",
+        "    model_name=get_job_name_with_datetime(prefix=\"sequence-classification-peft-serve\"),\n",
+        "    base_model_id=base_model_id,\n",
+        "    finetuned_lora_model_path=output_dir,\n",
+        "    service_account=SERVICE_ACCOUNT,\n",
+        "    task=\"sequence-classification-lora\",\n",
+        ")\n",
+        "print(\"endpoint_name:\", endpoint.name)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6dR3-Hc7oqu-"
+      },
+      "source": [
+        "NOTE: The model weights will be downloaded after the deployment succeeds. Thus additional 5 minutes of waiting time is needed after the above model deployment step succeeds and before you run the next step below. Otherwise you might see a ServiceUnavailable: 503 502:Bad Gateway error when you send requests to the endpoint.\n",
+        "\n",
+        "Once deployment succeeds, you can send requests to the endpoint with text prompts."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "xAmULPegoyXl"
+      },
+      "outputs": [],
+      "source": [
+        "# # Loads an existing endpoint as below.\n",
+        "# endpoint_name = endpoint.name\n",
+        "# aip_endpoint_name = (\n",
+        "#     f\"projects/{PROJECT_ID}/locations/{REGION}/endpoints/{endpoint_name}\"\n",
+        "# )\n",
+        "# endpoint = aiplatform.Endpoint(aip_endpoint_name)\n",
+        "instances = [\n",
+        "    {\"prompt\": \"The cat sat on the mat.\"},\n",
+        "]\n",
+        "response = endpoint.predict(instances=instances)\n",
+        "labels = [int(item) for item in response.predictions]\n",
+        "print(labels)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "1Vg_b46Jo6C1"
+      },
+      "source": [
+        "### Clean up resources"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "vNiTDL_VpAlp"
+      },
+      "outputs": [],
+      "source": [
+        "# Delete custom train jobs.\n",
+        "train_job.delete()\n",
+        "\n",
+        "# Undeploy model and delete endpoint.\n",
+        "endpoint.delete(force=True)\n",
+        "\n",
+        "# Delete models.\n",
+        "model.delete()"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "model_garden_pytorch_peft.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/community/model_garden/model_garden_pytorch_pic2word.ipynb

@@ -0,0 +1,633 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "ad41232f-4ac9-4607-8ee6-9b1d98d0f5c3",
+      "metadata": {
+        "id": "7d9bbf86da5e"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2023 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "id": "30e2cb63",
+      "metadata": {
+        "id": "99c1c3fc2ca5"
+      },
+      "source": [
+        "# Vertex AI Model Garden - Pic2Word\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pic2word.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_jpic2word.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>                                                                                               <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/notebooks/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/model_garden/model_garden_pic2word.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "  </td>\n",
+        "</table>"
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "id": "5d5af88c-a3c8-46fa-b90a-1b6737a91534",
+      "metadata": {
+        "id": "7e3e5205fbfd"
+      },
+      "source": [
+        "## Overview"
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "id": "87b71198-5c24-4e8c-aab1-44e080aa7fe7",
+      "metadata": {
+        "id": "523084ce4894"
+      },
+      "source": [
+        "This notebook demonstrates how to use the [Pic2Word](https://github.com/google-research/composed_image_retrieval) model in Vertex AI Model Garden. "
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "id": "0ba32ec8-e06f-474f-b753-20aaf74ebd12",
+      "metadata": {
+        "id": "f9cbeb1704e1"
+      },
+      "source": [
+        "## Objective"
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "id": "95694bc4-0626-4371-8b2a-6b89967742db",
+      "metadata": {
+        "id": "da71cd41e69e"
+      },
+      "source": [
+        "Following the notebook you will conduct experiments using the pre-built docker image on Vertex AI.\n",
+        "\n",
+        "- Deploy pretrained Pic2Word models in Google Cloud Vertex AI\n",
+        "\n",
+        "This tutorial uses the following Google Cloud ML services and resources:\n",
+        "\n",
+        "- Vertex AI Model Registry\n",
+        "- Vertex AI Online Prediction"
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "id": "b1893e0c-859b-4d42-b1d8-d133855d8767",
+      "metadata": {
+        "id": "8264c04e0f34"
+      },
+      "source": [
+        "## Dataset"
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "id": "a099439c-3122-429b-8215-caaf9aee69c5",
+      "metadata": {
+        "id": "52e4f9df87b2"
+      },
+      "source": [
+        "We use the [COCO](https://cocodataset.org/#home) validation set (5,000 images) for evaluation."
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "id": "56d2a9f8-2d99-407e-a378-57f99754185b",
+      "metadata": {
+        "id": "169ef74e43a6"
+      },
+      "source": [
+        "## Costs"
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "id": "bb62de0d-71ac-45ee-82ac-5f7465f2f0eb",
+      "metadata": {
+        "id": "61a64432ac87"
+      },
+      "source": [
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "- Vertex AI\n",
+        "- Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing?_ga=2.46650789.-341051769.1686949237) and [Cloud Storage pricing](https://cloud.google.com/storage/pricing?_ga=2.46650789.-341051769.1686949237), and use the [Pricing Calculator](https://cloud.google.com/products/calculator/?_ga=2.247379078.-341051769.1686949237) to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "id": "ea27403e",
+      "metadata": {
+        "id": "586c9147ba8a"
+      },
+      "source": [
+        "# Installation\n",
+        "\n",
+        "Install the following packages required to execute this notebook."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "f9c685d9",
+      "metadata": {
+        "id": "68f1241e68a7"
+      },
+      "outputs": [],
+      "source": [
+        "if \"google.colab\" in str(get_ipython()):\n",
+        "    # Configs for colab notebooks.\n",
+        "    ! pip3 install --upgrade --quiet google-cloud-aiplatform\n",
+        "\n",
+        "    # Automatically restart kernel after installs\n",
+        "    import IPython\n",
+        "\n",
+        "    app = IPython.Application.instance()\n",
+        "    app.kernel.do_shutdown(True)\n",
+        "\n",
+        "    from google.colab import auth as google_auth\n",
+        "\n",
+        "    google_auth.authenticate_user()"
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "id": "e75fe500-5dc1-4ab3-8829-3eeada5be7f7",
+      "metadata": {
+        "id": "d0c7bca32b78"
+      },
+      "source": [
+        "## Setup environment"
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "id": "e8b0ef65",
+      "metadata": {
+        "id": "BF1j6f9HApxa"
+      },
+      "source": [
+        "## Before you begin\n",
+        "\n",
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI API and Compute Engine API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component).\n",
+        "\n",
+        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "\n",
+        "1. [Create a service account](https://cloud.google.com/iam/docs/service-accounts-create#iam-service-accounts-create-console) with **Vertex AI User** and **Storage Object Admin** roles for deploying fine tuned model to Vertex AI endpoint.\n",
+        "\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with ! as shell commands, and it interpolates Python variables prefixed with $ into these commands."
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "id": "09490824",
+      "metadata": {
+        "id": "WReHDGG5g0XY"
+      },
+      "source": [
+        "#### Set your project ID\n",
+        "\n",
+        "**If you don't know your project ID**, try the following:\n",
+        "* Run `gcloud config list`.\n",
+        "* Run `gcloud projects list`.\n",
+        "* See the support page: [Locate the project ID](https://support.google.com/googleapi/answer/7014113)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "6ddc1d11",
+      "metadata": {
+        "id": "oM1iC_MfAts1"
+      },
+      "outputs": [],
+      "source": [
+        "PROJECT_ID = \"your-project-id\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# Set the project id\n",
+        "! gcloud config set project {PROJECT_ID}"
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "id": "9ed7d668",
+      "metadata": {
+        "id": "region"
+      },
+      "source": [
+        "#### Region\n",
+        "\n",
+        "You can also change the `REGION` variable used by Vertex AI. Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "90d0108f",
+      "metadata": {
+        "id": "twgKk-LsLmX3"
+      },
+      "outputs": [],
+      "source": [
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "id": "ca43938d",
+      "metadata": {
+        "id": "409f1effab6c"
+      },
+      "source": [
+        "### Buckets\n",
+        "\n",
+        "You can create a storage bucket to store model input and output images."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "889d10ac",
+      "metadata": {
+        "id": "bcdadd216c34"
+      },
+      "outputs": [],
+      "source": [
+        "# The form for BUCKET_URI is gs://.\n",
+        "BUCKET_URI = \"\"  # @param {type:\"string\"}\n",
+        "\n",
+        "import os\n",
+        "\n",
+        "INPUT_BUCKET = os.path.join(BUCKET_URI, \"/input\")\n",
+        "OUTPUT_BUCKET = os.path.join(BUCKET_URI, \"/output\")"
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "id": "15c0e671",
+      "metadata": {
+        "id": "sBCra4QMA2wR"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "Depending on your Jupyter environment, you may have to manually authenticate. Follow the relevant instructions below."
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "id": "623a03cc",
+      "metadata": {
+        "id": "74ccc9e52986"
+      },
+      "source": [
+        "**1. Vertex AI Workbench**\n",
+        "* Do nothing as you are already authenticated."
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "id": "fbee4160",
+      "metadata": {
+        "id": "de775a3773ba"
+      },
+      "source": [
+        "**2. Local JupyterLab instance, uncomment and run:**"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "3eabaf6b",
+      "metadata": {
+        "id": "254614fa0c46"
+      },
+      "outputs": [],
+      "source": [
+        "# ! gcloud auth login"
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "id": "90a615fe",
+      "metadata": {
+        "id": "ef21552ccea8"
+      },
+      "source": [
+        "**3. Colab, uncomment and run:**"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "3d4e50e1",
+      "metadata": {
+        "id": "603adbbf0532"
+      },
+      "outputs": [],
+      "source": [
+        "# from google.colab import auth\n",
+        "# auth.authenticate_user()"
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "id": "b0b40233",
+      "metadata": {
+        "id": "f6b2ccc891ed"
+      },
+      "source": [
+        "**4. Service account or other**\n",
+        "* See how to grant Cloud Storage permissions to your service account at https://cloud.google.com/storage/docs/gsutil/commands/iam#ch-examples."
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "id": "a8019fcc-26ce-4536-b394-aa1fb18794ee",
+      "metadata": {
+        "id": "4226467373ef"
+      },
+      "source": [
+        "If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk) and [gsutil](https://cloud.google.com/storage/docs/gsutil_install)."
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "id": "315598c1-3ff4-4bc8-9b82-7e2fe55d4678",
+      "metadata": {
+        "id": "8ba36d3a171d"
+      },
+      "source": [
+        "### Setup variables"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "6d26ddb2-04ad-4973-8965-9b25eb24993f",
+      "metadata": {
+        "id": "25ca675e59e7"
+      },
+      "outputs": [],
+      "source": [
+        "# Prediction constants.\n",
+        "PREDICTION_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai-restricted/vertex-vision-model-garden-dockers/pic2word_serve:latest\"\n",
+        "PREDICTION_ACCELERATOR_TYPE = \"NVIDIA_TESLA_T4\"\n",
+        "PREDICTION_MACHINE_TYPE = \"n1-standard-8\"\n",
+        "\n",
+        "# The serving port.\n",
+        "SERVE_PORT = 7080\n",
+        "\n",
+        "# The service account looks like:\n",
+        "# '@.iam.gserviceaccount.com'\n",
+        "# Please go to https://cloud.google.com/iam/docs/service-accounts-create#iam-service-accounts-create-console\n",
+        "# and create service account with `Vertex AI User` and `Storage Object Admin` roles.\n",
+        "SERVICE_ACCOUNT = \"\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "id": "0daec6e5-7642-458f-bd93-60a15431b0a1",
+      "metadata": {
+        "id": "71dd15118703"
+      },
+      "source": [
+        "## Deploy model for online prediction"
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "id": "b286ce6b-cb00-441b-8249-93497de17974",
+      "metadata": {
+        "id": "5c891f6352ad"
+      },
+      "source": [
+        "This section uploads the model to Vertex Model Registry and deploys it on an Endpoint resource. This will take around 15 minutes to finish."
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "id": "01f686b2-9227-496b-9c4d-a63becb59fb8",
+      "metadata": {
+        "id": "e6abf4ee450a"
+      },
+      "source": [
+        "### Upload and deploy model to Vertex AI"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "8885d5be-5007-4778-9562-976102fefe73",
+      "metadata": {
+        "id": "645d8d1df8d4"
+      },
+      "outputs": [],
+      "source": [
+        "from google.cloud import aiplatform\n",
+        "\n",
+        "# Init common setup.\n",
+        "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=BUCKET_URI)\n",
+        "\n",
+        "# Upload model.\n",
+        "serving_env = {}\n",
+        "model = aiplatform.Model.upload(\n",
+        "    display_name=\"pic2word-model\",\n",
+        "    serving_container_image_uri=PREDICTION_DOCKER_URI,\n",
+        "    serving_container_ports=[SERVE_PORT],\n",
+        "    serving_container_predict_route=\"/predictions/pic2word\",\n",
+        "    serving_container_health_route=\"/ping\",\n",
+        "    serving_container_environment_variables=serving_env,\n",
+        ")\n",
+        "# Or reuse a pre-uploaded model.\n",
+        "# model = aiplatform.Model('projects/123456789/locations/us-central1/models/123456789@1')\n",
+        "\n",
+        "# Create an endpoint.\n",
+        "endpoint = aiplatform.Endpoint.create(display_name=\"pytorch-pic2word-endpoint\")\n",
+        "# Or reuse a pre-created endpoint.\n",
+        "# endpoint = aiplatform.Endpoint('projects/123456789/locations/us-central1/endpoints/123456789')\n",
+        "\n",
+        "# Deploy model to endpoint.\n",
+        "model.deploy(\n",
+        "    endpoint=endpoint,\n",
+        "    machine_type=PREDICTION_MACHINE_TYPE,\n",
+        "    accelerator_type=PREDICTION_ACCELERATOR_TYPE,\n",
+        "    accelerator_count=1,\n",
+        "    traffic_percentage=100,\n",
+        "    service_account=SERVICE_ACCOUNT,\n",
+        ")"
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "id": "f0bd36bb-f988-479a-be40-18c54980ca1f",
+      "metadata": {
+        "id": "2bb47a2855a2"
+      },
+      "source": [
+        "You can manage your uploaded models in the [Model Registry](https://pantheon.corp.google.com/vertex-ai/models) and your endpoints in the [Endpoints](https://pantheon.corp.google.com/vertex-ai/endpoints)."
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "id": "95edbcdd-8246-444a-aef2-a9cadbc03000",
+      "metadata": {
+        "id": "f14ff5e7a476"
+      },
+      "source": [
+        "## Send a prediction request to the endpoint"
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "id": "3a83d6bb",
+      "metadata": {
+        "id": "c41c7d51d654"
+      },
+      "source": [
+        "Set the query text and upload your desired query image to the specified INPUT_BUCKET. Use an asterisk in the query text to specify which token gets replaced by the image token. "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "f7c6fc99-42cd-4544-86df-b14f49189bfd",
+      "metadata": {
+        "id": "58d4f8e84e02"
+      },
+      "outputs": [],
+      "source": [
+        "payload = json.dumps(\n",
+        "    {\n",
+        "        \"query\": \"a bunch of *\",\n",
+        "        \"image_path\": INPUT_BUCKET,\n",
+        "        \"image_file_name\": \"cat.png\",\n",
+        "        \"output_storage_dir\": OUTPUT_BUCKET,\n",
+        "    }\n",
+        ")\n",
+        "response = endpoint.predict(payload).predictions\n",
+        "print(response)"
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "id": "8e71ee23-967c-4871-802f-ccb39958386e",
+      "metadata": {
+        "id": "608b936179df"
+      },
+      "source": [
+        "## Cleaning Up\n",
+        "\n",
+        "To clean up all Google Cloud resources used in this project, you can delete the Google Cloud project you used for the tutorial.\n",
+        "\n",
+        "Otherwise, you can delete the individual resources you created in this tutorial:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "d9be2a18-db11-4108-9622-647b33be2594",
+      "metadata": {
+        "id": "7d17b385141d"
+      },
+      "outputs": [],
+      "source": [
+        "# Delete endpoint resource.\n",
+        "endpoint.delete(force=True)\n",
+        "\n",
+        "# Delete model resource.\n",
+        "model.delete()\n",
+        "\n",
+        "# Delete Cloud Storage objects that were created.\n",
+        "delete_bucket = False\n",
+        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "    ! gsutil -m rm -r $BUCKET_URI"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "model_garden_pytorch_pic2word.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/community/model_garden/model_garden_pytorch_sam.ipynb

@@ -0,0 +1,450 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6ad30fe2-1fc1-47e3-8a9f-624170b5aae6"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2023 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "cYYdTBQoKeCP"
+      },
+      "source": [
+        " # Vertex AI Model Garden - Segment Anything Model (SAM) Serving on Vertex AI\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_sam.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_sam.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/notebooks/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/model_garden/model_garden_pytorch_sam.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "    (a Python-3 CPU notebook is recommended)\n",
+        "  </td>\n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "JbmPgTp2LRCY"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "This notebook demonstrates using the [huggingface/transformers](https://github.com/huggingface/transformers) framework to serve Segment Anything Model (SAM) models and deploy them for online prediction on Vertex AI.\n",
+        "\n",
+        "Following the notebook you will conduct experiments using the pre-built docker image on Vertex AI.\n",
+        "\n",
+        "- With the pre-built docker images, you can **deploy** models for the following tasks:\n",
+        "    - Mask Generation\n",
+        "\n",
+        "### Objective\n",
+        "\n",
+        "- Upload the model to [Model Registry](https://cloud.google.com/vertex-ai/docs/model-registry/introduction).\n",
+        "- Deploy the model on [Endpoint](https://cloud.google.com/vertex-ai/docs/predictions/using-private-endpoints).\n",
+        "- Run online predictions for image captioning.\n",
+        "\n",
+        "### Costs\n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage pricing](https://cloud.google.com/storage/pricing), and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "78f72e0a-52e5-4de5-ac0f-2171b3493825"
+      },
+      "source": [
+        "## Setup environment\n",
+        "\n",
+        "**NOTE**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "G__krby2Mqmh"
+      },
+      "source": [
+        "### Colab only"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "RS5_6QVFMyc-"
+      },
+      "outputs": [],
+      "source": [
+        "!pip3 install --upgrade google-cloud-aiplatform"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Li2aj1n9NDYl"
+      },
+      "outputs": [],
+      "source": [
+        "from google.colab import auth as google_auth\n",
+        "\n",
+        "google_auth.authenticate_user()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "f6c1bc20-3495-448a-b242-01930ba8153c"
+      },
+      "source": [
+        "### Setup Google Cloud project\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI API and Compute Engine API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component).\n",
+        "\n",
+        "1. [Create a Cloud Storage bucket](https://cloud.google.com/storage/docs/creating-buckets) for storing experiment outputs."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5b1f2c08-c84e-4158-a976-e6b59d1055ca"
+      },
+      "source": [
+        "It's highly recommended to run this notebook on [Vertex AI workbench](https://cloud.google.com/vertex-ai-workbench)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "4fc1fc14-2d77-4bf7-8f6d-c1afc10c848a"
+      },
+      "source": [
+        "If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk) and [gsutil](https://cloud.google.com/storage/docs/gsutil_install)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e02b9811-b730-4573-83fa-9d47f2ce0436"
+      },
+      "source": [
+        "####Fill following variables for experiments environment:\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7c2ce2fa-5a9b-40f6-b99d-6c1325775b36"
+      },
+      "outputs": [],
+      "source": [
+        "# Cloud project id.\n",
+        "PROJECT_ID = \"\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# The region you want to launch jobs in.\n",
+        "REGION = \"us-central1\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# The Cloud Storage bucket for storing experiments output. Fill it without the 'gs://' prefix.\n",
+        "GCS_BUCKET = \"\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "fwgeVkFXOS64"
+      },
+      "source": [
+        "*Initialize* Vertex AI API:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "MxqnJMahPK8r"
+      },
+      "outputs": [],
+      "source": [
+        "from google.cloud import aiplatform\n",
+        "\n",
+        "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=GCS_BUCKET)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tSjMVyG8PYS-"
+      },
+      "source": [
+        "### Define constants"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "H05BzPO5Pcnh"
+      },
+      "outputs": [],
+      "source": [
+        "# The pre-built serving docker image.\n",
+        "# The model artifacts are embedded within the container, except for model weights which will be downloaded during deployment.\n",
+        "SERVE_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai-restricted/vertex-vision-model-garden-dockers/sam-serve\"\n",
+        "\n",
+        "# The serving port.\n",
+        "SERVE_PORT = 7080\n",
+        "\n",
+        "# The serving route.\n",
+        "SERVE_ROUTE = \"/predictions/sam_serving\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "82058975-23b5-4b97-9e14-dd9a29c578ed"
+      },
+      "source": [
+        "### Define common utility functions"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "a78f988d-e5e2-4a57-ba0f-569e970514c0"
+      },
+      "outputs": [],
+      "source": [
+        "import base64\n",
+        "from io import BytesIO\n",
+        "\n",
+        "import matplotlib.patches as patches\n",
+        "import matplotlib.pyplot as plt\n",
+        "import numpy as np\n",
+        "import pycocotools.mask as mask_util\n",
+        "import requests\n",
+        "from PIL import Image\n",
+        "\n",
+        "\n",
+        "def download_image(url):\n",
+        "    response = requests.get(url)\n",
+        "    return Image.open(BytesIO(response.content)).convert(\"RGB\")\n",
+        "\n",
+        "\n",
+        "def image_to_base64(image):\n",
+        "    buffer = BytesIO()\n",
+        "    image.save(buffer, format=\"JPEG\")\n",
+        "    image_str = base64.b64encode(buffer.getvalue()).decode(\"utf-8\")\n",
+        "    return image_str\n",
+        "\n",
+        "\n",
+        "def base64_to_image(image_str):\n",
+        "    image = Image.open(BytesIO(base64.b64decode(image_str)))\n",
+        "    return image\n",
+        "\n",
+        "\n",
+        "def image_grid(imgs, rows=2, cols=2):\n",
+        "    w, h = imgs[0].size\n",
+        "    grid = Image.new(\"RGB\", size=(cols * w, rows * h))\n",
+        "    for i, img in enumerate(imgs):\n",
+        "        grid.paste(img, box=(i % cols * w, i // cols * h))\n",
+        "    return grid\n",
+        "\n",
+        "\n",
+        "def draw_image_with_boxes(image, boxes):\n",
+        "    fig, ax = plt.subplots()\n",
+        "    ax.imshow(image)\n",
+        "    for box in boxes:\n",
+        "        x, y = box[\"xmin\"], box[\"ymin\"]\n",
+        "        width, height = box[\"xmax\"] - x, box[\"ymax\"] - y\n",
+        "        rect = patches.Rectangle(\n",
+        "            (x, y), width, height, linewidth=2, edgecolor=\"yellow\", facecolor=\"none\"\n",
+        "        )\n",
+        "        ax.add_patch(rect)\n",
+        "    plt.axis(\"off\")\n",
+        "    plt.show()\n",
+        "\n",
+        "\n",
+        "def decode_rle_masks(pred_masks_rle):\n",
+        "    return np.stack([mask_util.decode(rle) for rle in pred_masks_rle])\n",
+        "\n",
+        "\n",
+        "def show_mask(mask, ax, random_color=False):\n",
+        "    if random_color:\n",
+        "        color = np.concatenate([np.random.random(3), np.array([0.6])], axis=0)\n",
+        "    else:\n",
+        "        color = np.array([30 / 255, 144 / 255, 255 / 255, 0.6])\n",
+        "    h, w = mask.shape[-2:]\n",
+        "    mask_image = mask.reshape(h, w, 1) * color.reshape(1, 1, -1)\n",
+        "    ax.imshow(mask_image)\n",
+        "\n",
+        "\n",
+        "def show_predictions(preds):\n",
+        "    # create figure\n",
+        "    fig = plt.figure(figsize=(10, 7))\n",
+        "\n",
+        "    fig.add_subplot(1, 2, 1)\n",
+        "    plt.imshow(np.array(image1))\n",
+        "    ax = plt.gca()\n",
+        "    masks = decode_rle_masks(preds[0][\"masks_rle\"])\n",
+        "    for mask in masks:\n",
+        "        show_mask(mask, ax=ax, random_color=True)\n",
+        "    plt.axis(\"off\")\n",
+        "\n",
+        "    fig.add_subplot(1, 2, 2)\n",
+        "    plt.imshow(np.array(image2))\n",
+        "    ax = plt.gca()\n",
+        "    masks = decode_rle_masks(preds[1][\"masks_rle\"])\n",
+        "    for mask in masks:\n",
+        "        show_mask(mask, ax=ax, random_color=True)\n",
+        "    plt.axis(\"off\")\n",
+        "    plt.show()\n",
+        "\n",
+        "\n",
+        "def deploy_model(task, model_id):\n",
+        "    endpoint = aiplatform.Endpoint.create(display_name=f\"{task}-endpoint\")\n",
+        "    serving_env = {\n",
+        "        \"MODEL_ID\": model_id,\n",
+        "        \"TASK\": task,\n",
+        "    }\n",
+        "    model = aiplatform.Model.upload(\n",
+        "        display_name=task,\n",
+        "        serving_container_image_uri=SERVE_DOCKER_URI,\n",
+        "        serving_container_ports=[SERVE_PORT],\n",
+        "        serving_container_predict_route=SERVE_ROUTE,\n",
+        "        serving_container_health_route=\"/ping\",\n",
+        "        serving_container_environment_variables=serving_env,\n",
+        "    )\n",
+        "    model.deploy(\n",
+        "        endpoint=endpoint,\n",
+        "        machine_type=\"n1-standard-8\",\n",
+        "        accelerator_type=\"NVIDIA_TESLA_T4\",\n",
+        "        accelerator_count=1,\n",
+        "        deploy_request_timeout=1800,\n",
+        "    )\n",
+        "    return endpoint, model"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "242fc8cf-5a0e-483b-8c9d-77a474cedc4b"
+      },
+      "source": [
+        "## Mask Generation with Vertex AI Endpoint"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "f3f9ed94d69c"
+      },
+      "outputs": [],
+      "source": [
+        "endpoint, model = deploy_model(\n",
+        "    task=\"mask-generation\", model_id=\"facebook/sam-vit-large\"\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "d6e51c57-b5e2-4ae7-888a-5391cceee5fb"
+      },
+      "outputs": [],
+      "source": [
+        "image1 = download_image(\"http://images.cocodataset.org/val2017/000000039769.jpg\")\n",
+        "image2 = download_image(\"http://images.cocodataset.org/val2017/000000000285.jpg\")\n",
+        "grid = image_grid([image1, image2], 1, 2)\n",
+        "display(grid)\n",
+        "\n",
+        "instances = [\n",
+        "    {\"image\": image_to_base64(image1)},\n",
+        "    {\"image\": image_to_base64(image2)},\n",
+        "]\n",
+        "preds = endpoint.predict(instances=instances).predictions\n",
+        "show_predictions(preds)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5UdquXMWR1E4"
+      },
+      "source": [
+        "## Clean up"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "TGTWIJK8R136"
+      },
+      "outputs": [],
+      "source": [
+        "# Undeploy model and delete endpoint.\n",
+        "endpoint.delete(force=True)\n",
+        "\n",
+        "# Delete models.\n",
+        "model.delete()"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "model_garden_pytorch_sam.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}