Migrate gsutil usage to gcloud storage (#4293)

* Migrate gsutil usage to gcloud storage

* update

---------

Co-authored-by: gurusai-voleti <gvoleti@google.com>

community-content/pytorch_image_classification_single_gpu_with_vertex_sdk_and_torchserve/vertex_training_with_custom_container.ipynb

@@ -412,7 +412,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls $gcs_output_uri_prefix"
+        "! gcloud storage ls $gcs_output_uri_prefix"
       ]
     }
   ],

community-content/tf_keras_image_classification_distributed_multi_worker_with_vertex_sdk/multi_worker_vertex_training_on_cpu_with_custom_container.ipynb

@@ -324,7 +324,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls $gcs_output_uri_prefix"
+        "! gcloud storage ls $gcs_output_uri_prefix"
       ]
     },
     {
@@ -344,7 +344,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil rm -rf $gcs_output_uri_prefix"
+        "! gcloud storage rm --recursive --continue-on-error $gcs_output_uri_prefix"
       ]
     }
   ],

community-content/tf_keras_image_classification_distributed_multi_worker_with_vertex_sdk/multi_worker_vertex_training_on_gpu_with_custom_container.ipynb

@@ -328,7 +328,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls $gcs_output_uri_prefix"
+        "! gcloud storage ls $gcs_output_uri_prefix"
       ]
     },
     {
@@ -348,7 +348,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil rm -rf $gcs_output_uri_prefix"
+        "! gcloud storage rm --recursive --continue-on-error $gcs_output_uri_prefix"
       ]
     }
   ],

community-content/tf_keras_text_classification_distributed_single_worker_gpus_with_gcloud_local_run_and_vertex_sdk/vertex_training_with_local_mode_container.ipynb

@@ -341,7 +341,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls $gcs_output_uri_prefix"
+        "! gcloud storage ls $gcs_output_uri_prefix"
       ]
     },
     {
@@ -361,7 +361,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil rm -rf $gcs_output_uri_prefix"
+        "! gcloud storage rm --recursive --continue-on-error $gcs_output_uri_prefix"
       ]
     }
   ],

community-content/vertex_model_garden/model_oss/cloudnerf/local_colmap_and_resize.sh

@@ -66,7 +66,7 @@ mkdir -p "$local_folder"
 mkdir -p "$output_folder"
 
 # Download the content from the GCS URI
-gsutil -m cp -r "$gcs_dataset_path"/* "$local_folder/"
+gcloud storage cp --recursive "$gcs_dataset_path"/* "$local_folder/"
 
 # Process files in the local folder
 for file in "$local_folder"/*; do
@@ -122,23 +122,23 @@ cp -r "$output_folder" "$images_folder"/images_2
 pushd "$images_folder"/images_2
 ls | xargs -P 8 -I {} mogrify -resize 50% {}
 popd
-gsutil -m cp -r "$images_folder"/images_2/*  "$gcs_experiment_path"/data/images_2
+gcloud storage cp --recursive "$images_folder"/images_2/*  "$gcs_experiment_path"/data/images_2
 
 cp -r "$output_folder" "$images_folder"/images_4
 pushd "$images_folder"/images_4
 ls | xargs -P 8 -I {} mogrify -resize 25% {}
 popd
-gsutil -m cp -r "$images_folder"/images_4/*  "$gcs_experiment_path"/data/images_4
+gcloud storage cp --recursive "$images_folder"/images_4/*  "$gcs_experiment_path"/data/images_4
 
 cp -r "$output_folder" "$images_folder"/images_8
 pushd "$images_folder"/images_8
 ls | xargs -P 8 -I {} mogrify -resize 12.5% {}
 popd
-gsutil -m cp "$images_folder"/images_8/*  "$gcs_experiment_path"/data/images_8
+gcloud storage cp "$images_folder"/images_8/*  "$gcs_experiment_path"/data/images_8
 
 # Copy images and sparse reconstruction files to gcs experiment folder.
-gsutil -m cp "$images_folder"/images/*  "$gcs_experiment_path"/data/images
-gsutil -m cp -r "$local_folder"/sparse  "$gcs_experiment_path"/data
-gsutil -m cp "$local_folder"/database.db  "$gcs_experiment_path"/data
+gcloud storage cp "$images_folder"/images/*  "$gcs_experiment_path"/data/images
+gcloud storage cp --recursive "$local_folder"/sparse  "$gcs_experiment_path"/data
+gcloud storage cp "$local_folder"/database.db  "$gcs_experiment_path"/data
 
 echo "Processing complete."

notebooks/community/model_garden/docker_source_codes/model_oss/notebook_util/gcp_utils.py

@@ -117,24 +117,24 @@ def setup_buckets(bucket_uri: str, model_bucket_name: str) -> tuple[str, str]:
 
   # Check if bucket exists
   try:
-    run_command(["gsutil", "ls", "-b", bucket_uri])
+    run_command(["gcloud", "storage", "ls", "--buckets", bucket_uri])
     logger.info("Bucket %s already exists.", bucket_uri)
   except subprocess.CalledProcessError:
     logger.info("Creating bucket %s.", bucket_uri)
     # Create the bucket in the same region as the project
-    run_command(["gsutil", "mb", "-l", REGION, bucket_uri])
+    run_command(["gcloud", "storage", "buckets", "create", "--location", REGION, bucket_uri])
 
   # Construct the model bucket path
   model_bucket = os.path.join(bucket_uri, model_bucket_name)
 
   # Check if the model bucket exists (as a folder within the main bucket)
   try:
-    run_command(["gsutil", "ls", model_bucket])
+    run_command(["gcloud", "storage", "ls", model_bucket])
     logger.info("Model bucket %s already exists.", model_bucket)
   except subprocess.CalledProcessError:
     logger.info("Creating model bucket %s.", model_bucket)
     # Create the model bucket folder
-    run_command(["gsutil", "cp", "/dev/null", model_bucket + "/"])
+    run_command(["gcloud", "storage", "cp", "/dev/null", model_bucket + "/"])
 
   return bucket_name, model_bucket
 
@@ -174,11 +174,13 @@ def provision_permissions(service_account: str, bucket_name: str) -> None:
   """Provision permissions to the service account with the GCS bucket."""
   if bucket_name:
     run_command([
-        "gsutil",
-        "iam",
-        "ch",
-        f"serviceAccount:{service_account}:roles/storage.admin",
+        "gcloud",
+        "storage",
+        "buckets",
+        "add-iam-policy-binding",
         bucket_name,
+        f"--member=serviceAccount:{service_account}",
+        "--role=roles/storage.admin",
     ])
 
 
@@ -263,13 +265,13 @@ def delete_endpoint_and_model(
 
 
 def delete_gcs_bucket(bucket_name: str) -> str:
-  """Deletes a GCS bucket using gsutil."""
+  """Deletes a GCS bucket using gcloud storage."""
   try:
-    run_command(["gsutil", "-m", "rm", "-r", bucket_name])
-    logger.info("Bucket %s deleted using gsutil.", bucket_name)
+    run_command(["gcloud", "storage", "rm", "--recursive", bucket_name])
+    logger.info("Bucket %s deleted using gcloud storage.", bucket_name)
     return f"Bucket {bucket_name} deleted successfully!"
   except subprocess.CalledProcessError as e:
     logger.error(
-        "Error deleting bucket %s using gsutil: %s", bucket_name, str(e)
+        "Error deleting bucket %s using gcloud storage: %s", bucket_name, str(e)
     )
     return f"Bucket {bucket_name} could not be found or deleted. "

notebooks/community/model_garden/docker_source_codes/notebook_util/gcp_utils.py

@@ -117,24 +117,24 @@ def setup_buckets(bucket_uri: str, model_bucket_name: str) -> tuple[str, str]:
 
   # Check if bucket exists
   try:
-    run_command(["gsutil", "ls", "-b", bucket_uri])
+    run_command(["gcloud", "storage", "ls", "--buckets", bucket_uri])
     logger.info("Bucket %s already exists.", bucket_uri)
   except subprocess.CalledProcessError:
     logger.info("Creating bucket %s.", bucket_uri)
     # Create the bucket in the same region as the project
-    run_command(["gsutil", "mb", "-l", REGION, bucket_uri])
+    run_command(["gcloud", "storage", "buckets", "create", "--location", REGION, bucket_uri])
 
   # Construct the model bucket path
   model_bucket = os.path.join(bucket_uri, model_bucket_name)
 
   # Check if the model bucket exists (as a folder within the main bucket)
   try:
-    run_command(["gsutil", "ls", model_bucket])
+    run_command(["gcloud", "storage", "ls", model_bucket])
     logger.info("Model bucket %s already exists.", model_bucket)
   except subprocess.CalledProcessError:
     logger.info("Creating model bucket %s.", model_bucket)
     # Create the model bucket folder
-    run_command(["gsutil", "cp", "/dev/null", model_bucket + "/"])
+    run_command(["gcloud", "storage", "cp", "/dev/null", model_bucket + "/"])
 
   return bucket_name, model_bucket
 
@@ -174,11 +174,13 @@ def provision_permissions(service_account: str, bucket_name: str) -> None:
   """Provision permissions to the service account with the GCS bucket."""
   if bucket_name:
     run_command([
-        "gsutil",
-        "iam",
-        "ch",
-        f"serviceAccount:{service_account}:roles/storage.admin",
+        "gcloud",
+        "storage",
+        "buckets",
+        "add-iam-policy-binding",
         bucket_name,
+        f"--member=serviceAccount:{service_account}",
+        "--role=roles/storage.admin",
     ])
 
 
@@ -263,13 +265,13 @@ def delete_endpoint_and_model(
 
 
 def delete_gcs_bucket(bucket_name: str) -> str:
-  """Deletes a GCS bucket using gsutil."""
+  """Deletes a GCS bucket using gcloud storage."""
   try:
-    run_command(["gsutil", "-m", "rm", "-r", bucket_name])
-    logger.info("Bucket %s deleted using gsutil.", bucket_name)
+    run_command(["gcloud", "storage", "rm", "--recursive", bucket_name])
+    logger.info("Bucket %s deleted using gcloud storage.", bucket_name)
     return f"Bucket {bucket_name} deleted successfully!"
   except subprocess.CalledProcessError as e:
     logger.error(
-        "Error deleting bucket %s using gsutil: %s", bucket_name, str(e)
+        "Error deleting bucket %s using gcloud storage: %s", bucket_name, str(e)
     )
     return f"Bucket {bucket_name} could not be found or deleted. "

notebooks/community/model_garden/model_garden_pytorch_text_to_video_zero_shot.ipynb

@@ -124,10 +124,10 @@
         "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n",
         "    BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n",
         "    BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n",
-        "    ! gsutil mb -l {REGION} {BUCKET_URI}\n",
+        "    ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n",
         "else:\n",
         "    assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n",
-        "    shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n",
+        "    shell_output = ! gcloud storage buckets describe {BUCKET_NAME} --format=\"value(location)\"\n",
         "    bucket_region = shell_output[0].strip().lower()\n",
         "    if bucket_region != REGION:\n",
         "        raise ValueError(\n",
@@ -146,7 +146,7 @@
         "print(\"Using this default Service Account:\", SERVICE_ACCOUNT)\n",
         "\n",
         "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n",
-        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n",
+        "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n",
         "\n",
         "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=BUCKET_URI)\n",
         "\n",
@@ -271,7 +271,7 @@
         "# Delete bucket.\n",
         "delete_bucket = False  # @param {type:\"boolean\"}\n",
         "if delete_bucket:\n",
-        "    ! gsutil -m rm -r $BUCKET_NAME"
+        "    ! gcloud storage rm --recursive $BUCKET_NAME"
       ]
     }
   ],

notebooks/community/sdk/SDK_End_to_End_Tabular_Custom_Training.ipynb

@@ -133,7 +133,7 @@
       "outputs": [],
       "source": [
         "!wget https://storage.googleapis.com/download.tensorflow.org/data/abalone_train.csv\n",
-        "!gsutil cp abalone_train.csv {MY_STAGING_BUCKET}/data/\n",
+        "!gcloud storage cp abalone_train.csv {MY_STAGING_BUCKET}/data/\n",
         "\n",
         "gcs_csv_path = f\"{MY_STAGING_BUCKET}/data/abalone_train.csv\""
       ]
@@ -322,7 +322,7 @@
   "metadata": {
     "colab": {
       "collapsed_sections": [],
-      "name": "AI_Platform_(Unified)_SDK_End_to_End_Tabular_Custom_Training.ipynb",
+      "name": "SDK_End_to_End_Tabular_Custom_Training.ipynb",
       "toc_visible": true
     },
     "kernelspec": {

notebooks/community/sdk/SDK_Tabular_Custom_Model_Training_asynchronous.ipynb

@@ -133,7 +133,7 @@
       "outputs": [],
       "source": [
         "!wget https://storage.googleapis.com/download.tensorflow.org/data/abalone_train.csv\n",
-        "!gsutil cp abalone_train.csv {MY_STAGING_BUCKET}/data/\n",
+        "!gcloud storage cp abalone_train.csv {MY_STAGING_BUCKET}/data/\n",
         "\n",
         "gcs_csv_path = f\"{MY_STAGING_BUCKET}/data/abalone_train.csv\""
       ]
@@ -344,7 +344,7 @@
   "metadata": {
     "colab": {
       "collapsed_sections": [],
-      "name": "AI_Platform_(Unified)_SDK_Tabular_Custom_Model_Training_(asynchronous).ipynb",
+      "name": "SDK_Tabular_Custom_Model_Training_asynchronous.ipynb",
       "toc_visible": true
     },
     "kernelspec": {