From 1867536d76c0d284f1d3622a60a0111797aac2a2 Mon Sep 17 00:00:00 2001 From: Margubur Rahman <150442997+googlyrahman@users.noreply.github.com> Date: Thu, 18 Dec 2025 19:29:10 +0530 Subject: [PATCH] Migrate gsutil usage to gcloud storage (#4294) * Migrate gsutil usage to gcloud storage * updates * removed notes added by agent * removed note * linter changes --------- Co-authored-by: gurusai-voleti --- .../Upload_PyTorch_model_archive/component.yaml | 2 +- ...ediction_with_custom_torchserve_container.ipynb | 6 +++--- .../model_garden_pytorch_fill_mask.ipynb | 12 +++++------- ..._garden_pytorch_stable_diffusion_xl_turbo.ipynb | 8 ++++---- .../model_garden_pytorch_vilt_vqa.ipynb | 12 +++++++----- ...arden_pytorch_whisper_large_v3_deployment.ipynb | 14 ++++++++------ .../sdk/SDK_AutoML_Video_Action_Recognition.ipynb | 6 +++--- ...tom_Training_with_Unmanaged_Image_Dataset.ipynb | 8 ++++---- ..._embedding_api_semantic_search_with_scann.ipynb | 2 +- ...get_started_with_google_artifact_registry.ipynb | 2 +- 10 files changed, 37 insertions(+), 35 deletions(-) diff --git a/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/component.yaml b/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/component.yaml index 7044b6aec..bd6674274 100644 --- a/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/component.yaml +++ b/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/component.yaml @@ -148,7 +148,7 @@ implementation: # Downloading the model archive from GCS # TODO: Fix gsutil bugs (requires project ID, has auth issues) and use gsutil instead. - # gsutil cp "$model_archive_uri" "$model_archive_local_path" + # gcloud storage cp "$model_archive_uri" "$model_archive_local_path" pip install google-cloud-storage python -c ' import sys diff --git a/community-content/pytorch_image_classification_single_gpu_with_vertex_sdk_and_torchserve/vertex_prediction_with_custom_torchserve_container.ipynb b/community-content/pytorch_image_classification_single_gpu_with_vertex_sdk_and_torchserve/vertex_prediction_with_custom_torchserve_container.ipynb index db7ce69fe..30741c13f 100644 --- a/community-content/pytorch_image_classification_single_gpu_with_vertex_sdk_and_torchserve/vertex_prediction_with_custom_torchserve_container.ipynb +++ b/community-content/pytorch_image_classification_single_gpu_with_vertex_sdk_and_torchserve/vertex_prediction_with_custom_torchserve_container.ipynb @@ -110,7 +110,7 @@ }, "outputs": [], "source": [ - "! gsutil ls $gcs_output_uri_prefix" + "! gcloud storage ls $gcs_output_uri_prefix" ] }, { @@ -192,7 +192,7 @@ }, "outputs": [], "source": [ - "! gsutil cp -r $gcs_output_uri_prefix/model ./model_server/" + "! gcloud storage cp --recursive $gcs_output_uri_prefix/model ./model_server/" ] }, { @@ -556,7 +556,7 @@ }, "outputs": [], "source": [ - "! gsutil rm -rf $gcs_output_uri_prefix" + "! gcloud storage rm --recursive --continue-on-error $gcs_output_uri_prefix" ] }, { diff --git a/notebooks/community/model_garden/model_garden_pytorch_fill_mask.ipynb b/notebooks/community/model_garden/model_garden_pytorch_fill_mask.ipynb index 030529c6c..d759d9e8e 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_fill_mask.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_fill_mask.ipynb @@ -125,9 +125,9 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gsutil mb -l {REGION} {BUCKET_URI}\n", + " ! gcloud storage buckets create --location {REGION} {BUCKET_URI}\n", "else:\n", - " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", @@ -145,7 +145,7 @@ "print(\"Using this default Service Account:\", SERVICE_ACCOUNT)\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=\"serviceAccount:{SERVICE_ACCOUNT}\" --role=\"roles/storage.admin\"\n", "\n", "# The pre-built serving docker image. It contains serving scripts and models.\n", "SERVE_DOCKER_URI = \"us-docker.pkg.dev/deeplearning-platform-release/gcr.io/huggingface-pytorch-inference-cu121.2-2.transformers.4-41.ubuntu2204.py311\"\n", @@ -185,9 +185,7 @@ " accelerator_count=1,\n", " deploy_request_timeout=1800,\n", " service_account=SERVICE_ACCOUNT,\n", - " system_labels={\n", - " \"NOTEBOOK_NAME\": \"model_garden_pytorch_fill_mask.ipynb\"\n", - " },\n", + " system_labels={\"NOTEBOOK_NAME\": \"model_garden_pytorch_fill_mask.ipynb\"},\n", " )\n", " return model, endpoint" ] @@ -283,7 +281,7 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gsutil -m rm -r $BUCKET_NAME" + " ! gcloud storage rm --recursive $BUCKET_NAME" ] } ], diff --git a/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion_xl_turbo.ipynb b/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion_xl_turbo.ipynb index 6a59ec3de..a8ae6dfc3 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion_xl_turbo.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion_xl_turbo.ipynb @@ -148,10 +148,10 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gsutil mb -l {REGION} {BUCKET_URI}\n", + " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", @@ -170,7 +170,7 @@ "print(\"Using this default Service Account:\", SERVICE_ACCOUNT)\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=BUCKET_URI)\n", "\n", @@ -338,7 +338,7 @@ "# Delete bucket.\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gsutil -m rm -r $BUCKET_NAME" + " ! gcloud storage rm --recursive $BUCKET_NAME" ] } ], diff --git a/notebooks/community/model_garden/model_garden_pytorch_vilt_vqa.ipynb b/notebooks/community/model_garden/model_garden_pytorch_vilt_vqa.ipynb index df1728960..6fc498137 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_vilt_vqa.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_vilt_vqa.ipynb @@ -135,10 +135,10 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gsutil mb -l {REGION} {BUCKET_URI}\n", + " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", @@ -163,7 +163,7 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", @@ -271,7 +271,9 @@ "# print(\"Using this existing endpoint from a different session: {aip_endpoint_name}\")\n", "\n", "# @markdown ![](http://images.cocodataset.org/val2017/000000039769.jpg?w=1260&h=750)\n", - "image = \"http://images.cocodataset.org/val2017/000000039769.jpg\" # @param {type: \"string\"}\n", + "image = (\n", + " \"http://images.cocodataset.org/val2017/000000039769.jpg\" # @param {type: \"string\"}\n", + ")\n", "question = \"Which cat is bigger?\" # @param {type: \"string\"}\n", "\n", "instances = [\n", @@ -310,7 +312,7 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gsutil -m rm -r $BUCKET_NAME" + " ! gcloud storage rm --recursive $BUCKET_NAME" ] } ], diff --git a/notebooks/community/model_garden/model_garden_pytorch_whisper_large_v3_deployment.ipynb b/notebooks/community/model_garden/model_garden_pytorch_whisper_large_v3_deployment.ipynb index 283d8f354..53f1a7763 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_whisper_large_v3_deployment.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_whisper_large_v3_deployment.ipynb @@ -143,10 +143,10 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gsutil mb -l {REGION} {BUCKET_URI}\n", + " ! gcloud storage buckets create {BUCKET_URI} --location={REGION}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", @@ -171,7 +171,7 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", @@ -191,7 +191,9 @@ "\n", "# @markdown This section deploys the prebuilt whisper large v3 and whisper large v3 turbo model on a Vertex endpoint. It takes 15 minutes to deploy this model.\n", "\n", - "model_name = \"whisper-large-v3-turbo\" # @param [\"whisper-large-v3\", \"whisper-large-v3-turbo\"]\n", + "model_name = (\n", + " \"whisper-large-v3-turbo\" # @param [\"whisper-large-v3\", \"whisper-large-v3-turbo\"]\n", + ")\n", "model_id = \"openai/{name}\".format(name=model_name)\n", "task = \"audio2text\"\n", "\n", @@ -303,7 +305,7 @@ " print(\n", " f\"Provisioning roles/storage.objectViewer permission to the {SERVICE_ACCOUNT} with the {bucket}.\"\n", " )\n", - " ! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $bucket\n", + " ! gcloud storage buckets add-iam-policy-binding $bucket --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer\n", "\n", "instances = [{\"audio\": uri} for uri in gcs_uris]\n", "\n", @@ -341,7 +343,7 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gsutil -m rm -r $BUCKET_NAME" + " ! gcloud storage rm --recursive $BUCKET_NAME" ] } ], diff --git a/notebooks/community/sdk/SDK_AutoML_Video_Action_Recognition.ipynb b/notebooks/community/sdk/SDK_AutoML_Video_Action_Recognition.ipynb index 2a5b50ff5..426260a73 100644 --- a/notebooks/community/sdk/SDK_AutoML_Video_Action_Recognition.ipynb +++ b/notebooks/community/sdk/SDK_AutoML_Video_Action_Recognition.ipynb @@ -206,7 +206,7 @@ }, "outputs": [], "source": [ - "!gsutil cp $automl_video_demo_train_data $gcs_source_train" + "!gcloud storage cp $automl_video_demo_train_data $gcs_source_train" ] }, { @@ -416,7 +416,7 @@ }, "outputs": [], "source": [ - "!gsutil cp $automl_video_demo_batch_prediction_data $gcs_source_batch_prediction" + "!gcloud storage cp $automl_video_demo_batch_prediction_data $gcs_source_batch_prediction" ] }, { @@ -488,7 +488,7 @@ "metadata": { "colab": { "collapsed_sections": [], - "name": "AI_Platform_(Unified)_SDK_AutoML_Video_Action_Recognition.ipynb", + "name": "SDK_AutoML_Video_Action_Recognition.ipynb", "toc_visible": true }, "kernelspec": { diff --git a/notebooks/community/sdk/SDK_Custom_Training_with_Unmanaged_Image_Dataset.ipynb b/notebooks/community/sdk/SDK_Custom_Training_with_Unmanaged_Image_Dataset.ipynb index 7660cd7ce..605363d1b 100644 --- a/notebooks/community/sdk/SDK_Custom_Training_with_Unmanaged_Image_Dataset.ipynb +++ b/notebooks/community/sdk/SDK_Custom_Training_with_Unmanaged_Image_Dataset.ipynb @@ -338,9 +338,9 @@ }, "outputs": [], "source": [ - "!gsutil -m cp -R gs://cloud-ml-data/img/flower_photos/daisy/14221848160_7f0a37c395.jpg .\n", - "!gsutil -m cp -R gs://cloud-ml-data/img/flower_photos/tulips/13289268363_b9337d751e.jpg .\n", - "!gsutil -m cp -R gs://cloud-ml-data/img/flower_photos/sunflowers/14623719696_1bb7970208_n.jpg ." + "!gcloud storage cp --recursive gs://cloud-ml-data/img/flower_photos/daisy/14221848160_7f0a37c395.jpg .\n", + "!gcloud storage cp --recursive gs://cloud-ml-data/img/flower_photos/tulips/13289268363_b9337d751e.jpg .\n", + "!gcloud storage cp --recursive gs://cloud-ml-data/img/flower_photos/sunflowers/14623719696_1bb7970208_n.jpg ." ] }, { @@ -383,7 +383,7 @@ "metadata": { "colab": { "collapsed_sections": [], - "name": "AI_Platform_(Unified)_SDK_Custom_Training_with_Unmanaged_Image_Dataset.ipynb", + "name": "SDK_Custom_Training_with_Unmanaged_Image_Dataset.ipynb", "toc_visible": true }, "kernelspec": { diff --git a/notebooks/official/generative_ai/text_embedding_api_semantic_search_with_scann.ipynb b/notebooks/official/generative_ai/text_embedding_api_semantic_search_with_scann.ipynb index 5cb4a2638..dd93ef0b8 100644 --- a/notebooks/official/generative_ai/text_embedding_api_semantic_search_with_scann.ipynb +++ b/notebooks/official/generative_ai/text_embedding_api_semantic_search_with_scann.ipynb @@ -328,7 +328,7 @@ }, "outputs": [], "source": [ - "!gsutil cp gs://cloud-samples-data/vertex-ai/dataset-management/datasets/bert_finetuning/wide_and_deep_trainer_container_tests_input.jsonl ." + "!gcloud storage cp gs://cloud-samples-data/vertex-ai/dataset-management/datasets/bert_finetuning/wide_and_deep_trainer_container_tests_input.jsonl ." ] }, { diff --git a/notebooks/official/ml_metadata/get_started_with_google_artifact_registry.ipynb b/notebooks/official/ml_metadata/get_started_with_google_artifact_registry.ipynb index 61467f896..575d5c57b 100644 --- a/notebooks/official/ml_metadata/get_started_with_google_artifact_registry.ipynb +++ b/notebooks/official/ml_metadata/get_started_with_google_artifact_registry.ipynb @@ -257,7 +257,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}" + "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" ] }, {