xiaohei/model_garden
1
0
Fork 0
mirror of https://github.com/GoogleCloudPlatform/vertex-ai-samples.git synced 2026-09-26 14:42:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Migrate gsutil usage to gcloud storage (#4320)

Browse Source 
* Migrate gsutil usage to gcloud storage

* Changes for 4320

* fix linter issue for 4320

* removed chnages for model garden

---------

Co-authored-by: bhandarivijay <bhandarivijay@google.com>
Co-authored-by: gurusai-voleti <gvoleti@google.com>
This commit is contained in:
Margubur Rahman
2025-12-25 10:47:17 -05:00
committed by GitHub
co-authored by bhandarivijay gurusai-voleti
parent 5fc93c8bdb
commit 9ed3c2f83d
9 changed files with 41 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
notebooks/community/ml_ops/stage2/get_started_vertex_training_r.ipynb
+3 -3
View File
@@ -445,7 +445,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l $REGION $BUCKET_URI"
"! gcloud storage buckets create --location=$REGION $BUCKET_URI"
]
},
{
@@ -465,7 +465,7 @@
},
"outputs": [],
"source": [
"! gsutil ls -al $BUCKET_URI"
"! gcloud storage ls --all-versions --long $BUCKET_URI"
]
},
{
@@ -1427,7 +1427,7 @@
"delete_bucket = False\n",
"\n",
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
" ! gsutil rm -r $BUCKET_URI"
" ! gcloud storage rm --recursive $BUCKET_URI"
]
}
],
notebooks/community/ml_ops/stage3/get_started_with_automl_tabular_pipeline_workflow.ipynb
+5 -5
View File
@@ -450,7 +450,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l $REGION $BUCKET_URI"
"! gcloud storage buckets create --location=$REGION $BUCKET_URI"
]
},
{
@@ -470,7 +470,7 @@
},
"outputs": [],
"source": [
"! gsutil ls -al $BUCKET_URI"
"! gcloud storage ls --all-versions --long $BUCKET_URI"
]
},
{
@@ -496,7 +496,7 @@
"source": [
"# set GCS bucket object TTL to 7 days\n",
"! echo '{\"rule\":[{\"action\": {\"type\": \"Delete\"},\"condition\": {\"age\": 7}}]}' > gcs_lifecycle.tmp\n",
"! gsutil lifecycle set gcs_lifecycle.tmp {BUCKET_URI}\n",
"! gcloud storage buckets update --lifecycle-file=gcs_lifecycle.tmp {BUCKET_URI}\n",
"! rm gcs_lifecycle.tmp"
]
},
@@ -1047,7 +1047,7 @@
"\n",
"print(\"trained model without custom TF ops:\", model_artifact_no_custom_op)\n",
"\n",
"! gsutil ls {model_artifact_no_custom_op}\n",
"! gcloud storage ls {model_artifact_no_custom_op}\n",
"\n",
"model_evaluation = get_evaluation_metrics(pipeline_task_details)\n",
"print(\"Model evaluation artifacts:\", model_evaluation)"
@@ -1337,7 +1337,7 @@
"\n",
"delete_bucket = False\n",
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
" ! gsutil rm -rf $BUCKET_URI"
" ! gcloud storage rm --recursive --continue-on-error $BUCKET_URI"
]
}
],
notebooks/community/ml_ops/stage3/get_started_with_machine_management.ipynb
+10 -10
View File
@@ -450,7 +450,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l $REGION $BUCKET_URI"
"! gcloud storage buckets create --location=$REGION $BUCKET_URI"
]
},
{
@@ -470,7 +470,7 @@
},
"outputs": [],
"source": [
"! gsutil ls -al $BUCKET_URI"
"! gcloud storage ls --all-versions --long $BUCKET_URI"
]
},
{
@@ -541,9 +541,9 @@
},
"outputs": [],
"source": [
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n",
"\n",
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer"
]
},
{
@@ -1015,13 +1015,13 @@
" + \"/evaluation_metrics\"\n",
" )\n",
" if tf.io.gfile.exists(EXECUTE_OUTPUT):\n",
" ! gsutil cat $EXECUTE_OUTPUT\n",
" ! gcloud storage cat $EXECUTE_OUTPUT\n",
" return EXECUTE_OUTPUT\n",
" elif tf.io.gfile.exists(GCP_RESOURCES):\n",
" ! gsutil cat $GCP_RESOURCES\n",
" ! gcloud storage cat $GCP_RESOURCES\n",
" return GCP_RESOURCES\n",
" elif tf.io.gfile.exists(EVAL_METRICS):\n",
" ! gsutil cat $EVAL_METRICS\n",
" ! gcloud storage cat $EVAL_METRICS\n",
" return EVAL_METRICS\n",
"\n",
" return None\n",
@@ -1035,7 +1035,7 @@
"print(\"\\n\\n\")\n",
"print(\"model-upload\")\n",
"artifacts = print_pipeline_output(pipeline, \"model-upload\")\n",
"output = !gsutil cat $artifacts\n",
"output = !gcloud storage cat $artifacts\n",
"output = json.loads(output[0])\n",
"model_id = output[\"artifacts\"][\"model\"][\"artifacts\"][0][\"metadata\"][\"resourceName\"]\n",
"print(\"\\n\")\n",
@@ -1253,7 +1253,7 @@
"print(\"\\n\\n\")\n",
"print(\"model-upload\")\n",
"artifacts = print_pipeline_output(pipeline, \"model-upload\")\n",
"output = !gsutil cat $artifacts\n",
"output = !gcloud storage cat $artifacts\n",
"output = json.loads(output[0])\n",
"model_id = output[\"artifacts\"][\"model\"][\"artifacts\"][0][\"metadata\"][\"resourceName\"]\n",
"print(\"\\n\")\n",
@@ -1332,7 +1332,7 @@
"delete_bucket = False\n",
"\n",
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
" ! gsutil rm -r $BUCKET_URI"
" ! gcloud storage rm --recursive $BUCKET_URI"
]
}
],
notebooks/community/ml_ops/stage5/get_started_with_vertex_private_endpoints.ipynb
+4 -4
View File
@@ -458,7 +458,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l $REGION $BUCKET_URI"
"! gcloud storage buckets create --location=$REGION $BUCKET_URI"
]
},
{
@@ -478,7 +478,7 @@
},
"outputs": [],
"source": [
"! gsutil ls -al $BUCKET_URI"
"! gcloud storage ls --all-versions --long $BUCKET_URI"
]
},
{
@@ -1129,7 +1129,7 @@
},
"outputs": [],
"source": [
"! gsutil cp gs://cloud-ml-data/img/flower_photos/daisy/100080576_f52e8ee070_n.jpg test.jpg"
"! gcloud storage cp gs://cloud-ml-data/img/flower_photos/daisy/100080576_f52e8ee070_n.jpg test.jpg"
]
},
{
@@ -1314,7 +1314,7 @@
" print(e)\n",
"\n",
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
" ! gsutil rm -rf {BUCKET_URI}"
" ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}"
]
}
],
notebooks/official/experiments/get_started_with_vertex_experiments.ipynb
+3 -3
View File
@@ -321,7 +321,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l $LOCATION $BUCKET_URI"
"! gcloud storage buckets create --location $LOCATION $BUCKET_URI"
]
},
{
@@ -1238,7 +1238,7 @@
"! rm -f custom.tar custom.tar.gz\n",
"! tar cvf custom.tar custom\n",
"! gzip custom.tar\n",
"! gsutil cp custom.tar.gz $BUCKET_URI/trainer.tar.gz"
"! gcloud storage cp custom.tar.gz $BUCKET_URI/trainer.tar.gz"
]
},
{
@@ -1451,7 +1451,7 @@
"delete_bucket = False\n",
"\n",
"if delete_bucket:\n",
" ! gsutil rm -rf {BUCKET_URI}"
" ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}"
]
}
],
notebooks/official/pipelines/custom_tabular_train_batch_pred_bq_pipeline.ipynb
+5 -5
View File
@@ -308,7 +308,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l $LOCATION -p $PROJECT_ID $BUCKET_URI"
"! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI"
]
},
{
@@ -381,9 +381,9 @@
},
"outputs": [],
"source": [
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n",
"\n",
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer"
]
},
{
@@ -837,7 +837,7 @@
},
"outputs": [],
"source": [
"!gsutil cp -r python_package/dist/* $BUCKET_URI/training_package/"
"!gcloud storage cp --recursive python_package/dist/* $BUCKET_URI/training_package/"
]
},
{
@@ -1215,7 +1215,7 @@
"\n",
"# Delete Cloud Storage objects\n",
"if delete_bucket:\n",
" ! gsutil -m rm -r $BUCKET_URI\n",
" ! gcloud storage rm --recursive $BUCKET_URI\n",
"\n",
"! rm $PIPELINE_FILE_NAME"
]
notebooks/official/sdk/SDK_Custom_Training_Python_Package_Managed_Text_Dataset_Tensorflow_Serving_Container.ipynb
+4 -4
View File
@@ -324,7 +324,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l $LOCATION -p $PROJECT_ID $BUCKET_URI"
"! gcloud storage buckets create --location $LOCATION --project $PROJECT_ID $BUCKET_URI"
]
},
{
@@ -595,7 +595,7 @@
},
"outputs": [],
"source": [
"!gsutil ls gs://$BUCKET_NAME/$GCS_PREFIX/data"
"!gcloud storage ls gs://$BUCKET_NAME/$GCS_PREFIX/data"
]
},
{
@@ -1359,7 +1359,7 @@
},
"outputs": [],
"source": [
"!gsutil ls $gcs_source_test_url"
"!gcloud storage ls $gcs_source_test_url"
]
},
{
@@ -1488,7 +1488,7 @@
"batch_predict_job.delete()\n",
"\n",
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
" ! gsutil rm -r $BUCKET_URI"
" ! gcloud storage rm --recursive $BUCKET_URI"
]
}
],
notebooks/official/tabular_workflows/prophet_on_vertex_pipelines.ipynb
+5 -5
View File
@@ -310,7 +310,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l $LOCATION -p $PROJECT_ID $BUCKET_URI"
"! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI"
]
},
{
@@ -381,9 +381,9 @@
},
"outputs": [],
"source": [
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI\n",
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/dataflow.developer"
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n",
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer\n",
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer"
]
},
{
@@ -1165,7 +1165,7 @@
"# Delete Cloud Storage objects that were created\n",
"delete_bucket = False # Set True for deletion\n",
"if delete_bucket:\n",
" ! gsutil -m rm -r $BUCKET_URI"
" ! gcloud storage rm --recursive $BUCKET_URI"
]
}
],
notebooks/official/training/tpuv5e_gemma_peft_finetuning_and_serving.ipynb
+2 -2
View File
@@ -371,7 +371,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}"
"! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}"
]
},
{
@@ -1423,7 +1423,7 @@
"# Delete Cloud Storage objects that were created\n",
"delete_bucket = False\n",
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
" ! gsutil -m rm -r $BUCKET_URI"
" ! gcloud storage rm --recursive $BUCKET_URI"
]
}
],