xiaohei/model_garden
1
0
Fork 0
mirror of https://github.com/GoogleCloudPlatform/vertex-ai-samples.git synced 2026-09-26 14:42:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Migrate gsutil usage to gcloud storage (#4321)

Browse Source 
* Migrate gsutil usage to gcloud storage

* Changes for 4321

* fix the linter issue for br 4321

* removed changes for model garden

---------

Co-authored-by: bhandarivijay <bhandarivijay@google.com>
Co-authored-by: gurusai-voleti <gvoleti@google.com>
This commit is contained in:
Margubur Rahman
2025-12-25 10:46:04 -05:00
committed by GitHub
co-authored by bhandarivijay gurusai-voleti
parent 439f6a0cae
commit 3cf46226e9
8 changed files with 41 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files
notebooks/community/gapic/custom/showcase_custom_text_classification_online_exported_ds.ipynb
+6 -15
View File
@@ -426,8 +426,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l $REGION $BUCKET_NAME"
]
"! gcloud storage buckets create --location=$REGION $BUCKET_NAME" ]
},
{
"cell_type": "markdown",
@@ -446,8 +445,7 @@
},
"outputs": [],
"source": [
"! gsutil ls -al $BUCKET_NAME"
]
"! gcloud storage ls --all-versions --long $BUCKET_NAME" ]
},
{
"cell_type": "markdown",
@@ -483,9 +481,6 @@
"import time\n",
"\n",
"from google.cloud.aiplatform import gapic as aip\n",
"from google.protobuf import json_format\n",
"from google.protobuf.json_format import MessageToJson, ParseDict\n",
"from google.protobuf.struct_pb2 import Struct, Value"
]
},
{
@@ -991,12 +986,10 @@
"else:\n",
" FILE = IMPORT_FILE\n",
"\n",
"count = ! gsutil cat $FILE | wc -l\n",
"print(\"Number of Examples\", int(count[0]))\n",
"count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
"\n",
"print(\"First 10 rows\")\n",
"! gsutil cat $FILE | head"
]
"! gcloud storage cat $FILE | head" ]
},
{
"cell_type": "markdown",
@@ -1153,8 +1146,7 @@
"source": [
"jsonl_index = result.exported_files[0]\n",
"\n",
"! gsutil cat $jsonl_index | head"
]
"! gcloud storage cat $jsonl_index | head" ]
},
{
"cell_type": "markdown",
@@ -1286,8 +1278,7 @@
" print(e)\n",
"\n",
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
" ! gsutil rm -r $BUCKET_NAME"
]
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
}
],
"metadata": {
notebooks/community/migration/UJ7 legacy AutoML Natural Language Text Entity Extraction.ipynb
+7 -17
View File
@@ -326,8 +326,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l $REGION gs://$BUCKET_NAME"
]
"! gcloud storage buckets create --location=$REGION gs://$BUCKET_NAME" ]
},
{
"cell_type": "markdown",
@@ -346,8 +345,7 @@
},
"outputs": [],
"source": [
"! gsutil ls -al gs://$BUCKET_NAME"
]
"! gcloud storage ls --all-versions --long gs://$BUCKET_NAME" ]
},
{
"cell_type": "markdown",
@@ -380,14 +378,11 @@
},
"outputs": [],
"source": [
"import json\n",
"import os\n",
"import sys\n",
"import time\n",
"\n",
"from google.cloud import automl\n",
"from google.protobuf.json_format import MessageToJson\n",
"from google.protobuf.struct_pb2 import Value"
"from google.protobuf.json_format import MessageToJson\n"
]
},
{
@@ -476,8 +471,7 @@
},
"outputs": [],
"source": [
"! gsutil cat $IMPORT_FILE | head -n 10"
]
"! gcloud storage cat $IMPORT_FILE | head -n 10" ]
},
{
"cell_type": "markdown",
@@ -1132,8 +1126,7 @@
" data = {\"id\": 0, \"text_snippet\": {\"content\": test_item}}\n",
" f.write(json.dumps(data) + \"\\n\")\n",
"\n",
"! gsutil cat $gcs_input_uri"
]
"! gcloud storage cat $gcs_input_uri" ]
},
{
"cell_type": "markdown",
@@ -1280,9 +1273,7 @@
"source": [
"destination_uri = output_config[\"gcs_destination\"][\"output_uri_prefix\"][:-1]\n",
"\n",
"! gsutil ls $destination_uri/*\n",
"! gsutil cat $destination_uri/prediction*/*.jsonl"
]
"! gcloud storage ls $destination_uri/*\n", "! gcloud storage cat $destination_uri/prediction*/*.jsonl" ]
},
{
"cell_type": "markdown",
@@ -1614,8 +1605,7 @@
"\n",
"\n",
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
" ! gsutil rm -r gs://$BUCKET_NAME"
]
" ! gcloud storage rm --recursive gs://$BUCKET_NAME" ]
}
],
"metadata": {
notebooks/community/ml_ops/stage2/get_started_vertex_tensorboard.ipynb
+4 -8
View File
@@ -511,8 +511,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l $REGION $BUCKET_URI"
]
"! gcloud storage buckets create --location $REGION $BUCKET_URI" ]
},
{
"cell_type": "markdown",
@@ -531,8 +530,7 @@
},
"outputs": [],
"source": [
"! gsutil ls -al $BUCKET_URI"
]
"! gcloud storage ls --all-versions --long $BUCKET_URI" ]
},
{
"cell_type": "markdown",
@@ -1147,8 +1145,7 @@
"! rm -f custom.tar custom.tar.gz\n",
"! tar cvf custom.tar custom\n",
"! gzip custom.tar\n",
"! gsutil cp custom.tar.gz $BUCKET_URI/trainer_example.tar.gz"
]
"! gcloud storage cp custom.tar.gz $BUCKET_URI/trainer_example.tar.gz" ]
},
{
"cell_type": "markdown",
@@ -1357,8 +1354,7 @@
"delete_bucket = False\n",
"\n",
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
" ! gsutil rm -r $BUCKET_URI"
]
" ! gcloud storage rm --recursive $BUCKET_URI" ]
}
],
"metadata": {
notebooks/community/ml_ops/stage2/get_started_with_tfhub_models.ipynb
+4 -8
View File
@@ -448,8 +448,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l $REGION $BUCKET_URI"
]
"! gcloud storage buckets create --location=$REGION $BUCKET_URI" ]
},
{
"cell_type": "markdown",
@@ -468,8 +467,7 @@
},
"outputs": [],
"source": [
"! gsutil ls -al $BUCKET_URI"
]
"! gcloud storage ls --all-versions --long $BUCKET_URI" ]
},
{
"cell_type": "markdown",
@@ -1132,8 +1130,7 @@
"! rm -f custom.tar custom.tar.gz\n",
"! tar cvf custom.tar custom\n",
"! gzip custom.tar\n",
"! gsutil cp custom.tar.gz $BUCKET_URI/trainer_cifar10.tar.gz"
]
"! gcloud storage cp custom.tar.gz $BUCKET_URI/trainer_cifar10.tar.gz" ]
},
{
"cell_type": "markdown",
@@ -1381,8 +1378,7 @@
"delete_bucket = False\n",
"\n",
"if delete_bucket and \"BUCKET_URI\" in globals():\n",
" ! gsutil rm -r $BUCKET_URI"
]
" ! gcloud storage rm --recursive $BUCKET_URI" ]
}
],
"metadata": {
notebooks/community/ml_ops/stage5/get_started_with_autoscaling.ipynb
+3 -6
View File
@@ -437,8 +437,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l $REGION $BUCKET_URI"
]
"! gcloud storage buckets create --location $REGION $BUCKET_URI" ]
},
{
"cell_type": "markdown",
@@ -457,8 +456,7 @@
},
"outputs": [],
"source": [
"! gsutil ls -al $BUCKET_URI"
]
"! gcloud storage ls --all-versions --long $BUCKET_URI" ]
},
{
"cell_type": "markdown",
@@ -1357,8 +1355,7 @@
"delete_bucket = True\n",
"\n",
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
" ! gsutil rm -r $BUCKET_URI"
]
" ! gcloud storage rm --recursive $BUCKET_URI" ]
}
],
"metadata": {
notebooks/community/ml_ops/stage6/get_started_with_xgboost_model_online.ipynb
+9 -18
View File
@@ -484,8 +484,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l $REGION $BUCKET_URI"
]
"! gcloud storage buckets create --location=$REGION $BUCKET_URI" ]
},
{
"cell_type": "markdown",
@@ -504,8 +503,7 @@
},
"outputs": [],
"source": [
"! gsutil ls -al $BUCKET_URI"
]
"! gcloud storage ls --all-versions --long $BUCKET_URI" ]
},
{
"cell_type": "markdown",
@@ -717,8 +715,7 @@
"with open(\"instance.yaml\", \"w\") as f:\n",
" f.write(yaml)\n",
"\n",
"! gsutil cp instance.yaml {BUCKET_URI}/instance.yaml"
]
"! gcloud storage cp instance.yaml {BUCKET_URI}/instance.yaml" ]
},
{
"cell_type": "markdown",
@@ -1274,14 +1271,11 @@
" + \"/evaluation_metrics\"\n",
" )\n",
" if tf.io.gfile.exists(EXECUTE_OUTPUT):\n",
" ! gsutil cat $EXECUTE_OUTPUT\n",
" return EXECUTE_OUTPUT\n",
" ! gcloud storage cat $EXECUTE_OUTPUT\n", " return EXECUTE_OUTPUT\n",
" elif tf.io.gfile.exists(GCP_RESOURCES):\n",
" ! gsutil cat $GCP_RESOURCES\n",
" return GCP_RESOURCES\n",
" ! gcloud storage cat $GCP_RESOURCES\n", " return GCP_RESOURCES\n",
" elif tf.io.gfile.exists(EVAL_METRICS):\n",
" ! gsutil cat $EVAL_METRICS\n",
" return EVAL_METRICS\n",
" ! gcloud storage cat $EVAL_METRICS\n", " return EVAL_METRICS\n",
"\n",
" return None\n",
"\n",
@@ -1289,15 +1283,13 @@
"print(\"get-vertex-model\")\n",
"artifacts = print_pipeline_output(pipeline, \"get-vertex-model\")\n",
"print(\"\\n\\n\")\n",
"output = !gsutil cat $artifacts\n",
"output = json.loads(output[0])\n",
"output = !gcloud storage cat $artifacts\n", "output = json.loads(output[0])\n",
"model_id = output[\"artifacts\"][\"model\"][\"artifacts\"][0][\"metadata\"][\"resourceName\"]\n",
"print(\"\\n\\n\")\n",
"print(\"endpoint-create\")\n",
"artifacts = print_pipeline_output(pipeline, \"endpoint-create\")\n",
"print(\"\\n\\n\")\n",
"output = !gsutil cat $artifacts\n",
"output = json.loads(output[0])\n",
"output = !gcloud storage cat $artifacts\n", "output = json.loads(output[0])\n",
"endpoint_id = output[\"artifacts\"][\"endpoint\"][\"artifacts\"][0][\"metadata\"][\n",
" \"resourceName\"\n",
"]\n",
@@ -1404,8 +1396,7 @@
"delete_bucket = True\n",
"\n",
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
" ! gsutil rm -rf {BUCKET_URI}"
]
" ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}" ]
}
],
"metadata": {
notebooks/official/custom/get_started_with_vertex_endpoint_and_shared_vm.ipynb
+3 -5
View File
@@ -295,8 +295,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l $LOCATION $BUCKET_URI"
]
"! gcloud storage buckets create --location $LOCATION $BUCKET_URI" ]
},
{
"cell_type": "markdown",
@@ -1203,7 +1202,7 @@
},
"outputs": [],
"source": [
"! gsutil cp gs://cloud-ml-data/img/flower_photos/daisy/100080576_f52e8ee070_n.jpg test.jpg\n",
"! gcloud storage cp gs://cloud-ml-data/img/flower_photos/daisy/100080576_f52e8ee070_n.jpg test.jpg\n",
"\n",
"import base64\n",
"\n",
@@ -1311,8 +1310,7 @@
"delete_bucket = False\n",
"\n",
"if delete_bucket:\n",
" ! gsutil rm -r $BUCKET_URI\n",
"\n",
" ! gcloud storage rm --recursive $BUCKET_URI\n", "\n",
"# Undeploy the models\n",
"# When you're done doing predictions, you undeploy the model from the `Endpoint` resouce. \n",
"# This deprovisions all compute resources and ends billing for the deployed model.\n",
notebooks/official/ray_on_vertex_ai/get_started_with_pytorch_rov.ipynb
+5 -10
View File
@@ -259,8 +259,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}"
]
"! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" ]
},
{
"cell_type": "markdown",
@@ -1025,8 +1024,7 @@
},
"outputs": [],
"source": [
"! gsutil ls -l {TRAINING_URI}"
]
"! gcloud storage ls --long {TRAINING_URI}" ]
},
{
"cell_type": "markdown",
@@ -1064,8 +1062,7 @@
},
"outputs": [],
"source": [
"! gsutil -q cp -r {TRAINING_URI} {DELIVERABLES_PATH}"
]
"! gcloud storage cp --recursive {TRAINING_URI} {DELIVERABLES_PATH}" ]
},
{
"cell_type": "markdown",
@@ -1348,8 +1345,7 @@
},
"outputs": [],
"source": [
"!gsutil cp {str(build_path)}/cifar10.mar {BUILD_URI}/model.mar"
]
"!gcloud storage cp {str(build_path)}/cifar10.mar {BUILD_URI}/model.mar" ]
},
{
"cell_type": "markdown",
@@ -1523,8 +1519,7 @@
"\n",
"# Delete Cloud Storage objects that were created\n",
"if delete_bucket:\n",
" ! gsutil -q -m rm -r $BUCKET_URI"
]
" ! gcloud -q storage rm --recursive $BUCKET_URI" ]
}
],
"metadata": {