xiaohei/model_garden
1
0
Fork 0
mirror of https://github.com/GoogleCloudPlatform/vertex-ai-samples.git synced 2026-09-26 14:42:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Migrate gsutil usage to gcloud storage (#4315)

Browse Source 
* Migrate gsutil usage to gcloud storage

* changes for 4315

* Apply automated linter fixes

* removed model_garden folder changes

---------

Co-authored-by: gurusai-voleti <gvoleti@google.com>
This commit is contained in:
Margubur Rahman
2025-12-24 09:41:28 -05:00
committed by GitHub
co-authored by gurusai-voleti
parent acb10d14b8
commit 3239b301f2
8 changed files with 1526 additions and 1538 deletions
Download Patch File Download Diff File Expand all files Collapse all files
community-content/prediction_featurestore_integration/prediction_featurestore_integration.ipynb
+1492 -1505
View File
File diff suppressed because it is too large Load Diff
notebooks/community/ml_ops/stage3/get_started_with_model_registry.ipynb
+3 -3
View File
@@ -441,7 +441,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l $REGION $BUCKET_URI"
"! gcloud storage buckets create --location=$REGION $BUCKET_URI"
]
},
{
@@ -461,7 +461,7 @@
},
"outputs": [],
"source": [
"! gsutil ls -al $BUCKET_URI"
"! gcloud storage ls --all-versions --long $BUCKET_URI"
]
},
{
@@ -1176,7 +1176,7 @@
" print(e)\n",
"\n",
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
" ! gsutil rm -rf {BUCKET_URI}"
" ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}"
]
}
],
notebooks/community/vertex_endpoints/tf_hub_obj_detection/deploy_tfhub_object_detection_on_vertex_endpoints.ipynb
+8 -7
View File
@@ -445,7 +445,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -p $PROJECT_ID -l $REGION $BUCKET_NAME"
"! gcloud storage buckets create --project=$PROJECT_ID --location=$REGION $BUCKET_NAME"
]
},
{
@@ -465,7 +465,7 @@
},
"outputs": [],
"source": [
"! gsutil ls -al $BUCKET_NAME"
"! gcloud storage ls --all-versions --long $BUCKET_NAME"
]
},
{
@@ -512,11 +512,12 @@
},
"outputs": [],
"source": [
"from io import BytesIO\n",
"\n",
"import matplotlib.pyplot as plt\n",
"import numpy as np\n",
"import tensorflow as tf\n",
"from PIL import Image\n",
"from six import BytesIO"
"from PIL import Image"
]
},
{
@@ -950,7 +951,7 @@
},
"outputs": [],
"source": [
"!gsutil cp -r $VERTEX_MODEL_PATH $BUCKET_NAME/obj_detection_model_vertex"
"!gcloud storage cp --recursive $VERTEX_MODEL_PATH $BUCKET_NAME/obj_detection_model_vertex"
]
},
{
@@ -961,7 +962,7 @@
},
"outputs": [],
"source": [
"!gsutil ls $BUCKET_NAME"
"!gcloud storage ls $BUCKET_NAME"
]
},
{
@@ -1297,7 +1298,7 @@
"--quiet\n",
"\n",
"# Delete Cloud Storage objects that were created\n",
"#! gsutil -m rm -r $BUCKET_NAME"
"#! gcloud storage rm --recursive $BUCKET_NAME"
]
}
],
notebooks/official/feature_store_legacy/sdk-feature-store-pandas.ipynb
+5 -5
View File
@@ -303,7 +303,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}"
"! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}"
]
},
{
@@ -564,9 +564,9 @@
"# set the local file names\n",
"USERS_AVRO_FN = \"users.avro\"\n",
"MOVIES_AVRO_FN = \"movies.avro\"\n",
"# copy the files using gsutil\n",
"! gsutil cp $GCS_USERS_AVRO_URI $USERS_AVRO_FN\n",
"! gsutil cp $GCS_MOVIES_AVRO_URI $MOVIES_AVRO_FN"
"# copy the files using gcloud storage\n",
"! gcloud storage cp $GCS_USERS_AVRO_URI $USERS_AVRO_FN\n",
"! gcloud storage cp $GCS_MOVIES_AVRO_URI $MOVIES_AVRO_FN"
]
},
{
@@ -1127,7 +1127,7 @@
"# Delete Cloud Storage objects that were created\n",
"delete_bucket = False # Set True for deletion\n",
"if delete_bucket:\n",
" ! gsutil -m rm -r $BUCKET_URI"
" ! gcloud storage rm --recursive $BUCKET_URI"
]
}
],
notebooks/official/pipelines/custom_model_training_and_batch_prediction.ipynb
+9 -9
View File
@@ -296,7 +296,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}"
"! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}"
]
},
{
@@ -369,9 +369,9 @@
},
"outputs": [],
"source": [
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n",
"\n",
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer"
]
},
{
@@ -1068,13 +1068,13 @@
" + \"/evaluation_metrics\"\n",
" )\n",
" if tf.io.gfile.exists(EXECUTE_OUTPUT):\n",
" ! gsutil cat $EXECUTE_OUTPUT\n",
" ! gcloud storage cat $EXECUTE_OUTPUT\n",
" return EXECUTE_OUTPUT\n",
" elif tf.io.gfile.exists(GCP_RESOURCES):\n",
" ! gsutil cat $GCP_RESOURCES\n",
" ! gcloud storage cat $GCP_RESOURCES\n",
" return GCP_RESOURCES\n",
" elif tf.io.gfile.exists(EVAL_METRICS):\n",
" ! gsutil cat $EVAL_METRICS\n",
" ! gcloud storage cat $EVAL_METRICS\n",
" return EVAL_METRICS\n",
"\n",
" return None\n",
@@ -1083,14 +1083,14 @@
"print(\"model-upload\")\n",
"artifacts = print_pipeline_output(job, \"model-upload\")\n",
"print(\"\\n\")\n",
"output = !gsutil cat $artifacts\n",
"output = !gcloud storage cat $artifacts\n",
"print(output)\n",
"output = json.loads(output[0])\n",
"model_id = output[\"artifacts\"][\"model\"][\"artifacts\"][0][\"metadata\"][\"resourceName\"]\n",
"print(\"model-batch-predict\")\n",
"artifacts = print_pipeline_output(job, \"model-batch-predict\")\n",
"print(\"\\n\")\n",
"output = !gsutil cat $artifacts\n",
"output = !gcloud storage cat $artifacts\n",
"output = json.loads(output[0])\n",
"batch_job_id = output[\"artifacts\"][\"batchpredictionjob\"][\"artifacts\"][0][\"metadata\"][\n",
" \"resourceName\"\n",
@@ -1133,7 +1133,7 @@
"# Delete the Cloud Storage bucket\n",
"delete_bucket = False # Set True for deletion\n",
"if delete_bucket:\n",
" ! gsutil rm -r $BUCKET_URI\n",
" ! gcloud storage rm --recursive $BUCKET_URI\n",
"\n",
"# Remove the locally generated files\n",
"! rm custom_model_training_spec.yaml\n",
notebooks/official/prediction/get_started_with_vertex_private_endpoints.ipynb
+3 -3
View File
@@ -307,7 +307,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}"
"! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}"
]
},
{
@@ -975,7 +975,7 @@
},
"outputs": [],
"source": [
"! gsutil cp gs://cloud-ml-data/img/flower_photos/daisy/100080576_f52e8ee070_n.jpg test.jpg"
"! gcloud storage cp gs://cloud-ml-data/img/flower_photos/daisy/100080576_f52e8ee070_n.jpg test.jpg"
]
},
{
@@ -1171,7 +1171,7 @@
" print(e)\n",
"\n",
"if delete_bucket:\n",
" ! gsutil rm -rf {BUCKET_URI}\n",
" ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}\n",
"\n",
"if delete_generated_files:\n",
" ! rm -rf \"test.jpg\" \"instances.json\""
notebooks/official/tabular_workflows/wide_and_deep_on_vertex_pipelines.ipynb
+2 -2
View File
@@ -297,7 +297,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l $LOCATION -p $PROJECT_ID $BUCKET_URI"
"! gcloud storage buckets create --location $LOCATION --project $PROJECT_ID $BUCKET_URI"
]
},
{
@@ -957,7 +957,7 @@
"# Delete bucket\n",
"delete_bucket = False\n",
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
" ! gsutil -m rm -r $BUCKET_URI"
" ! gcloud storage rm --recursive $BUCKET_URI"
]
}
],
notebooks/official/training/multi_node_ddp_gloo_vertex_training_with_custom_container.ipynb
+4 -4
View File
@@ -282,7 +282,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l $LOCATION -p $PROJECT_ID $BUCKET_URI"
"! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI"
]
},
{
@@ -1203,7 +1203,7 @@
},
"outputs": [],
"source": [
"! gsutil ls $gcs_output_uri_prefix"
"! gcloud storage ls $gcs_output_uri_prefix"
]
},
{
@@ -1242,7 +1242,7 @@
"# Set this to true only if you'd like to delete your artifact repository\n",
"delete_artifact_repository = False\n",
"\n",
"! gsutil rm -rf $gcs_output_uri_prefix\n",
"! gcloud storage rm --recursive --continue-on-error $gcs_output_uri_prefix\n",
"\n",
"! rm -rf ./trainer\n",
"\n",
@@ -1250,7 +1250,7 @@
" !gcloud artifacts repositories delete {PRIVATE_REPO} --location={LOCATION} --quiet\n",
"\n",
"if delete_bucket:\n",
" ! gsutil rm -r $BUCKET_URI\n",
" ! gcloud storage rm --recursive $BUCKET_URI\n",
"\n",
"if delete_tensorboard:\n",
" tensorboard.delete()"