xiaohei/model_garden
1
0
Fork 0
mirror of https://github.com/GoogleCloudPlatform/vertex-ai-samples.git synced 2026-09-26 14:42:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Migrate gsutil usage to gcloud storage (#4328)

Browse Source 
* Migrate gsutil usage to gcloud storage

* PR changes for 4328

* Fix Linter issue for 4328

* changes removed model garden

* Update model_garden_jax_fvlm.ipynb

* Update model_garden_jax_fvlm.ipynb

---------

Co-authored-by: bhandarivijay <bhandarivijay@google.com>
Co-authored-by: gurusai-voleti <gvoleti@google.com>
This commit is contained in:
Margubur Rahman
2025-12-25 10:46:52 -05:00
committed by GitHub
co-authored by bhandarivijay gurusai-voleti
parent 3cf46226e9
commit 5fc93c8bdb
9 changed files with 51 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files
notebooks/community/gapic/automl/showcase_automl_image_classification_export_edge.ipynb
+8 -10
View File
@@ -416,7 +416,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l $REGION $BUCKET_NAME"
"! gcloud storage buckets create --location=$REGION $BUCKET_NAME"
]
},
{
@@ -436,7 +436,7 @@
},
"outputs": [],
"source": [
"! gsutil ls -al $BUCKET_NAME"
"! gcloud storage ls --all-versions --long $BUCKET_NAME"
]
},
{
@@ -474,7 +474,6 @@
"\n",
"from google.cloud.aiplatform import gapic as aip\n",
"from google.protobuf import json_format\n",
"from google.protobuf.json_format import MessageToJson, ParseDict\n",
"from google.protobuf.struct_pb2 import Struct, Value"
]
},
@@ -784,11 +783,11 @@
"else:\n",
" FILE = IMPORT_FILE\n",
"\n",
"count = ! gsutil cat $FILE | wc -l\n",
"count = ! gcloud storage cat $FILE | wc -l\n",
"print(\"Number of Examples\", int(count[0]))\n",
"\n",
"print(\"First 10 rows\")\n",
"! gsutil cat $FILE | head"
"! gcloud storage cat $FILE | head"
]
},
{
@@ -1215,7 +1214,6 @@
" }\n",
" response = clients[\"model\"].export_model(name=name, output_config=output_config)\n",
" print(\"Long running operation:\", response.operation.name)\n",
" result = response.result(timeout=1800)\n",
" metadata = response.operation.metadata\n",
" artifact_uri = str(metadata.value).split(\"\\\\\")[-1][4:-1]\n",
" print(\"Artifact Uri\", artifact_uri)\n",
@@ -1244,9 +1242,9 @@
},
"outputs": [],
"source": [
"! gsutil ls $model_package\n",
"! gcloud storage ls $model_package\n",
"# Download the model artifacts\n",
"! gsutil cp -r $model_package tflite\n",
"! gcloud storage cp --recursive $model_package tflite\n",
"\n",
"tflite_path = \"tflite/model.tflite\""
]
@@ -1305,7 +1303,7 @@
},
"outputs": [],
"source": [
"test_items = ! gsutil cat $IMPORT_FILE | head -n1\n",
"test_items = ! gcloud storage cat $IMPORT_FILE | head -n1\n",
"test_item = test_items[0].split(\",\")[0]\n",
"\n",
"with tf.io.gfile.GFile(test_item, \"rb\") as f:\n",
@@ -1449,7 +1447,7 @@
" print(e)\n",
"\n",
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
" ! gsutil rm -r $BUCKET_NAME"
" ! gcloud storage rm --recursive $BUCKET_NAME"
]
}
],
notebooks/community/gapic/automl/showcase_automl_image_object_detection_export_edge.ipynb
+9 -12
View File
@@ -416,7 +416,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l $REGION $BUCKET_NAME"
"! gcloud storage buckets create --location $REGION $BUCKET_NAME"
]
},
{
@@ -436,7 +436,7 @@
},
"outputs": [],
"source": [
"! gsutil ls -al $BUCKET_NAME"
"! gcloud storage ls --all-versions --long $BUCKET_NAME"
]
},
{
@@ -473,9 +473,7 @@
"import time\n",
"\n",
"from google.cloud.aiplatform import gapic as aip\n",
"from google.protobuf import json_format\n",
"from google.protobuf.json_format import MessageToJson, ParseDict\n",
"from google.protobuf.struct_pb2 import Struct, Value"
"from google.protobuf import json_format\n"
]
},
{
@@ -785,11 +783,11 @@
"else:\n",
" FILE = IMPORT_FILE\n",
"\n",
"count = ! gsutil cat $FILE | wc -l\n",
"count = ! gcloud storage cat $FILE | wc -l\n",
"print(\"Number of Examples\", int(count[0]))\n",
"\n",
"print(\"First 10 rows\")\n",
"! gsutil cat $FILE | head"
"! gcloud storage cat $FILE | head"
]
},
{
@@ -1218,7 +1216,6 @@
" }\n",
" response = clients[\"model\"].export_model(name=name, output_config=output_config)\n",
" print(\"Long running operation:\", response.operation.name)\n",
" result = response.result(timeout=1800)\n",
" metadata = response.operation.metadata\n",
" artifact_uri = str(metadata.value).split(\"\\\\\")[-1][4:-1]\n",
" print(\"Artifact Uri\", artifact_uri)\n",
@@ -1247,9 +1244,9 @@
},
"outputs": [],
"source": [
"! gsutil ls $model_package\n",
"! gcloud storage ls $model_package\n",
"# Download the model artifacts\n",
"! gsutil cp -r $model_package tflite\n",
"! gcloud storage cp --recursive $model_package tflite\n",
"\n",
"tflite_path = \"tflite/model.tflite\""
]
@@ -1308,7 +1305,7 @@
},
"outputs": [],
"source": [
"test_items = ! gsutil cat $IMPORT_FILE | head -n1\n",
"test_items = ! gcloud storage cat $IMPORT_FILE | head -n1\n",
"test_item = test_items[0].split(\",\")[0]\n",
"\n",
"with tf.io.gfile.GFile(test_item, \"rb\") as f:\n",
@@ -1452,7 +1449,7 @@
" print(e)\n",
"\n",
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
" ! gsutil rm -r $BUCKET_NAME"
" ! gcloud storage rm --recursive $BUCKET_NAME"
]
}
],
notebooks/community/ml_ops/stage3/get_started_with_kubeflow_pipelines.ipynb
+9 -12
View File
@@ -430,7 +430,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l $REGION $BUCKET_URI"
"! gcloud storage buckets create --location=$REGION $BUCKET_URI"
]
},
{
@@ -450,7 +450,7 @@
},
"outputs": [],
"source": [
"! gsutil ls -al $BUCKET_URI"
"! gcloud storage ls --all-versions --long $BUCKET_URI"
]
},
{
@@ -520,9 +520,8 @@
},
"outputs": [],
"source": [
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
"\n",
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n","\n",
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer"
]
},
{
@@ -807,13 +806,13 @@
" + \"/evaluation_metrics\"\n",
" )\n",
" if tf.io.gfile.exists(EXECUTE_OUTPUT):\n",
" ! gsutil cat $EXECUTE_OUTPUT\n",
" ! gcloud storage cat $EXECUTE_OUTPUT\n",
" return EXECUTE_OUTPUT\n",
" elif tf.io.gfile.exists(GCP_RESOURCES):\n",
" ! gsutil cat $GCP_RESOURCES\n",
" ! gcloud storage cat $GCP_RESOURCES\n",
" return GCP_RESOURCES\n",
" elif tf.io.gfile.exists(EVAL_METRICS):\n",
" ! gsutil cat $EVAL_METRICS\n",
" ! gcloud storage cat $EVAL_METRICS\n",
" return EVAL_METRICS\n",
"\n",
" return None\n",
@@ -1167,9 +1166,7 @@
"\n",
"\n",
"@component()\n",
"def multi_output(\n",
" text1: str, text2: str\n",
") -> NamedTuple(\n",
"def multi_output(text1: str, text2: str) -> NamedTuple(\n",
" \"Outputs\",\n",
" [\n",
" (\"output_1\", str), # Return parameters\n",
@@ -1587,7 +1584,7 @@
"delete_bucket = False\n",
"\n",
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
" ! gsutil rm -r $BUCKET_URI"
" ! gcloud storage rm --recursive $BUCKET_URI"
]
}
],
notebooks/community/ml_ops/stage6/get_started_with_custom_tabular_model_batch.ipynb
+6 -6
View File
@@ -464,7 +464,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l $REGION $BUCKET_URI"
"! gcloud storage buckets create --location=$REGION $BUCKET_URI"
]
},
{
@@ -484,7 +484,7 @@
},
"outputs": [],
"source": [
"! gsutil ls -al $BUCKET_URI"
"! gcloud storage ls --all-versions --long $BUCKET_URI"
]
},
{
@@ -892,7 +892,7 @@
" f.write(json.dumps(instance) + \"\\n\")\n",
"\n",
"# Upload to Cloud Storage bucket\n",
"! gsutil cp $BATCH_PREDICTION_INSTANCES_FILE $BATCH_PREDICTION_GCS_SOURCE\n",
"! gcloud storage cp $BATCH_PREDICTION_INSTANCES_FILE $BATCH_PREDICTION_GCS_SOURCE\n",
"\n",
"print(\"Uploaded instances to: \", BATCH_PREDICTION_GCS_SOURCE)"
]
@@ -1108,7 +1108,7 @@
"outputs": [],
"source": [
"data_file = \"gs://cloud-samples-data/ai-platform/iris/iris_data.csv\"\n",
"! gsutil cat $data_file | head -n 10 > test.csv\n",
"! gcloud storage cat $data_file | head -n 10 > test.csv\n",
"\n",
"! cat test.csv\n",
"\n",
@@ -1118,7 +1118,7 @@
" BUCKET_URI + \"/batch_prediction_instances/\" + BATCH_PREDICTION_INSTANCES_FILE\n",
")\n",
"\n",
"! gsutil cp test.csv $BATCH_PREDICTION_GCS_SOURCE"
"! gcloud storage cp test.csv $BATCH_PREDICTION_GCS_SOURCE"
]
},
{
@@ -1497,7 +1497,7 @@
"delete_bucket = False\n",
"\n",
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
" ! gsutil rm -rf {BUCKET_URI}"
" ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}"
]
}
],
notebooks/community/ml_ops/stage6/get_started_with_xai_and_custom_server.ipynb
+3 -3
View File
@@ -504,7 +504,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l $REGION $BUCKET_URI"
"! gcloud storage buckets create --location=$REGION $BUCKET_URI"
]
},
{
@@ -524,7 +524,7 @@
},
"outputs": [],
"source": [
"! gsutil ls -al $BUCKET_URI"
"! gcloud storage ls --all-versions --long $BUCKET_URI"
]
},
{
@@ -1734,7 +1734,7 @@
"! rm -rf app\n",
"\n",
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
" ! gsutil rm -rf {BUCKET_URI}"
" ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}"
]
}
],
notebooks/community/model_evaluation/automl_tabular_regression_model_evaluation.ipynb
+6 -6
View File
@@ -537,7 +537,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
"! gcloud storage buckets create --location=$REGION --project=$PROJECT_ID $BUCKET_URI"
]
},
{
@@ -557,7 +557,7 @@
},
"outputs": [],
"source": [
"! gsutil ls -al $BUCKET_URI"
"! gcloud storage ls --all-versions --long $BUCKET_URI"
]
},
{
@@ -627,9 +627,9 @@
},
"outputs": [],
"source": [
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n",
"\n",
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer"
]
},
{
@@ -1383,7 +1383,7 @@
"outputs": [],
"source": [
"# Load the results\n",
"attributions = !gsutil cat $feat_attrs_gcs_uri\n",
"attributions = !gcloud storage cat $feat_attrs_gcs_uri\n",
"\n",
"# Print the results obtained\n",
"attributions = json.loads(attributions[0])\n",
@@ -1463,7 +1463,7 @@
"# Delete Cloud Storage objects\n",
"delete_bucket = False\n",
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
" ! gsutil -m rm -r $BUCKET_URI"
" ! gcloud storage rm --recursive $BUCKET_URI"
]
}
],
notebooks/official/explainable_ai/sdk_custom_tabular_regression_batch_explain.ipynb
+4 -4
View File
@@ -288,7 +288,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}"
"! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}"
]
},
{
@@ -707,7 +707,7 @@
"! rm -f custom.tar custom.tar.gz\n",
"! tar cvf custom.tar custom\n",
"! gzip custom.tar\n",
"! gsutil cp custom.tar.gz $BUCKET_URI/trainer_boston.tar.gz"
"! gcloud storage cp custom.tar.gz $BUCKET_URI/trainer_boston.tar.gz"
]
},
{
@@ -1260,7 +1260,7 @@
"\n",
"gcs_input_uri = BUCKET_URI + \"/test.csv\"\n",
"\n",
"! gsutil cp batch.csv $gcs_input_uri"
"! gcloud storage cp batch.csv $gcs_input_uri"
]
},
{
@@ -1398,7 +1398,7 @@
"# Delete the Cloud Storage bucket\n",
"delete_bucket = False # Set True for deletion\n",
"if delete_bucket:\n",
" ! gsutil rm -r $BUCKET_URI\n",
" ! gcloud storage rm --recursive $BUCKET_URI\n",
"\n",
"# Remove the locally generated files\n",
"! rm -rf batch.csv custom.tar.gz custom"
notebooks/official/explainable_ai/sdk_custom_tabular_regression_online_explain_get_metadata.ipynb
+3 -3
View File
@@ -293,7 +293,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}"
"! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}"
]
},
{
@@ -706,7 +706,7 @@
"! rm -f custom.tar custom.tar.gz\n",
"! tar cvf custom.tar custom\n",
"! gzip custom.tar\n",
"! gsutil cp custom.tar.gz $BUCKET_URI/trainer_boston.tar.gz"
"! gcloud storage cp custom.tar.gz $BUCKET_URI/trainer_boston.tar.gz"
]
},
{
@@ -1444,7 +1444,7 @@
"\n",
"delete_bucket = False # Set True for deletion\n",
"if delete_bucket:\n",
" ! gsutil rm -r $BUCKET_URI"
" ! gcloud storage rm --recursive $BUCKET_URI"
]
}
],
notebooks/official/training/tpuv5e_llama2_pytorch_finetuning_and_serving.ipynb
+3 -3
View File
@@ -279,7 +279,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}"
"! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}"
]
},
{
@@ -1264,7 +1264,7 @@
},
"outputs": [],
"source": [
"!gsutil ls $GCS_MODEL_PATH"
"!gcloud storage ls $GCS_MODEL_PATH"
]
},
{
@@ -1517,7 +1517,7 @@
"# Delete Cloud Storage objects that were created\n",
"delete_bucket = False\n",
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
" ! gsutil -m rm -r $BUCKET_URI"
" ! gcloud storage rm --recursive $BUCKET_URI"
]
}
],