xiaohei/model_garden
1
0
Fork 0
mirror of https://github.com/GoogleCloudPlatform/vertex-ai-samples.git synced 2026-09-26 14:42:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Migrate gsutil usage to gcloud storage (#4328)

Browse Source 
* Migrate gsutil usage to gcloud storage

* PR changes for 4328

* Fix Linter issue for 4328

* changes removed model garden

* Update model_garden_jax_fvlm.ipynb

* Update model_garden_jax_fvlm.ipynb

---------

Co-authored-by: bhandarivijay <bhandarivijay@google.com>
Co-authored-by: gurusai-voleti <gvoleti@google.com>
This commit is contained in:
Margubur Rahman
2025-12-25 10:46:52 -05:00
committed by GitHub
co-authored by bhandarivijay gurusai-voleti
parent 3cf46226e9
commit 5fc93c8bdb
9 changed files with 51 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files
notebooks/community/gapic/automl/showcase_automl_image_classification_export_edge.ipynb
+8 -10
View File
@@ -416,7 +416,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil mb -l $REGION $BUCKET_NAME" "! gcloud storage buckets create --location=$REGION $BUCKET_NAME"
] ]
}, },
{ {
@@ -436,7 +436,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil ls -al $BUCKET_NAME" "! gcloud storage ls --all-versions --long $BUCKET_NAME"
] ]
}, },
{ {
@@ -474,7 +474,6 @@
"\n", "\n",
"from google.cloud.aiplatform import gapic as aip\n", "from google.cloud.aiplatform import gapic as aip\n",
"from google.protobuf import json_format\n", "from google.protobuf import json_format\n",
"from google.protobuf.json_format import MessageToJson, ParseDict\n",
"from google.protobuf.struct_pb2 import Struct, Value" "from google.protobuf.struct_pb2 import Struct, Value"
] ]
}, },
@@ -784,11 +783,11 @@
"else:\n", "else:\n",
" FILE = IMPORT_FILE\n", " FILE = IMPORT_FILE\n",
"\n", "\n",
"count = ! gsutil cat $FILE | wc -l\n", "count = ! gcloud storage cat $FILE | wc -l\n",
"print(\"Number of Examples\", int(count[0]))\n", "print(\"Number of Examples\", int(count[0]))\n",
"\n", "\n",
"print(\"First 10 rows\")\n", "print(\"First 10 rows\")\n",
"! gsutil cat $FILE | head" "! gcloud storage cat $FILE | head"
] ]
}, },
{ {
@@ -1215,7 +1214,6 @@
" }\n", " }\n",
" response = clients[\"model\"].export_model(name=name, output_config=output_config)\n", " response = clients[\"model\"].export_model(name=name, output_config=output_config)\n",
" print(\"Long running operation:\", response.operation.name)\n", " print(\"Long running operation:\", response.operation.name)\n",
" result = response.result(timeout=1800)\n",
" metadata = response.operation.metadata\n", " metadata = response.operation.metadata\n",
" artifact_uri = str(metadata.value).split(\"\\\\\")[-1][4:-1]\n", " artifact_uri = str(metadata.value).split(\"\\\\\")[-1][4:-1]\n",
" print(\"Artifact Uri\", artifact_uri)\n", " print(\"Artifact Uri\", artifact_uri)\n",
@@ -1244,9 +1242,9 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil ls $model_package\n", "! gcloud storage ls $model_package\n",
"# Download the model artifacts\n", "# Download the model artifacts\n",
"! gsutil cp -r $model_package tflite\n", "! gcloud storage cp --recursive $model_package tflite\n",
"\n", "\n",
"tflite_path = \"tflite/model.tflite\"" "tflite_path = \"tflite/model.tflite\""
] ]
@@ -1305,7 +1303,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"test_items = ! gsutil cat $IMPORT_FILE | head -n1\n", "test_items = ! gcloud storage cat $IMPORT_FILE | head -n1\n",
"test_item = test_items[0].split(\",\")[0]\n", "test_item = test_items[0].split(\",\")[0]\n",
"\n", "\n",
"with tf.io.gfile.GFile(test_item, \"rb\") as f:\n", "with tf.io.gfile.GFile(test_item, \"rb\") as f:\n",
@@ -1449,7 +1447,7 @@
" print(e)\n", " print(e)\n",
"\n", "\n",
"if delete_bucket and \"BUCKET_NAME\" in globals():\n", "if delete_bucket and \"BUCKET_NAME\" in globals():\n",
" ! gsutil rm -r $BUCKET_NAME" " ! gcloud storage rm --recursive $BUCKET_NAME"
] ]
} }
], ],
notebooks/community/gapic/automl/showcase_automl_image_object_detection_export_edge.ipynb
+9 -12
View File
@@ -416,7 +416,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil mb -l $REGION $BUCKET_NAME" "! gcloud storage buckets create --location $REGION $BUCKET_NAME"
] ]
}, },
{ {
@@ -436,7 +436,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil ls -al $BUCKET_NAME" "! gcloud storage ls --all-versions --long $BUCKET_NAME"
] ]
}, },
{ {
@@ -473,9 +473,7 @@
"import time\n", "import time\n",
"\n", "\n",
"from google.cloud.aiplatform import gapic as aip\n", "from google.cloud.aiplatform import gapic as aip\n",
"from google.protobuf import json_format\n", "from google.protobuf import json_format\n"
"from google.protobuf.json_format import MessageToJson, ParseDict\n",
"from google.protobuf.struct_pb2 import Struct, Value"
] ]
}, },
{ {
@@ -785,11 +783,11 @@
"else:\n", "else:\n",
" FILE = IMPORT_FILE\n", " FILE = IMPORT_FILE\n",
"\n", "\n",
"count = ! gsutil cat $FILE | wc -l\n", "count = ! gcloud storage cat $FILE | wc -l\n",
"print(\"Number of Examples\", int(count[0]))\n", "print(\"Number of Examples\", int(count[0]))\n",
"\n", "\n",
"print(\"First 10 rows\")\n", "print(\"First 10 rows\")\n",
"! gsutil cat $FILE | head" "! gcloud storage cat $FILE | head"
] ]
}, },
{ {
@@ -1218,7 +1216,6 @@
" }\n", " }\n",
" response = clients[\"model\"].export_model(name=name, output_config=output_config)\n", " response = clients[\"model\"].export_model(name=name, output_config=output_config)\n",
" print(\"Long running operation:\", response.operation.name)\n", " print(\"Long running operation:\", response.operation.name)\n",
" result = response.result(timeout=1800)\n",
" metadata = response.operation.metadata\n", " metadata = response.operation.metadata\n",
" artifact_uri = str(metadata.value).split(\"\\\\\")[-1][4:-1]\n", " artifact_uri = str(metadata.value).split(\"\\\\\")[-1][4:-1]\n",
" print(\"Artifact Uri\", artifact_uri)\n", " print(\"Artifact Uri\", artifact_uri)\n",
@@ -1247,9 +1244,9 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil ls $model_package\n", "! gcloud storage ls $model_package\n",
"# Download the model artifacts\n", "# Download the model artifacts\n",
"! gsutil cp -r $model_package tflite\n", "! gcloud storage cp --recursive $model_package tflite\n",
"\n", "\n",
"tflite_path = \"tflite/model.tflite\"" "tflite_path = \"tflite/model.tflite\""
] ]
@@ -1308,7 +1305,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"test_items = ! gsutil cat $IMPORT_FILE | head -n1\n", "test_items = ! gcloud storage cat $IMPORT_FILE | head -n1\n",
"test_item = test_items[0].split(\",\")[0]\n", "test_item = test_items[0].split(\",\")[0]\n",
"\n", "\n",
"with tf.io.gfile.GFile(test_item, \"rb\") as f:\n", "with tf.io.gfile.GFile(test_item, \"rb\") as f:\n",
@@ -1452,7 +1449,7 @@
" print(e)\n", " print(e)\n",
"\n", "\n",
"if delete_bucket and \"BUCKET_NAME\" in globals():\n", "if delete_bucket and \"BUCKET_NAME\" in globals():\n",
" ! gsutil rm -r $BUCKET_NAME" " ! gcloud storage rm --recursive $BUCKET_NAME"
] ]
} }
], ],
notebooks/community/ml_ops/stage3/get_started_with_kubeflow_pipelines.ipynb
+9 -12
View File
@@ -430,7 +430,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil mb -l $REGION $BUCKET_URI" "! gcloud storage buckets create --location=$REGION $BUCKET_URI"
] ]
}, },
{ {
@@ -450,7 +450,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil ls -al $BUCKET_URI" "! gcloud storage ls --all-versions --long $BUCKET_URI"
] ]
}, },
{ {
@@ -520,9 +520,8 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n", "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n","\n",
"\n", "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer"
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
] ]
}, },
{ {
@@ -807,13 +806,13 @@
" + \"/evaluation_metrics\"\n", " + \"/evaluation_metrics\"\n",
" )\n", " )\n",
" if tf.io.gfile.exists(EXECUTE_OUTPUT):\n", " if tf.io.gfile.exists(EXECUTE_OUTPUT):\n",
" ! gsutil cat $EXECUTE_OUTPUT\n", " ! gcloud storage cat $EXECUTE_OUTPUT\n",
" return EXECUTE_OUTPUT\n", " return EXECUTE_OUTPUT\n",
" elif tf.io.gfile.exists(GCP_RESOURCES):\n", " elif tf.io.gfile.exists(GCP_RESOURCES):\n",
" ! gsutil cat $GCP_RESOURCES\n", " ! gcloud storage cat $GCP_RESOURCES\n",
" return GCP_RESOURCES\n", " return GCP_RESOURCES\n",
" elif tf.io.gfile.exists(EVAL_METRICS):\n", " elif tf.io.gfile.exists(EVAL_METRICS):\n",
" ! gsutil cat $EVAL_METRICS\n", " ! gcloud storage cat $EVAL_METRICS\n",
" return EVAL_METRICS\n", " return EVAL_METRICS\n",
"\n", "\n",
" return None\n", " return None\n",
@@ -1167,9 +1166,7 @@
"\n", "\n",
"\n", "\n",
"@component()\n", "@component()\n",
"def multi_output(\n", "def multi_output(text1: str, text2: str) -> NamedTuple(\n",
" text1: str, text2: str\n",
") -> NamedTuple(\n",
" \"Outputs\",\n", " \"Outputs\",\n",
" [\n", " [\n",
" (\"output_1\", str), # Return parameters\n", " (\"output_1\", str), # Return parameters\n",
@@ -1587,7 +1584,7 @@
"delete_bucket = False\n", "delete_bucket = False\n",
"\n", "\n",
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
" ! gsutil rm -r $BUCKET_URI" " ! gcloud storage rm --recursive $BUCKET_URI"
] ]
} }
], ],
notebooks/community/ml_ops/stage6/get_started_with_custom_tabular_model_batch.ipynb
+6 -6
View File
@@ -464,7 +464,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil mb -l $REGION $BUCKET_URI" "! gcloud storage buckets create --location=$REGION $BUCKET_URI"
] ]
}, },
{ {
@@ -484,7 +484,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil ls -al $BUCKET_URI" "! gcloud storage ls --all-versions --long $BUCKET_URI"
] ]
}, },
{ {
@@ -892,7 +892,7 @@
" f.write(json.dumps(instance) + \"\\n\")\n", " f.write(json.dumps(instance) + \"\\n\")\n",
"\n", "\n",
"# Upload to Cloud Storage bucket\n", "# Upload to Cloud Storage bucket\n",
"! gsutil cp $BATCH_PREDICTION_INSTANCES_FILE $BATCH_PREDICTION_GCS_SOURCE\n", "! gcloud storage cp $BATCH_PREDICTION_INSTANCES_FILE $BATCH_PREDICTION_GCS_SOURCE\n",
"\n", "\n",
"print(\"Uploaded instances to: \", BATCH_PREDICTION_GCS_SOURCE)" "print(\"Uploaded instances to: \", BATCH_PREDICTION_GCS_SOURCE)"
] ]
@@ -1108,7 +1108,7 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"data_file = \"gs://cloud-samples-data/ai-platform/iris/iris_data.csv\"\n", "data_file = \"gs://cloud-samples-data/ai-platform/iris/iris_data.csv\"\n",
"! gsutil cat $data_file | head -n 10 > test.csv\n", "! gcloud storage cat $data_file | head -n 10 > test.csv\n",
"\n", "\n",
"! cat test.csv\n", "! cat test.csv\n",
"\n", "\n",
@@ -1118,7 +1118,7 @@
" BUCKET_URI + \"/batch_prediction_instances/\" + BATCH_PREDICTION_INSTANCES_FILE\n", " BUCKET_URI + \"/batch_prediction_instances/\" + BATCH_PREDICTION_INSTANCES_FILE\n",
")\n", ")\n",
"\n", "\n",
"! gsutil cp test.csv $BATCH_PREDICTION_GCS_SOURCE" "! gcloud storage cp test.csv $BATCH_PREDICTION_GCS_SOURCE"
] ]
}, },
{ {
@@ -1497,7 +1497,7 @@
"delete_bucket = False\n", "delete_bucket = False\n",
"\n", "\n",
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
" ! gsutil rm -rf {BUCKET_URI}" " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}"
] ]
} }
], ],
notebooks/community/ml_ops/stage6/get_started_with_xai_and_custom_server.ipynb
+3 -3
View File
@@ -504,7 +504,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil mb -l $REGION $BUCKET_URI" "! gcloud storage buckets create --location=$REGION $BUCKET_URI"
] ]
}, },
{ {
@@ -524,7 +524,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil ls -al $BUCKET_URI" "! gcloud storage ls --all-versions --long $BUCKET_URI"
] ]
}, },
{ {
@@ -1734,7 +1734,7 @@
"! rm -rf app\n", "! rm -rf app\n",
"\n", "\n",
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
" ! gsutil rm -rf {BUCKET_URI}" " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}"
] ]
} }
], ],
notebooks/community/model_evaluation/automl_tabular_regression_model_evaluation.ipynb
+6 -6
View File
@@ -537,7 +537,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI" "! gcloud storage buckets create --location=$REGION --project=$PROJECT_ID $BUCKET_URI"
] ]
}, },
{ {
@@ -557,7 +557,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil ls -al $BUCKET_URI" "! gcloud storage ls --all-versions --long $BUCKET_URI"
] ]
}, },
{ {
@@ -627,9 +627,9 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n", "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n",
"\n", "\n",
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI" "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer"
] ]
}, },
{ {
@@ -1383,7 +1383,7 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"# Load the results\n", "# Load the results\n",
"attributions = !gsutil cat $feat_attrs_gcs_uri\n", "attributions = !gcloud storage cat $feat_attrs_gcs_uri\n",
"\n", "\n",
"# Print the results obtained\n", "# Print the results obtained\n",
"attributions = json.loads(attributions[0])\n", "attributions = json.loads(attributions[0])\n",
@@ -1463,7 +1463,7 @@
"# Delete Cloud Storage objects\n", "# Delete Cloud Storage objects\n",
"delete_bucket = False\n", "delete_bucket = False\n",
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
" ! gsutil -m rm -r $BUCKET_URI" " ! gcloud storage rm --recursive $BUCKET_URI"
] ]
} }
], ],
notebooks/official/explainable_ai/sdk_custom_tabular_regression_batch_explain.ipynb
+4 -4
View File
@@ -288,7 +288,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}" "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}"
] ]
}, },
{ {
@@ -707,7 +707,7 @@
"! rm -f custom.tar custom.tar.gz\n", "! rm -f custom.tar custom.tar.gz\n",
"! tar cvf custom.tar custom\n", "! tar cvf custom.tar custom\n",
"! gzip custom.tar\n", "! gzip custom.tar\n",
"! gsutil cp custom.tar.gz $BUCKET_URI/trainer_boston.tar.gz" "! gcloud storage cp custom.tar.gz $BUCKET_URI/trainer_boston.tar.gz"
] ]
}, },
{ {
@@ -1260,7 +1260,7 @@
"\n", "\n",
"gcs_input_uri = BUCKET_URI + \"/test.csv\"\n", "gcs_input_uri = BUCKET_URI + \"/test.csv\"\n",
"\n", "\n",
"! gsutil cp batch.csv $gcs_input_uri" "! gcloud storage cp batch.csv $gcs_input_uri"
] ]
}, },
{ {
@@ -1398,7 +1398,7 @@
"# Delete the Cloud Storage bucket\n", "# Delete the Cloud Storage bucket\n",
"delete_bucket = False # Set True for deletion\n", "delete_bucket = False # Set True for deletion\n",
"if delete_bucket:\n", "if delete_bucket:\n",
" ! gsutil rm -r $BUCKET_URI\n", " ! gcloud storage rm --recursive $BUCKET_URI\n",
"\n", "\n",
"# Remove the locally generated files\n", "# Remove the locally generated files\n",
"! rm -rf batch.csv custom.tar.gz custom" "! rm -rf batch.csv custom.tar.gz custom"
notebooks/official/explainable_ai/sdk_custom_tabular_regression_online_explain_get_metadata.ipynb
+3 -3
View File
@@ -293,7 +293,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}" "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}"
] ]
}, },
{ {
@@ -706,7 +706,7 @@
"! rm -f custom.tar custom.tar.gz\n", "! rm -f custom.tar custom.tar.gz\n",
"! tar cvf custom.tar custom\n", "! tar cvf custom.tar custom\n",
"! gzip custom.tar\n", "! gzip custom.tar\n",
"! gsutil cp custom.tar.gz $BUCKET_URI/trainer_boston.tar.gz" "! gcloud storage cp custom.tar.gz $BUCKET_URI/trainer_boston.tar.gz"
] ]
}, },
{ {
@@ -1444,7 +1444,7 @@
"\n", "\n",
"delete_bucket = False # Set True for deletion\n", "delete_bucket = False # Set True for deletion\n",
"if delete_bucket:\n", "if delete_bucket:\n",
" ! gsutil rm -r $BUCKET_URI" " ! gcloud storage rm --recursive $BUCKET_URI"
] ]
} }
], ],
notebooks/official/training/tpuv5e_llama2_pytorch_finetuning_and_serving.ipynb
+3 -3
View File
@@ -279,7 +279,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}" "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}"
] ]
}, },
{ {
@@ -1264,7 +1264,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"!gsutil ls $GCS_MODEL_PATH" "!gcloud storage ls $GCS_MODEL_PATH"
] ]
}, },
{ {
@@ -1517,7 +1517,7 @@
"# Delete Cloud Storage objects that were created\n", "# Delete Cloud Storage objects that were created\n",
"delete_bucket = False\n", "delete_bucket = False\n",
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
" ! gsutil -m rm -r $BUCKET_URI" " ! gcloud storage rm --recursive $BUCKET_URI"
] ]
} }
], ],