xiaohei/model_garden
1
0
Fork 0
mirror of https://github.com/GoogleCloudPlatform/vertex-ai-samples.git synced 2026-09-26 14:42:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Migrate gsutil usage to gcloud storage (#4332)

Browse Source
This commit is contained in:
Margubur Rahman
2025-12-25 10:42:12 -05:00
committed by GitHub
parent 7a613785b9
commit 814827ac19
10 changed files with 40 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files
notebooks/community/gapic/automl/showcase_automl_text_classification_online.ipynb
+4 -7
View File
@@ -716,12 +716,10 @@
"else:\n",
" FILE = IMPORT_FILE\n",
"\n",
"count = ! gsutil cat $FILE | wc -l\n",
"print(\"Number of Examples\", int(count[0]))\n",
"count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
"\n",
"print(\"First 10 rows\")\n",
"! gsutil cat $FILE | head"
]
"! gcloud storage cat $FILE | head" ]
},
{
"cell_type": "markdown",
@@ -1327,7 +1325,7 @@
},
"outputs": [],
"source": [
"test_item = ! gsutil cat $IMPORT_FILE | head -n1\n",
"test_item = ! gcloud storage cat $IMPORT_FILE | head -n1\n",
"if len(test_item[0]) == 3:\n",
" _, test_item, test_label = str(test_item[0]).split(\",\")\n",
"else:\n",
@@ -1533,8 +1531,7 @@
" print(e)\n",
"\n",
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
" ! gsutil rm -r $BUCKET_NAME"
]
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
}
],
"metadata": {
notebooks/community/gapic/automl/showcase_automl_text_multi-label_classification_online.ipynb
+4 -8
View File
@@ -716,12 +716,10 @@
"else:\n",
" FILE = IMPORT_FILE\n",
"\n",
"count = ! gsutil cat $FILE | wc -l\n",
"print(\"Number of Examples\", int(count[0]))\n",
"count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
"\n",
"print(\"First 10 rows\")\n",
"! gsutil cat $FILE | head"
]
"! gcloud storage cat $FILE | head" ]
},
{
"cell_type": "markdown",
@@ -1327,8 +1325,7 @@
},
"outputs": [],
"source": [
"test_item = ! gsutil cat $IMPORT_FILE | head -n1\n",
"\n",
"test_item = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "\n",
"cols = str(test_item[0]).split(\",\")\n",
"test_item = cols[0]\n",
"test_label = cols[1:]\n",
@@ -1533,8 +1530,7 @@
" print(e)\n",
"\n",
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
" ! gsutil rm -r $BUCKET_NAME"
]
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
}
],
"metadata": {
notebooks/community/gapic/automl/showcase_automl_text_sentiment_analysis_online.ipynb
+4 -8
View File
@@ -718,12 +718,10 @@
"else:\n",
" FILE = IMPORT_FILE\n",
"\n",
"count = ! gsutil cat $FILE | wc -l\n",
"print(\"Number of Examples\", int(count[0]))\n",
"count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
"\n",
"print(\"First 10 rows\")\n",
"! gsutil cat $FILE | head"
]
"! gcloud storage cat $FILE | head" ]
},
{
"cell_type": "markdown",
@@ -1329,8 +1327,7 @@
},
"outputs": [],
"source": [
"test_item = ! gsutil cat $IMPORT_FILE | head -n1\n",
"if len(test_item[0]) == 3:\n",
"test_item = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "if len(test_item[0]) == 3:\n",
" _, test_item, test_label, max = str(test_item[0]).split(\",\")\n",
"else:\n",
" test_item, test_label, max = str(test_item[0]).split(\",\")\n",
@@ -1534,8 +1531,7 @@
" print(e)\n",
"\n",
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
" ! gsutil rm -r $BUCKET_NAME"
]
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
}
],
"metadata": {
notebooks/community/gapic/custom/showcase_tfhub_image_classification_online.ipynb
+4 -8
View File
@@ -422,8 +422,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l $REGION $BUCKET_NAME"
]
"! gcloud storage buckets create --location=$REGION $BUCKET_NAME" ]
},
{
"cell_type": "markdown",
@@ -442,8 +441,7 @@
},
"outputs": [],
"source": [
"! gsutil ls -al $BUCKET_NAME"
]
"! gcloud storage ls --all-versions --long $BUCKET_NAME" ]
},
{
"cell_type": "markdown",
@@ -1244,8 +1242,7 @@
"source": [
"FLOWERS_CSV = \"gs://cloud-ml-data/img/flower_photos/all_data.csv\"\n",
"\n",
"test_images = ! gsutil cat $FLOWERS_CSV | head -n1\n",
"test_image = test_images[0].split(\",\")[0]\n",
"test_images = ! gcloud storage cat $FLOWERS_CSV | head -n1\n", "test_image = test_images[0].split(\",\")[0]\n",
"print(test_image)"
]
},
@@ -1471,8 +1468,7 @@
" print(e)\n",
"\n",
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
" ! gsutil rm -r $BUCKET_NAME"
]
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
}
],
"metadata": {
notebooks/community/ml_ops/stage7/get_started_with_model_monitoring_custom.ipynb
+4 -8
View File
@@ -561,8 +561,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l $REGION $BUCKET_URI"
]
"! gcloud storage buckets create --location $REGION $BUCKET_URI" ]
},
{
"cell_type": "markdown",
@@ -581,8 +580,7 @@
},
"outputs": [],
"source": [
"! gsutil ls -al $BUCKET_URI"
]
"! gcloud storage ls --all-versions --long $BUCKET_URI" ]
},
{
"cell_type": "markdown",
@@ -1145,8 +1143,7 @@
"with open(\"schema.yaml\", \"w\") as f:\n",
" f.write(yaml)\n",
"\n",
"! gsutil cp schema.yaml {BUCKET_URI}/schema.yaml"
]
"! gcloud storage cp schema.yaml {BUCKET_URI}/schema.yaml" ]
},
{
"cell_type": "markdown",
@@ -1633,8 +1630,7 @@
"delete_bucket = False\n",
"\n",
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
" ! gsutil rm -rf {BUCKET_URI}\n",
"\n",
" ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}\n", "\n",
"! rm -f schema.yaml\n",
"\n",
"! bq rm -f {PROJECT_ID}.model_deployment_monitoring_{ENDPOINT_ID}"
notebooks/community/ml_ops/stage7/get_started_with_model_monitoring_setup.ipynb
+4 -8
View File
@@ -561,8 +561,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l $REGION $BUCKET_URI"
]
"! gcloud storage buckets create --location $REGION $BUCKET_URI" ]
},
{
"cell_type": "markdown",
@@ -581,8 +580,7 @@
},
"outputs": [],
"source": [
"! gsutil ls -al $BUCKET_URI"
]
"! gcloud storage ls --all-versions --long $BUCKET_URI" ]
},
{
"cell_type": "markdown",
@@ -1556,8 +1554,7 @@
"with open(\"schema.yaml\", \"w\") as f:\n",
" f.write(yaml)\n",
"\n",
"! gsutil cp schema.yaml {BUCKET_URI}/schema.yaml"
]
"! gcloud storage cp schema.yaml {BUCKET_URI}/schema.yaml" ]
},
{
"cell_type": "markdown",
@@ -1688,8 +1685,7 @@
"delete_bucket = True\n",
"\n",
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
" ! gsutil rm -rf {BUCKET_URI}\n",
"\n",
" ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}\n", "\n",
"! rm -f schema.yaml\n",
"\n",
"! bq rm -r -f {PROJECT_ID}.model_deployment_monitoring_{ENDPOINT_ID}"
notebooks/community/prediction/custom_prediction_routines/SDK_Custom_Predict_and_Handler_SDK_Integration.ipynb
+4 -9
View File
@@ -505,8 +505,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
]
"! gcloud storage buckets create --location $REGION --project $PROJECT_ID $BUCKET_URI" ]
},
{
"cell_type": "markdown",
@@ -525,8 +524,7 @@
},
"outputs": [],
"source": [
"! gsutil ls -al $BUCKET_URI"
]
"! gcloud storage ls --all-versions --long $BUCKET_URI" ]
},
{
"cell_type": "markdown",
@@ -696,9 +694,7 @@
"outputs": [],
"source": [
"%cd ..\n",
"!gsutil cp {LOCAL_MODEL_ARTIFACTS_DIR}/* {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/\n",
"!gsutil ls {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/"
]
"!gcloud storage cp {LOCAL_MODEL_ARTIFACTS_DIR}/* {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/\n", "!gcloud storage ls {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/" ]
},
{
"cell_type": "markdown",
@@ -1899,8 +1895,7 @@
"delete_bucket = False\n",
"\n",
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
" ! gsutil rm -r $BUCKET_URI"
]
" ! gcloud storage rm --recursive $BUCKET_URI" ]
}
],
"metadata": {
notebooks/community/prediction/custom_prediction_routines/SDK_Pytorch_Custom_Predict.ipynb
+4 -9
View File
@@ -503,8 +503,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
]
"! gcloud storage buckets create --location=$REGION --project=$PROJECT_ID $BUCKET_URI" ]
},
{
"cell_type": "markdown",
@@ -523,8 +522,7 @@
},
"outputs": [],
"source": [
"! gsutil ls -al $BUCKET_URI"
]
"! gcloud storage ls --all-versions --long $BUCKET_URI" ]
},
{
"cell_type": "markdown",
@@ -878,9 +876,7 @@
},
"outputs": [],
"source": [
"!gsutil cp {LOCAL_MODEL_ARTIFACTS_DIR}/* {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/\n",
"!gsutil ls {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/"
]
"!gcloud storage cp {LOCAL_MODEL_ARTIFACTS_DIR}/* {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/\n", "!gcloud storage ls {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/" ]
},
{
"cell_type": "markdown",
@@ -1952,8 +1948,7 @@
"delete_bucket = False\n",
"\n",
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
" ! gsutil rm -r $BUCKET_URI"
]
" ! gcloud storage rm --recursive $BUCKET_URI" ]
}
],
"metadata": {
notebooks/official/explainable_ai/sdk_custom_image_classification_batch_explain.ipynb
+4 -9
View File
@@ -294,8 +294,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l $LOCATION -p $PROJECT_ID $BUCKET_URI"
]
"! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI" ]
},
{
"cell_type": "markdown",
@@ -700,8 +699,7 @@
"! rm -f custom.tar custom.tar.gz\n",
"! tar cvf custom.tar custom\n",
"! gzip custom.tar\n",
"! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_cifar10.tar.gz"
]
"! gcloud storage cp custom.tar.gz $BUCKET_NAME/trainer_cifar10.tar.gz" ]
},
{
"cell_type": "markdown",
@@ -1326,9 +1324,7 @@
},
"outputs": [],
"source": [
"! gsutil cp tmp1.jpg $BUCKET_URI/tmp1.jpg\n",
"! gsutil cp tmp2.jpg $BUCKET_URI/tmp2.jpg\n",
"\n",
"! gcloud storage cp tmp1.jpg $BUCKET_URI/tmp1.jpg\n", "! gcloud storage cp tmp2.jpg $BUCKET_URI/tmp2.jpg\n", "\n",
"test_item_1 = BUCKET_URI + \"/tmp1.jpg\"\n",
"test_item_2 = BUCKET_URI + \"/tmp2.jpg\""
]
@@ -1529,8 +1525,7 @@
" print(e)\n",
"\n",
"if delete_bucket:\n",
" ! gsutil rm -r $BUCKET_URI\n",
"\n",
" ! gcloud storage rm --recursive $BUCKET_URI\n", "\n",
"if delete_application_directory:\n",
" ! rm -rf custom custom.tar.gz temp1.jpg temp2.jpg"
]
notebooks/official/model_monitoring/get_started_with_model_monitoring_custom_tf_serving.ipynb
+4 -8
View File
@@ -363,8 +363,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l $LOCATION -p $PROJECT_ID $BUCKET_URI"
]
"! gcloud storage buckets create --location $LOCATION --project $PROJECT_ID $BUCKET_URI" ]
},
{
"cell_type": "markdown",
@@ -747,8 +746,7 @@
"MODEL_ARTIFACT_URI = \"gs://mco-mm/churn\"\n",
"MODEL_DIR = BUCKET_URI + \"/model/1\"\n",
"\n",
"! gsutil cp -r $MODEL_ARTIFACT_URI $MODEL_DIR"
]
"! gcloud storage cp --recursive $MODEL_ARTIFACT_URI $MODEL_DIR" ]
},
{
"cell_type": "markdown",
@@ -1132,8 +1130,7 @@
"with open(\"schema.yaml\", \"w\") as f:\n",
" f.write(yaml)\n",
"\n",
"! gsutil cp schema.yaml {BUCKET_URI}/schema.yaml"
]
"! gcloud storage cp schema.yaml {BUCKET_URI}/schema.yaml" ]
},
{
"cell_type": "markdown",
@@ -1620,8 +1617,7 @@
"delete_bucket = False\n",
"\n",
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
" ! gsutil rm -rf {BUCKET_URI}\n",
"\n",
" ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}\n", "\n",
"! rm -f schema.yaml\n",
"\n",
"! bq rm -f {PROJECT_ID}.model_deployment_monitoring_{ENDPOINT_ID}\n",