xiaohei/model_garden
1
0
Fork 0
mirror of https://github.com/GoogleCloudPlatform/vertex-ai-samples.git synced 2026-09-26 14:42:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Migrate gsutil usage to gcloud storage (#4310)

Browse Source 
* Migrate gsutil usage to gcloud storage

* changes for 4310

* Apply automated linter fixes

* removed model_garden folder changes

---------

Co-authored-by: gurusai-voleti <gvoleti@google.com>
This commit is contained in:
Margubur Rahman
2025-12-24 09:38:53 -05:00
committed by GitHub
co-authored by gurusai-voleti
parent f4b56e92ae
commit c53f392c5a
7 changed files with 47 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
notebooks/community/ml_ops/stage1/get_started_bq_datasets.ipynb
+4 -4
View File
@@ -419,13 +419,13 @@
"\n",
"! bq --location=us extract --destination_format CSV $BQ_PROJECT_DATASET_TABLE $BUCKET_URI/mydata*.csv\n",
"\n",
"IMPORT_FILES = ! gsutil ls $BUCKET_URI/mydata*.csv\n",
"IMPORT_FILES = ! gcloud storage ls $BUCKET_URI/mydata*.csv\n",
"\n",
"print(IMPORT_FILES)\n",
"\n",
"EXAMPLE_FILE = IMPORT_FILES[0]\n",
"\n",
"! gsutil cat $EXAMPLE_FILE | head"
"! gcloud storage cat $EXAMPLE_FILE | head"
]
},
{
@@ -926,7 +926,7 @@
},
"outputs": [],
"source": [
"! gsutil cp $EXAMPLE_FILE data.csv\n",
"! gcloud storage cp $EXAMPLE_FILE data.csv\n",
"\n",
"dtrain = xgb.DMatrix(\"data.csv?format=csv&label_column=4\")"
]
@@ -970,7 +970,7 @@
"delete_storage = False\n",
"if delete_storage or os.getenv(\"IS_TESTING\"):\n",
" # Delete the created GCS bucket\n",
" ! gsutil rm -r $BUCKET_URI\n",
" ! gcloud storage rm --recursive $BUCKET_URI\n",
" # Delete the created BigQuery datasets\n",
" ! bq rm -r -f $PROJECT_ID:$BQ_MY_DATASET"
]
notebooks/community/sdk/sdk_automl_image_classification_online_export_edge.ipynb
+8 -8
View File
@@ -472,7 +472,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l $REGION $BUCKET_NAME"
"! gcloud storage buckets create --location=$REGION $BUCKET_NAME"
]
},
{
@@ -492,7 +492,7 @@
},
"outputs": [],
"source": [
"! gsutil ls -al $BUCKET_NAME"
"! gcloud storage ls --all-versions --long $BUCKET_NAME"
]
},
{
@@ -601,11 +601,11 @@
"else:\n",
" FILE = IMPORT_FILE\n",
"\n",
"count = ! gsutil cat $FILE | wc -l\n",
"count = ! gcloud storage cat $FILE | wc -l\n",
"print(\"Number of Examples\", int(count[0]))\n",
"\n",
"print(\"First 10 rows\")\n",
"! gsutil cat $FILE | head"
"! gcloud storage cat $FILE | head"
]
},
{
@@ -823,9 +823,9 @@
},
"outputs": [],
"source": [
"! gsutil ls $model_package\n",
"! gcloud storage ls $model_package\n",
"# Download the model artifacts\n",
"! gsutil cp -r $model_package tflite\n",
"! gcloud storage cp --recursive $model_package tflite\n",
"\n",
"tflite_path = \"tflite/model.tflite\""
]
@@ -884,7 +884,7 @@
},
"outputs": [],
"source": [
"test_items = ! gsutil cat $IMPORT_FILE | head -n1\n",
"test_items = ! gcloud storage cat $IMPORT_FILE | head -n1\n",
"test_item = test_items[0].split(\",\")[0]\n",
"\n",
"with tf.io.gfile.GFile(test_item, \"rb\") as f:\n",
@@ -1023,7 +1023,7 @@
" print(e)\n",
"\n",
" if \"BUCKET_NAME\" in globals():\n",
" ! gsutil rm -r $BUCKET_NAME"
" ! gcloud storage rm --recursive $BUCKET_NAME"
]
}
],
notebooks/community/sdk/sdk_automl_image_object_detection_online_export_edge.ipynb
+8 -8
View File
@@ -472,7 +472,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l $REGION $BUCKET_NAME"
"! gcloud storage buckets create --location=$REGION $BUCKET_NAME"
]
},
{
@@ -492,7 +492,7 @@
},
"outputs": [],
"source": [
"! gsutil ls -al $BUCKET_NAME"
"! gcloud storage ls --all-versions --long $BUCKET_NAME"
]
},
{
@@ -599,11 +599,11 @@
"else:\n",
" FILE = IMPORT_FILE\n",
"\n",
"count = ! gsutil cat $FILE | wc -l\n",
"count = ! gcloud storage cat $FILE | wc -l\n",
"print(\"Number of Examples\", int(count[0]))\n",
"\n",
"print(\"First 10 rows\")\n",
"! gsutil cat $FILE | head"
"! gcloud storage cat $FILE | head"
]
},
{
@@ -821,9 +821,9 @@
},
"outputs": [],
"source": [
"! gsutil ls $model_package\n",
"! gcloud storage ls $model_package\n",
"# Download the model artifacts\n",
"! gsutil cp -r $model_package tflite\n",
"! gcloud storage cp --recursive $model_package tflite\n",
"\n",
"tflite_path = \"tflite/model.tflite\""
]
@@ -882,7 +882,7 @@
},
"outputs": [],
"source": [
"test_items = ! gsutil cat $IMPORT_FILE | head -n1\n",
"test_items = ! gcloud storage cat $IMPORT_FILE | head -n1\n",
"test_item = test_items[0].split(\",\")[0]\n",
"\n",
"with tf.io.gfile.GFile(test_item, \"rb\") as f:\n",
@@ -1021,7 +1021,7 @@
" print(e)\n",
"\n",
" if \"BUCKET_NAME\" in globals():\n",
" ! gsutil rm -r $BUCKET_NAME"
" ! gcloud storage rm --recursive $BUCKET_NAME"
]
}
],
notebooks/community/sdk/sdk_automl_tabular_binary_classification_batch_explain.ipynb
+9 -9
View File
@@ -477,7 +477,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l $REGION $BUCKET_NAME"
"! gcloud storage buckets create --location=$REGION $BUCKET_NAME"
]
},
{
@@ -497,7 +497,7 @@
},
"outputs": [],
"source": [
"! gsutil ls -al $BUCKET_NAME"
"! gcloud storage ls --all-versions --long $BUCKET_NAME"
]
},
{
@@ -601,13 +601,13 @@
},
"outputs": [],
"source": [
"count = ! gsutil cat $IMPORT_FILE | wc -l\n",
"count = ! gcloud storage cat $IMPORT_FILE | wc -l\n",
"print(\"Number of Examples\", int(count[0]))\n",
"\n",
"print(\"First 10 rows\")\n",
"! gsutil cat $IMPORT_FILE | head\n",
"! gcloud storage cat $IMPORT_FILE | head\n",
"\n",
"heading = ! gsutil cat $IMPORT_FILE | head -n1\n",
"heading = ! gcloud storage cat $IMPORT_FILE | head -n1\n",
"label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n",
"print(\"Label Column Name\", label_column)\n",
"if label_column is None:\n",
@@ -826,14 +826,14 @@
},
"outputs": [],
"source": [
"! gsutil cat $IMPORT_FILE | head -n 1 > tmp.csv\n",
"! gsutil cat $IMPORT_FILE | tail -n 10 >> tmp.csv\n",
"! gcloud storage cat $IMPORT_FILE | head -n 1 > tmp.csv\n",
"! gcloud storage cat $IMPORT_FILE | tail -n 10 >> tmp.csv\n",
"\n",
"! cut -d, -f1-16 tmp.csv > batch.csv\n",
"\n",
"gcs_input_uri = BUCKET_NAME + \"/test.csv\"\n",
"\n",
"! gsutil cp batch.csv $gcs_input_uri"
"! gcloud storage cp batch.csv $gcs_input_uri"
]
},
{
@@ -1024,7 +1024,7 @@
" print(e)\n",
"\n",
" if \"BUCKET_NAME\" in globals():\n",
" ! gsutil rm -r $BUCKET_NAME"
" ! gcloud storage rm --recursive $BUCKET_NAME"
]
}
],
notebooks/community/sdk/sdk_automl_text_classification_batch.ipynb
+7 -7
View File
@@ -477,7 +477,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l $REGION $BUCKET_NAME"
"! gcloud storage buckets create --location=$REGION $BUCKET_NAME"
]
},
{
@@ -497,7 +497,7 @@
},
"outputs": [],
"source": [
"! gsutil ls -al $BUCKET_NAME"
"! gcloud storage ls --all-versions --long $BUCKET_NAME"
]
},
{
@@ -604,11 +604,11 @@
"else:\n",
" FILE = IMPORT_FILE\n",
"\n",
"count = ! gsutil cat $FILE | wc -l\n",
"count = ! gcloud storage cat $FILE | wc -l\n",
"print(\"Number of Examples\", int(count[0]))\n",
"\n",
"print(\"First 10 rows\")\n",
"! gsutil cat $FILE | head"
"! gcloud storage cat $FILE | head"
]
},
{
@@ -789,7 +789,7 @@
},
"outputs": [],
"source": [
"test_items = ! gsutil cat $IMPORT_FILE | head -n2\n",
"test_items = ! gcloud storage cat $IMPORT_FILE | head -n2\n",
"if len(test_items[0]) == 3:\n",
" _, test_item_1, test_label_1 = str(test_items[0]).split(\",\")\n",
" _, test_item_2, test_label_2 = str(test_items[1]).split(\",\")\n",
@@ -846,7 +846,7 @@
" f.write(json.dumps(data) + \"\\n\")\n",
"\n",
"print(gcs_input_uri)\n",
"! gsutil cat $gcs_input_uri"
"! gcloud storage cat $gcs_input_uri"
]
},
{
@@ -1038,7 +1038,7 @@
" print(e)\n",
"\n",
" if \"BUCKET_NAME\" in globals():\n",
" ! gsutil rm -r $BUCKET_NAME"
" ! gcloud storage rm --recursive $BUCKET_NAME"
]
}
],
notebooks/official/prediction/pytorch_train_deploy_models_with_prebuilt_containers.ipynb
+9 -9
View File
@@ -296,7 +296,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}"
"! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}"
]
},
{
@@ -619,7 +619,7 @@
"outputs": [],
"source": [
"! cd {PYTHON_PACKAGE_APPLICATION_DIR} && python3 setup.py sdist --formats=gztar\n",
"! gsutil cp {source_package_file_name} {python_package_gcs_uri}\n",
"! gcloud storage cp {source_package_file_name} {python_package_gcs_uri}\n",
"\n",
"print(f\"Python source distribution package location: {python_package_gcs_uri}\")"
]
@@ -735,7 +735,7 @@
"outputs": [],
"source": [
"print(f\"Model artifacts are available at {MODEL_DIR}\")\n",
"! gsutil ls -l {MODEL_DIR}"
"! gcloud storage ls --long {MODEL_DIR}"
]
},
{
@@ -776,7 +776,7 @@
"model_name = \"pytorch-mnist.pt\"\n",
"model_file = f\"{model_path}/{model_name}\"\n",
"\n",
"! gsutil cp {MODEL_DIR}/{model_name} {model_file}"
"! gcloud storage cp {MODEL_DIR}/{model_name} {model_file}"
]
},
{
@@ -896,7 +896,7 @@
"source": [
"#### Copy the model artifacts to Cloud Storage\n",
"\n",
"Next, use `gsutil` to copy the model artifacts to your Cloud Storage bucket."
"Next, use `gcloud storage` to copy the model artifacts to your Cloud Storage bucket."
]
},
{
@@ -909,9 +909,9 @@
"source": [
"MODEL_URI = f\"{BUCKET_URI}/model\"\n",
"\n",
"! gsutil -m rm -r -f $MODEL_URI\n",
"! gsutil -m cp -r $model_path $MODEL_URI\n",
"! gsutil ls $MODEL_URI"
"! gcloud storage rm --recursive --continue-on-error $MODEL_URI\n",
"! gcloud storage cp --recursive $model_path $MODEL_URI\n",
"! gcloud storage ls $MODEL_URI"
]
},
{
@@ -1088,7 +1088,7 @@
"# Delete Cloud Storage objects that were created\n",
"delete_bucket = True\n",
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
" ! gsutil -m rm -r $BUCKET_URI"
" ! gcloud storage rm --recursive $BUCKET_URI"
]
}
],
notebooks/official/workbench/fraud_detection/fraud-detection-model.ipynb
+2 -2
View File
@@ -312,7 +312,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}"
"! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}"
]
},
{
@@ -1077,7 +1077,7 @@
"# delete the bucket\n",
"delete_bucket = False\n",
"if delete_bucket:\n",
" ! gsutil rm -r $BUCKET_URI\n",
" ! gcloud storage rm --recursive $BUCKET_URI\n",
"\n",
"# delete the local files\n",
"! rm $LOCAL_FILE_NAME"