xiaohei/model_garden
1
0
Fork 0
mirror of https://github.com/GoogleCloudPlatform/vertex-ai-samples.git synced 2026-09-26 14:42:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Migrate gsutil usage to gcloud storage (#4309)

Browse Source 
* Migrate gsutil usage to gcloud storage

* changes for 4309

* changes for 4309

* Apply automated linter fixes

---------

Co-authored-by: gurusai-voleti <gvoleti@google.com>
This commit is contained in:
Margubur Rahman
2025-12-20 11:34:42 -05:00
committed by GitHub
co-authored by gurusai-voleti
parent 300fce6b9f
commit 46e0ea4f1c
10 changed files with 51 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
notebooks/community/ml_ops/stage1/get_started_dataflow.ipynb
+3 -3
View File
@@ -1052,8 +1052,8 @@
"run_pipeline(args)\n", "run_pipeline(args)\n",
"print(\"Data preprocessing completed.\")\n", "print(\"Data preprocessing completed.\")\n",
"\n", "\n",
"! gsutil ls $EXPORTED_DATA_PREFIX/train\n", "! gcloud storage ls $EXPORTED_DATA_PREFIX/train\n",
"! gsutil ls $EXPORTED_DATA_PREFIX/eval" "! gcloud storage ls $EXPORTED_DATA_PREFIX/eval"
] ]
}, },
{ {
@@ -1082,7 +1082,7 @@
"\n", "\n",
"if delete_storage or os.getenv(\"IS_TESTING\"):\n", "if delete_storage or os.getenv(\"IS_TESTING\"):\n",
" if \"BUCKET_URI\" in globals():\n", " if \"BUCKET_URI\" in globals():\n",
" ! gsutil rm -r $BUCKET_URI" " ! gcloud storage rm --recursive $BUCKET_URI"
] ]
} }
], ],
notebooks/community/ml_ops/stage1/get_started_with_visionapi_and_vertex_datasets.ipynb
+6 -6
View File
@@ -539,7 +539,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil mb -l $VERTEX_AI_REGION -p $PROJECT_ID $BUCKET_URI" "! gcloud storage buckets create --location=$VERTEX_AI_REGION --project=$PROJECT_ID $BUCKET_URI"
] ]
}, },
{ {
@@ -559,7 +559,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil ls -al $BUCKET_URI" "! gcloud storage ls --all-versions --long $BUCKET_URI"
] ]
}, },
{ {
@@ -775,10 +775,10 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"json_files = ! gsutil ls {gcs_destination_uri}\n", "json_files = ! gcloud storage ls {gcs_destination_uri}\n",
"\n", "\n",
"example = json_files[0]\n", "example = json_files[0]\n",
"! gsutil cat {example} | head -n 1" "! gcloud storage cat {example} | head -n 1"
] ]
}, },
{ {
@@ -863,7 +863,7 @@
"source": [ "source": [
"example = import_file_entries[0][\"textGcsUri\"]\n", "example = import_file_entries[0][\"textGcsUri\"]\n",
"\n", "\n",
"! gsutil cat {example}" "! gcloud storage cat {example}"
] ]
}, },
{ {
@@ -984,7 +984,7 @@
"\n", "\n",
"# Delete the bucket created\n", "# Delete the bucket created\n",
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
" ! gsutil rm -r $BUCKET_URI" " ! gcloud storage rm --recursive $BUCKET_URI"
] ]
} }
], ],
notebooks/community/sdk/sdk_automl_image_object_detection_online.ipynb
+6 -6
View File
@@ -473,7 +473,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil mb -l $REGION $BUCKET_NAME" "! gcloud storage buckets create --location=$REGION $BUCKET_NAME"
] ]
}, },
{ {
@@ -493,7 +493,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil ls -al $BUCKET_NAME" "! gcloud storage ls --all-versions --long $BUCKET_NAME"
] ]
}, },
{ {
@@ -600,11 +600,11 @@
"else:\n", "else:\n",
" FILE = IMPORT_FILE\n", " FILE = IMPORT_FILE\n",
"\n", "\n",
"count = ! gsutil cat $FILE | wc -l\n", "count = ! gcloud storage cat $FILE | wc -l\n",
"print(\"Number of Examples\", int(count[0]))\n", "print(\"Number of Examples\", int(count[0]))\n",
"\n", "\n",
"print(\"First 10 rows\")\n", "print(\"First 10 rows\")\n",
"! gsutil cat $FILE | head" "! gcloud storage cat $FILE | head"
] ]
}, },
{ {
@@ -819,7 +819,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"test_items = !gsutil cat $IMPORT_FILE | head -n1\n", "test_items = !gcloud storage cat $IMPORT_FILE | head -n1\n",
"cols = str(test_items[0]).split(\",\")\n", "cols = str(test_items[0]).split(\",\")\n",
"if len(cols) == 11:\n", "if len(cols) == 11:\n",
" test_item = str(cols[1])\n", " test_item = str(cols[1])\n",
@@ -992,7 +992,7 @@
" print(e)\n", " print(e)\n",
"\n", "\n",
" if \"BUCKET_NAME\" in globals():\n", " if \"BUCKET_NAME\" in globals():\n",
" ! gsutil rm -r $BUCKET_NAME" " ! gcloud storage rm --recursive $BUCKET_NAME"
] ]
} }
], ],
notebooks/community/sdk/sdk_automl_tabular_binary_classification_batch.ipynb
+9 -9
View File
@@ -477,7 +477,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil mb -l $REGION $BUCKET_NAME" "! gcloud storage buckets create --location=$REGION $BUCKET_NAME"
] ]
}, },
{ {
@@ -497,7 +497,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil ls -al $BUCKET_NAME" "! gcloud storage ls --all-versions --long $BUCKET_NAME"
] ]
}, },
{ {
@@ -601,13 +601,13 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"count = ! gsutil cat $IMPORT_FILE | wc -l\n", "count = ! gcloud storage cat $IMPORT_FILE | wc -l\n",
"print(\"Number of Examples\", int(count[0]))\n", "print(\"Number of Examples\", int(count[0]))\n",
"\n", "\n",
"print(\"First 10 rows\")\n", "print(\"First 10 rows\")\n",
"! gsutil cat $IMPORT_FILE | head\n", "! gcloud storage cat $IMPORT_FILE | head\n",
"\n", "\n",
"heading = ! gsutil cat $IMPORT_FILE | head -n1\n", "heading = ! gcloud storage cat $IMPORT_FILE | head -n1\n",
"label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n", "label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n",
"print(\"Label Column Name\", label_column)\n", "print(\"Label Column Name\", label_column)\n",
"if label_column is None:\n", "if label_column is None:\n",
@@ -826,14 +826,14 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil cat $IMPORT_FILE | head -n 1 > tmp.csv\n", "! gcloud storage cat $IMPORT_FILE | head -n 1 > tmp.csv\n",
"! gsutil cat $IMPORT_FILE | tail -n 10 >> tmp.csv\n", "! gcloud storage cat $IMPORT_FILE | tail -n 10 >> tmp.csv\n",
"\n", "\n",
"! cut -d, -f1-16 tmp.csv > batch.csv\n", "! cut -d, -f1-16 tmp.csv > batch.csv\n",
"\n", "\n",
"gcs_input_uri = BUCKET_NAME + \"/test.csv\"\n", "gcs_input_uri = BUCKET_NAME + \"/test.csv\"\n",
"\n", "\n",
"! gsutil cp batch.csv $gcs_input_uri" "! gcloud storage cp batch.csv $gcs_input_uri"
] ]
}, },
{ {
@@ -1022,7 +1022,7 @@
" print(e)\n", " print(e)\n",
"\n", "\n",
" if \"BUCKET_NAME\" in globals():\n", " if \"BUCKET_NAME\" in globals():\n",
" ! gsutil rm -r $BUCKET_NAME" " ! gcloud storage rm --recursive $BUCKET_NAME"
] ]
} }
], ],
notebooks/community/sdk/sdk_automl_text_sentiment_analysis_batch.ipynb
+7 -7
View File
@@ -477,7 +477,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil mb -l $REGION $BUCKET_NAME" "! gcloud storage buckets create --location=$REGION $BUCKET_NAME"
] ]
}, },
{ {
@@ -497,7 +497,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil ls -al $BUCKET_NAME" "! gcloud storage ls --all-versions --long $BUCKET_NAME"
] ]
}, },
{ {
@@ -605,11 +605,11 @@
"else:\n", "else:\n",
" FILE = IMPORT_FILE\n", " FILE = IMPORT_FILE\n",
"\n", "\n",
"count = ! gsutil cat $FILE | wc -l\n", "count = ! gcloud storage cat $FILE | wc -l\n",
"print(\"Number of Examples\", int(count[0]))\n", "print(\"Number of Examples\", int(count[0]))\n",
"\n", "\n",
"print(\"First 10 rows\")\n", "print(\"First 10 rows\")\n",
"! gsutil cat $FILE | head" "! gcloud storage cat $FILE | head"
] ]
}, },
{ {
@@ -790,7 +790,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"test_items = ! gsutil cat $IMPORT_FILE | head -n2\n", "test_items = ! gcloud storage cat $IMPORT_FILE | head -n2\n",
"\n", "\n",
"if len(test_items[0]) == 4:\n", "if len(test_items[0]) == 4:\n",
" _, test_item_1, test_label_1, _ = str(test_items[0]).split(\",\")\n", " _, test_item_1, test_label_1, _ = str(test_items[0]).split(\",\")\n",
@@ -849,7 +849,7 @@
" f.write(json.dumps(data) + \"\\n\")\n", " f.write(json.dumps(data) + \"\\n\")\n",
"\n", "\n",
"print(gcs_input_uri)\n", "print(gcs_input_uri)\n",
"! gsutil cat $gcs_input_uri" "! gcloud storage cat $gcs_input_uri"
] ]
}, },
{ {
@@ -1039,7 +1039,7 @@
" print(e)\n", " print(e)\n",
"\n", "\n",
" if \"BUCKET_NAME\" in globals():\n", " if \"BUCKET_NAME\" in globals():\n",
" ! gsutil rm -r $BUCKET_NAME" " ! gcloud storage rm --recursive $BUCKET_NAME"
] ]
} }
], ],
notebooks/official/automl/automl_tabular_on_vertex_pipelines.ipynb
+4 -4
View File
@@ -281,7 +281,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil mb -l $LOCATION $BUCKET_URI" "! gcloud storage buckets create --location=$LOCATION $BUCKET_URI"
] ]
}, },
{ {
@@ -353,9 +353,9 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n", "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n",
"\n", "\n",
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI" "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer"
] ]
}, },
{ {
@@ -915,7 +915,7 @@
"# Delete Cloud Storage objects that were created\n", "# Delete Cloud Storage objects that were created\n",
"delete_bucket = False # Set True for deletion\n", "delete_bucket = False # Set True for deletion\n",
"if delete_bucket:\n", "if delete_bucket:\n",
" ! gsutil -m rm -r $BUCKET_URI" " ! gcloud storage rm --recursive $BUCKET_URI"
] ]
} }
], ],
notebooks/official/custom/SDK_Custom_Container_Prediction.ipynb
+3 -3
View File
@@ -321,7 +321,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}" "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}"
] ]
}, },
{ {
@@ -533,7 +533,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"!gsutil cp model.joblib preprocessor.pkl {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/\n", "!gcloud storage cp model.joblib preprocessor.pkl {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/\n",
"%cd .." "%cd .."
] ]
}, },
@@ -1112,7 +1112,7 @@
" \n", " \n",
"# Delete the Cloud Storage bucket\n", "# Delete the Cloud Storage bucket\n",
"if delete_bucket:\n", "if delete_bucket:\n",
" ! gsutil -m rm -r $BUCKET_URI" " ! gcloud storage rm --recursive $BUCKET_URI"
] ]
}, },
{ {
notebooks/official/custom/sdk-custom-image-classification-batch.ipynb
+6 -6
View File
@@ -289,7 +289,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil mb -l $LOCATION -p $PROJECT_ID $BUCKET_URI" "! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI"
] ]
}, },
{ {
@@ -648,7 +648,7 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"# Download the images\n", "# Download the images\n",
"! gsutil -m cp -r gs://cloud-samples-data/ai-platform-unified/cifar_test_images ." "! gcloud storage cp --recursive gs://cloud-samples-data/ai-platform-unified/cifar_test_images ."
] ]
}, },
{ {
@@ -734,7 +734,7 @@
" f.write(json.dumps(x) + \"\\n\")\n", " f.write(json.dumps(x) + \"\\n\")\n",
"\n", "\n",
"# Upload to Cloud Storage bucket\n", "# Upload to Cloud Storage bucket\n",
"! gsutil cp $BATCH_PREDICTION_INSTANCES_FILE $BATCH_PREDICTION_GCS_SOURCE\n", "! gcloud storage cp $BATCH_PREDICTION_INSTANCES_FILE $BATCH_PREDICTION_GCS_SOURCE\n",
"\n", "\n",
"print(\"Uploaded instances to: \", BATCH_PREDICTION_GCS_SOURCE)" "print(\"Uploaded instances to: \", BATCH_PREDICTION_GCS_SOURCE)"
] ]
@@ -829,7 +829,7 @@
"os.makedirs(RESULTS_DIRECTORY, exist_ok=True)\n", "os.makedirs(RESULTS_DIRECTORY, exist_ok=True)\n",
"\n", "\n",
"# Get the Cloud Storage paths for each result\n", "# Get the Cloud Storage paths for each result\n",
"! gsutil -m cp -r $BATCH_PREDICTION_GCS_DEST_PREFIX $RESULTS_DIRECTORY\n", "! gcloud storage cp --recursive $BATCH_PREDICTION_GCS_DEST_PREFIX $RESULTS_DIRECTORY\n",
"\n", "\n",
"# Get most recently modified directory\n", "# Get most recently modified directory\n",
"latest_directory = max(\n", "latest_directory = max(\n",
@@ -851,7 +851,7 @@
"results = []\n", "results = []\n",
"for results_file in results_files:\n", "for results_file in results_files:\n",
" # Download each result\n", " # Download each result\n",
" with open(results_file, \"r\") as file:\n", " with open(results_file) as file:\n",
" results.extend([json.loads(line) for line in file.readlines()])" " results.extend([json.loads(line) for line in file.readlines()])"
] ]
}, },
@@ -924,7 +924,7 @@
"model.delete()\n", "model.delete()\n",
"\n", "\n",
"if delete_bucket:\n", "if delete_bucket:\n",
" ! gsutil -m rm -r $BUCKET_URI" " ! gcloud storage rm --recursive $BUCKET_URI"
] ]
} }
], ],
notebooks/official/experiments/build_model_experimentation_lineage_with_prebuild_code.ipynb
+4 -4
View File
@@ -297,7 +297,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil mb -l {LOCATION} {BUCKET_URI}" "! gcloud storage buckets create --location={LOCATION} {BUCKET_URI}"
] ]
}, },
{ {
@@ -377,7 +377,7 @@
"! unzip {DATA_PATH}/*.zip -d {DATA_PATH}/temp\n", "! unzip {DATA_PATH}/*.zip -d {DATA_PATH}/temp\n",
"! mv {DATA_PATH}/temp/*.csv {DATA_PATH}/raw && rm -Rf {DATA_PATH}/temp && rm -f {DATA_PATH}/*.zip\n", "! mv {DATA_PATH}/temp/*.csv {DATA_PATH}/raw && rm -Rf {DATA_PATH}/temp && rm -f {DATA_PATH}/*.zip\n",
"\n", "\n",
"! gsutil -m cp -R {DATA_PATH}/raw $BUCKET_URI/{DATA_PATH}/raw" "! gcloud storage cp --recursive {DATA_PATH}/raw $BUCKET_URI/{DATA_PATH}/raw"
] ]
}, },
{ {
@@ -814,7 +814,7 @@
" try:\n", " try:\n",
" with tempfile.NamedTemporaryFile() as tmp:\n", " with tempfile.NamedTemporaryFile() as tmp:\n",
" joblib.dump(trained_pipeline, filename=tmp.name)\n", " joblib.dump(trained_pipeline, filename=tmp.name)\n",
" ! gsutil cp {tmp.name} {save_path}/model.joblib\n", " ! gcloud storage cp {tmp.name} {save_path}/model.joblib\n",
" except RuntimeError as error:\n", " except RuntimeError as error:\n",
" print(error)\n", " print(error)\n",
" return 1" " return 1"
@@ -1008,7 +1008,7 @@
"delete_bucket = True\n", "delete_bucket = True\n",
"\n", "\n",
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
" ! gsutil -m rm -r $BUCKET_URI\n", " ! gcloud storage rm --recursive $BUCKET_URI\n",
"\n", "\n",
"!rm -Rf {DATA_PATH}" "!rm -Rf {DATA_PATH}"
] ]
notebooks/official/vector_search/sdk_vector_search_create_stack_overflow_embeddings.ipynb
+3 -3
View File
@@ -276,7 +276,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}" "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}"
] ]
}, },
{ {
@@ -624,7 +624,7 @@
"source": [ "source": [
"UNIQUE_FOLDER_NAME = \"embeddings_folder_unique\"\n", "UNIQUE_FOLDER_NAME = \"embeddings_folder_unique\"\n",
"remote_folder = f\"{BUCKET_URI}/{UNIQUE_FOLDER_NAME}/\"\n", "remote_folder = f\"{BUCKET_URI}/{UNIQUE_FOLDER_NAME}/\"\n",
"! gsutil cp {embeddings_file.name} {remote_folder}" "! gcloud storage cp {embeddings_file.name} {remote_folder}"
] ]
}, },
{ {
@@ -1123,7 +1123,7 @@
"# Delete cloud storage bucket\n", "# Delete cloud storage bucket\n",
"delete_bucket = False\n", "delete_bucket = False\n",
"if delete_bucket:\n", "if delete_bucket:\n",
" ! gsutil rm -rf {BUCKET_URI}\n", " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}\n",
"\n", "\n",
"# Delete redis instance\n", "# Delete redis instance\n",
"! gcloud redis instances delete '{REDIS_INSTANCE_NAME}' --region {LOCATION} --quiet" "! gcloud redis instances delete '{REDIS_INSTANCE_NAME}' --region {LOCATION} --quiet"