xiaohei/model_garden
1
0
Fork 0
mirror of https://github.com/GoogleCloudPlatform/vertex-ai-samples.git synced 2026-09-26 14:42:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Migrate gsutil usage to gcloud storage (#4299)

Browse Source 
* Migrate gsutil usage to gcloud storage

* changes for 4299

* Apply automated linter fixes

* update

* remove model_garden changes

* revert to main

* revert model garden file

* Update model_garden_weather_prediction_on_vertex.ipynb

---------

Co-authored-by: gurusai-voleti <gvoleti@google.com>
This commit is contained in:
Margubur Rahman
2025-12-31 08:37:14 -05:00
committed by GitHub
co-authored by gurusai-voleti
parent 20d19fb11c
commit e79a45358c
7 changed files with 36 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
notebooks/community/ml_ops/stage3/get_started_with_dataflow_flex_template_component.ipynb
+7 -7
View File
@@ -456,7 +456,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil mb -l $REGION $BUCKET_URI" "! gcloud storage buckets create --location $REGION $BUCKET_URI"
] ]
}, },
{ {
@@ -476,7 +476,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil ls -al $BUCKET_URI" "! gcloud storage ls --all-versions --long $BUCKET_URI"
] ]
}, },
{ {
@@ -546,9 +546,9 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n", "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n",
"\n", "\n",
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI" "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer"
] ]
}, },
{ {
@@ -678,7 +678,7 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"GCS_AVRO_SCHEMA = BUCKET_URI + \"/gaming_schema.avsc\"\n", "GCS_AVRO_SCHEMA = BUCKET_URI + \"/gaming_schema.avsc\"\n",
"! gsutil cp gaming_schema.avsc $GCS_AVRO_SCHEMA\n", "! gcloud storage cp gaming_schema.avsc $GCS_AVRO_SCHEMA\n",
"\n", "\n",
"GCS_FLEX_TEMPLATE_PATH = \"gs://dataflow-templates/latest/flex/File_Format_Conversion\"\n", "GCS_FLEX_TEMPLATE_PATH = \"gs://dataflow-templates/latest/flex/File_Format_Conversion\"\n",
"GCS_CONVERT_IN = \"gs://dataflow-samples/game/5000_gaming_data.csv\"\n", "GCS_CONVERT_IN = \"gs://dataflow-samples/game/5000_gaming_data.csv\"\n",
@@ -757,7 +757,7 @@
"\n", "\n",
"pipeline.run()\n", "pipeline.run()\n",
"\n", "\n",
"! gsutil ls $GCS_CONVERT_OUT\n", "! gcloud storage ls $GCS_CONVERT_OUT\n",
"\n", "\n",
"! rm -f dataflow_file_conversion.yaml gaming_schema.avsc" "! rm -f dataflow_file_conversion.yaml gaming_schema.avsc"
] ]
@@ -813,7 +813,7 @@
"delete_bucket = False\n", "delete_bucket = False\n",
"\n", "\n",
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
" ! gsutil rm -r $BUCKET_URI" " ! gcloud storage rm --recursive $BUCKET_URI"
] ]
} }
], ],
notebooks/official/automl/automl_image_classification_batch_prediction.ipynb
+8 -8
View File
@@ -287,7 +287,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}" "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}"
] ]
}, },
{ {
@@ -394,11 +394,11 @@
"else:\n", "else:\n",
" FILE = IMPORT_FILE\n", " FILE = IMPORT_FILE\n",
"\n", "\n",
"count = ! gsutil cat $FILE | wc -l\n", "count = ! gcloud storage cat $FILE | wc -l\n",
"print(\"Number of Examples\", int(count[0]))\n", "print(\"Number of Examples\", int(count[0]))\n",
"\n", "\n",
"print(\"First 10 rows\")\n", "print(\"First 10 rows\")\n",
"! gsutil cat $FILE | head" "! gcloud storage cat $FILE | head"
] ]
}, },
{ {
@@ -580,7 +580,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"test_items = !gsutil cat $IMPORT_FILE | head -n2\n", "test_items = !gcloud storage cat $IMPORT_FILE | head -n2\n",
"if len(str(test_items[0]).split(\",\")) == 3:\n", "if len(str(test_items[0]).split(\",\")) == 3:\n",
" _, test_item_1, test_label_1 = str(test_items[0]).split(\",\")\n", " _, test_item_1, test_label_1 = str(test_items[0]).split(\",\")\n",
" _, test_item_2, test_label_2 = str(test_items[1]).split(\",\")\n", " _, test_item_2, test_label_2 = str(test_items[1]).split(\",\")\n",
@@ -614,8 +614,8 @@
"file_1 = test_item_1.split(\"/\")[-1]\n", "file_1 = test_item_1.split(\"/\")[-1]\n",
"file_2 = test_item_2.split(\"/\")[-1]\n", "file_2 = test_item_2.split(\"/\")[-1]\n",
"\n", "\n",
"! gsutil cp $test_item_1 $BUCKET_URI/$file_1\n", "! gcloud storage cp $test_item_1 $BUCKET_URI/$file_1\n",
"! gsutil cp $test_item_2 $BUCKET_URI/$file_2\n", "! gcloud storage cp $test_item_2 $BUCKET_URI/$file_2\n",
"\n", "\n",
"test_item_1 = BUCKET_URI + \"/\" + file_1\n", "test_item_1 = BUCKET_URI + \"/\" + file_1\n",
"test_item_2 = BUCKET_URI + \"/\" + file_2" "test_item_2 = BUCKET_URI + \"/\" + file_2"
@@ -659,7 +659,7 @@
" f.write(json.dumps(data) + \"\\n\")\n", " f.write(json.dumps(data) + \"\\n\")\n",
"\n", "\n",
"print(gcs_input_uri)\n", "print(gcs_input_uri)\n",
"! gsutil cat $gcs_input_uri" "! gcloud storage cat $gcs_input_uri"
] ]
}, },
{ {
@@ -803,7 +803,7 @@
"# Delete the cloud storage bucket\n", "# Delete the cloud storage bucket\n",
"delete_bucket = False # set True for deletion\n", "delete_bucket = False # set True for deletion\n",
"if delete_bucket:\n", "if delete_bucket:\n",
" ! gsutil rm -r $BUCKET_URI" " ! gcloud storage rm --recursive $BUCKET_URI"
] ]
} }
], ],
notebooks/official/automl/automl_image_object_detection_online_prediction.ipynb
+7 -7
View File
@@ -297,7 +297,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil mb -l $LOCATION $BUCKET_URI" "! gcloud storage buckets create --location=$LOCATION $BUCKET_URI"
] ]
}, },
{ {
@@ -386,7 +386,7 @@
"source": [ "source": [
"#### Copying data between Google Cloud Storage Buckets\n", "#### Copying data between Google Cloud Storage Buckets\n",
"\n", "\n",
"In this step, you prevent access issues for the images in your original dataset. The code below extracts folder paths from image paths, constructs destination paths for Cloud Storage, copies images using gsutil commands, updates image paths in the DataFrame, and finally saves the modified DataFrame back to Cloud Storage as a CSV file." "In this step, you prevent access issues for the images in your original dataset. The code below extracts folder paths from image paths, constructs destination paths for Cloud Storage, copies images using gcloud storage commands, updates image paths in the DataFrame, and finally saves the modified DataFrame back to Cloud Storage as a CSV file."
] ]
}, },
{ {
@@ -415,7 +415,7 @@
"\n", "\n",
"# Copy images using gsutil commands directly\n", "# Copy images using gsutil commands directly\n",
"for src, dest in zip(df.iloc[:, 0], df[\"destination_path\"]):\n", "for src, dest in zip(df.iloc[:, 0], df[\"destination_path\"]):\n",
" ! gsutil -m cp {src} {dest}\n", " ! gcloud storage cp {src} {dest}\n",
"\n", "\n",
"print(f\"Files copied to {BUCKET_URI}\")" "print(f\"Files copied to {BUCKET_URI}\")"
] ]
@@ -496,11 +496,11 @@
"else:\n", "else:\n",
" FILE = IMPORT_FILE\n", " FILE = IMPORT_FILE\n",
"\n", "\n",
"count = ! gsutil cat $FILE | wc -l\n", "count = ! gcloud storage cat $FILE | wc -l\n",
"print(\"Number of Examples\", int(count[0]))\n", "print(\"Number of Examples\", int(count[0]))\n",
"\n", "\n",
"print(\"First 10 rows\")\n", "print(\"First 10 rows\")\n",
"! gsutil cat $FILE | head" "! gcloud storage cat $FILE | head"
] ]
}, },
{ {
@@ -706,7 +706,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"test_items = !gsutil cat $IMPORT_FILE | head -n1\n", "test_items = !gcloud storage cat $IMPORT_FILE | head -n1\n",
"cols = str(test_items[0]).split(\",\")\n", "cols = str(test_items[0]).split(\",\")\n",
"if len(cols) == 11:\n", "if len(cols) == 11:\n",
" test_item = str(cols[1])\n", " test_item = str(cols[1])\n",
@@ -834,7 +834,7 @@
"dag.delete()\n", "dag.delete()\n",
"\n", "\n",
"if delete_bucket:\n", "if delete_bucket:\n",
" ! gsutil rm -r $BUCKET_URI" " ! gcloud storage rm --recursive $BUCKET_URI"
] ]
} }
], ],
notebooks/official/model_evaluation/get_started_with_custom_model_evaluation_import.ipynb
+2 -2
View File
@@ -304,7 +304,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}" "! gcloud storage buckets create {BUCKET_URI} --location={LOCATION} --project={PROJECT_ID}"
] ]
}, },
{ {
@@ -778,7 +778,7 @@
"# Delete Cloud Storage objects that were created\n", "# Delete Cloud Storage objects that were created\n",
"delete_bucket = False # set True for deletion\n", "delete_bucket = False # set True for deletion\n",
"if delete_bucket:\n", "if delete_bucket:\n",
" ! gsutil -m rm -r $BUCKET_URI" " ! gcloud storage rm --recursive $BUCKET_URI"
] ]
} }
], ],
notebooks/official/model_evaluation/model_based_llm_evaluation/autosxs_llm_evaluation_for_summarization_task.ipynb
+3 -3
View File
@@ -355,7 +355,7 @@
"):\n", "):\n",
" BUCKET_URI = \"gs://\" + PROJECT_ID + \"-aip-\" + UUID\n", " BUCKET_URI = \"gs://\" + PROJECT_ID + \"-aip-\" + UUID\n",
"\n", "\n",
"! gsutil mb -l $LOCATION -p $PROJECT_ID $BUCKET_URI" "! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI"
] ]
}, },
{ {
@@ -519,7 +519,7 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"examples.to_json(\"evaluation_dataset.json\", orient=\"records\", lines=True)\n", "examples.to_json(\"evaluation_dataset.json\", orient=\"records\", lines=True)\n",
"! gsutil cp evaluation_dataset.json $BUCKET_URI/input/evaluation_dataset.json\n", "! gcloud storage cp evaluation_dataset.json $BUCKET_URI/input/evaluation_dataset.json\n",
"DATASET = f\"{BUCKET_URI}/input/evaluation_dataset.json\"" "DATASET = f\"{BUCKET_URI}/input/evaluation_dataset.json\""
] ]
}, },
@@ -765,7 +765,7 @@
"# Delete Cloud Storage objects that were created\n", "# Delete Cloud Storage objects that were created\n",
"delete_bucket = False\n", "delete_bucket = False\n",
"if delete_bucket:\n", "if delete_bucket:\n",
" ! gsutil -m rm -r $BUCKET_URI" " ! gcloud storage rm --recursive $BUCKET_URI"
] ]
} }
], ],
notebooks/official/tensorboard/tensorboard_custom_training_with_prebuilt_container.ipynb
+4 -4
View File
@@ -316,7 +316,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil mb -l $LOCATION -p $PROJECT_ID $BUCKET_URI" "! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI"
] ]
}, },
{ {
@@ -480,7 +480,7 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"# Download the sample code\n", "# Download the sample code\n",
"! gsutil cp gs://cloud-samples-data/ai-platform/hello-custom/hello-custom-sample-v1.tar.gz - | tar -xzv\n", "! gcloud storage cp gs://cloud-samples-data/ai-platform/hello-custom/hello-custom-sample-v1.tar.gz - | tar -xzv\n",
"%cd hello-custom-sample/" "%cd hello-custom-sample/"
] ]
}, },
@@ -664,7 +664,7 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"GCS_BUCKET_TRAINING = f\"{BUCKET_URI}/data/\"\n", "GCS_BUCKET_TRAINING = f\"{BUCKET_URI}/data/\"\n",
"! gsutil cp dist/hello-custom-training-3.0.tar.gz {GCS_BUCKET_TRAINING}" "! gcloud storage cp dist/hello-custom-training-3.0.tar.gz {GCS_BUCKET_TRAINING}"
] ]
}, },
{ {
@@ -787,7 +787,7 @@
"# Delete GCS bucket.\n", "# Delete GCS bucket.\n",
"delete_bucket = False\n", "delete_bucket = False\n",
"if delete_bucket:\n", "if delete_bucket:\n",
" ! gsutil -m rm -r $BUCKET_URI\n", " ! gcloud storage rm --recursive $BUCKET_URI\n",
"\n", "\n",
"!rm -rf ../hello-custom-sample/" "!rm -rf ../hello-custom-sample/"
] ]
notebooks/official/tensorboard/tensorboard_profiler_custom_training_with_prebuilt_container.ipynb
+5 -5
View File
@@ -356,7 +356,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}" "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}"
] ]
}, },
{ {
@@ -696,11 +696,11 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"!cd {PYTHON_PACKAGE_APPLICATION_DIR} && python3 setup.py sdist --formats=gztar\n", "! cd {PYTHON_PACKAGE_APPLICATION_DIR} && python3 setup.py sdist --formats=gztar\n",
"\n", "\n",
"!gsutil cp {source_package_file_name} {python_package_gcs_uri}\n", "! gcloud storage cp {source_package_file_name} {python_package_gcs_uri}\n",
"\n", "\n",
"!gsutil ls -l {python_package_gcs_uri}" "! gcloud storage ls --long {python_package_gcs_uri}"
] ]
}, },
{ {
@@ -833,7 +833,7 @@
"# delete the bucket\n", "# delete the bucket\n",
"delete_bucket = False # set True for deletion\n", "delete_bucket = False # set True for deletion\n",
"if delete_bucket:\n", "if delete_bucket:\n",
" ! gsutil -m rm -r $BUCKET_URI" " ! gcloud storage rm --recursive $BUCKET_URI"
] ]
} }
], ],