xiaohei/model_garden
1
0
Fork 0
mirror of https://github.com/GoogleCloudPlatform/vertex-ai-samples.git synced 2026-09-26 14:42:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Migrate gsutil usage to gcloud storage (#4296)

Browse Source 
* Migrate gsutil usage to gcloud storage

* updates

* update

* Update llm_streaming_prediction.ipynb

* revert change

* removed model_garden folder changes

* removed model_garden folder changes

---------

Co-authored-by: gurusai-voleti <gvoleti@google.com>
This commit is contained in:
Margubur Rahman
2025-12-19 11:57:20 -05:00
committed by GitHub
co-authored by gurusai-voleti
parent ca7d4e153e
commit 27ebf52198
5 changed files with 16 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.cloud-build/execute_changed_notebooks_helper.py
+1 -1
View File
@@ -365,7 +365,7 @@ def process_and_execute_notebook(
# Use gcloud to get tail # Use gcloud to get tail
try: try:
result.error_message = subprocess.check_output( result.error_message = subprocess.check_output(
["gsutil", "cat", "-r", "-1000", log_file_uri], encoding="UTF-8" ["gcloud", "storage", "cat", "--range", "-1000", log_file_uri], encoding="UTF-8"
) )
except Exception as error: except Exception as error:
result.error_message = str(error) result.error_message = str(error)
notebooks/official/generative_ai/distillation.ipynb
+4 -4
View File
@@ -323,7 +323,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}" "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}"
] ]
}, },
{ {
@@ -347,8 +347,8 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil cp gs://cloud-samples-data/vertex-ai/model-evaluation/peft_eval_sample.jsonl {BUCKET_URI}/peft_eval_sample.jsonl\n", "! gcloud storage cp gs://cloud-samples-data/vertex-ai/model-evaluation/peft_eval_sample.jsonl {BUCKET_URI}/peft_eval_sample.jsonl\n",
"! gsutil cp gs://cloud-samples-data/vertex-ai/model-evaluation/peft_train_sample.jsonl {BUCKET_URI}/peft_train_sample.jsonl" "! gcloud storage cp gs://cloud-samples-data/vertex-ai/model-evaluation/peft_train_sample.jsonl {BUCKET_URI}/peft_train_sample.jsonl"
] ]
}, },
{ {
@@ -554,7 +554,7 @@
"# Delete the Cloud Storage bucket\n", "# Delete the Cloud Storage bucket\n",
"delete_bucket = True\n", "delete_bucket = True\n",
"if delete_bucket:\n", "if delete_bucket:\n",
" ! gsutil rm -rf {BUCKET_URI}" " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}"
] ]
} }
], ],
notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb
+3 -3
View File
@@ -274,7 +274,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}" "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}"
] ]
}, },
{ {
@@ -390,7 +390,7 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"!wget https://storage.googleapis.com/download.tensorflow.org/data/abalone_train.csv\n", "!wget https://storage.googleapis.com/download.tensorflow.org/data/abalone_train.csv\n",
"!gsutil cp abalone_train.csv {BUCKET_URI}/data/\n", "!gcloud storage cp abalone_train.csv {BUCKET_URI}/data/\n",
"\n", "\n",
"gcs_csv_path = f\"{BUCKET_URI}/data/abalone_train.csv\"" "gcs_csv_path = f\"{BUCKET_URI}/data/abalone_train.csv\""
] ]
@@ -784,7 +784,7 @@
"\n", "\n",
"\n", "\n",
"if delete_bucket:\n", "if delete_bucket:\n",
" ! gsutil -m rm -r $BUCKET_URI" " ! gcloud storage rm --recursive $BUCKET_URI"
] ]
} }
], ],
notebooks/official/prediction/get_started_with_psc_private_endpoint.ipynb
+4 -4
View File
@@ -201,7 +201,7 @@
"source": [ "source": [
"# Create GCS Bucket\n", "# Create GCS Bucket\n",
"BUCKET_URI = \"gs://your-bucket-name-unique\" # @param {type:\"string\"}\n", "BUCKET_URI = \"gs://your-bucket-name-unique\" # @param {type:\"string\"}\n",
"! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}" "! gcloud storage buckets create --location {LOCATION} --project {PROJECT_ID} {BUCKET_URI}"
] ]
}, },
{ {
@@ -237,7 +237,7 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"# Copy Models to the Bucket\n", "# Copy Models to the Bucket\n",
"! gsutil cp -r \"gs://cloud-samples-data/vertex-ai/prediction/test-models-requests/*\" {BUCKET_URI}" "! gcloud storage cp --recursive \"gs://cloud-samples-data/vertex-ai/prediction/test-models-requests/*\" {BUCKET_URI}"
] ]
}, },
{ {
@@ -508,7 +508,7 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"# Download the requests files:\n", "# Download the requests files:\n",
"! gsutil cp {BUCKET_URI}/requests/* ./" "! gcloud storage cp {BUCKET_URI}/requests/* ./"
] ]
}, },
{ {
@@ -716,7 +716,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil rm -r {BUCKET_URI}" "! gcloud storage rm --recursive {BUCKET_URI}"
] ]
}, },
{ {
notebooks/official/prediction/llm_streaming_prediction.ipynb
+4 -4
View File
@@ -286,7 +286,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}" "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}"
] ]
}, },
{ {
@@ -500,7 +500,7 @@
"source": [ "source": [
"dataset = \"gs://cloud-samples-data/vertex-ai/prediction/llm/test_table.jsonl\"\n", "dataset = \"gs://cloud-samples-data/vertex-ai/prediction/llm/test_table.jsonl\"\n",
"destination_uri_prefix = f\"{BUCKET_URI}/text-bison@001_\"\n", "destination_uri_prefix = f\"{BUCKET_URI}/text-bison@001_\"\n",
"! gsutil cp -r gs://cloud-samples-data/vertex-ai/prediction/llm/text-bison@001_/ {destination_uri_prefix}\n", "! gcloud storage cp --recursive gs://cloud-samples-data/vertex-ai/prediction/llm/text-bison@001_/ {destination_uri_prefix}\n",
"\n", "\n",
"\n", "\n",
"from vertexai.language_models import TextGenerationModel\n", "from vertexai.language_models import TextGenerationModel\n",
@@ -580,7 +580,7 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"# Tuning model\n", "# Tuning model\n",
"! gsutil cp gs://cloud-samples-data/vertex-ai/prediction/llm/q_a_train_with_context.jsonl {BUCKET_URI}/q_a_train_with_context.jsonl\n", "! gcloud storage cp gs://cloud-samples-data/vertex-ai/prediction/llm/q_a_train_with_context.jsonl {BUCKET_URI}/q_a_train_with_context.jsonl\n",
"\n", "\n",
"tuning_job = model3.tune_model(\n", "tuning_job = model3.tune_model(\n",
" training_data=f\"{BUCKET_URI}/q_a_train_with_context.jsonl\",\n", " training_data=f\"{BUCKET_URI}/q_a_train_with_context.jsonl\",\n",
@@ -740,7 +740,7 @@
"batch_job_2.delete()\n", "batch_job_2.delete()\n",
"\n", "\n",
"if delete_bucket:\n", "if delete_bucket:\n",
" ! gsutil -m rm -r $BUCKET_URI" " ! gcloud storage rm --recursive $BUCKET_URI"
] ]
} }
], ],