xiaohei/model_garden
1
0
Fork 0
mirror of https://github.com/GoogleCloudPlatform/vertex-ai-samples.git synced 2026-09-26 14:42:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Migrate gsutil usage to gcloud storage (#4293)

Browse Source 
* Migrate gsutil usage to gcloud storage

* update

---------

Co-authored-by: gurusai-voleti <gvoleti@google.com>
This commit is contained in:
Margubur Rahman
2025-12-18 13:58:03 +00:00
committed by GitHub
co-authored by gurusai-voleti
parent 8471b5cb6f
commit 0edae683e7
10 changed files with 50 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
community-content/pytorch_image_classification_single_gpu_with_vertex_sdk_and_torchserve/vertex_training_with_custom_container.ipynb
+1 -1
View File
@@ -412,7 +412,7 @@
},
"outputs": [],
"source": [
"! gsutil ls $gcs_output_uri_prefix"
"! gcloud storage ls $gcs_output_uri_prefix"
]
}
],
community-content/tf_keras_image_classification_distributed_multi_worker_with_vertex_sdk/multi_worker_vertex_training_on_cpu_with_custom_container.ipynb
+2 -2
View File
@@ -324,7 +324,7 @@
},
"outputs": [],
"source": [
"! gsutil ls $gcs_output_uri_prefix"
"! gcloud storage ls $gcs_output_uri_prefix"
]
},
{
@@ -344,7 +344,7 @@
},
"outputs": [],
"source": [
"! gsutil rm -rf $gcs_output_uri_prefix"
"! gcloud storage rm --recursive --continue-on-error $gcs_output_uri_prefix"
]
}
],
community-content/tf_keras_image_classification_distributed_multi_worker_with_vertex_sdk/multi_worker_vertex_training_on_gpu_with_custom_container.ipynb
+2 -2
View File
@@ -328,7 +328,7 @@
},
"outputs": [],
"source": [
"! gsutil ls $gcs_output_uri_prefix"
"! gcloud storage ls $gcs_output_uri_prefix"
]
},
{
@@ -348,7 +348,7 @@
},
"outputs": [],
"source": [
"! gsutil rm -rf $gcs_output_uri_prefix"
"! gcloud storage rm --recursive --continue-on-error $gcs_output_uri_prefix"
]
}
],
community-content/tf_keras_text_classification_distributed_single_worker_gpus_with_gcloud_local_run_and_vertex_sdk/vertex_training_with_local_mode_container.ipynb
+2 -2
View File
@@ -341,7 +341,7 @@
},
"outputs": [],
"source": [
"! gsutil ls $gcs_output_uri_prefix"
"! gcloud storage ls $gcs_output_uri_prefix"
]
},
{
@@ -361,7 +361,7 @@
},
"outputs": [],
"source": [
"! gsutil rm -rf $gcs_output_uri_prefix"
"! gcloud storage rm --recursive --continue-on-error $gcs_output_uri_prefix"
]
}
],
community-content/vertex_model_garden/model_oss/cloudnerf/local_colmap_and_resize.sh
+7 -7
View File
@@ -66,7 +66,7 @@ mkdir -p "$local_folder"
mkdir -p "$output_folder"
# Download the content from the GCS URI
gsutil -m cp -r "$gcs_dataset_path"/* "$local_folder/"
gcloud storage cp --recursive "$gcs_dataset_path"/* "$local_folder/"
# Process files in the local folder
for file in "$local_folder"/*; do
@@ -122,23 +122,23 @@ cp -r "$output_folder" "$images_folder"/images_2
pushd "$images_folder"/images_2
ls | xargs -P 8 -I {} mogrify -resize 50% {}
popd
gsutil -m cp -r "$images_folder"/images_2/* "$gcs_experiment_path"/data/images_2
gcloud storage cp --recursive "$images_folder"/images_2/* "$gcs_experiment_path"/data/images_2
cp -r "$output_folder" "$images_folder"/images_4
pushd "$images_folder"/images_4
ls | xargs -P 8 -I {} mogrify -resize 25% {}
popd
gsutil -m cp -r "$images_folder"/images_4/* "$gcs_experiment_path"/data/images_4
gcloud storage cp --recursive "$images_folder"/images_4/* "$gcs_experiment_path"/data/images_4
cp -r "$output_folder" "$images_folder"/images_8
pushd "$images_folder"/images_8
ls | xargs -P 8 -I {} mogrify -resize 12.5% {}
popd
gsutil -m cp "$images_folder"/images_8/* "$gcs_experiment_path"/data/images_8
gcloud storage cp "$images_folder"/images_8/* "$gcs_experiment_path"/data/images_8
# Copy images and sparse reconstruction files to gcs experiment folder.
gsutil -m cp "$images_folder"/images/* "$gcs_experiment_path"/data/images
gsutil -m cp -r "$local_folder"/sparse "$gcs_experiment_path"/data
gsutil -m cp "$local_folder"/database.db "$gcs_experiment_path"/data
gcloud storage cp "$images_folder"/images/* "$gcs_experiment_path"/data/images
gcloud storage cp --recursive "$local_folder"/sparse "$gcs_experiment_path"/data
gcloud storage cp "$local_folder"/database.db "$gcs_experiment_path"/data
echo "Processing complete."