diff --git a/.cloud-build/execute_notebook_helper.py b/.cloud-build/execute_notebook_helper.py index dfbc96184..bf87b237b 100644 --- a/.cloud-build/execute_notebook_helper.py +++ b/.cloud-build/execute_notebook_helper.py @@ -56,8 +56,8 @@ def execute_notebook( print("\n=== DOWNLOAD EXECUTED NOTEBOOK ===\n") print(f"Please debug the executed notebook by downloading the executed notebook:") - print("Option 1. Using gsutil. Run the following command in your terminal.") - print(f'\tgsutil cp "{output_file_or_uri}" .') + print("Option 1. Using gcloud storage. Run the following command in your terminal.") + print(f'\tgcloud storage cp "{output_file_or_uri}" .') print("Option 2. Using this link.") print(f"\thttps://storage.googleapis.com/{output_file_or_uri[5:]}") diff --git a/.cloud-build/utils/NotebookProcessors.py b/.cloud-build/utils/NotebookProcessors.py index 1f39fd361..7fe130fc2 100644 --- a/.cloud-build/utils/NotebookProcessors.py +++ b/.cloud-build/utils/NotebookProcessors.py @@ -108,7 +108,7 @@ class VertexAIInstallProprocessor(Preprocessor): if "google-cloud-aiplatform" not in content: return content return ( - f"gsutil cp {self.vertex_ai_wheel} google-cloud-aiplatform.whl\n" + + f"gcloud storage cp {self.vertex_ai_wheel} google-cloud-aiplatform.whl\n" + content.replace("google-cloud-aiplatform\n", "google-cloud-aiplatform.whl\n") .replace("google-cloud-aiplatform ", "google-cloud-aiplatform.whl ") ) diff --git a/.cloud-build/utils/util.py b/.cloud-build/utils/util.py index a6f8f1297..dc3d8664b 100644 --- a/.cloud-build/utils/util.py +++ b/.cloud-build/utils/util.py @@ -15,7 +15,7 @@ def download_file(bucket_name: str, blob_name: str, destination_file: str) -> st remote_file_path = "".join(["gs://", "/".join([bucket_name, blob_name])]) subprocess.check_output( - ["gsutil", "cp", remote_file_path, destination_file], encoding="UTF-8" + ["gcloud", "storage", "cp", remote_file_path, destination_file], encoding="UTF-8" ) return destination_file @@ -27,7 +27,7 @@ def upload_file( ) -> str: """Copies a local file to a GCS path""" subprocess.check_output( - ["gsutil", "cp", local_file_path, remote_file_path], encoding="UTF-8" + ["gcloud", "storage", "cp", local_file_path, remote_file_path], encoding="UTF-8" ) return remote_file_path diff --git a/community-content/pipeline_components/google-cloud/storage/download/component.yaml b/community-content/pipeline_components/google-cloud/storage/download/component.yaml index f1be5576b..94840de36 100644 --- a/community-content/pipeline_components/google-cloud/storage/download/component.yaml +++ b/community-content/pipeline_components/google-cloud/storage/download/component.yaml @@ -24,12 +24,12 @@ implementation: # Checking whether the URI points to a single blob, a directory or a URI pattern # URI points to a blob when that URI does not end with slash and listing that URI only yields the same URI - if [[ "$uri" != */ ]] && (gsutil ls "$uri" | grep --fixed-strings --line-regexp "$uri"); then + if [[ "$uri" != */ ]] && (gcloud storage ls "$uri" | grep --fixed-strings --line-regexp "$uri"); then mkdir -p "$(dirname "$output_path")" - gsutil -m cp -r "$uri" "$output_path" + gcloud storage cp --recursive "$uri" "$output_path" else mkdir -p "$output_path" # When source path is a directory, gsutil requires the destination to also be a directory - gsutil -m rsync -r "$uri" "$output_path" # gsutil cp has different path handling than Linux cp. It always puts the source directory (name) inside the destination directory. gsutil rsync does not have that problem. + gcloud storage rsync --recursive "$uri" "$output_path" # gsutil cp has different path handling than Linux cp. It always puts the source directory (name) inside the destination directory. gsutil rsync does not have that problem. fi - inputValue: GCS path - outputPath: Data diff --git a/community-content/pytorch_text_classification_using_vertex_sdk_and_gcloud/custom_container/scripts/train-cloud.sh b/community-content/pytorch_text_classification_using_vertex_sdk_and_gcloud/custom_container/scripts/train-cloud.sh index 795ebf246..9162fd20a 100755 --- a/community-content/pytorch_text_classification_using_vertex_sdk_and_gcloud/custom_container/scripts/train-cloud.sh +++ b/community-content/pytorch_text_classification_using_vertex_sdk_and_gcloud/custom_container/scripts/train-cloud.sh @@ -77,4 +77,4 @@ echo "After the job is completed successfully, model files will be saved at $JOB # # Verify the model was exported # echo "Verify the model was exported:" -# gsutil ls ${JOB_DIR}/ \ No newline at end of file +# gcloud storage ls ${JOB_DIR}/ diff --git a/community-content/pytorch_text_classification_using_vertex_sdk_and_gcloud/predictor/Dockerfile.serve b/community-content/pytorch_text_classification_using_vertex_sdk_and_gcloud/predictor/Dockerfile.serve index e10ec89a2..ac3584081 100644 --- a/community-content/pytorch_text_classification_using_vertex_sdk_and_gcloud/predictor/Dockerfile.serve +++ b/community-content/pytorch_text_classification_using_vertex_sdk_and_gcloud/predictor/Dockerfile.serve @@ -34,4 +34,4 @@ RUN echo "service_envelope=json\n" "inference_address=http://0.0.0.0:${AIP_H USER model-server # run Torchserve HTTP serve to respond to prediction requests -CMD ["echo", "AIP_STORAGE_URI=${AIP_STORAGE_URI}", ";", "gsutil", "cp", "-r", "${AIP_STORAGE_URI}/${MODEL_NAME}.mar", "/home/model-server/model-store/", ";", "ls", "-ltr", "/home/model-server/model-store/", ";", "torchserve", "--start", "--ts-config=/home/model-server/config.properties", "--models", "${MODEL_NAME}=${MODEL_NAME}.mar", "--model-store", "/home/model-server/model-store"] +CMD ["echo", "AIP_STORAGE_URI=${AIP_STORAGE_URI}", ";", "gcloud", "storage", "cp", "--recursive", "${AIP_STORAGE_URI}/${MODEL_NAME}.mar", "/home/model-server/model-store/", ";", "ls", "-ltr", "/home/model-server/model-store/", ";", "torchserve", "--start", "--ts-config=/home/model-server/config.properties", "--models", "${MODEL_NAME}=${MODEL_NAME}.mar", "--model-store", "/home/model-server/model-store"] diff --git a/community-content/pytorch_text_classification_using_vertex_sdk_and_gcloud/python_package/scripts/train-cloud.sh b/community-content/pytorch_text_classification_using_vertex_sdk_and_gcloud/python_package/scripts/train-cloud.sh index a0ef3c36c..edaa111a5 100755 --- a/community-content/pytorch_text_classification_using_vertex_sdk_and_gcloud/python_package/scripts/train-cloud.sh +++ b/community-content/pytorch_text_classification_using_vertex_sdk_and_gcloud/python_package/scripts/train-cloud.sh @@ -67,4 +67,4 @@ echo "After the job is completed successfully, model files will be saved at $JOB # # Verify the model was exported # echo "Verify the model was exported:" -# gsutil ls ${JOB_DIR}/ +# gcloud storage ls ${JOB_DIR}/ diff --git a/community-content/vertex_model_garden/model_oss/cloudnerf/render.sh b/community-content/vertex_model_garden/model_oss/cloudnerf/render.sh index 3d7ebf338..50d296ce1 100644 --- a/community-content/vertex_model_garden/model_oss/cloudnerf/render.sh +++ b/community-content/vertex_model_garden/model_oss/cloudnerf/render.sh @@ -99,14 +99,14 @@ create_dir_if_not_exists "$CHECKPOINTS_PATH" touch "$local_experiment_path/$exp_folder_name/log_render.txt" # Copy experiment from GCS bucket to local -gsutil -m cp -r "${args[-gcs_experiment_path]}/data" "$local_experiment_path/$exp_folder_name" || exit 1 -gsutil -m cp -r "${args[-gcs_experiment_path]}/checkpoints/${training_job_name}/*" "$CHECKPOINTS_PATH" || exit 1 +gcloud storage cp --recursive "${args[-gcs_experiment_path]}/data" "$local_experiment_path/$exp_folder_name" || exit 1 +gcloud storage cp --recursive "${args[-gcs_experiment_path]}/checkpoints/${training_job_name}/*" "$CHECKPOINTS_PATH" || exit 1 # Check and copy keyframes file. if [[ -n ${args[-gcs_keyframes_file]} ]]; then keyframes_file_basename=$(basename "${args[-gcs_keyframes_file]}") local_keyframes_file="$local_dataset_path/$keyframes_file_basename" - gsutil cp "${args[-gcs_keyframes_file]}" "$local_keyframes_file" || exit 1 + gcloud storage cp "${args[-gcs_keyframes_file]}" "$local_keyframes_file" || exit 1 echo "Local keyframe file: $local_keyframes_file" launch_rendering "$local_keyframes_file" else @@ -114,4 +114,4 @@ else fi # Copy rendered data back to GCS. -gsutil -m cp -r "$OUTPUT_RENDER_PATH" "${args[-gcs_experiment_path]}/render/${rendering_job_name}" \ No newline at end of file +gcloud storage cp --recursive "$OUTPUT_RENDER_PATH" "${args[-gcs_experiment_path]}/render/${rendering_job_name}" \ No newline at end of file diff --git a/community-content/vertex_model_garden/model_oss/cloudnerf/train.sh b/community-content/vertex_model_garden/model_oss/cloudnerf/train.sh index 7165594f6..c34f49403 100644 --- a/community-content/vertex_model_garden/model_oss/cloudnerf/train.sh +++ b/community-content/vertex_model_garden/model_oss/cloudnerf/train.sh @@ -74,7 +74,7 @@ create_dir_if_not_exists "$local_experiment_path" create_dir_if_not_exists "$local_experiment_path/$scene_folder_name" # Copy experiment from GCS bucket to local. -gsutil -m cp -r "${gcs_experiment_path}/data" "$local_experiment_path/$scene_folder_name" || exit 1 +gcloud storage cp --recursive "${gcs_experiment_path}/data" "$local_experiment_path/$scene_folder_name" || exit 1 echo "GCS Experiment: $gcs_experiment_path" echo "Gin Config File: $gin_config_file" @@ -89,6 +89,6 @@ accelerate launch train.py --gin_configs="$gin_config_file" \ --gin_bindings="Config.factor = ${factor}" \ --gin_bindings="Config.max_steps = ${max_training_steps}" -gsutil -m rm -r "${gcs_experiment_path}/checkpoints/${training_job_name}" -gsutil -m cp -r "$local_experiment_path/$scene_folder_name/config.gin" "${gcs_experiment_path}/${training_job_name}_config.gin" -gsutil -m cp -r "$local_experiment_path/$scene_folder_name/checkpoints/*/*" "${gcs_experiment_path}/checkpoints/${training_job_name}" \ No newline at end of file +gcloud storage rm --recursive "${gcs_experiment_path}/checkpoints/${training_job_name}" +gcloud storage cp --recursive "$local_experiment_path/$scene_folder_name/config.gin" "${gcs_experiment_path}/${training_job_name}_config.gin" +gcloud storage cp --recursive "$local_experiment_path/$scene_folder_name/checkpoints/*/*" "${gcs_experiment_path}/checkpoints/${training_job_name}" \ No newline at end of file diff --git a/notebooks/community/ml_ops/setup.py b/notebooks/community/ml_ops/setup.py index c9efef720..96205cebd 100644 --- a/notebooks/community/ml_ops/setup.py +++ b/notebooks/community/ml_ops/setup.py @@ -89,7 +89,7 @@ print("UUID", UUID) if args.bucket_required: BUCKET_NAME = PROJECT_ID + "aip-" + UUID BUCKET_URI = f"gs://{BUCKET_NAME}" - os.system(f"gsutil mb -l {REGION} {BUCKET_URI}") + os.system(f"gcloud storage buckets create --location={REGION} {BUCKET_URI}") print("BUCKET_URI", BUCKET_URI)