diff --git a/community-content/pytorch_image_classification_single_gpu_with_vertex_sdk_and_torchserve/vertex_training_with_custom_container.ipynb b/community-content/pytorch_image_classification_single_gpu_with_vertex_sdk_and_torchserve/vertex_training_with_custom_container.ipynb index e78c2f03c..117504ee8 100644 --- a/community-content/pytorch_image_classification_single_gpu_with_vertex_sdk_and_torchserve/vertex_training_with_custom_container.ipynb +++ b/community-content/pytorch_image_classification_single_gpu_with_vertex_sdk_and_torchserve/vertex_training_with_custom_container.ipynb @@ -412,7 +412,7 @@ }, "outputs": [], "source": [ - "! gsutil ls $gcs_output_uri_prefix" + "! gcloud storage ls $gcs_output_uri_prefix" ] } ], diff --git a/community-content/tf_keras_image_classification_distributed_multi_worker_with_vertex_sdk/multi_worker_vertex_training_on_cpu_with_custom_container.ipynb b/community-content/tf_keras_image_classification_distributed_multi_worker_with_vertex_sdk/multi_worker_vertex_training_on_cpu_with_custom_container.ipynb index 564c8fd77..19eaed8c5 100644 --- a/community-content/tf_keras_image_classification_distributed_multi_worker_with_vertex_sdk/multi_worker_vertex_training_on_cpu_with_custom_container.ipynb +++ b/community-content/tf_keras_image_classification_distributed_multi_worker_with_vertex_sdk/multi_worker_vertex_training_on_cpu_with_custom_container.ipynb @@ -324,7 +324,7 @@ }, "outputs": [], "source": [ - "! gsutil ls $gcs_output_uri_prefix" + "! gcloud storage ls $gcs_output_uri_prefix" ] }, { @@ -344,7 +344,7 @@ }, "outputs": [], "source": [ - "! gsutil rm -rf $gcs_output_uri_prefix" + "! gcloud storage rm --recursive --continue-on-error $gcs_output_uri_prefix" ] } ], diff --git a/community-content/tf_keras_image_classification_distributed_multi_worker_with_vertex_sdk/multi_worker_vertex_training_on_gpu_with_custom_container.ipynb b/community-content/tf_keras_image_classification_distributed_multi_worker_with_vertex_sdk/multi_worker_vertex_training_on_gpu_with_custom_container.ipynb index 966fcb083..b1b514686 100644 --- a/community-content/tf_keras_image_classification_distributed_multi_worker_with_vertex_sdk/multi_worker_vertex_training_on_gpu_with_custom_container.ipynb +++ b/community-content/tf_keras_image_classification_distributed_multi_worker_with_vertex_sdk/multi_worker_vertex_training_on_gpu_with_custom_container.ipynb @@ -328,7 +328,7 @@ }, "outputs": [], "source": [ - "! gsutil ls $gcs_output_uri_prefix" + "! gcloud storage ls $gcs_output_uri_prefix" ] }, { @@ -348,7 +348,7 @@ }, "outputs": [], "source": [ - "! gsutil rm -rf $gcs_output_uri_prefix" + "! gcloud storage rm --recursive --continue-on-error $gcs_output_uri_prefix" ] } ], diff --git a/community-content/tf_keras_text_classification_distributed_single_worker_gpus_with_gcloud_local_run_and_vertex_sdk/vertex_training_with_local_mode_container.ipynb b/community-content/tf_keras_text_classification_distributed_single_worker_gpus_with_gcloud_local_run_and_vertex_sdk/vertex_training_with_local_mode_container.ipynb index f6bf594b2..adf4fd769 100644 --- a/community-content/tf_keras_text_classification_distributed_single_worker_gpus_with_gcloud_local_run_and_vertex_sdk/vertex_training_with_local_mode_container.ipynb +++ b/community-content/tf_keras_text_classification_distributed_single_worker_gpus_with_gcloud_local_run_and_vertex_sdk/vertex_training_with_local_mode_container.ipynb @@ -341,7 +341,7 @@ }, "outputs": [], "source": [ - "! gsutil ls $gcs_output_uri_prefix" + "! gcloud storage ls $gcs_output_uri_prefix" ] }, { @@ -361,7 +361,7 @@ }, "outputs": [], "source": [ - "! gsutil rm -rf $gcs_output_uri_prefix" + "! gcloud storage rm --recursive --continue-on-error $gcs_output_uri_prefix" ] } ], diff --git a/community-content/vertex_model_garden/model_oss/cloudnerf/local_colmap_and_resize.sh b/community-content/vertex_model_garden/model_oss/cloudnerf/local_colmap_and_resize.sh index de01ce5cd..17638fe7a 100644 --- a/community-content/vertex_model_garden/model_oss/cloudnerf/local_colmap_and_resize.sh +++ b/community-content/vertex_model_garden/model_oss/cloudnerf/local_colmap_and_resize.sh @@ -66,7 +66,7 @@ mkdir -p "$local_folder" mkdir -p "$output_folder" # Download the content from the GCS URI -gsutil -m cp -r "$gcs_dataset_path"/* "$local_folder/" +gcloud storage cp --recursive "$gcs_dataset_path"/* "$local_folder/" # Process files in the local folder for file in "$local_folder"/*; do @@ -122,23 +122,23 @@ cp -r "$output_folder" "$images_folder"/images_2 pushd "$images_folder"/images_2 ls | xargs -P 8 -I {} mogrify -resize 50% {} popd -gsutil -m cp -r "$images_folder"/images_2/* "$gcs_experiment_path"/data/images_2 +gcloud storage cp --recursive "$images_folder"/images_2/* "$gcs_experiment_path"/data/images_2 cp -r "$output_folder" "$images_folder"/images_4 pushd "$images_folder"/images_4 ls | xargs -P 8 -I {} mogrify -resize 25% {} popd -gsutil -m cp -r "$images_folder"/images_4/* "$gcs_experiment_path"/data/images_4 +gcloud storage cp --recursive "$images_folder"/images_4/* "$gcs_experiment_path"/data/images_4 cp -r "$output_folder" "$images_folder"/images_8 pushd "$images_folder"/images_8 ls | xargs -P 8 -I {} mogrify -resize 12.5% {} popd -gsutil -m cp "$images_folder"/images_8/* "$gcs_experiment_path"/data/images_8 +gcloud storage cp "$images_folder"/images_8/* "$gcs_experiment_path"/data/images_8 # Copy images and sparse reconstruction files to gcs experiment folder. -gsutil -m cp "$images_folder"/images/* "$gcs_experiment_path"/data/images -gsutil -m cp -r "$local_folder"/sparse "$gcs_experiment_path"/data -gsutil -m cp "$local_folder"/database.db "$gcs_experiment_path"/data +gcloud storage cp "$images_folder"/images/* "$gcs_experiment_path"/data/images +gcloud storage cp --recursive "$local_folder"/sparse "$gcs_experiment_path"/data +gcloud storage cp "$local_folder"/database.db "$gcs_experiment_path"/data echo "Processing complete." \ No newline at end of file diff --git a/notebooks/community/model_garden/docker_source_codes/model_oss/notebook_util/gcp_utils.py b/notebooks/community/model_garden/docker_source_codes/model_oss/notebook_util/gcp_utils.py index 34345df99..54810a8e5 100644 --- a/notebooks/community/model_garden/docker_source_codes/model_oss/notebook_util/gcp_utils.py +++ b/notebooks/community/model_garden/docker_source_codes/model_oss/notebook_util/gcp_utils.py @@ -117,24 +117,24 @@ def setup_buckets(bucket_uri: str, model_bucket_name: str) -> tuple[str, str]: # Check if bucket exists try: - run_command(["gsutil", "ls", "-b", bucket_uri]) + run_command(["gcloud", "storage", "ls", "--buckets", bucket_uri]) logger.info("Bucket %s already exists.", bucket_uri) except subprocess.CalledProcessError: logger.info("Creating bucket %s.", bucket_uri) # Create the bucket in the same region as the project - run_command(["gsutil", "mb", "-l", REGION, bucket_uri]) + run_command(["gcloud", "storage", "buckets", "create", "--location", REGION, bucket_uri]) # Construct the model bucket path model_bucket = os.path.join(bucket_uri, model_bucket_name) # Check if the model bucket exists (as a folder within the main bucket) try: - run_command(["gsutil", "ls", model_bucket]) + run_command(["gcloud", "storage", "ls", model_bucket]) logger.info("Model bucket %s already exists.", model_bucket) except subprocess.CalledProcessError: logger.info("Creating model bucket %s.", model_bucket) # Create the model bucket folder - run_command(["gsutil", "cp", "/dev/null", model_bucket + "/"]) + run_command(["gcloud", "storage", "cp", "/dev/null", model_bucket + "/"]) return bucket_name, model_bucket @@ -174,11 +174,13 @@ def provision_permissions(service_account: str, bucket_name: str) -> None: """Provision permissions to the service account with the GCS bucket.""" if bucket_name: run_command([ - "gsutil", - "iam", - "ch", - f"serviceAccount:{service_account}:roles/storage.admin", + "gcloud", + "storage", + "buckets", + "add-iam-policy-binding", bucket_name, + f"--member=serviceAccount:{service_account}", + "--role=roles/storage.admin", ]) @@ -263,13 +265,13 @@ def delete_endpoint_and_model( def delete_gcs_bucket(bucket_name: str) -> str: - """Deletes a GCS bucket using gsutil.""" + """Deletes a GCS bucket using gcloud storage.""" try: - run_command(["gsutil", "-m", "rm", "-r", bucket_name]) - logger.info("Bucket %s deleted using gsutil.", bucket_name) + run_command(["gcloud", "storage", "rm", "--recursive", bucket_name]) + logger.info("Bucket %s deleted using gcloud storage.", bucket_name) return f"Bucket {bucket_name} deleted successfully!" except subprocess.CalledProcessError as e: logger.error( - "Error deleting bucket %s using gsutil: %s", bucket_name, str(e) + "Error deleting bucket %s using gcloud storage: %s", bucket_name, str(e) ) return f"Bucket {bucket_name} could not be found or deleted. " diff --git a/notebooks/community/model_garden/docker_source_codes/notebook_util/gcp_utils.py b/notebooks/community/model_garden/docker_source_codes/notebook_util/gcp_utils.py index 34345df99..54810a8e5 100644 --- a/notebooks/community/model_garden/docker_source_codes/notebook_util/gcp_utils.py +++ b/notebooks/community/model_garden/docker_source_codes/notebook_util/gcp_utils.py @@ -117,24 +117,24 @@ def setup_buckets(bucket_uri: str, model_bucket_name: str) -> tuple[str, str]: # Check if bucket exists try: - run_command(["gsutil", "ls", "-b", bucket_uri]) + run_command(["gcloud", "storage", "ls", "--buckets", bucket_uri]) logger.info("Bucket %s already exists.", bucket_uri) except subprocess.CalledProcessError: logger.info("Creating bucket %s.", bucket_uri) # Create the bucket in the same region as the project - run_command(["gsutil", "mb", "-l", REGION, bucket_uri]) + run_command(["gcloud", "storage", "buckets", "create", "--location", REGION, bucket_uri]) # Construct the model bucket path model_bucket = os.path.join(bucket_uri, model_bucket_name) # Check if the model bucket exists (as a folder within the main bucket) try: - run_command(["gsutil", "ls", model_bucket]) + run_command(["gcloud", "storage", "ls", model_bucket]) logger.info("Model bucket %s already exists.", model_bucket) except subprocess.CalledProcessError: logger.info("Creating model bucket %s.", model_bucket) # Create the model bucket folder - run_command(["gsutil", "cp", "/dev/null", model_bucket + "/"]) + run_command(["gcloud", "storage", "cp", "/dev/null", model_bucket + "/"]) return bucket_name, model_bucket @@ -174,11 +174,13 @@ def provision_permissions(service_account: str, bucket_name: str) -> None: """Provision permissions to the service account with the GCS bucket.""" if bucket_name: run_command([ - "gsutil", - "iam", - "ch", - f"serviceAccount:{service_account}:roles/storage.admin", + "gcloud", + "storage", + "buckets", + "add-iam-policy-binding", bucket_name, + f"--member=serviceAccount:{service_account}", + "--role=roles/storage.admin", ]) @@ -263,13 +265,13 @@ def delete_endpoint_and_model( def delete_gcs_bucket(bucket_name: str) -> str: - """Deletes a GCS bucket using gsutil.""" + """Deletes a GCS bucket using gcloud storage.""" try: - run_command(["gsutil", "-m", "rm", "-r", bucket_name]) - logger.info("Bucket %s deleted using gsutil.", bucket_name) + run_command(["gcloud", "storage", "rm", "--recursive", bucket_name]) + logger.info("Bucket %s deleted using gcloud storage.", bucket_name) return f"Bucket {bucket_name} deleted successfully!" except subprocess.CalledProcessError as e: logger.error( - "Error deleting bucket %s using gsutil: %s", bucket_name, str(e) + "Error deleting bucket %s using gcloud storage: %s", bucket_name, str(e) ) return f"Bucket {bucket_name} could not be found or deleted. " diff --git a/notebooks/community/model_garden/model_garden_pytorch_text_to_video_zero_shot.ipynb b/notebooks/community/model_garden/model_garden_pytorch_text_to_video_zero_shot.ipynb index 0017e8e2c..03861cb54 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_text_to_video_zero_shot.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_text_to_video_zero_shot.ipynb @@ -124,10 +124,10 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gsutil mb -l {REGION} {BUCKET_URI}\n", + " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", + " shell_output = ! gcloud storage buckets describe {BUCKET_NAME} --format=\"value(location)\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", @@ -146,7 +146,7 @@ "print(\"Using this default Service Account:\", SERVICE_ACCOUNT)\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=BUCKET_URI)\n", "\n", @@ -271,7 +271,7 @@ "# Delete bucket.\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gsutil -m rm -r $BUCKET_NAME" + " ! gcloud storage rm --recursive $BUCKET_NAME" ] } ], diff --git a/notebooks/community/sdk/SDK_End_to_End_Tabular_Custom_Training.ipynb b/notebooks/community/sdk/SDK_End_to_End_Tabular_Custom_Training.ipynb index a9456d10d..cf09081f9 100644 --- a/notebooks/community/sdk/SDK_End_to_End_Tabular_Custom_Training.ipynb +++ b/notebooks/community/sdk/SDK_End_to_End_Tabular_Custom_Training.ipynb @@ -133,7 +133,7 @@ "outputs": [], "source": [ "!wget https://storage.googleapis.com/download.tensorflow.org/data/abalone_train.csv\n", - "!gsutil cp abalone_train.csv {MY_STAGING_BUCKET}/data/\n", + "!gcloud storage cp abalone_train.csv {MY_STAGING_BUCKET}/data/\n", "\n", "gcs_csv_path = f\"{MY_STAGING_BUCKET}/data/abalone_train.csv\"" ] @@ -322,7 +322,7 @@ "metadata": { "colab": { "collapsed_sections": [], - "name": "AI_Platform_(Unified)_SDK_End_to_End_Tabular_Custom_Training.ipynb", + "name": "SDK_End_to_End_Tabular_Custom_Training.ipynb", "toc_visible": true }, "kernelspec": { diff --git a/notebooks/community/sdk/SDK_Tabular_Custom_Model_Training_asynchronous.ipynb b/notebooks/community/sdk/SDK_Tabular_Custom_Model_Training_asynchronous.ipynb index e744054a4..24eb04cdc 100644 --- a/notebooks/community/sdk/SDK_Tabular_Custom_Model_Training_asynchronous.ipynb +++ b/notebooks/community/sdk/SDK_Tabular_Custom_Model_Training_asynchronous.ipynb @@ -133,7 +133,7 @@ "outputs": [], "source": [ "!wget https://storage.googleapis.com/download.tensorflow.org/data/abalone_train.csv\n", - "!gsutil cp abalone_train.csv {MY_STAGING_BUCKET}/data/\n", + "!gcloud storage cp abalone_train.csv {MY_STAGING_BUCKET}/data/\n", "\n", "gcs_csv_path = f\"{MY_STAGING_BUCKET}/data/abalone_train.csv\"" ] @@ -344,7 +344,7 @@ "metadata": { "colab": { "collapsed_sections": [], - "name": "AI_Platform_(Unified)_SDK_Tabular_Custom_Model_Training_(asynchronous).ipynb", + "name": "SDK_Tabular_Custom_Model_Training_asynchronous.ipynb", "toc_visible": true }, "kernelspec": {