diff --git a/notebooks/community/ml_ops/stage2/get_started_vertex_training_r.ipynb b/notebooks/community/ml_ops/stage2/get_started_vertex_training_r.ipynb index bb241cf14..be45a5d59 100644 --- a/notebooks/community/ml_ops/stage2/get_started_vertex_training_r.ipynb +++ b/notebooks/community/ml_ops/stage2/get_started_vertex_training_r.ipynb @@ -445,7 +445,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_URI" + "! gcloud storage buckets create --location=$REGION $BUCKET_URI" ] }, { @@ -465,7 +465,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_URI" + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { @@ -1427,7 +1427,7 @@ "delete_bucket = False\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gsutil rm -r $BUCKET_URI" + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], diff --git a/notebooks/community/ml_ops/stage3/get_started_with_automl_tabular_pipeline_workflow.ipynb b/notebooks/community/ml_ops/stage3/get_started_with_automl_tabular_pipeline_workflow.ipynb index 8fd1e29d8..796c17a66 100644 --- a/notebooks/community/ml_ops/stage3/get_started_with_automl_tabular_pipeline_workflow.ipynb +++ b/notebooks/community/ml_ops/stage3/get_started_with_automl_tabular_pipeline_workflow.ipynb @@ -450,7 +450,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_URI" + "! gcloud storage buckets create --location=$REGION $BUCKET_URI" ] }, { @@ -470,7 +470,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_URI" + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { @@ -496,7 +496,7 @@ "source": [ "# set GCS bucket object TTL to 7 days\n", "! echo '{\"rule\":[{\"action\": {\"type\": \"Delete\"},\"condition\": {\"age\": 7}}]}' > gcs_lifecycle.tmp\n", - "! gsutil lifecycle set gcs_lifecycle.tmp {BUCKET_URI}\n", + "! gcloud storage buckets update --lifecycle-file=gcs_lifecycle.tmp {BUCKET_URI}\n", "! rm gcs_lifecycle.tmp" ] }, @@ -1047,7 +1047,7 @@ "\n", "print(\"trained model without custom TF ops:\", model_artifact_no_custom_op)\n", "\n", - "! gsutil ls {model_artifact_no_custom_op}\n", + "! gcloud storage ls {model_artifact_no_custom_op}\n", "\n", "model_evaluation = get_evaluation_metrics(pipeline_task_details)\n", "print(\"Model evaluation artifacts:\", model_evaluation)" @@ -1337,7 +1337,7 @@ "\n", "delete_bucket = False\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gsutil rm -rf $BUCKET_URI" + " ! gcloud storage rm --recursive --continue-on-error $BUCKET_URI" ] } ], diff --git a/notebooks/community/ml_ops/stage3/get_started_with_machine_management.ipynb b/notebooks/community/ml_ops/stage3/get_started_with_machine_management.ipynb index 872db849e..4861f834f 100644 --- a/notebooks/community/ml_ops/stage3/get_started_with_machine_management.ipynb +++ b/notebooks/community/ml_ops/stage3/get_started_with_machine_management.ipynb @@ -450,7 +450,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_URI" + "! gcloud storage buckets create --location=$REGION $BUCKET_URI" ] }, { @@ -470,7 +470,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_URI" + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { @@ -541,9 +541,9 @@ }, "outputs": [], "source": [ - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI" + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] }, { @@ -1015,13 +1015,13 @@ " + \"/evaluation_metrics\"\n", " )\n", " if tf.io.gfile.exists(EXECUTE_OUTPUT):\n", - " ! gsutil cat $EXECUTE_OUTPUT\n", + " ! gcloud storage cat $EXECUTE_OUTPUT\n", " return EXECUTE_OUTPUT\n", " elif tf.io.gfile.exists(GCP_RESOURCES):\n", - " ! gsutil cat $GCP_RESOURCES\n", + " ! gcloud storage cat $GCP_RESOURCES\n", " return GCP_RESOURCES\n", " elif tf.io.gfile.exists(EVAL_METRICS):\n", - " ! gsutil cat $EVAL_METRICS\n", + " ! gcloud storage cat $EVAL_METRICS\n", " return EVAL_METRICS\n", "\n", " return None\n", @@ -1035,7 +1035,7 @@ "print(\"\\n\\n\")\n", "print(\"model-upload\")\n", "artifacts = print_pipeline_output(pipeline, \"model-upload\")\n", - "output = !gsutil cat $artifacts\n", + "output = !gcloud storage cat $artifacts\n", "output = json.loads(output[0])\n", "model_id = output[\"artifacts\"][\"model\"][\"artifacts\"][0][\"metadata\"][\"resourceName\"]\n", "print(\"\\n\")\n", @@ -1253,7 +1253,7 @@ "print(\"\\n\\n\")\n", "print(\"model-upload\")\n", "artifacts = print_pipeline_output(pipeline, \"model-upload\")\n", - "output = !gsutil cat $artifacts\n", + "output = !gcloud storage cat $artifacts\n", "output = json.loads(output[0])\n", "model_id = output[\"artifacts\"][\"model\"][\"artifacts\"][0][\"metadata\"][\"resourceName\"]\n", "print(\"\\n\")\n", @@ -1332,7 +1332,7 @@ "delete_bucket = False\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gsutil rm -r $BUCKET_URI" + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], diff --git a/notebooks/community/ml_ops/stage5/get_started_with_vertex_private_endpoints.ipynb b/notebooks/community/ml_ops/stage5/get_started_with_vertex_private_endpoints.ipynb index 156541774..b53e14095 100644 --- a/notebooks/community/ml_ops/stage5/get_started_with_vertex_private_endpoints.ipynb +++ b/notebooks/community/ml_ops/stage5/get_started_with_vertex_private_endpoints.ipynb @@ -458,7 +458,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_URI" + "! gcloud storage buckets create --location=$REGION $BUCKET_URI" ] }, { @@ -478,7 +478,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_URI" + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { @@ -1129,7 +1129,7 @@ }, "outputs": [], "source": [ - "! gsutil cp gs://cloud-ml-data/img/flower_photos/daisy/100080576_f52e8ee070_n.jpg test.jpg" + "! gcloud storage cp gs://cloud-ml-data/img/flower_photos/daisy/100080576_f52e8ee070_n.jpg test.jpg" ] }, { @@ -1314,7 +1314,7 @@ " print(e)\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gsutil rm -rf {BUCKET_URI}" + " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}" ] } ], diff --git a/notebooks/official/experiments/get_started_with_vertex_experiments.ipynb b/notebooks/official/experiments/get_started_with_vertex_experiments.ipynb index 12ddd233e..b2330642e 100644 --- a/notebooks/official/experiments/get_started_with_vertex_experiments.ipynb +++ b/notebooks/official/experiments/get_started_with_vertex_experiments.ipynb @@ -321,7 +321,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $LOCATION $BUCKET_URI" + "! gcloud storage buckets create --location $LOCATION $BUCKET_URI" ] }, { @@ -1238,7 +1238,7 @@ "! rm -f custom.tar custom.tar.gz\n", "! tar cvf custom.tar custom\n", "! gzip custom.tar\n", - "! gsutil cp custom.tar.gz $BUCKET_URI/trainer.tar.gz" + "! gcloud storage cp custom.tar.gz $BUCKET_URI/trainer.tar.gz" ] }, { @@ -1451,7 +1451,7 @@ "delete_bucket = False\n", "\n", "if delete_bucket:\n", - " ! gsutil rm -rf {BUCKET_URI}" + " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}" ] } ], diff --git a/notebooks/official/pipelines/custom_tabular_train_batch_pred_bq_pipeline.ipynb b/notebooks/official/pipelines/custom_tabular_train_batch_pred_bq_pipeline.ipynb index 47c90c10b..0025b0b18 100644 --- a/notebooks/official/pipelines/custom_tabular_train_batch_pred_bq_pipeline.ipynb +++ b/notebooks/official/pipelines/custom_tabular_train_batch_pred_bq_pipeline.ipynb @@ -308,7 +308,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $LOCATION -p $PROJECT_ID $BUCKET_URI" + "! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI" ] }, { @@ -381,9 +381,9 @@ }, "outputs": [], "source": [ - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI" + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] }, { @@ -837,7 +837,7 @@ }, "outputs": [], "source": [ - "!gsutil cp -r python_package/dist/* $BUCKET_URI/training_package/" + "!gcloud storage cp --recursive python_package/dist/* $BUCKET_URI/training_package/" ] }, { @@ -1215,7 +1215,7 @@ "\n", "# Delete Cloud Storage objects\n", "if delete_bucket:\n", - " ! gsutil -m rm -r $BUCKET_URI\n", + " ! gcloud storage rm --recursive $BUCKET_URI\n", "\n", "! rm $PIPELINE_FILE_NAME" ] diff --git a/notebooks/official/sdk/SDK_Custom_Training_Python_Package_Managed_Text_Dataset_Tensorflow_Serving_Container.ipynb b/notebooks/official/sdk/SDK_Custom_Training_Python_Package_Managed_Text_Dataset_Tensorflow_Serving_Container.ipynb index bf799918c..809844ae4 100644 --- a/notebooks/official/sdk/SDK_Custom_Training_Python_Package_Managed_Text_Dataset_Tensorflow_Serving_Container.ipynb +++ b/notebooks/official/sdk/SDK_Custom_Training_Python_Package_Managed_Text_Dataset_Tensorflow_Serving_Container.ipynb @@ -324,7 +324,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $LOCATION -p $PROJECT_ID $BUCKET_URI" + "! gcloud storage buckets create --location $LOCATION --project $PROJECT_ID $BUCKET_URI" ] }, { @@ -595,7 +595,7 @@ }, "outputs": [], "source": [ - "!gsutil ls gs://$BUCKET_NAME/$GCS_PREFIX/data" + "!gcloud storage ls gs://$BUCKET_NAME/$GCS_PREFIX/data" ] }, { @@ -1359,7 +1359,7 @@ }, "outputs": [], "source": [ - "!gsutil ls $gcs_source_test_url" + "!gcloud storage ls $gcs_source_test_url" ] }, { @@ -1488,7 +1488,7 @@ "batch_predict_job.delete()\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gsutil rm -r $BUCKET_URI" + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], diff --git a/notebooks/official/tabular_workflows/prophet_on_vertex_pipelines.ipynb b/notebooks/official/tabular_workflows/prophet_on_vertex_pipelines.ipynb index 489016bf8..959a31964 100644 --- a/notebooks/official/tabular_workflows/prophet_on_vertex_pipelines.ipynb +++ b/notebooks/official/tabular_workflows/prophet_on_vertex_pipelines.ipynb @@ -310,7 +310,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $LOCATION -p $PROJECT_ID $BUCKET_URI" + "! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI" ] }, { @@ -381,9 +381,9 @@ }, "outputs": [], "source": [ - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/dataflow.developer" + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] }, { @@ -1165,7 +1165,7 @@ "# Delete Cloud Storage objects that were created\n", "delete_bucket = False # Set True for deletion\n", "if delete_bucket:\n", - " ! gsutil -m rm -r $BUCKET_URI" + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], diff --git a/notebooks/official/training/tpuv5e_gemma_peft_finetuning_and_serving.ipynb b/notebooks/official/training/tpuv5e_gemma_peft_finetuning_and_serving.ipynb index 2b057f43c..28127a8e9 100644 --- a/notebooks/official/training/tpuv5e_gemma_peft_finetuning_and_serving.ipynb +++ b/notebooks/official/training/tpuv5e_gemma_peft_finetuning_and_serving.ipynb @@ -371,7 +371,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}" + "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" ] }, { @@ -1423,7 +1423,7 @@ "# Delete Cloud Storage objects that were created\n", "delete_bucket = False\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gsutil -m rm -r $BUCKET_URI" + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ],