diff --git a/notebooks/community/ml_ops/stage6/get_started_with_automl_tabular_exported_deploy.ipynb b/notebooks/community/ml_ops/stage6/get_started_with_automl_tabular_exported_deploy.ipynb index e344869da..191169186 100644 --- a/notebooks/community/ml_ops/stage6/get_started_with_automl_tabular_exported_deploy.ipynb +++ b/notebooks/community/ml_ops/stage6/get_started_with_automl_tabular_exported_deploy.ipynb @@ -436,7 +436,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$REGION $BUCKET_URI" ] + "! gcloud storage buckets create --location=$REGION $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -455,7 +456,8 @@ }, "outputs": [], "source": [ - "! gcloud storage ls --all-versions --long $BUCKET_URI" ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -586,7 +588,8 @@ }, "outputs": [], "source": [ - "! gcloud storage ls {MODEL_PACKAGE}" ] + "! gcloud storage ls {MODEL_PACKAGE}" + ] }, { "cell_type": "markdown", @@ -630,7 +633,8 @@ "source": [ "import json\n", "\n", - "output = !gcloud storage cat {MODEL_PACKAGE}/environment.json\n", "\n", + "output = !gcloud storage cat {MODEL_PACKAGE}/environment.json\n", + "\n", "MODEL_SERVER = json.loads(output[0])[\"container_uri\"]" ] }, @@ -856,7 +860,8 @@ "delete_bucket = False\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { diff --git a/notebooks/community/model_garden/model_garden_camp_zipnerf.ipynb b/notebooks/community/model_garden/model_garden_camp_zipnerf.ipynb index 5a02c6825..5e6b415c3 100644 --- a/notebooks/community/model_garden/model_garden_camp_zipnerf.ipynb +++ b/notebooks/community/model_garden/model_garden_camp_zipnerf.ipynb @@ -150,8 +150,10 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", + " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", + "else:\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", + " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -161,7 +163,8 @@ "print(f\"Using this GCS Bucket: {BUCKET_URI}\")\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", + "\n", "! gcloud config set project $PROJECT_ID\n", "\n", "staging_bucket = os.path.join(BUCKET_URI, \"zipnerf_staging\")\n", @@ -189,7 +192,9 @@ "\n", "\n", "def get_mp4_video_link(mp4_rendering_path: str) -> str:\n", - " # Define the gcloud storage command.\n", " command = f\"gcloud storage ls {mp4_rendering_path}\"\n", "\n", + " # Define the gcloud storage command.\n", + " command = f\"gcloud storage ls {mp4_rendering_path}\"\n", + "\n", " # Run the command and capture the output.\n", " try:\n", " result = subprocess.check_output(command, shell=True, text=True)\n", @@ -206,7 +211,8 @@ " bucket_path: str, output_gcs_file: str, max_files: int = 10\n", ") -> List[Any]:\n", " # Get the list of files in the GCS bucket.\n", - " cmd = f\"gcloud storage ls {bucket_path}\"\n", " result = subprocess.run(cmd, shell=True, capture_output=True, text=True)\n", + " cmd = f\"gcloud storage ls {bucket_path}\"\n", + " result = subprocess.run(cmd, shell=True, capture_output=True, text=True)\n", "\n", " if result.returncode != 0:\n", " print(\"Error listing GCS bucket:\", result.stderr)\n", @@ -223,7 +229,8 @@ " for name in image_files[:max_files]:\n", " file.write(name + \"\\n\")\n", "\n", - " cmd = f\"gcloud storage cp {output_file} {output_gcs_file}\"\n", " result = subprocess.run(cmd, shell=True, capture_output=True, text=True)\n", + " cmd = f\"gcloud storage cp {output_file} {output_gcs_file}\"\n", + " result = subprocess.run(cmd, shell=True, capture_output=True, text=True)\n", "\n", " if result.returncode != 0:\n", " print(\"Error listing GCS bucket:\", result.stderr)\n", @@ -268,7 +275,8 @@ "\n", "# Move mipnerf360 data from local directory to Cloud Storage.\n", "# This step takes a few minutes to finish.\n", - "! gcloud storage cp --recursive $mipnerf_dataset_directory/* $MIPNERF_DATA_GCS_PATH" ] + "! gcloud storage cp --recursive $mipnerf_dataset_directory/* $MIPNERF_DATA_GCS_PATH" + ] }, { "cell_type": "markdown", @@ -651,7 +659,8 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_NAME\n", "\n", + " ! gcloud storage rm --recursive $BUCKET_NAME\n", + "\n", "# Delete pose estimation, training and rendering custom jobs.\n", "if data_calibration_custom_job.list(\n", " filter=f'display_name=\"{data_calibration_job_name}\"'\n", diff --git a/notebooks/community/model_garden/model_garden_proprietary_image_object_detection.ipynb b/notebooks/community/model_garden/model_garden_proprietary_image_object_detection.ipynb index 2f237ca4d..2af412383 100644 --- a/notebooks/community/model_garden/model_garden_proprietary_image_object_detection.ipynb +++ b/notebooks/community/model_garden/model_garden_proprietary_image_object_detection.ipynb @@ -366,10 +366,12 @@ }, "outputs": [], "source": [ - "count = ! gcloud storage cat $DATASET_FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n", + "count = ! gcloud storage cat $DATASET_FILE | wc -l\n", + "print(\"Number of Examples\", int(count[0]))\n", "\n", "print(\"First 10 rows\")\n", - "! gcloud storage cat $DATASET_FILE | head" ] + "! gcloud storage cat $DATASET_FILE | head" + ] }, { "cell_type": "markdown", diff --git a/notebooks/community/model_garden/model_garden_pytorch_falcon_instruct_finetuning.ipynb b/notebooks/community/model_garden/model_garden_pytorch_falcon_instruct_finetuning.ipynb index a95519e8c..5339fae53 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_falcon_instruct_finetuning.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_falcon_instruct_finetuning.ipynb @@ -216,7 +216,7 @@ " serving_container_predict_route=\"/predictions/peft_serving\",\n", " serving_container_health_route=\"/ping\",\n", " serving_container_environment_variables=serving_env,\n", - " model_garden_source_model_name=\"publishers/tiiuae/models/falcon-instruct-7b-peft\"\n", + " model_garden_source_model_name=\"publishers/tiiuae/models/falcon-instruct-7b-peft\",\n", " )\n", " model.deploy(\n", " endpoint=endpoint,\n", @@ -266,7 +266,7 @@ " serving_container_ports=[7080],\n", " serving_container_predict_route=\"/generate\",\n", " serving_container_health_route=\"/ping\",\n", - " model_garden_source_model_name=\"publishers/tiiuae/models/falcon-instruct-7b-peft\"\n", + " model_garden_source_model_name=\"publishers/tiiuae/models/falcon-instruct-7b-peft\",\n", " )\n", "\n", " model.deploy(\n", diff --git a/notebooks/official/automl/automl_text_entity_extraction_batch_prediction.ipynb b/notebooks/official/automl/automl_text_entity_extraction_batch_prediction.ipynb index 06ff827b4..8fe5b035e 100644 --- a/notebooks/official/automl/automl_text_entity_extraction_batch_prediction.ipynb +++ b/notebooks/official/automl/automl_text_entity_extraction_batch_prediction.ipynb @@ -331,7 +331,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location $REGION $BUCKET_URI" ] + "! gcloud storage buckets create --location $REGION $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -437,10 +438,12 @@ "else:\n", " FILE = IMPORT_FILE\n", "\n", - "count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n", + "count = ! gcloud storage cat $FILE | wc -l\n", + "print(\"Number of Examples\", int(count[0]))\n", "\n", "print(\"First 10 rows\")\n", - "! gcloud storage cat $FILE | head" ] + "! gcloud storage cat $FILE | head" + ] }, { "cell_type": "markdown", @@ -658,7 +661,8 @@ " f.write(json.dumps(data) + \"\\n\")\n", "\n", "print(gcs_input_uri)\n", - "! gcloud storage cat $gcs_input_uri" ] + "! gcloud storage cat $gcs_input_uri" + ] }, { "cell_type": "markdown", @@ -793,7 +797,8 @@ "delete_bucket = False\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gcloud storage rm --recursive $BUCKET_URI\n", "\n", + " ! gcloud storage rm --recursive $BUCKET_URI\n", + "\n", "# Delete batch\n", "batch_predict_job.delete()\n", "\n", diff --git a/notebooks/official/custom/custom_training_tensorboard_profiler.ipynb b/notebooks/official/custom/custom_training_tensorboard_profiler.ipynb index 5120c4f06..23a6b580c 100644 --- a/notebooks/official/custom/custom_training_tensorboard_profiler.ipynb +++ b/notebooks/official/custom/custom_training_tensorboard_profiler.ipynb @@ -281,7 +281,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location $LOCATION --project $PROJECT_ID $BUCKET_URI" ] + "! gcloud storage buckets create --location $LOCATION --project $PROJECT_ID $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -377,8 +378,10 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=\"serviceAccount:{SERVICE_ACCOUNT}\" --role=\"roles/storage.objectCreator\"\n", "\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=\"serviceAccount:{SERVICE_ACCOUNT}\" --role=\"roles/storage.objectCreator\"\n", + "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" + ] }, { "cell_type": "markdown", @@ -902,7 +905,8 @@ " tensorboard.delete()\n", "\n", "if delete_bucket and \"BUCKET_URI\" in globals():\n", - " ! gcloud storage rm --recursive $BUCKET_URI\n", "\n", + " ! gcloud storage rm --recursive $BUCKET_URI\n", + "\n", "if delete_generated_files_after_execution:\n", " ! rm -rf $PYTHON_PACKAGE_APPLICATION_DIR Dockerfile Dockerfile" ] diff --git a/notebooks/official/experiments/comparing_pipeline_runs.ipynb b/notebooks/official/experiments/comparing_pipeline_runs.ipynb index cd806fd04..8c9c6111b 100644 --- a/notebooks/official/experiments/comparing_pipeline_runs.ipynb +++ b/notebooks/official/experiments/comparing_pipeline_runs.ipynb @@ -284,7 +284,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$LOCATION $BUCKET_URI" ] + "! gcloud storage buckets create --location=$LOCATION $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -359,7 +360,8 @@ "source": [ "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" + ] }, { "cell_type": "markdown", @@ -380,7 +382,8 @@ "source": [ "DATASET_URI = \"gs://cloud-samples-data/ai-platform/iris\"\n", "\n", - "!gcloud storage cp --recursive $DATASET_URI $BUCKET_URI" ] + "!gcloud storage cp --recursive $DATASET_URI $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -885,7 +888,8 @@ "# Delete the Cloud Storage bucket\n", "delete_bucket = False # Set True for deletion\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}\n", "\n", + " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}\n", + "\n", "# Remove local files\n", "!rm {PIPELINE_TEMPLATE_FILE}" ] diff --git a/notebooks/official/pipelines/lightweight_functions_component_io_kfp.ipynb b/notebooks/official/pipelines/lightweight_functions_component_io_kfp.ipynb index 831118b1c..2c9985456 100644 --- a/notebooks/official/pipelines/lightweight_functions_component_io_kfp.ipynb +++ b/notebooks/official/pipelines/lightweight_functions_component_io_kfp.ipynb @@ -282,7 +282,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" ] + "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" + ] }, { "cell_type": "markdown", @@ -354,8 +355,10 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", + "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" + ] }, { "cell_type": "markdown", @@ -777,7 +780,8 @@ "delete_bucket = False\n", "\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_URI\n", "\n", + " ! gcloud storage rm --recursive $BUCKET_URI\n", + "\n", "! rm lightweight_pipeline.yaml" ] }