mirror of
https://github.com/GoogleCloudPlatform/vertex-ai-samples.git
synced 2026-09-26 14:42:04 +00:00
linter changes
This commit is contained in:
+10
-5
@@ -436,7 +436,8 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gcloud storage buckets create --location=$REGION $BUCKET_URI" ]
|
||||
"! gcloud storage buckets create --location=$REGION $BUCKET_URI"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -455,7 +456,8 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gcloud storage ls --all-versions --long $BUCKET_URI" ]
|
||||
"! gcloud storage ls --all-versions --long $BUCKET_URI"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -586,7 +588,8 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gcloud storage ls {MODEL_PACKAGE}" ]
|
||||
"! gcloud storage ls {MODEL_PACKAGE}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -630,7 +633,8 @@
|
||||
"source": [
|
||||
"import json\n",
|
||||
"\n",
|
||||
"output = !gcloud storage cat {MODEL_PACKAGE}/environment.json\n", "\n",
|
||||
"output = !gcloud storage cat {MODEL_PACKAGE}/environment.json\n",
|
||||
"\n",
|
||||
"MODEL_SERVER = json.loads(output[0])[\"container_uri\"]"
|
||||
]
|
||||
},
|
||||
@@ -856,7 +860,8 @@
|
||||
"delete_bucket = False\n",
|
||||
"\n",
|
||||
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
|
||||
" ! gcloud storage rm --recursive $BUCKET_URI" ]
|
||||
" ! gcloud storage rm --recursive $BUCKET_URI"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
|
||||
@@ -150,8 +150,10 @@
|
||||
"if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n",
|
||||
" BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n",
|
||||
" BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n",
|
||||
" ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n",
|
||||
" shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n",
|
||||
" ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n",
|
||||
"else:\n",
|
||||
" shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n",
|
||||
" bucket_region = shell_output[0].strip().lower()\n",
|
||||
" if bucket_region != REGION:\n",
|
||||
" raise ValueError(\n",
|
||||
" \"Bucket region %s is different from notebook region %s\"\n",
|
||||
@@ -161,7 +163,8 @@
|
||||
"print(f\"Using this GCS Bucket: {BUCKET_URI}\")\n",
|
||||
"\n",
|
||||
"# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n",
|
||||
"! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n",
|
||||
"! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n",
|
||||
"\n",
|
||||
"! gcloud config set project $PROJECT_ID\n",
|
||||
"\n",
|
||||
"staging_bucket = os.path.join(BUCKET_URI, \"zipnerf_staging\")\n",
|
||||
@@ -189,7 +192,9 @@
|
||||
"\n",
|
||||
"\n",
|
||||
"def get_mp4_video_link(mp4_rendering_path: str) -> str:\n",
|
||||
" # Define the gcloud storage command.\n", " command = f\"gcloud storage ls {mp4_rendering_path}\"\n", "\n",
|
||||
" # Define the gcloud storage command.\n",
|
||||
" command = f\"gcloud storage ls {mp4_rendering_path}\"\n",
|
||||
"\n",
|
||||
" # Run the command and capture the output.\n",
|
||||
" try:\n",
|
||||
" result = subprocess.check_output(command, shell=True, text=True)\n",
|
||||
@@ -206,7 +211,8 @@
|
||||
" bucket_path: str, output_gcs_file: str, max_files: int = 10\n",
|
||||
") -> List[Any]:\n",
|
||||
" # Get the list of files in the GCS bucket.\n",
|
||||
" cmd = f\"gcloud storage ls {bucket_path}\"\n", " result = subprocess.run(cmd, shell=True, capture_output=True, text=True)\n",
|
||||
" cmd = f\"gcloud storage ls {bucket_path}\"\n",
|
||||
" result = subprocess.run(cmd, shell=True, capture_output=True, text=True)\n",
|
||||
"\n",
|
||||
" if result.returncode != 0:\n",
|
||||
" print(\"Error listing GCS bucket:\", result.stderr)\n",
|
||||
@@ -223,7 +229,8 @@
|
||||
" for name in image_files[:max_files]:\n",
|
||||
" file.write(name + \"\\n\")\n",
|
||||
"\n",
|
||||
" cmd = f\"gcloud storage cp {output_file} {output_gcs_file}\"\n", " result = subprocess.run(cmd, shell=True, capture_output=True, text=True)\n",
|
||||
" cmd = f\"gcloud storage cp {output_file} {output_gcs_file}\"\n",
|
||||
" result = subprocess.run(cmd, shell=True, capture_output=True, text=True)\n",
|
||||
"\n",
|
||||
" if result.returncode != 0:\n",
|
||||
" print(\"Error listing GCS bucket:\", result.stderr)\n",
|
||||
@@ -268,7 +275,8 @@
|
||||
"\n",
|
||||
"# Move mipnerf360 data from local directory to Cloud Storage.\n",
|
||||
"# This step takes a few minutes to finish.\n",
|
||||
"! gcloud storage cp --recursive $mipnerf_dataset_directory/* $MIPNERF_DATA_GCS_PATH" ]
|
||||
"! gcloud storage cp --recursive $mipnerf_dataset_directory/* $MIPNERF_DATA_GCS_PATH"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -651,7 +659,8 @@
|
||||
"\n",
|
||||
"delete_bucket = False # @param {type:\"boolean\"}\n",
|
||||
"if delete_bucket:\n",
|
||||
" ! gcloud storage rm --recursive $BUCKET_NAME\n", "\n",
|
||||
" ! gcloud storage rm --recursive $BUCKET_NAME\n",
|
||||
"\n",
|
||||
"# Delete pose estimation, training and rendering custom jobs.\n",
|
||||
"if data_calibration_custom_job.list(\n",
|
||||
" filter=f'display_name=\"{data_calibration_job_name}\"'\n",
|
||||
|
||||
+4
-2
@@ -366,10 +366,12 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"count = ! gcloud storage cat $DATASET_FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
|
||||
"count = ! gcloud storage cat $DATASET_FILE | wc -l\n",
|
||||
"print(\"Number of Examples\", int(count[0]))\n",
|
||||
"\n",
|
||||
"print(\"First 10 rows\")\n",
|
||||
"! gcloud storage cat $DATASET_FILE | head" ]
|
||||
"! gcloud storage cat $DATASET_FILE | head"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
|
||||
+2
-2
@@ -216,7 +216,7 @@
|
||||
" serving_container_predict_route=\"/predictions/peft_serving\",\n",
|
||||
" serving_container_health_route=\"/ping\",\n",
|
||||
" serving_container_environment_variables=serving_env,\n",
|
||||
" model_garden_source_model_name=\"publishers/tiiuae/models/falcon-instruct-7b-peft\"\n",
|
||||
" model_garden_source_model_name=\"publishers/tiiuae/models/falcon-instruct-7b-peft\",\n",
|
||||
" )\n",
|
||||
" model.deploy(\n",
|
||||
" endpoint=endpoint,\n",
|
||||
@@ -266,7 +266,7 @@
|
||||
" serving_container_ports=[7080],\n",
|
||||
" serving_container_predict_route=\"/generate\",\n",
|
||||
" serving_container_health_route=\"/ping\",\n",
|
||||
" model_garden_source_model_name=\"publishers/tiiuae/models/falcon-instruct-7b-peft\"\n",
|
||||
" model_garden_source_model_name=\"publishers/tiiuae/models/falcon-instruct-7b-peft\",\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" model.deploy(\n",
|
||||
|
||||
@@ -331,7 +331,8 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gcloud storage buckets create --location $REGION $BUCKET_URI" ]
|
||||
"! gcloud storage buckets create --location $REGION $BUCKET_URI"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -437,10 +438,12 @@
|
||||
"else:\n",
|
||||
" FILE = IMPORT_FILE\n",
|
||||
"\n",
|
||||
"count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
|
||||
"count = ! gcloud storage cat $FILE | wc -l\n",
|
||||
"print(\"Number of Examples\", int(count[0]))\n",
|
||||
"\n",
|
||||
"print(\"First 10 rows\")\n",
|
||||
"! gcloud storage cat $FILE | head" ]
|
||||
"! gcloud storage cat $FILE | head"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -658,7 +661,8 @@
|
||||
" f.write(json.dumps(data) + \"\\n\")\n",
|
||||
"\n",
|
||||
"print(gcs_input_uri)\n",
|
||||
"! gcloud storage cat $gcs_input_uri" ]
|
||||
"! gcloud storage cat $gcs_input_uri"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -793,7 +797,8 @@
|
||||
"delete_bucket = False\n",
|
||||
"\n",
|
||||
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
|
||||
" ! gcloud storage rm --recursive $BUCKET_URI\n", "\n",
|
||||
" ! gcloud storage rm --recursive $BUCKET_URI\n",
|
||||
"\n",
|
||||
"# Delete batch\n",
|
||||
"batch_predict_job.delete()\n",
|
||||
"\n",
|
||||
|
||||
@@ -281,7 +281,8 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gcloud storage buckets create --location $LOCATION --project $PROJECT_ID $BUCKET_URI" ]
|
||||
"! gcloud storage buckets create --location $LOCATION --project $PROJECT_ID $BUCKET_URI"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -377,8 +378,10 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=\"serviceAccount:{SERVICE_ACCOUNT}\" --role=\"roles/storage.objectCreator\"\n", "\n",
|
||||
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ]
|
||||
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=\"serviceAccount:{SERVICE_ACCOUNT}\" --role=\"roles/storage.objectCreator\"\n",
|
||||
"\n",
|
||||
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -902,7 +905,8 @@
|
||||
" tensorboard.delete()\n",
|
||||
"\n",
|
||||
"if delete_bucket and \"BUCKET_URI\" in globals():\n",
|
||||
" ! gcloud storage rm --recursive $BUCKET_URI\n", "\n",
|
||||
" ! gcloud storage rm --recursive $BUCKET_URI\n",
|
||||
"\n",
|
||||
"if delete_generated_files_after_execution:\n",
|
||||
" ! rm -rf $PYTHON_PACKAGE_APPLICATION_DIR Dockerfile Dockerfile"
|
||||
]
|
||||
|
||||
@@ -284,7 +284,8 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gcloud storage buckets create --location=$LOCATION $BUCKET_URI" ]
|
||||
"! gcloud storage buckets create --location=$LOCATION $BUCKET_URI"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -359,7 +360,8 @@
|
||||
"source": [
|
||||
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n",
|
||||
"\n",
|
||||
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ]
|
||||
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -380,7 +382,8 @@
|
||||
"source": [
|
||||
"DATASET_URI = \"gs://cloud-samples-data/ai-platform/iris\"\n",
|
||||
"\n",
|
||||
"!gcloud storage cp --recursive $DATASET_URI $BUCKET_URI" ]
|
||||
"!gcloud storage cp --recursive $DATASET_URI $BUCKET_URI"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -885,7 +888,8 @@
|
||||
"# Delete the Cloud Storage bucket\n",
|
||||
"delete_bucket = False # Set True for deletion\n",
|
||||
"if delete_bucket:\n",
|
||||
" ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}\n", "\n",
|
||||
" ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}\n",
|
||||
"\n",
|
||||
"# Remove local files\n",
|
||||
"!rm {PIPELINE_TEMPLATE_FILE}"
|
||||
]
|
||||
|
||||
@@ -282,7 +282,8 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" ]
|
||||
"! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -354,8 +355,10 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n",
|
||||
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ]
|
||||
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n",
|
||||
"\n",
|
||||
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -777,7 +780,8 @@
|
||||
"delete_bucket = False\n",
|
||||
"\n",
|
||||
"if delete_bucket:\n",
|
||||
" ! gcloud storage rm --recursive $BUCKET_URI\n", "\n",
|
||||
" ! gcloud storage rm --recursive $BUCKET_URI\n",
|
||||
"\n",
|
||||
"! rm lightweight_pipeline.yaml"
|
||||
]
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user