xiaohei/model_garden
1
0
Fork 0
mirror of https://github.com/GoogleCloudPlatform/vertex-ai-samples.git synced 2026-09-26 14:42:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix: Resolved linter errors and formatted notebooks

Browse Source
This commit is contained in:
bhandarivijay
2025-12-16 19:14:56 +00:00
parent f497568138
commit 7534760fe1
11 changed files with 516 additions and 372 deletions
Download Patch File Download Diff File Expand all files Collapse all files
notebooks/community/gapic/automl/showcase_automl_tabular_binary_classification_batch.ipynb
+21 -11
View File
@@ -421,7 +421,8 @@
},
"outputs": [],
"source": [
"! gcloud storage buckets create --location $REGION $BUCKET_NAME" ]
"! gcloud storage buckets create --location $REGION $BUCKET_NAME"
]
},
{
"cell_type": "markdown",
@@ -440,7 +441,8 @@
},
"outputs": [],
"source": [
"! gcloud storage ls --all-versions --long $BUCKET_NAME" ]
"! gcloud storage ls --all-versions --long $BUCKET_NAME"
]
},
{
"cell_type": "markdown",
@@ -807,11 +809,14 @@
},
"outputs": [],
"source": [
"count = ! gcloud storage cat $IMPORT_FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
"count = ! gcloud storage cat $IMPORT_FILE | wc -l\n",
"print(\"Number of Examples\", int(count[0]))\n",
"\n",
"print(\"First 10 rows\")\n",
"! gcloud storage cat $IMPORT_FILE | head\n", "\n",
"heading = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n",
"! gcloud storage cat $IMPORT_FILE | head\n",
"\n",
"heading = ! gcloud storage cat $IMPORT_FILE | head -n1\n",
"label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n",
"print(\"Label Column Name\", label_column)\n",
"if label_column is None:\n",
" raise Exception(\"label column missing\")"
@@ -1372,7 +1377,8 @@
" f.write(str(INSTANCE_2) + \"\\n\")\n",
"\n",
"print(gcs_input_uri)\n",
"! gcloud storage cat $gcs_input_uri" ]
"! gcloud storage cat $gcs_input_uri"
]
},
{
"cell_type": "markdown",
@@ -1639,8 +1645,9 @@
"outputs": [],
"source": [
"def get_latest_predictions(gcs_out_dir):\n",
" \"\"\" Get the latest prediction subfolder using the timestamp in the subfolder name\"\"\"\n",
" folders = !gcloud storage ls $gcs_out_dir\n", " latest = \"\"\n",
" \"\"\"Get the latest prediction subfolder using the timestamp in the subfolder name\"\"\"\n",
" folders = !gcloud storage ls $gcs_out_dir\n",
" latest = \"\"\n",
" for folder in folders:\n",
" subfolder = folder.split(\"/\")[-2]\n",
" if subfolder.startswith(\"prediction-\"):\n",
@@ -1657,8 +1664,10 @@
" raise Exception(\"Batch Job Failed\")\n",
" else:\n",
" folder = get_latest_predictions(predictions)\n",
" ! gcloud storage ls $folder/prediction*.csv\n", "\n",
" ! gcloud storage cat $folder/prediction*.csv\n", " break\n",
" ! gcloud storage ls $folder/prediction*.csv\n",
"\n",
" ! gcloud storage cat $folder/prediction*.csv\n",
" break\n",
" time.sleep(60)"
]
},
@@ -1752,7 +1761,8 @@
" print(e)\n",
"\n",
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
" ! gcloud storage rm --recursive $BUCKET_NAME"
]
}
],
"metadata": {
notebooks/community/gapic/automl/showcase_automl_tabular_classification_batch_explain.ipynb
+21 -12
View File
@@ -421,7 +421,8 @@
},
"outputs": [],
"source": [
"! gcloud storage buckets create --location $REGION $BUCKET_NAME" ]
"! gcloud storage buckets create --location $REGION $BUCKET_NAME"
]
},
{
"cell_type": "markdown",
@@ -440,7 +441,8 @@
},
"outputs": [],
"source": [
"! gcloud storage ls --all-versions --long $BUCKET_NAME" ]
"! gcloud storage ls --all-versions --long $BUCKET_NAME"
]
},
{
"cell_type": "markdown",
@@ -477,7 +479,6 @@
"\n",
"import google.cloud.aiplatform_v1beta1 as aip\n",
"from google.protobuf import json_format\n",
"from google.protobuf.json_format import MessageToJson, ParseDict\n",
"from google.protobuf.struct_pb2 import Struct, Value"
]
},
@@ -808,11 +809,14 @@
},
"outputs": [],
"source": [
"count = ! gcloud storage cat $IMPORT_FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
"count = ! gcloud storage cat $IMPORT_FILE | wc -l\n",
"print(\"Number of Examples\", int(count[0]))\n",
"\n",
"print(\"First 10 rows\")\n",
"! gcloud storage cat $IMPORT_FILE | head\n", "\n",
"heading = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n",
"! gcloud storage cat $IMPORT_FILE | head\n",
"\n",
"heading = ! gcloud storage cat $IMPORT_FILE | head -n1\n",
"label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n",
"print(\"Label Column Name\", label_column)\n",
"if label_column is None:\n",
" raise Exception(\"label column missing\")"
@@ -1358,7 +1362,8 @@
" f.write(str(INSTANCE_2) + \"\\n\")\n",
"\n",
"print(gcs_input_uri)\n",
"! gcloud storage cat $gcs_input_uri" ]
"! gcloud storage cat $gcs_input_uri"
]
},
{
"cell_type": "markdown",
@@ -1624,8 +1629,9 @@
"outputs": [],
"source": [
"def get_latest_predictions(gcs_out_dir):\n",
" \"\"\" Get the latest prediction subfolder using the timestamp in the subfolder name\"\"\"\n",
" folders = !gcloud storage ls $gcs_out_dir\n", " latest = \"\"\n",
" \"\"\"Get the latest prediction subfolder using the timestamp in the subfolder name\"\"\"\n",
" folders = !gcloud storage ls $gcs_out_dir\n",
" latest = \"\"\n",
" for folder in folders:\n",
" subfolder = folder.split(\"/\")[-2]\n",
" if subfolder.startswith(\"prediction-\"):\n",
@@ -1642,8 +1648,10 @@
" raise Exception(\"Batch Job Failed\")\n",
" else:\n",
" folder = get_latest_predictions(predictions)\n",
" ! gcloud storage ls $folder/explanation*.csv\n", "\n",
" ! gcloud storage cat $folder/explanation*.csv\n", " break\n",
" ! gcloud storage ls $folder/explanation*.csv\n",
"\n",
" ! gcloud storage cat $folder/explanation*.csv\n",
" break\n",
" time.sleep(60)"
]
},
@@ -1737,7 +1745,8 @@
" print(e)\n",
"\n",
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
" ! gcloud storage rm --recursive $BUCKET_NAME"
]
}
],
"metadata": {
notebooks/community/gapic/automl/showcase_automl_video_action_recognition_batch.ipynb
+144 -106
View File
@@ -138,9 +138,9 @@
"\n",
"# Google Cloud Notebook\n",
"if os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
" USER_FLAG = '--user'\n",
" USER_FLAG = \"--user\"\n",
"else:\n",
" USER_FLAG = ''\n",
" USER_FLAG = \"\"\n",
"\n",
"! pip3 install -U google-cloud-aiplatform $USER_FLAG"
]
@@ -187,6 +187,7 @@
"if not os.getenv(\"IS_TESTING\"):\n",
" # Automatically restart kernel after installs\n",
" import IPython\n",
"\n",
" app = IPython.Application.instance()\n",
" app.kernel.do_shutdown(True)"
]
@@ -229,7 +230,7 @@
},
"outputs": [],
"source": [
"PROJECT_ID = \"[your-project-id]\" #@param {type:\"string\"}"
"PROJECT_ID = \"[your-project-id]\" # @param {type:\"string\"}"
]
},
{
@@ -284,7 +285,7 @@
},
"outputs": [],
"source": [
"REGION = 'us-central1' #@param {type: \"string\"}"
"REGION = \"us-central1\" # @param {type: \"string\"}"
]
},
{
@@ -388,7 +389,7 @@
},
"outputs": [],
"source": [
"BUCKET_NAME = \"gs://[your-bucket-name]\" #@param {type:\"string\"}"
"BUCKET_NAME = \"gs://[your-bucket-name]\" # @param {type:\"string\"}"
]
},
{
@@ -420,7 +421,8 @@
},
"outputs": [],
"source": [
"! gcloud storage buckets create --location=$REGION $BUCKET_NAME" ]
"! gcloud storage buckets create --location=$REGION $BUCKET_NAME"
]
},
{
"cell_type": "markdown",
@@ -439,7 +441,8 @@
},
"outputs": [],
"source": [
"! gcloud storage ls --all-versions --long $BUCKET_NAME" ]
"! gcloud storage ls --all-versions --long $BUCKET_NAME"
]
},
{
"cell_type": "markdown",
@@ -475,9 +478,7 @@
"import time\n",
"\n",
"from google.cloud.aiplatform import gapic as aip\n",
"from google.protobuf import json_format\n",
"from google.protobuf.json_format import MessageToJson, ParseDict\n",
"from google.protobuf.struct_pb2 import Struct, Value"
"from google.protobuf import json_format"
]
},
{
@@ -533,7 +534,7 @@
"outputs": [],
"source": [
"# Video Dataset type\n",
"DATA_SCHEMA = 'gs://google-cloud-aiplatform/schema/dataset/metadata/video_1.0.0.yaml'\n",
"DATA_SCHEMA = \"gs://google-cloud-aiplatform/schema/dataset/metadata/video_1.0.0.yaml\"\n",
"# Video Labeling type\n",
"LABEL_SCHEMA = \"gs://google-cloud-aiplatform/schema/dataset/ioformat/video_action_recognition_io_format_1.0.0.yaml\"\n",
"# Video Training task\n",
@@ -573,7 +574,10 @@
"outputs": [],
"source": [
"if os.getenv(\"IS_TESTING_DEPOLY_GPU\"):\n",
" DEPLOY_GPU, DEPLOY_NGPU = (aip.AcceleratorType.NVIDIA_TESLA_K80, int(os.getenv(\"IS_TESTING_DEPOLY_GPU\")))\n",
" DEPLOY_GPU, DEPLOY_NGPU = (\n",
" aip.AcceleratorType.NVIDIA_TESLA_K80,\n",
" int(os.getenv(\"IS_TESTING_DEPOLY_GPU\")),\n",
" )\n",
"else:\n",
" DEPLOY_GPU, DEPLOY_NGPU = (aip.AcceleratorType.NVIDIA_TESLA_K80, 1)"
]
@@ -620,11 +624,11 @@
"if os.getenv(\"IS_TESTING_DEPLOY_MACHINE\"):\n",
" MACHINE_TYPE = os.getenv(\"IS_TESTING_DEPLOY_MACHINE\")\n",
"else:\n",
" MACHINE_TYPE = 'n1-standard'\n",
" MACHINE_TYPE = \"n1-standard\"\n",
"\n",
"VCPU = '4'\n",
"DEPLOY_COMPUTE = MACHINE_TYPE + '-' + VCPU\n",
"print('Deploy machine type', DEPLOY_COMPUTE)"
"VCPU = \"4\"\n",
"DEPLOY_COMPUTE = MACHINE_TYPE + \"-\" + VCPU\n",
"print(\"Deploy machine type\", DEPLOY_COMPUTE)"
]
},
{
@@ -669,38 +673,30 @@
"\n",
"\n",
"def create_dataset_client():\n",
" client = aip.DatasetServiceClient(\n",
" client_options=client_options\n",
" )\n",
" client = aip.DatasetServiceClient(client_options=client_options)\n",
" return client\n",
"\n",
"\n",
"def create_model_client():\n",
" client = aip.ModelServiceClient(\n",
" client_options=client_options\n",
" )\n",
" client = aip.ModelServiceClient(client_options=client_options)\n",
" return client\n",
"\n",
"\n",
"def create_pipeline_client():\n",
" client = aip.PipelineServiceClient(\n",
" client_options=client_options\n",
" )\n",
" client = aip.PipelineServiceClient(client_options=client_options)\n",
" return client\n",
"\n",
"\n",
"def create_job_client():\n",
" client = aip.JobServiceClient(\n",
" client_options=client_options\n",
" )\n",
" client = aip.JobServiceClient(client_options=client_options)\n",
" return client\n",
"\n",
"\n",
"clients = {}\n",
"clients['dataset'] = create_dataset_client()\n",
"clients['model'] = create_model_client()\n",
"clients['pipeline'] = create_pipeline_client()\n",
"clients['job'] = create_job_client()\n",
"clients[\"dataset\"] = create_dataset_client()\n",
"clients[\"model\"] = create_model_client()\n",
"clients[\"pipeline\"] = create_pipeline_client()\n",
"clients[\"job\"] = create_job_client()\n",
"\n",
"for client in clients.items():\n",
" print(client)"
@@ -752,12 +748,15 @@
"source": [
"TIMEOUT = 90\n",
"\n",
"\n",
"def create_dataset(name, schema, labels=None, timeout=TIMEOUT):\n",
" start_time = time.time()\n",
" try:\n",
" dataset = aip.Dataset(display_name=name, metadata_schema_uri=schema, labels=labels)\n",
" dataset = aip.Dataset(\n",
" display_name=name, metadata_schema_uri=schema, labels=labels\n",
" )\n",
"\n",
" operation = clients['dataset'].create_dataset(parent=PARENT, dataset=dataset)\n",
" operation = clients[\"dataset\"].create_dataset(parent=PARENT, dataset=dataset)\n",
" print(\"Long running operation:\", operation.operation.name)\n",
" result = operation.result(timeout=TIMEOUT)\n",
" print(\"time:\", time.time() - start_time)\n",
@@ -799,7 +798,7 @@
"# The full unique ID for the dataset\n",
"dataset_id = result.name\n",
"# The short numeric ID for the dataset\n",
"dataset_short_id = dataset_id.split('/')[-1]\n",
"dataset_short_id = dataset_id.split(\"/\")[-1]\n",
"\n",
"print(dataset_id)"
]
@@ -857,7 +856,10 @@
},
"outputs": [],
"source": [
"IMPORT_FILES = ['gs://automl-video-demo-data/hmdb_golf_swing_train.csv', 'gs://automl-video-demo-data/hmdb_golf_swing_test.csv']"
"IMPORT_FILES = [\n",
" \"gs://automl-video-demo-data/hmdb_golf_swing_train.csv\",\n",
" \"gs://automl-video-demo-data/hmdb_golf_swing_test.csv\",\n",
"]"
]
},
{
@@ -881,15 +883,17 @@
},
"outputs": [],
"source": [
"if 'IMPORT_FILES' in globals():\n",
"if \"IMPORT_FILES\" in globals():\n",
" FILE = IMPORT_FILES[0]\n",
"else:\n",
" FILE = IMPORT_FILE\n",
"\n",
"count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
"count = ! gcloud storage cat $FILE | wc -l\n",
"print(\"Number of Examples\", int(count[0]))\n",
"\n",
"print(\"First 10 rows\")\n",
"! gcloud storage cat $FILE | head" ]
"! gcloud storage cat $FILE | head"
]
},
{
"cell_type": "markdown",
@@ -922,14 +926,13 @@
"outputs": [],
"source": [
"def import_data(dataset, gcs_sources, schema):\n",
" config = [{\n",
" 'gcs_source': {'uris': gcs_sources},\n",
" 'import_schema_uri': schema\n",
" }]\n",
" config = [{\"gcs_source\": {\"uris\": gcs_sources}, \"import_schema_uri\": schema}]\n",
" print(\"dataset:\", dataset_id)\n",
" start_time = time.time()\n",
" try:\n",
" operation = clients['dataset'].import_data(name=dataset_id, import_configs=config)\n",
" operation = clients[\"dataset\"].import_data(\n",
" name=dataset_id, import_configs=config\n",
" )\n",
" print(\"Long running operation:\", operation.operation.name)\n",
"\n",
" result = operation.result()\n",
@@ -937,7 +940,14 @@
" print(\"time:\", int(time.time() - start_time), \"secs\")\n",
" print(\"error:\", operation.exception())\n",
" print(\"meta :\", operation.metadata)\n",
" print(\"after: running:\", operation.running(), \"done:\", operation.done(), \"cancelled:\", operation.cancelled())\n",
" print(\n",
" \"after: running:\",\n",
" operation.running(),\n",
" \"done:\",\n",
" operation.done(),\n",
" \"cancelled:\",\n",
" operation.cancelled(),\n",
" )\n",
"\n",
" return operation\n",
" except Exception as e:\n",
@@ -1010,15 +1020,14 @@
},
"outputs": [],
"source": [
" def create_pipeline(pipeline_name, model_name, dataset, schema, task):\n",
"def create_pipeline(pipeline_name, model_name, dataset, schema, task):\n",
"\n",
" dataset_id = dataset.split('/')[-1]\n",
" dataset_id = dataset.split(\"/\")[-1]\n",
"\n",
" input_config = {'dataset_id': dataset_id,\n",
" 'fraction_split': {\n",
" 'training_fraction': 0.8,\n",
" 'test_fraction': 0.2\n",
" }}\n",
" input_config = {\n",
" \"dataset_id\": dataset_id,\n",
" \"fraction_split\": {\"training_fraction\": 0.8, \"test_fraction\": 0.2},\n",
" }\n",
"\n",
" training_pipeline = {\n",
" \"display_name\": pipeline_name,\n",
@@ -1029,7 +1038,9 @@
" }\n",
"\n",
" try:\n",
" pipeline = clients['pipeline'].create_training_pipeline(parent=PARENT, training_pipeline=training_pipeline)\n",
" pipeline = clients[\"pipeline\"].create_training_pipeline(\n",
" parent=PARENT, training_pipeline=training_pipeline\n",
" )\n",
" print(pipeline)\n",
" except Exception as e:\n",
" print(\"exception:\", e)\n",
@@ -1065,8 +1076,12 @@
"PIPE_NAME = \"golf_pipe-\" + TIMESTAMP\n",
"MODEL_NAME = \"golf_model-\" + TIMESTAMP\n",
"\n",
"task = json_format.ParseDict({'model_type': \"CLOUD\",\n",
" }, Value())\n",
"task = json_format.ParseDict(\n",
" {\n",
" \"model_type\": \"CLOUD\",\n",
" },\n",
" Value(),\n",
")\n",
"\n",
"response = create_pipeline(PIPE_NAME, MODEL_NAME, dataset_id, TRAINING_SCHEMA, task)"
]
@@ -1091,7 +1106,7 @@
"# The full unique ID for the pipeline\n",
"pipeline_id = response.name\n",
"# The short numeric ID for the pipeline\n",
"pipeline_short_id = pipeline_id.split('/')[-1]\n",
"pipeline_short_id = pipeline_id.split(\"/\")[-1]\n",
"\n",
"print(pipeline_id)"
]
@@ -1120,7 +1135,7 @@
"outputs": [],
"source": [
"def get_training_pipeline(name, silent=False):\n",
" response = clients['pipeline'].get_training_pipeline(name=name)\n",
" response = clients[\"pipeline\"].get_training_pipeline(name=name)\n",
" if silent:\n",
" return response\n",
"\n",
@@ -1227,7 +1242,7 @@
"outputs": [],
"source": [
"def list_model_evaluations(name):\n",
" response = clients['model'].list_model_evaluations(parent=name)\n",
" response = clients[\"model\"].list_model_evaluations(parent=name)\n",
" for evaluation in response:\n",
" print(\"model_evaluation\")\n",
" print(\" name:\", evaluation.name)\n",
@@ -1235,8 +1250,7 @@
" metrics = json_format.MessageToDict(evaluation._pb.metrics)\n",
" for metric in metrics.keys():\n",
" print(metric)\n",
" print('videoActionMetrics', metrics['videoActionMetrics'])\n",
"\n",
" print(\"videoActionMetrics\", metrics[\"videoActionMetrics\"])\n",
"\n",
" return evaluation.name\n",
"\n",
@@ -1303,15 +1317,14 @@
},
"outputs": [],
"source": [
"import json\n",
"\n",
"import_file = IMPORT_FILES[0]\n",
"test_items = ! gcloud storage cat $import_file | head -n2\n", "\n",
"cols = str(test_items[0]).split(',')\n",
"test_items = ! gcloud storage cat $import_file | head -n2\n",
"\n",
"cols = str(test_items[0]).split(\",\")\n",
"test_item_1 = str(cols[0])\n",
"test_label_1 = str(cols[-1])\n",
"\n",
"cols = str(test_items[1]).split(',')\n",
"cols = str(test_items[1]).split(\",\")\n",
"test_item_2 = str(cols[0])\n",
"test_label_2 = str(cols[-1])\n",
"\n",
@@ -1347,15 +1360,26 @@
"\n",
"import tensorflow as tf\n",
"\n",
"gcs_input_uri = BUCKET_NAME + '/test.jsonl'\n",
"with tf.io.gfile.GFile(gcs_input_uri, 'w') as f:\n",
" data = { \"content\": test_item_1, \"mimeType\": \"video/avi\", \"timeSegmentStart\": \"0.0s\", 'timeSegmentEnd': '5.0s' }\n",
" f.write(json.dumps(data) + '\\n')\n",
" data = { \"content\": test_item_2, \"mimeType\": \"video/avi\", \"timeSegmentStart\": \"0.0s\", 'timeSegmentEnd': '5.0s' }\n",
" f.write(json.dumps(data) + '\\n')\n",
"gcs_input_uri = BUCKET_NAME + \"/test.jsonl\"\n",
"with tf.io.gfile.GFile(gcs_input_uri, \"w\") as f:\n",
" data = {\n",
" \"content\": test_item_1,\n",
" \"mimeType\": \"video/avi\",\n",
" \"timeSegmentStart\": \"0.0s\",\n",
" \"timeSegmentEnd\": \"5.0s\",\n",
" }\n",
" f.write(json.dumps(data) + \"\\n\")\n",
" data = {\n",
" \"content\": test_item_2,\n",
" \"mimeType\": \"video/avi\",\n",
" \"timeSegmentStart\": \"0.0s\",\n",
" \"timeSegmentEnd\": \"5.0s\",\n",
" }\n",
" f.write(json.dumps(data) + \"\\n\")\n",
"\n",
"print(gcs_input_uri)\n",
"! gcloud storage cat $gcs_input_uri" ]
"! gcloud storage cat $gcs_input_uri"
]
},
{
"cell_type": "markdown",
@@ -1462,7 +1486,13 @@
"BATCH_MODEL = \"golf_batch-\" + TIMESTAMP\n",
"\n",
"\n",
"def create_batch_prediction_job(display_name, model_name, gcs_source_uri, gcs_destination_output_uri_prefix, parameters=None):\n",
"def create_batch_prediction_job(\n",
" display_name,\n",
" model_name,\n",
" gcs_source_uri,\n",
" gcs_destination_output_uri_prefix,\n",
" parameters=None,\n",
"):\n",
"\n",
" if DEPLOY_GPU:\n",
" machine_spec = {\n",
@@ -1492,11 +1522,10 @@
" \"dedicated_resources\": {\n",
" \"machine_spec\": machine_spec,\n",
" \"starting_replica_count\": MIN_NODES,\n",
" \"max_replica_count\": MAX_NODES\n",
" }\n",
"\n",
" \"max_replica_count\": MAX_NODES,\n",
" },\n",
" }\n",
" response = clients['job'].create_batch_prediction_job(\n",
" response = clients[\"job\"].create_batch_prediction_job(\n",
" parent=PARENT, batch_prediction_job=batch_prediction_job\n",
" )\n",
" print(\"response\")\n",
@@ -1516,11 +1545,16 @@
" return response\n",
"\n",
"\n",
"IN_FORMAT = 'jsonl'\n",
"OUT_FORMAT = 'jsonl' # [jsonl]\n",
"IN_FORMAT = \"jsonl\"\n",
"OUT_FORMAT = \"jsonl\" # [jsonl]\n",
"\n",
"response = create_batch_prediction_job(BATCH_MODEL, model_to_deploy_id, gcs_input_uri, BUCKET_NAME,\n",
" {'confidenceThreshold': 0.5, 'maxPredictions': 2})"
"response = create_batch_prediction_job(\n",
" BATCH_MODEL,\n",
" model_to_deploy_id,\n",
" gcs_input_uri,\n",
" BUCKET_NAME,\n",
" {\"confidenceThreshold\": 0.5, \"maxPredictions\": 2},\n",
")"
]
},
{
@@ -1543,7 +1577,7 @@
"# The full unique ID for the batch job\n",
"batch_job_id = response.name\n",
"# The short numeric ID for the batch job\n",
"batch_job_short_id = batch_job_id.split('/')[-1]\n",
"batch_job_short_id = batch_job_id.split(\"/\")[-1]\n",
"\n",
"print(batch_job_id)"
]
@@ -1576,7 +1610,7 @@
"outputs": [],
"source": [
"def get_batch_prediction_job(job_name, silent=False):\n",
" response = clients['job'].get_batch_prediction_job(name=job_name)\n",
" response = clients[\"job\"].get_batch_prediction_job(name=job_name)\n",
" if silent:\n",
" return response.output_config.gcs_destination.output_uri_prefix, response.state\n",
"\n",
@@ -1584,7 +1618,7 @@
" print(\" name:\", response.name)\n",
" print(\" display_name:\", response.display_name)\n",
" print(\" model:\", response.model)\n",
" try: # not all data types support explanations\n",
" try: # not all data types support explanations\n",
" print(\" generate_explanation:\", response.generate_explanation)\n",
" except:\n",
" pass\n",
@@ -1630,11 +1664,12 @@
"outputs": [],
"source": [
"def get_latest_predictions(gcs_out_dir):\n",
" ''' Get the latest prediction subfolder using the timestamp in the subfolder name'''\n",
" folders = !gcloud storage ls $gcs_out_dir\n", " latest = \"\"\n",
" \"\"\"Get the latest prediction subfolder using the timestamp in the subfolder name\"\"\"\n",
" folders = !gcloud storage ls $gcs_out_dir\n",
" latest = \"\"\n",
" for folder in folders:\n",
" subfolder = folder.split('/')[-2]\n",
" if subfolder.startswith('prediction-'):\n",
" subfolder = folder.split(\"/\")[-2]\n",
" if subfolder.startswith(\"prediction-\"):\n",
" if subfolder > latest:\n",
" latest = folder[:-1]\n",
" return latest\n",
@@ -1648,8 +1683,10 @@
" raise Exception(\"Batch Job Failed\")\n",
" else:\n",
" folder = get_latest_predictions(predictions)\n",
" ! gcloud storage ls $folder/prediction*.jsonl\n", "\n",
" ! gcloud storage cat $folder/prediction*.jsonl\n", " break\n",
" ! gcloud storage ls $folder/prediction*.jsonl\n",
"\n",
" ! gcloud storage cat $folder/prediction*.jsonl\n",
" break\n",
" time.sleep(60)"
]
},
@@ -1695,55 +1732,56 @@
"\n",
"# Delete the dataset using the Vertex fully qualified identifier for the dataset\n",
"try:\n",
" if delete_dataset and 'dataset_id' in globals():\n",
" clients['dataset'].delete_dataset(name=dataset_id)\n",
" if delete_dataset and \"dataset_id\" in globals():\n",
" clients[\"dataset\"].delete_dataset(name=dataset_id)\n",
"except Exception as e:\n",
" print(e)\n",
"\n",
"# Delete the training pipeline using the Vertex fully qualified identifier for the pipeline\n",
"try:\n",
" if delete_pipeline and 'pipeline_id' in globals():\n",
" clients['pipeline'].delete_training_pipeline(name=pipeline_id)\n",
" if delete_pipeline and \"pipeline_id\" in globals():\n",
" clients[\"pipeline\"].delete_training_pipeline(name=pipeline_id)\n",
"except Exception as e:\n",
" print(e)\n",
"\n",
"# Delete the model using the Vertex fully qualified identifier for the model\n",
"try:\n",
" if delete_model and 'model_to_deploy_id' in globals():\n",
" clients['model'].delete_model(name=model_to_deploy_id)\n",
" if delete_model and \"model_to_deploy_id\" in globals():\n",
" clients[\"model\"].delete_model(name=model_to_deploy_id)\n",
"except Exception as e:\n",
" print(e)\n",
"\n",
"# Delete the endpoint using the Vertex fully qualified identifier for the endpoint\n",
"try:\n",
" if delete_endpoint and 'endpoint_id' in globals():\n",
" clients['endpoint'].delete_endpoint(name=endpoint_id)\n",
" if delete_endpoint and \"endpoint_id\" in globals():\n",
" clients[\"endpoint\"].delete_endpoint(name=endpoint_id)\n",
"except Exception as e:\n",
" print(e)\n",
"\n",
"# Delete the batch job using the Vertex fully qualified identifier for the batch job\n",
"try:\n",
" if delete_batchjob and 'batch_job_id' in globals():\n",
" clients['job'].delete_batch_prediction_job(name=batch_job_id)\n",
" if delete_batchjob and \"batch_job_id\" in globals():\n",
" clients[\"job\"].delete_batch_prediction_job(name=batch_job_id)\n",
"except Exception as e:\n",
" print(e)\n",
"\n",
"# Delete the custom job using the Vertex fully qualified identifier for the custom job\n",
"try:\n",
" if delete_customjob and 'job_id' in globals():\n",
" clients['job'].delete_custom_job(name=job_id)\n",
" if delete_customjob and \"job_id\" in globals():\n",
" clients[\"job\"].delete_custom_job(name=job_id)\n",
"except Exception as e:\n",
" print(e)\n",
"\n",
"# Delete the hyperparameter tuning job using the Vertex fully qualified identifier for the hyperparameter tuning job\n",
"try:\n",
" if delete_hptjob and 'hpt_job_id' in globals():\n",
" clients['job'].delete_hyperparameter_tuning_job(name=hpt_job_id)\n",
" if delete_hptjob and \"hpt_job_id\" in globals():\n",
" clients[\"job\"].delete_hyperparameter_tuning_job(name=hpt_job_id)\n",
"except Exception as e:\n",
" print(e)\n",
"\n",
"if delete_bucket and 'BUCKET_NAME' in globals():\n",
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
" ! gcloud storage rm --recursive $BUCKET_NAME"
]
}
],
"metadata": {
notebooks/community/gapic/automl/showcase_automl_video_classification_batch.ipynb
+132 -103
View File
@@ -138,9 +138,9 @@
"\n",
"# Google Cloud Notebook\n",
"if os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
" USER_FLAG = '--user'\n",
" USER_FLAG = \"--user\"\n",
"else:\n",
" USER_FLAG = ''\n",
" USER_FLAG = \"\"\n",
"\n",
"! pip3 install -U google-cloud-aiplatform $USER_FLAG"
]
@@ -187,6 +187,7 @@
"if not os.getenv(\"IS_TESTING\"):\n",
" # Automatically restart kernel after installs\n",
" import IPython\n",
"\n",
" app = IPython.Application.instance()\n",
" app.kernel.do_shutdown(True)"
]
@@ -229,7 +230,7 @@
},
"outputs": [],
"source": [
"PROJECT_ID = \"[your-project-id]\" #@param {type:\"string\"}"
"PROJECT_ID = \"[your-project-id]\" # @param {type:\"string\"}"
]
},
{
@@ -284,7 +285,7 @@
},
"outputs": [],
"source": [
"REGION = 'us-central1' #@param {type: \"string\"}"
"REGION = \"us-central1\" # @param {type: \"string\"}"
]
},
{
@@ -388,7 +389,7 @@
},
"outputs": [],
"source": [
"BUCKET_NAME = \"gs://[your-bucket-name]\" #@param {type:\"string\"}"
"BUCKET_NAME = \"gs://[your-bucket-name]\" # @param {type:\"string\"}"
]
},
{
@@ -420,7 +421,8 @@
},
"outputs": [],
"source": [
"! gcloud storage buckets create --location $REGION $BUCKET_NAME" ]
"! gcloud storage buckets create --location $REGION $BUCKET_NAME"
]
},
{
"cell_type": "markdown",
@@ -439,7 +441,8 @@
},
"outputs": [],
"source": [
"! gcloud storage ls --all-versions --long $BUCKET_NAME" ]
"! gcloud storage ls --all-versions --long $BUCKET_NAME"
]
},
{
"cell_type": "markdown",
@@ -475,9 +478,7 @@
"import time\n",
"\n",
"from google.cloud.aiplatform import gapic as aip\n",
"from google.protobuf import json_format\n",
"from google.protobuf.json_format import MessageToJson, ParseDict\n",
"from google.protobuf.struct_pb2 import Struct, Value"
"from google.protobuf import json_format"
]
},
{
@@ -533,7 +534,7 @@
"outputs": [],
"source": [
"# Video Dataset type\n",
"DATA_SCHEMA = 'gs://google-cloud-aiplatform/schema/dataset/metadata/video_1.0.0.yaml'\n",
"DATA_SCHEMA = \"gs://google-cloud-aiplatform/schema/dataset/metadata/video_1.0.0.yaml\"\n",
"# Video Labeling type\n",
"LABEL_SCHEMA = \"gs://google-cloud-aiplatform/schema/dataset/ioformat/video_classification_io_format_1.0.0.yaml\"\n",
"# Video Training task\n",
@@ -573,7 +574,10 @@
"outputs": [],
"source": [
"if os.getenv(\"IS_TESTING_DEPOLY_GPU\"):\n",
" DEPLOY_GPU, DEPLOY_NGPU = (aip.AcceleratorType.NVIDIA_TESLA_K80, int(os.getenv(\"IS_TESTING_DEPOLY_GPU\")))\n",
" DEPLOY_GPU, DEPLOY_NGPU = (\n",
" aip.AcceleratorType.NVIDIA_TESLA_K80,\n",
" int(os.getenv(\"IS_TESTING_DEPOLY_GPU\")),\n",
" )\n",
"else:\n",
" DEPLOY_GPU, DEPLOY_NGPU = (aip.AcceleratorType.NVIDIA_TESLA_K80, 1)"
]
@@ -620,11 +624,11 @@
"if os.getenv(\"IS_TESTING_DEPLOY_MACHINE\"):\n",
" MACHINE_TYPE = os.getenv(\"IS_TESTING_DEPLOY_MACHINE\")\n",
"else:\n",
" MACHINE_TYPE = 'n1-standard'\n",
" MACHINE_TYPE = \"n1-standard\"\n",
"\n",
"VCPU = '4'\n",
"DEPLOY_COMPUTE = MACHINE_TYPE + '-' + VCPU\n",
"print('Deploy machine type', DEPLOY_COMPUTE)"
"VCPU = \"4\"\n",
"DEPLOY_COMPUTE = MACHINE_TYPE + \"-\" + VCPU\n",
"print(\"Deploy machine type\", DEPLOY_COMPUTE)"
]
},
{
@@ -669,38 +673,30 @@
"\n",
"\n",
"def create_dataset_client():\n",
" client = aip.DatasetServiceClient(\n",
" client_options=client_options\n",
" )\n",
" client = aip.DatasetServiceClient(client_options=client_options)\n",
" return client\n",
"\n",
"\n",
"def create_model_client():\n",
" client = aip.ModelServiceClient(\n",
" client_options=client_options\n",
" )\n",
" client = aip.ModelServiceClient(client_options=client_options)\n",
" return client\n",
"\n",
"\n",
"def create_pipeline_client():\n",
" client = aip.PipelineServiceClient(\n",
" client_options=client_options\n",
" )\n",
" client = aip.PipelineServiceClient(client_options=client_options)\n",
" return client\n",
"\n",
"\n",
"def create_job_client():\n",
" client = aip.JobServiceClient(\n",
" client_options=client_options\n",
" )\n",
" client = aip.JobServiceClient(client_options=client_options)\n",
" return client\n",
"\n",
"\n",
"clients = {}\n",
"clients['dataset'] = create_dataset_client()\n",
"clients['model'] = create_model_client()\n",
"clients['pipeline'] = create_pipeline_client()\n",
"clients['job'] = create_job_client()\n",
"clients[\"dataset\"] = create_dataset_client()\n",
"clients[\"model\"] = create_model_client()\n",
"clients[\"pipeline\"] = create_pipeline_client()\n",
"clients[\"job\"] = create_job_client()\n",
"\n",
"for client in clients.items():\n",
" print(client)"
@@ -752,12 +748,15 @@
"source": [
"TIMEOUT = 90\n",
"\n",
"\n",
"def create_dataset(name, schema, labels=None, timeout=TIMEOUT):\n",
" start_time = time.time()\n",
" try:\n",
" dataset = aip.Dataset(display_name=name, metadata_schema_uri=schema, labels=labels)\n",
" dataset = aip.Dataset(\n",
" display_name=name, metadata_schema_uri=schema, labels=labels\n",
" )\n",
"\n",
" operation = clients['dataset'].create_dataset(parent=PARENT, dataset=dataset)\n",
" operation = clients[\"dataset\"].create_dataset(parent=PARENT, dataset=dataset)\n",
" print(\"Long running operation:\", operation.operation.name)\n",
" result = operation.result(timeout=TIMEOUT)\n",
" print(\"time:\", time.time() - start_time)\n",
@@ -799,7 +798,7 @@
"# The full unique ID for the dataset\n",
"dataset_id = result.name\n",
"# The short numeric ID for the dataset\n",
"dataset_short_id = dataset_id.split('/')[-1]\n",
"dataset_short_id = dataset_id.split(\"/\")[-1]\n",
"\n",
"print(dataset_id)"
]
@@ -854,7 +853,7 @@
},
"outputs": [],
"source": [
"IMPORT_FILE = 'gs://automl-video-demo-data/hmdb_split1_5classes_train_inf.csv'"
"IMPORT_FILE = \"gs://automl-video-demo-data/hmdb_split1_5classes_train_inf.csv\""
]
},
{
@@ -878,15 +877,17 @@
},
"outputs": [],
"source": [
"if 'IMPORT_FILES' in globals():\n",
"if \"IMPORT_FILES\" in globals():\n",
" FILE = IMPORT_FILES[0]\n",
"else:\n",
" FILE = IMPORT_FILE\n",
"\n",
"count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
"count = ! gcloud storage cat $FILE | wc -l\n",
"print(\"Number of Examples\", int(count[0]))\n",
"\n",
"print(\"First 10 rows\")\n",
"! gcloud storage cat $FILE | head" ]
"! gcloud storage cat $FILE | head"
]
},
{
"cell_type": "markdown",
@@ -919,14 +920,13 @@
"outputs": [],
"source": [
"def import_data(dataset, gcs_sources, schema):\n",
" config = [{\n",
" 'gcs_source': {'uris': gcs_sources},\n",
" 'import_schema_uri': schema\n",
" }]\n",
" config = [{\"gcs_source\": {\"uris\": gcs_sources}, \"import_schema_uri\": schema}]\n",
" print(\"dataset:\", dataset_id)\n",
" start_time = time.time()\n",
" try:\n",
" operation = clients['dataset'].import_data(name=dataset_id, import_configs=config)\n",
" operation = clients[\"dataset\"].import_data(\n",
" name=dataset_id, import_configs=config\n",
" )\n",
" print(\"Long running operation:\", operation.operation.name)\n",
"\n",
" result = operation.result()\n",
@@ -934,7 +934,14 @@
" print(\"time:\", int(time.time() - start_time), \"secs\")\n",
" print(\"error:\", operation.exception())\n",
" print(\"meta :\", operation.metadata)\n",
" print(\"after: running:\", operation.running(), \"done:\", operation.done(), \"cancelled:\", operation.cancelled())\n",
" print(\n",
" \"after: running:\",\n",
" operation.running(),\n",
" \"done:\",\n",
" operation.done(),\n",
" \"cancelled:\",\n",
" operation.cancelled(),\n",
" )\n",
"\n",
" return operation\n",
" except Exception as e:\n",
@@ -1007,15 +1014,14 @@
},
"outputs": [],
"source": [
" def create_pipeline(pipeline_name, model_name, dataset, schema, task):\n",
"def create_pipeline(pipeline_name, model_name, dataset, schema, task):\n",
"\n",
" dataset_id = dataset.split('/')[-1]\n",
" dataset_id = dataset.split(\"/\")[-1]\n",
"\n",
" input_config = {'dataset_id': dataset_id,\n",
" 'fraction_split': {\n",
" 'training_fraction': 0.8,\n",
" 'test_fraction': 0.2\n",
" }}\n",
" input_config = {\n",
" \"dataset_id\": dataset_id,\n",
" \"fraction_split\": {\"training_fraction\": 0.8, \"test_fraction\": 0.2},\n",
" }\n",
"\n",
" training_pipeline = {\n",
" \"display_name\": pipeline_name,\n",
@@ -1026,7 +1032,9 @@
" }\n",
"\n",
" try:\n",
" pipeline = clients['pipeline'].create_training_pipeline(parent=PARENT, training_pipeline=training_pipeline)\n",
" pipeline = clients[\"pipeline\"].create_training_pipeline(\n",
" parent=PARENT, training_pipeline=training_pipeline\n",
" )\n",
" print(pipeline)\n",
" except Exception as e:\n",
" print(\"exception:\", e)\n",
@@ -1085,7 +1093,7 @@
"# The full unique ID for the pipeline\n",
"pipeline_id = response.name\n",
"# The short numeric ID for the pipeline\n",
"pipeline_short_id = pipeline_id.split('/')[-1]\n",
"pipeline_short_id = pipeline_id.split(\"/\")[-1]\n",
"\n",
"print(pipeline_id)"
]
@@ -1114,7 +1122,7 @@
"outputs": [],
"source": [
"def get_training_pipeline(name, silent=False):\n",
" response = clients['pipeline'].get_training_pipeline(name=name)\n",
" response = clients[\"pipeline\"].get_training_pipeline(name=name)\n",
" if silent:\n",
" return response\n",
"\n",
@@ -1163,12 +1171,12 @@
" model_to_deploy_id = None\n",
" if response.state == aip.PipelineState.PIPELINE_STATE_FAILED:\n",
" raise Exception(\"Training Job Failed\")\n",
" time.sleep(60)\n",
" else:\n",
" model_to_deploy = response.model_to_upload\n",
" model_to_deploy_id = model_to_deploy.name\n",
" print(\"Training Time:\", response.end_time - response.start_time)\n",
" break\n",
" time.sleep(60)\n",
"\n",
"print(\"model to deploy:\", model_to_deploy_id)"
]
@@ -1221,7 +1229,7 @@
"outputs": [],
"source": [
"def list_model_evaluations(name):\n",
" response = clients['model'].list_model_evaluations(parent=name)\n",
" response = clients[\"model\"].list_model_evaluations(parent=name)\n",
" for evaluation in response:\n",
" print(\"model_evaluation\")\n",
" print(\" name:\", evaluation.name)\n",
@@ -1229,8 +1237,7 @@
" metrics = json_format.MessageToDict(evaluation._pb.metrics)\n",
" for metric in metrics.keys():\n",
" print(metric)\n",
" print('auPrc', metrics['auPrc'])\n",
"\n",
" print(\"auPrc\", metrics[\"auPrc\"])\n",
"\n",
" return evaluation.name\n",
"\n",
@@ -1297,13 +1304,14 @@
},
"outputs": [],
"source": [
"test_items = ! gcloud storage cat $IMPORT_FILE | head -n2\n", "\n",
"test_items = ! gcloud storage cat $IMPORT_FILE | head -n2\n",
"\n",
"if len(test_items[0]) == 5:\n",
" _, test_item_1, test_label_1, _, _ = str(test_items[0]).split(',')\n",
" _, test_item_2, test_label_2, _, _ = str(test_items[1]).split(',')\n",
" _, test_item_1, test_label_1, _, _ = str(test_items[0]).split(\",\")\n",
" _, test_item_2, test_label_2, _, _ = str(test_items[1]).split(\",\")\n",
"else:\n",
" test_item_1, test_label_1, _, _ = str(test_items[0]).split(',')\n",
" test_item_2, test_label_2, _, _ = str(test_items[1]).split(',')\n",
" test_item_1, test_label_1, _, _ = str(test_items[0]).split(\",\")\n",
" test_item_2, test_label_2, _, _ = str(test_items[1]).split(\",\")\n",
"\n",
"\n",
"print(test_item_1, test_label_1)\n",
@@ -1338,15 +1346,26 @@
"\n",
"import tensorflow as tf\n",
"\n",
"gcs_input_uri = BUCKET_NAME + '/test.jsonl'\n",
"with tf.io.gfile.GFile(gcs_input_uri, 'w') as f:\n",
" data = { \"content\": test_item_1, \"mimeType\": \"video/avi\", \"timeSegmentStart\": \"0.0s\", 'timeSegmentEnd': '5.0s' }\n",
" f.write(json.dumps(data) + '\\n')\n",
" data = { \"content\": test_item_2, \"mimeType\": \"video/avi\", \"timeSegmentStart\": \"0.0s\", 'timeSegmentEnd': '5.0s' }\n",
" f.write(json.dumps(data) + '\\n')\n",
"gcs_input_uri = BUCKET_NAME + \"/test.jsonl\"\n",
"with tf.io.gfile.GFile(gcs_input_uri, \"w\") as f:\n",
" data = {\n",
" \"content\": test_item_1,\n",
" \"mimeType\": \"video/avi\",\n",
" \"timeSegmentStart\": \"0.0s\",\n",
" \"timeSegmentEnd\": \"5.0s\",\n",
" }\n",
" f.write(json.dumps(data) + \"\\n\")\n",
" data = {\n",
" \"content\": test_item_2,\n",
" \"mimeType\": \"video/avi\",\n",
" \"timeSegmentStart\": \"0.0s\",\n",
" \"timeSegmentEnd\": \"5.0s\",\n",
" }\n",
" f.write(json.dumps(data) + \"\\n\")\n",
"\n",
"print(gcs_input_uri)\n",
"! gcloud storage cat $gcs_input_uri" ]
"! gcloud storage cat $gcs_input_uri"
]
},
{
"cell_type": "markdown",
@@ -1456,7 +1475,13 @@
"BATCH_MODEL = \"hmdb,tst_batch-\" + TIMESTAMP\n",
"\n",
"\n",
"def create_batch_prediction_job(display_name, model_name, gcs_source_uri, gcs_destination_output_uri_prefix, parameters=None):\n",
"def create_batch_prediction_job(\n",
" display_name,\n",
" model_name,\n",
" gcs_source_uri,\n",
" gcs_destination_output_uri_prefix,\n",
" parameters=None,\n",
"):\n",
"\n",
" if DEPLOY_GPU:\n",
" machine_spec = {\n",
@@ -1486,11 +1511,10 @@
" \"dedicated_resources\": {\n",
" \"machine_spec\": machine_spec,\n",
" \"starting_replica_count\": MIN_NODES,\n",
" \"max_replica_count\": MAX_NODES\n",
" }\n",
"\n",
" \"max_replica_count\": MAX_NODES,\n",
" },\n",
" }\n",
" response = clients['job'].create_batch_prediction_job(\n",
" response = clients[\"job\"].create_batch_prediction_job(\n",
" parent=PARENT, batch_prediction_job=batch_prediction_job\n",
" )\n",
" print(\"response\")\n",
@@ -1510,10 +1534,12 @@
" return response\n",
"\n",
"\n",
"IN_FORMAT = 'jsonl'\n",
"OUT_FORMAT = 'jsonl' # [jsonl]\n",
"IN_FORMAT = \"jsonl\"\n",
"OUT_FORMAT = \"jsonl\" # [jsonl]\n",
"\n",
"response = create_batch_prediction_job(BATCH_MODEL, model_to_deploy_id, gcs_input_uri, BUCKET_NAME, None)"
"response = create_batch_prediction_job(\n",
" BATCH_MODEL, model_to_deploy_id, gcs_input_uri, BUCKET_NAME, None\n",
")"
]
},
{
@@ -1536,7 +1562,7 @@
"# The full unique ID for the batch job\n",
"batch_job_id = response.name\n",
"# The short numeric ID for the batch job\n",
"batch_job_short_id = batch_job_id.split('/')[-1]\n",
"batch_job_short_id = batch_job_id.split(\"/\")[-1]\n",
"\n",
"print(batch_job_id)"
]
@@ -1569,7 +1595,7 @@
"outputs": [],
"source": [
"def get_batch_prediction_job(job_name, silent=False):\n",
" response = clients['job'].get_batch_prediction_job(name=job_name)\n",
" response = clients[\"job\"].get_batch_prediction_job(name=job_name)\n",
" if silent:\n",
" return response.output_config.gcs_destination.output_uri_prefix, response.state\n",
"\n",
@@ -1577,7 +1603,7 @@
" print(\" name:\", response.name)\n",
" print(\" display_name:\", response.display_name)\n",
" print(\" model:\", response.model)\n",
" try: # not all data types support explanations\n",
" try: # not all data types support explanations\n",
" print(\" generate_explanation:\", response.generate_explanation)\n",
" except:\n",
" pass\n",
@@ -1624,7 +1650,8 @@
"source": [
"def get_latest_predictions(gcs_out_dir):\n",
" ''' Get the latest prediction subfolder using the timestamp in the subfolder name'''\n",
" folders = !gcloud storage ls $gcs_out_dir\n", " latest = \"\"\n",
" folders = !gcloud storage ls $gcs_out_dir\n",
" latest = \"\"\n",
" for folder in folders:\n",
" subfolder = folder.split('/')[-2]\n",
" if subfolder.startswith('prediction-'):\n",
@@ -1639,11 +1666,12 @@
" print(\"The job has not completed:\", state)\n",
" if state == aip.JobState.JOB_STATE_FAILED:\n",
" raise Exception(\"Batch Job Failed\")\n",
" else:\n",
" time.sleep(60) else:\n",
" folder = get_latest_predictions(predictions)\n",
" ! gcloud storage ls $folder/prediction*.jsonl\n", "\n",
"! gcloud storage cat $folder/prediction*.jsonl\n", " break\n",
" time.sleep(60)"
" ! gcloud storage ls $folder/prediction*.jsonl\n",
"\n",
" ! gcloud storage cat $folder/prediction*.jsonl\n",
" break"
]
},
{
@@ -1688,55 +1716,56 @@
"\n",
"# Delete the dataset using the Vertex fully qualified identifier for the dataset\n",
"try:\n",
" if delete_dataset and 'dataset_id' in globals():\n",
" clients['dataset'].delete_dataset(name=dataset_id)\n",
" if delete_dataset and \"dataset_id\" in globals():\n",
" clients[\"dataset\"].delete_dataset(name=dataset_id)\n",
"except Exception as e:\n",
" print(e)\n",
"\n",
"# Delete the training pipeline using the Vertex fully qualified identifier for the pipeline\n",
"try:\n",
" if delete_pipeline and 'pipeline_id' in globals():\n",
" clients['pipeline'].delete_training_pipeline(name=pipeline_id)\n",
" if delete_pipeline and \"pipeline_id\" in globals():\n",
" clients[\"pipeline\"].delete_training_pipeline(name=pipeline_id)\n",
"except Exception as e:\n",
" print(e)\n",
"\n",
"# Delete the model using the Vertex fully qualified identifier for the model\n",
"try:\n",
" if delete_model and 'model_to_deploy_id' in globals():\n",
" clients['model'].delete_model(name=model_to_deploy_id)\n",
" if delete_model and \"model_to_deploy_id\" in globals():\n",
" clients[\"model\"].delete_model(name=model_to_deploy_id)\n",
"except Exception as e:\n",
" print(e)\n",
"\n",
"# Delete the endpoint using the Vertex fully qualified identifier for the endpoint\n",
"try:\n",
" if delete_endpoint and 'endpoint_id' in globals():\n",
" clients['endpoint'].delete_endpoint(name=endpoint_id)\n",
" if delete_endpoint and \"endpoint_id\" in globals():\n",
" clients[\"endpoint\"].delete_endpoint(name=endpoint_id)\n",
"except Exception as e:\n",
" print(e)\n",
"\n",
"# Delete the batch job using the Vertex fully qualified identifier for the batch job\n",
"try:\n",
" if delete_batchjob and 'batch_job_id' in globals():\n",
" clients['job'].delete_batch_prediction_job(name=batch_job_id)\n",
" if delete_batchjob and \"batch_job_id\" in globals():\n",
" clients[\"job\"].delete_batch_prediction_job(name=batch_job_id)\n",
"except Exception as e:\n",
" print(e)\n",
"\n",
"# Delete the custom job using the Vertex fully qualified identifier for the custom job\n",
"try:\n",
" if delete_customjob and 'job_id' in globals():\n",
" clients['job'].delete_custom_job(name=job_id)\n",
" if delete_customjob and \"job_id\" in globals():\n",
" clients[\"job\"].delete_custom_job(name=job_id)\n",
"except Exception as e:\n",
" print(e)\n",
"\n",
"# Delete the hyperparameter tuning job using the Vertex fully qualified identifier for the hyperparameter tuning job\n",
"try:\n",
" if delete_hptjob and 'hpt_job_id' in globals():\n",
" clients['job'].delete_hyperparameter_tuning_job(name=hpt_job_id)\n",
" if delete_hptjob and \"hpt_job_id\" in globals():\n",
" clients[\"job\"].delete_hyperparameter_tuning_job(name=hpt_job_id)\n",
"except Exception as e:\n",
" print(e)\n",
"\n",
"if delete_bucket and 'BUCKET_NAME' in globals():\n",
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
" ! gcloud storage rm --recursive $BUCKET_NAME"
]
}
],
"metadata": {
notebooks/community/gapic/automl/showcase_automl_video_object_tracking_batch.ipynb
+141 -104
View File
@@ -138,9 +138,9 @@
"\n",
"# Google Cloud Notebook\n",
"if os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
" USER_FLAG = '--user'\n",
" USER_FLAG = \"--user\"\n",
"else:\n",
" USER_FLAG = ''\n",
" USER_FLAG = \"\"\n",
"\n",
"! pip3 install -U google-cloud-aiplatform $USER_FLAG"
]
@@ -187,6 +187,7 @@
"if not os.getenv(\"IS_TESTING\"):\n",
" # Automatically restart kernel after installs\n",
" import IPython\n",
"\n",
" app = IPython.Application.instance()\n",
" app.kernel.do_shutdown(True)"
]
@@ -229,7 +230,7 @@
},
"outputs": [],
"source": [
"PROJECT_ID = \"[your-project-id]\" #@param {type:\"string\"}"
"PROJECT_ID = \"[your-project-id]\" # @param {type:\"string\"}"
]
},
{
@@ -284,7 +285,7 @@
},
"outputs": [],
"source": [
"REGION = 'us-central1' #@param {type: \"string\"}"
"REGION = \"us-central1\" # @param {type: \"string\"}"
]
},
{
@@ -388,7 +389,7 @@
},
"outputs": [],
"source": [
"BUCKET_NAME = \"gs://[your-bucket-name]\" #@param {type:\"string\"}"
"BUCKET_NAME = \"gs://[your-bucket-name]\" # @param {type:\"string\"}"
]
},
{
@@ -420,7 +421,8 @@
},
"outputs": [],
"source": [
"! gcloud storage buckets create --location $REGION $BUCKET_NAME" ]
"! gcloud storage buckets create --location $REGION $BUCKET_NAME"
]
},
{
"cell_type": "markdown",
@@ -439,7 +441,8 @@
},
"outputs": [],
"source": [
"! gcloud storage ls --all-versions --long $BUCKET_NAME" ]
"! gcloud storage ls --all-versions --long $BUCKET_NAME"
]
},
{
"cell_type": "markdown",
@@ -475,9 +478,7 @@
"import time\n",
"\n",
"from google.cloud.aiplatform import gapic as aip\n",
"from google.protobuf import json_format\n",
"from google.protobuf.json_format import MessageToJson, ParseDict\n",
"from google.protobuf.struct_pb2 import Struct, Value"
"from google.protobuf import json_format"
]
},
{
@@ -533,7 +534,7 @@
"outputs": [],
"source": [
"# Video Dataset type\n",
"DATA_SCHEMA = 'gs://google-cloud-aiplatform/schema/dataset/metadata/video_1.0.0.yaml'\n",
"DATA_SCHEMA = \"gs://google-cloud-aiplatform/schema/dataset/metadata/video_1.0.0.yaml\"\n",
"# Video Labeling type\n",
"LABEL_SCHEMA = \"gs://google-cloud-aiplatform/schema/dataset/ioformat/video_object_tracking_io_format_1.0.0.yaml\"\n",
"# Video Training task\n",
@@ -573,7 +574,10 @@
"outputs": [],
"source": [
"if os.getenv(\"IS_TESTING_DEPOLY_GPU\"):\n",
" DEPLOY_GPU, DEPLOY_NGPU = (aip.AcceleratorType.NVIDIA_TESLA_K80, int(os.getenv(\"IS_TESTING_DEPOLY_GPU\")))\n",
" DEPLOY_GPU, DEPLOY_NGPU = (\n",
" aip.AcceleratorType.NVIDIA_TESLA_K80,\n",
" int(os.getenv(\"IS_TESTING_DEPOLY_GPU\")),\n",
" )\n",
"else:\n",
" DEPLOY_GPU, DEPLOY_NGPU = (aip.AcceleratorType.NVIDIA_TESLA_K80, 1)"
]
@@ -620,11 +624,11 @@
"if os.getenv(\"IS_TESTING_DEPLOY_MACHINE\"):\n",
" MACHINE_TYPE = os.getenv(\"IS_TESTING_DEPLOY_MACHINE\")\n",
"else:\n",
" MACHINE_TYPE = 'n1-standard'\n",
" MACHINE_TYPE = \"n1-standard\"\n",
"\n",
"VCPU = '4'\n",
"DEPLOY_COMPUTE = MACHINE_TYPE + '-' + VCPU\n",
"print('Deploy machine type', DEPLOY_COMPUTE)"
"VCPU = \"4\"\n",
"DEPLOY_COMPUTE = MACHINE_TYPE + \"-\" + VCPU\n",
"print(\"Deploy machine type\", DEPLOY_COMPUTE)"
]
},
{
@@ -669,38 +673,30 @@
"\n",
"\n",
"def create_dataset_client():\n",
" client = aip.DatasetServiceClient(\n",
" client_options=client_options\n",
" )\n",
" client = aip.DatasetServiceClient(client_options=client_options)\n",
" return client\n",
"\n",
"\n",
"def create_model_client():\n",
" client = aip.ModelServiceClient(\n",
" client_options=client_options\n",
" )\n",
" client = aip.ModelServiceClient(client_options=client_options)\n",
" return client\n",
"\n",
"\n",
"def create_pipeline_client():\n",
" client = aip.PipelineServiceClient(\n",
" client_options=client_options\n",
" )\n",
" client = aip.PipelineServiceClient(client_options=client_options)\n",
" return client\n",
"\n",
"\n",
"def create_job_client():\n",
" client = aip.JobServiceClient(\n",
" client_options=client_options\n",
" )\n",
" client = aip.JobServiceClient(client_options=client_options)\n",
" return client\n",
"\n",
"\n",
"clients = {}\n",
"clients['dataset'] = create_dataset_client()\n",
"clients['model'] = create_model_client()\n",
"clients['pipeline'] = create_pipeline_client()\n",
"clients['job'] = create_job_client()\n",
"clients[\"dataset\"] = create_dataset_client()\n",
"clients[\"model\"] = create_model_client()\n",
"clients[\"pipeline\"] = create_pipeline_client()\n",
"clients[\"job\"] = create_job_client()\n",
"\n",
"for client in clients.items():\n",
" print(client)"
@@ -752,12 +748,15 @@
"source": [
"TIMEOUT = 90\n",
"\n",
"\n",
"def create_dataset(name, schema, labels=None, timeout=TIMEOUT):\n",
" start_time = time.time()\n",
" try:\n",
" dataset = aip.Dataset(display_name=name, metadata_schema_uri=schema, labels=labels)\n",
" dataset = aip.Dataset(\n",
" display_name=name, metadata_schema_uri=schema, labels=labels\n",
" )\n",
"\n",
" operation = clients['dataset'].create_dataset(parent=PARENT, dataset=dataset)\n",
" operation = clients[\"dataset\"].create_dataset(parent=PARENT, dataset=dataset)\n",
" print(\"Long running operation:\", operation.operation.name)\n",
" result = operation.result(timeout=TIMEOUT)\n",
" print(\"time:\", time.time() - start_time)\n",
@@ -799,7 +798,7 @@
"# The full unique ID for the dataset\n",
"dataset_id = result.name\n",
"# The short numeric ID for the dataset\n",
"dataset_short_id = dataset_id.split('/')[-1]\n",
"dataset_short_id = dataset_id.split(\"/\")[-1]\n",
"\n",
"print(dataset_id)"
]
@@ -859,7 +858,7 @@
},
"outputs": [],
"source": [
"IMPORT_FILE = 'gs://automl-video-demo-data/traffic_videos/traffic_videos_labels.csv'"
"IMPORT_FILE = \"gs://automl-video-demo-data/traffic_videos/traffic_videos_labels.csv\""
]
},
{
@@ -883,15 +882,17 @@
},
"outputs": [],
"source": [
"if 'IMPORT_FILES' in globals():\n",
"if \"IMPORT_FILES\" in globals():\n",
" FILE = IMPORT_FILES[0]\n",
"else:\n",
" FILE = IMPORT_FILE\n",
"\n",
"count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
"count = ! gcloud storage cat $FILE | wc -l\n",
"print(\"Number of Examples\", int(count[0]))\n",
"\n",
"print(\"First 10 rows\")\n",
"! gcloud storage cat $FILE | head" ]
"! gcloud storage cat $FILE | head"
]
},
{
"cell_type": "markdown",
@@ -924,14 +925,13 @@
"outputs": [],
"source": [
"def import_data(dataset, gcs_sources, schema):\n",
" config = [{\n",
" 'gcs_source': {'uris': gcs_sources},\n",
" 'import_schema_uri': schema\n",
" }]\n",
" config = [{\"gcs_source\": {\"uris\": gcs_sources}, \"import_schema_uri\": schema}]\n",
" print(\"dataset:\", dataset_id)\n",
" start_time = time.time()\n",
" try:\n",
" operation = clients['dataset'].import_data(name=dataset_id, import_configs=config)\n",
" operation = clients[\"dataset\"].import_data(\n",
" name=dataset_id, import_configs=config\n",
" )\n",
" print(\"Long running operation:\", operation.operation.name)\n",
"\n",
" result = operation.result()\n",
@@ -939,7 +939,14 @@
" print(\"time:\", int(time.time() - start_time), \"secs\")\n",
" print(\"error:\", operation.exception())\n",
" print(\"meta :\", operation.metadata)\n",
" print(\"after: running:\", operation.running(), \"done:\", operation.done(), \"cancelled:\", operation.cancelled())\n",
" print(\n",
" \"after: running:\",\n",
" operation.running(),\n",
" \"done:\",\n",
" operation.done(),\n",
" \"cancelled:\",\n",
" operation.cancelled(),\n",
" )\n",
"\n",
" return operation\n",
" except Exception as e:\n",
@@ -1012,15 +1019,14 @@
},
"outputs": [],
"source": [
" def create_pipeline(pipeline_name, model_name, dataset, schema, task):\n",
"def create_pipeline(pipeline_name, model_name, dataset, schema, task):\n",
"\n",
" dataset_id = dataset.split('/')[-1]\n",
" dataset_id = dataset.split(\"/\")[-1]\n",
"\n",
" input_config = {'dataset_id': dataset_id,\n",
" 'fraction_split': {\n",
" 'training_fraction': 0.8,\n",
" 'test_fraction': 0.2\n",
" }}\n",
" input_config = {\n",
" \"dataset_id\": dataset_id,\n",
" \"fraction_split\": {\"training_fraction\": 0.8, \"test_fraction\": 0.2},\n",
" }\n",
"\n",
" training_pipeline = {\n",
" \"display_name\": pipeline_name,\n",
@@ -1031,7 +1037,9 @@
" }\n",
"\n",
" try:\n",
" pipeline = clients['pipeline'].create_training_pipeline(parent=PARENT, training_pipeline=training_pipeline)\n",
" pipeline = clients[\"pipeline\"].create_training_pipeline(\n",
" parent=PARENT, training_pipeline=training_pipeline\n",
" )\n",
" print(pipeline)\n",
" except Exception as e:\n",
" print(\"exception:\", e)\n",
@@ -1067,8 +1075,12 @@
"PIPE_NAME = \"traffic_pipe-\" + TIMESTAMP\n",
"MODEL_NAME = \"traffic_model-\" + TIMESTAMP\n",
"\n",
"task = json_format.ParseDict({'model_type': \"CLOUD\",\n",
" }, Value())\n",
"task = json_format.ParseDict(\n",
" {\n",
" \"model_type\": \"CLOUD\",\n",
" },\n",
" Value(),\n",
")\n",
"\n",
"response = create_pipeline(PIPE_NAME, MODEL_NAME, dataset_id, TRAINING_SCHEMA, task)"
]
@@ -1093,7 +1105,7 @@
"# The full unique ID for the pipeline\n",
"pipeline_id = response.name\n",
"# The short numeric ID for the pipeline\n",
"pipeline_short_id = pipeline_id.split('/')[-1]\n",
"pipeline_short_id = pipeline_id.split(\"/\")[-1]\n",
"\n",
"print(pipeline_id)"
]
@@ -1122,7 +1134,7 @@
"outputs": [],
"source": [
"def get_training_pipeline(name, silent=False):\n",
" response = clients['pipeline'].get_training_pipeline(name=name)\n",
" response = clients[\"pipeline\"].get_training_pipeline(name=name)\n",
" if silent:\n",
" return response\n",
"\n",
@@ -1229,7 +1241,7 @@
"outputs": [],
"source": [
"def list_model_evaluations(name):\n",
" response = clients['model'].list_model_evaluations(parent=name)\n",
" response = clients[\"model\"].list_model_evaluations(parent=name)\n",
" for evaluation in response:\n",
" print(\"model_evaluation\")\n",
" print(\" name:\", evaluation.name)\n",
@@ -1237,8 +1249,7 @@
" metrics = json_format.MessageToDict(evaluation._pb.metrics)\n",
" for metric in metrics.keys():\n",
" print(metric)\n",
" print('boundingBoxMetrics', metrics['boundingBoxMetrics'])\n",
"\n",
" print(\"boundingBoxMetrics\", metrics[\"boundingBoxMetrics\"])\n",
"\n",
" return evaluation.name\n",
"\n",
@@ -1305,9 +1316,10 @@
},
"outputs": [],
"source": [
"test_items = ! gcloud storage cat $IMPORT_FILE | head -n2\n", "\n",
"cols_1 = test_items[0].split(',')\n",
"cols_2 = test_items[1].split(',')\n",
"test_items = ! gcloud storage cat $IMPORT_FILE | head -n2\n",
"\n",
"cols_1 = test_items[0].split(\",\")\n",
"cols_2 = test_items[1].split(\",\")\n",
"if len(cols_1) > 12:\n",
" test_item_1 = str(cols_1[1])\n",
" test_item_2 = str(cols_2[1])\n",
@@ -1352,15 +1364,26 @@
"\n",
"import tensorflow as tf\n",
"\n",
"gcs_input_uri = BUCKET_NAME + '/test.jsonl'\n",
"with tf.io.gfile.GFile(gcs_input_uri, 'w') as f:\n",
" data = { \"content\": test_item_1, \"mimeType\": \"video/avi\", \"timeSegmentStart\": \"0.0s\", 'timeSegmentEnd': '5.0s' }\n",
" f.write(json.dumps(data) + '\\n')\n",
" data = { \"content\": test_item_2, \"mimeType\": \"video/avi\", \"timeSegmentStart\": \"0.0s\", 'timeSegmentEnd': '5.0s' }\n",
" f.write(json.dumps(data) + '\\n')\n",
"gcs_input_uri = BUCKET_NAME + \"/test.jsonl\"\n",
"with tf.io.gfile.GFile(gcs_input_uri, \"w\") as f:\n",
" data = {\n",
" \"content\": test_item_1,\n",
" \"mimeType\": \"video/avi\",\n",
" \"timeSegmentStart\": \"0.0s\",\n",
" \"timeSegmentEnd\": \"5.0s\",\n",
" }\n",
" f.write(json.dumps(data) + \"\\n\")\n",
" data = {\n",
" \"content\": test_item_2,\n",
" \"mimeType\": \"video/avi\",\n",
" \"timeSegmentStart\": \"0.0s\",\n",
" \"timeSegmentEnd\": \"5.0s\",\n",
" }\n",
" f.write(json.dumps(data) + \"\\n\")\n",
"\n",
"print(gcs_input_uri)\n",
"! gcloud storage cat $gcs_input_uri" ]
"! gcloud storage cat $gcs_input_uri"
]
},
{
"cell_type": "markdown",
@@ -1468,7 +1491,13 @@
"BATCH_MODEL = \"traffic_batch-\" + TIMESTAMP\n",
"\n",
"\n",
"def create_batch_prediction_job(display_name, model_name, gcs_source_uri, gcs_destination_output_uri_prefix, parameters=None):\n",
"def create_batch_prediction_job(\n",
" display_name,\n",
" model_name,\n",
" gcs_source_uri,\n",
" gcs_destination_output_uri_prefix,\n",
" parameters=None,\n",
"):\n",
"\n",
" if DEPLOY_GPU:\n",
" machine_spec = {\n",
@@ -1498,11 +1527,10 @@
" \"dedicated_resources\": {\n",
" \"machine_spec\": machine_spec,\n",
" \"starting_replica_count\": MIN_NODES,\n",
" \"max_replica_count\": MAX_NODES\n",
" }\n",
"\n",
" \"max_replica_count\": MAX_NODES,\n",
" },\n",
" }\n",
" response = clients['job'].create_batch_prediction_job(\n",
" response = clients[\"job\"].create_batch_prediction_job(\n",
" parent=PARENT, batch_prediction_job=batch_prediction_job\n",
" )\n",
" print(\"response\")\n",
@@ -1522,11 +1550,16 @@
" return response\n",
"\n",
"\n",
"IN_FORMAT = 'jsonl'\n",
"OUT_FORMAT = 'jsonl' # [jsonl]\n",
"IN_FORMAT = \"jsonl\"\n",
"OUT_FORMAT = \"jsonl\" # [jsonl]\n",
"\n",
"response = create_batch_prediction_job(BATCH_MODEL, model_to_deploy_id, gcs_input_uri, BUCKET_NAME,\n",
" {'confidenceThreshold': 0.5, 'maxPredictions': 2})"
"response = create_batch_prediction_job(\n",
" BATCH_MODEL,\n",
" model_to_deploy_id,\n",
" gcs_input_uri,\n",
" BUCKET_NAME,\n",
" {\"confidenceThreshold\": 0.5, \"maxPredictions\": 2},\n",
")"
]
},
{
@@ -1549,7 +1582,7 @@
"# The full unique ID for the batch job\n",
"batch_job_id = response.name\n",
"# The short numeric ID for the batch job\n",
"batch_job_short_id = batch_job_id.split('/')[-1]\n",
"batch_job_short_id = batch_job_id.split(\"/\")[-1]\n",
"\n",
"print(batch_job_id)"
]
@@ -1582,7 +1615,7 @@
"outputs": [],
"source": [
"def get_batch_prediction_job(job_name, silent=False):\n",
" response = clients['job'].get_batch_prediction_job(name=job_name)\n",
" response = clients[\"job\"].get_batch_prediction_job(name=job_name)\n",
" if silent:\n",
" return response.output_config.gcs_destination.output_uri_prefix, response.state\n",
"\n",
@@ -1590,7 +1623,7 @@
" print(\" name:\", response.name)\n",
" print(\" display_name:\", response.display_name)\n",
" print(\" model:\", response.model)\n",
" try: # not all data types support explanations\n",
" try: # not all data types support explanations\n",
" print(\" generate_explanation:\", response.generate_explanation)\n",
" except:\n",
" pass\n",
@@ -1631,11 +1664,12 @@
"outputs": [],
"source": [
"def get_latest_predictions(gcs_out_dir):\n",
" ''' Get the latest prediction subfolder using the timestamp in the subfolder name'''\n",
" folders = !gcloud storage ls $gcs_out_dir\n", " latest = \"\"\n",
" \"\"\"Get the latest prediction subfolder using the timestamp in the subfolder name\"\"\"\n",
" folders = !gcloud storage ls $gcs_out_dir\n",
" latest = \"\"\n",
" for folder in folders:\n",
" subfolder = folder.split('/')[-2]\n",
" if subfolder.startswith('prediction-'):\n",
" subfolder = folder.split(\"/\")[-2]\n",
" if subfolder.startswith(\"prediction-\"):\n",
" if subfolder > latest:\n",
" latest = folder[:-1]\n",
" return latest\n",
@@ -1649,8 +1683,10 @@
" raise Exception(\"Batch Job Failed\")\n",
" else:\n",
" folder = get_latest_predictions(predictions)\n",
" ! gcloud storage ls $folder/prediction*.jsonl\n", "\n",
" ! gcloud storage cat $folder/prediction*.jsonl\n", " break\n",
" ! gcloud storage ls $folder/prediction*.jsonl\n",
"\n",
" ! gcloud storage cat $folder/prediction*.jsonl\n",
" break\n",
" time.sleep(60)"
]
},
@@ -1696,55 +1732,56 @@
"\n",
"# Delete the dataset using the Vertex fully qualified identifier for the dataset\n",
"try:\n",
" if delete_dataset and 'dataset_id' in globals():\n",
" clients['dataset'].delete_dataset(name=dataset_id)\n",
" if delete_dataset and \"dataset_id\" in globals():\n",
" clients[\"dataset\"].delete_dataset(name=dataset_id)\n",
"except Exception as e:\n",
" print(e)\n",
"\n",
"# Delete the training pipeline using the Vertex fully qualified identifier for the pipeline\n",
"try:\n",
" if delete_pipeline and 'pipeline_id' in globals():\n",
" clients['pipeline'].delete_training_pipeline(name=pipeline_id)\n",
" if delete_pipeline and \"pipeline_id\" in globals():\n",
" clients[\"pipeline\"].delete_training_pipeline(name=pipeline_id)\n",
"except Exception as e:\n",
" print(e)\n",
"\n",
"# Delete the model using the Vertex fully qualified identifier for the model\n",
"try:\n",
" if delete_model and 'model_to_deploy_id' in globals():\n",
" clients['model'].delete_model(name=model_to_deploy_id)\n",
" if delete_model and \"model_to_deploy_id\" in globals():\n",
" clients[\"model\"].delete_model(name=model_to_deploy_id)\n",
"except Exception as e:\n",
" print(e)\n",
"\n",
"# Delete the endpoint using the Vertex fully qualified identifier for the endpoint\n",
"try:\n",
" if delete_endpoint and 'endpoint_id' in globals():\n",
" clients['endpoint'].delete_endpoint(name=endpoint_id)\n",
" if delete_endpoint and \"endpoint_id\" in globals():\n",
" clients[\"endpoint\"].delete_endpoint(name=endpoint_id)\n",
"except Exception as e:\n",
" print(e)\n",
"\n",
"# Delete the batch job using the Vertex fully qualified identifier for the batch job\n",
"try:\n",
" if delete_batchjob and 'batch_job_id' in globals():\n",
" clients['job'].delete_batch_prediction_job(name=batch_job_id)\n",
" if delete_batchjob and \"batch_job_id\" in globals():\n",
" clients[\"job\"].delete_batch_prediction_job(name=batch_job_id)\n",
"except Exception as e:\n",
" print(e)\n",
"\n",
"# Delete the custom job using the Vertex fully qualified identifier for the custom job\n",
"try:\n",
" if delete_customjob and 'job_id' in globals():\n",
" clients['job'].delete_custom_job(name=job_id)\n",
" if delete_customjob and \"job_id\" in globals():\n",
" clients[\"job\"].delete_custom_job(name=job_id)\n",
"except Exception as e:\n",
" print(e)\n",
"\n",
"# Delete the hyperparameter tuning job using the Vertex fully qualified identifier for the hyperparameter tuning job\n",
"try:\n",
" if delete_hptjob and 'hpt_job_id' in globals():\n",
" clients['job'].delete_hyperparameter_tuning_job(name=hpt_job_id)\n",
" if delete_hptjob and \"hpt_job_id\" in globals():\n",
" clients[\"job\"].delete_hyperparameter_tuning_job(name=hpt_job_id)\n",
"except Exception as e:\n",
" print(e)\n",
"\n",
"if delete_bucket and 'BUCKET_NAME' in globals():\n",
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
" ! gcloud storage rm --recursive $BUCKET_NAME"
]
}
],
"metadata": {
notebooks/community/migration/UJ2,12 legacy Custom Training Prebuilt Container TF Keras.ipynb
+15 -12
View File
@@ -308,7 +308,8 @@
},
"outputs": [],
"source": [
"! gcloud storage buckets create --location=$REGION gs://$BUCKET_NAME" ]
"! gcloud storage buckets create --location=$REGION gs://$BUCKET_NAME"
]
},
{
"cell_type": "markdown",
@@ -327,7 +328,8 @@
},
"outputs": [],
"source": [
"! gcloud storage ls --all-versions --long gs://$BUCKET_NAME" ]
"! gcloud storage ls --all-versions --long gs://$BUCKET_NAME"
]
},
{
"cell_type": "markdown",
@@ -350,11 +352,7 @@
"outputs": [],
"source": [
"import json\n",
"import time\n",
"\n",
"from google.protobuf.json_format import MessageToJson\n",
"from google.protobuf.struct_pb2 import Value\n",
"from googleapiclient import discovery, errors"
"import time"
]
},
{
@@ -584,7 +582,8 @@
"! rm -f cifar.tar cifar.tar.gz\n",
"! tar cvf cifar.tar cifar\n",
"! gzip cifar.tar\n",
"! gcloud storage cp cifar.tar.gz gs://$BUCKET_NAME/trainer_cifar.tar.gz" ]
"! gcloud storage cp cifar.tar.gz gs://$BUCKET_NAME/trainer_cifar.tar.gz"
]
},
{
"cell_type": "markdown",
@@ -1068,7 +1067,8 @@
" b64str = base64.b64encode(bytes.numpy()).decode(\"utf-8\")\n",
" f.write(json.dumps({\"key\": img, input_name: {\"b64\": b64str}}) + \"\\n\")\n",
"\n",
"! gcloud storage cat $gcs_input_uri" ]
"! gcloud storage cat $gcs_input_uri"
]
},
{
"cell_type": "markdown",
@@ -1357,8 +1357,10 @@
" break\n",
" else:\n",
" folder = response[\"predictionInput\"][\"outputPath\"][:-1]\n",
" ! gcloud storage ls $folder/prediction*\n", "\n",
" ! gcloud storage cat $folder/prediction*\n", " break\n",
" ! gcloud storage ls $folder/prediction*\n",
"\n",
" ! gcloud storage cat $folder/prediction*\n",
" break\n",
" time.sleep(60)"
]
},
@@ -2040,7 +2042,8 @@
" print(e)\n",
"\n",
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
" ! gcloud storage rm --recursive gs://$BUCKET_NAME" ]
" ! gcloud storage rm --recursive gs://$BUCKET_NAME"
]
}
],
"metadata": {
notebooks/community/ml_ops/stage7/get_started_with_model_monitoring_custom_tf_serving.ipynb
+10 -5
View File
@@ -561,7 +561,8 @@
},
"outputs": [],
"source": [
"! gcloud storage buckets create --location=$REGION $BUCKET_URI" ]
"! gcloud storage buckets create --location=$REGION $BUCKET_URI"
]
},
{
"cell_type": "markdown",
@@ -580,7 +581,8 @@
},
"outputs": [],
"source": [
"! gcloud storage ls --all-versions --long $BUCKET_URI" ]
"! gcloud storage ls --all-versions --long $BUCKET_URI"
]
},
{
"cell_type": "markdown",
@@ -974,7 +976,8 @@
"MODEL_ARTIFACT_URI = \"gs://mco-mm/churn\"\n",
"MODEL_DIR = BUCKET_URI + \"/model/1\"\n",
"\n",
"! gcloud storage cp --recursive $MODEL_ARTIFACT_URI $MODEL_DIR" ]
"! gcloud storage cp --recursive $MODEL_ARTIFACT_URI $MODEL_DIR"
]
},
{
"cell_type": "markdown",
@@ -1358,7 +1361,8 @@
"with open(\"schema.yaml\", \"w\") as f:\n",
" f.write(yaml)\n",
"\n",
"! gcloud storage cp schema.yaml {BUCKET_URI}/schema.yaml" ]
"! gcloud storage cp schema.yaml {BUCKET_URI}/schema.yaml"
]
},
{
"cell_type": "markdown",
@@ -1845,7 +1849,8 @@
"delete_bucket = False\n",
"\n",
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
" ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}\n", "\n",
" ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}\n",
"\n",
"! rm -f schema.yaml\n",
"\n",
"! bq rm -f {PROJECT_ID}.model_deployment_monitoring_{ENDPOINT_ID}"
notebooks/community/model_garden/model_garden_axolotl_qwen3_finetuning.ipynb
+14 -7
View File
@@ -343,9 +343,11 @@
"if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n",
" BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n",
" BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n",
" ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n",
" ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n",
"else:\n",
" assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n",
" shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n",
" shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n",
" bucket_region = shell_output[0].strip().lower()\n",
" if bucket_region != REGION:\n",
" raise ValueError(\n",
" \"Bucket region %s is different from notebook region %s\"\n",
@@ -365,10 +367,12 @@
"shell_output = ! gcloud projects describe $PROJECT_ID\n",
"project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n",
"SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n",
"print(\"Using this default Service Account:\", SERVICE_ACCOUNT)\n","\n",
"print(\"Using this default Service Account:\", SERVICE_ACCOUNT)\n",
"\n",
"\n",
"# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n",
"! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n",
"! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n",
"\n",
"! gcloud config set project $PROJECT_ID\n",
"! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n",
"! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\""
@@ -709,7 +713,8 @@
"! accelerate launch -m axolotl.cli.train $axolotl_args $local_config_path\n",
"\n",
"# @markdown 4. Check the output in the bucket.\n",
"! gcloud storage ls $AXOLOTL_OUTPUT_GCS_URI" ]
"! gcloud storage ls $AXOLOTL_OUTPUT_GCS_URI"
]
},
{
"cell_type": "code",
@@ -847,7 +852,8 @@
"vertex_ai_config_path = AXOLOTL_CONFIG_PATH\n",
"# Copy the config file to the bucket.\n",
"if AXOLOTL_SOURCE == \"LOCAL\":\n",
" ! gcloud storage cp $AXOLOTL_CONFIG_PATH $MODEL_BUCKET/config/\n", " vertex_ai_config_path = f\"{common_util.gcs_fuse_path(MODEL_BUCKET)}/config/{pathlib.Path(AXOLOTL_CONFIG_PATH).name}\"\n",
" ! gcloud storage cp $AXOLOTL_CONFIG_PATH $MODEL_BUCKET/config/\n",
" vertex_ai_config_path = f\"{common_util.gcs_fuse_path(MODEL_BUCKET)}/config/{pathlib.Path(AXOLOTL_CONFIG_PATH).name}\"\n",
"\n",
"job_name = common_util.get_job_name_with_datetime(\"axolotl-train\")\n",
"AXOLOTL_OUTPUT_GCS_URI = f\"{BASE_AXOLOTL_OUTPUT_GCS_URI}/{job_name}\"\n",
@@ -1375,7 +1381,8 @@
"\n",
"delete_bucket = False # @param {type:\"boolean\"}\n",
"if delete_bucket:\n",
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
" ! gcloud storage rm --recursive $BUCKET_NAME"
]
}
],
"metadata": {
notebooks/community/persistent_resource/training_code/mpg_container/trainer/train.py
-3
View File
@@ -14,10 +14,7 @@
import argparse
import logging
import os
import numpy as np
import pandas as pd
import pathlib
import tensorflow as tf
from tensorflow import keras
notebooks/community/vertex_endpoints/optimized_tensorflow_runtime/bert_optimized_online_prediction.ipynb
+10 -5
View File
@@ -493,7 +493,8 @@
},
"outputs": [],
"source": [
"! gcloud storage buckets create --location=$REGION --project=$PROJECT_ID $BUCKET_URI" ]
"! gcloud storage buckets create --location=$REGION --project=$PROJECT_ID $BUCKET_URI"
]
},
{
"cell_type": "markdown",
@@ -512,7 +513,8 @@
},
"outputs": [],
"source": [
"! gcloud storage ls --all-versions --long $BUCKET_URI" ]
"! gcloud storage ls --all-versions --long $BUCKET_URI"
]
},
{
"cell_type": "markdown",
@@ -1146,7 +1148,8 @@
},
"outputs": [],
"source": [
"!gcloud storage rm --recursive $BUCKET_URI/*" ]
"!gcloud storage rm --recursive $BUCKET_URI/*"
]
},
{
"cell_type": "code",
@@ -1156,7 +1159,8 @@
},
"outputs": [],
"source": [
"!gcloud storage cp --recursive $LOCAL_DIRECTORY_FULL/* $BUCKET_URI" ]
"!gcloud storage cp --recursive $LOCAL_DIRECTORY_FULL/* $BUCKET_URI"
]
},
{
"cell_type": "markdown",
@@ -2100,7 +2104,8 @@
"# Set this to true only if you'd like to delete your bucket\n",
"delete_bucket = False\n",
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
" !gcloud storage rm --recursive $BUCKET_URI" ]
" !gcloud storage rm --recursive $BUCKET_URI"
]
}
],
"metadata": {
notebooks/official/training/sdk_pytorch_torchrun_custom_container_training_imagenet.ipynb
+8 -4
View File
@@ -280,7 +280,8 @@
},
"outputs": [],
"source": [
"! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" ]
"! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}"
]
},
{
"cell_type": "markdown",
@@ -689,12 +690,14 @@
" HOST_IP=$(hostname -i)\n",
" echo \"HOST_IP=\"$HOST_IP\n",
" echo \"Writing host IP address to \"$gcsfilepath\n",
" echo $HOST_IP| gcloud storage cp - $gcsfilepath\n", " setup_etcd $HOST_IP\n",
" echo $HOST_IP| gcloud storage cp - $gcsfilepath\n",
" setup_etcd $HOST_IP\n",
"else\n",
" echo \"Wait 60s for the host server to come online\"\n",
" sleep 60\n",
" echo \"reading host IP address from \"$gcsfilepath\n",
" HOST_IP=$(gcloud storage cat $gcsfilepath)\n", " echo \"HOST_IP=\"$HOST_IP\n",
" HOST_IP=$(gcloud storage cat $gcsfilepath)\n",
" echo \"HOST_IP=\"$HOST_IP\n",
"fi\n",
"\n",
"env=\"env://\"\n",
@@ -1809,7 +1812,8 @@
"# Delete Cloud Storage objects that were created\n",
"delete_bucket = False\n",
"if delete_bucket:\n",
" ! gcloud storage rm --recursive $BUCKET_URI" ]
" ! gcloud storage rm --recursive $BUCKET_URI"
]
}
],
"metadata": {