mirror of
https://github.com/GoogleCloudPlatform/vertex-ai-samples.git
synced 2026-09-26 14:42:04 +00:00
AutoML Video Classificaton (#871)
* changes according to andrew review comments * changes according to andrew review comments Co-authored-by: Andrew Ferlitsch <aferlitsch@google.com>
This commit is contained in:
udaypunna
co-authored by
Andrew Ferlitsch
Andrew Ferlitsch
parent
80770188ec
commit
9c10899db8
@@ -32,17 +32,24 @@
|
||||
"# Vertex AI: Vertex AI Migration: AutoML Video Classificaton\n",
|
||||
"\n",
|
||||
"<table align=\"left\">\n",
|
||||
"\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/ai-platform-samples/blob/master/vertex-ai-samples/tree/master/notebooks/official/migration/UJ14%20Vertex%20SDK%20AutoML%20Video%20Classification.ipynb\">\n",
|
||||
" <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ14 Vertex SDK AutoML Video Classification.ipynb\">\n",
|
||||
" <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/ai-platform-samples/blob/master/vertex-ai-samples/tree/master/notebooks/official/migration/UJ14%20Vertex%20SDK%20AutoML%20Video%20Classification.ipynb\">\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ14 Vertex SDK AutoML Video Classification.ipynb\">\n",
|
||||
" <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ14 Vertex SDK AutoML Video Classification.ipynb\">\n",
|
||||
" <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
|
||||
" Open in Vertex AI Workbench\n",
|
||||
" </a>\n",
|
||||
" </td> \n",
|
||||
"</table>\n",
|
||||
"<br/><br/><br/>"
|
||||
]
|
||||
@@ -150,17 +157,6 @@
|
||||
"Install the latest GA version of *google-cloud-storage* library as well."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "install_storage"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! pip3 install -U google-cloud-storage $USER_FLAG"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
@@ -170,7 +166,7 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"if os.getenv(\"IS_TESTING\"):\n",
|
||||
" ! pip3 install --upgrade tensorflow $USER_FLAG"
|
||||
" ! pip3 install -U google-cloud-storage --upgrade tensorflow $USER_FLAG"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -297,7 +293,10 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"REGION = \"us-central1\" # @param {type: \"string\"}"
|
||||
"REGION = \"[your-region]\" # @param {type: \"string\"}\n",
|
||||
"\n",
|
||||
"if REGION == \"[your-region]\":\n",
|
||||
" REGION = \"us-central1\""
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -306,9 +305,9 @@
|
||||
"id": "timestamp"
|
||||
},
|
||||
"source": [
|
||||
"#### Timestamp\n",
|
||||
"#### UUID\n",
|
||||
"\n",
|
||||
"If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
|
||||
"If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -319,9 +318,16 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from datetime import datetime\n",
|
||||
"import random\n",
|
||||
"import string\n",
|
||||
"\n",
|
||||
"TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
|
||||
"\n",
|
||||
"# Generate a uuid of a specifed length(default=8)\n",
|
||||
"def generate_uuid(length: int = 8) -> str:\n",
|
||||
" return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"UUID = generate_uuid()"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -332,7 +338,7 @@
|
||||
"source": [
|
||||
"### Authenticate your Google Cloud account\n",
|
||||
"\n",
|
||||
"**If you are using Google Cloud Notebooks**, your environment is already authenticated. Skip this step.\n",
|
||||
"**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
|
||||
"\n",
|
||||
"**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
|
||||
"\n",
|
||||
@@ -404,7 +410,8 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"BUCKET_NAME = \"gs://[your-bucket-name]\" # @param {type:\"string\"}"
|
||||
"BUCKET_NAME = \"[your-bucket-name]\" # @param {type:\"string\"}\n",
|
||||
"BUCKET_URI = f\"gs://{BUCKET_NAME}\""
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -415,8 +422,9 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
|
||||
" BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
|
||||
"if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
|
||||
" BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
|
||||
" BUCKET_URI = f\"gs://{BUCKET_NAME}\""
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -436,7 +444,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil mb -l $REGION $BUCKET_NAME"
|
||||
"! gsutil mb -l $REGION $BUCKET_URI"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -456,7 +464,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil ls -al $BUCKET_NAME"
|
||||
"! gsutil ls -al $BUCKET_URI"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -501,7 +509,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"aip.init(project=PROJECT_ID, staging_bucket=BUCKET_NAME)"
|
||||
"aip.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -602,7 +610,7 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"dataset = aip.VideoDataset.create(\n",
|
||||
" display_name=\"MIT Human Motion\" + \"_\" + TIMESTAMP,\n",
|
||||
" display_name=\"MIT Human Motion\" + \"_\" + UUID,\n",
|
||||
" gcs_source=[IMPORT_FILE],\n",
|
||||
" import_schema_uri=aip.schema.dataset.ioformat.video.classification,\n",
|
||||
")\n",
|
||||
@@ -677,7 +685,7 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"dag = aip.AutoMLVideoTrainingJob(\n",
|
||||
" display_name=\"hmdb_\" + TIMESTAMP,\n",
|
||||
" display_name=\"hmdb_\" + UUID,\n",
|
||||
" prediction_type=\"classification\",\n",
|
||||
")\n",
|
||||
"\n",
|
||||
@@ -725,7 +733,7 @@
|
||||
"source": [
|
||||
"model = dag.run(\n",
|
||||
" dataset=dataset,\n",
|
||||
" model_display_name=\"hmdb_\" + TIMESTAMP,\n",
|
||||
" model_display_name=\"hmdb_\" + UUID,\n",
|
||||
" training_fraction_split=0.8,\n",
|
||||
" test_fraction_split=0.2,\n",
|
||||
")"
|
||||
@@ -800,7 +808,7 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Get model resource ID\n",
|
||||
"models = aip.Model.list(filter=\"display_name=hmdb_\" + TIMESTAMP)\n",
|
||||
"models = aip.Model.list(filter=\"display_name=hmdb_\" + UUID)\n",
|
||||
"\n",
|
||||
"# Get a reference to the Model Service client\n",
|
||||
"client_options = {\"api_endpoint\": f\"{REGION}-aiplatform.googleapis.com\"}\n",
|
||||
@@ -932,7 +940,7 @@
|
||||
"\n",
|
||||
"import tensorflow as tf\n",
|
||||
"\n",
|
||||
"gcs_input_uri = BUCKET_NAME + \"/test.jsonl\"\n",
|
||||
"gcs_input_uri = BUCKET_URI + \"/test.jsonl\"\n",
|
||||
"with tf.io.gfile.GFile(gcs_input_uri, \"w\") as f:\n",
|
||||
" data = {\n",
|
||||
" \"content\": test_item_1,\n",
|
||||
@@ -978,9 +986,9 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"batch_predict_job = model.batch_predict(\n",
|
||||
" job_display_name=\"hmdb_\" + TIMESTAMP,\n",
|
||||
" job_display_name=\"hmdb_\" + UUID,\n",
|
||||
" gcs_source=gcs_input_uri,\n",
|
||||
" gcs_destination_prefix=BUCKET_NAME,\n",
|
||||
" gcs_destination_prefix=BUCKET_URI,\n",
|
||||
" sync=False,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
@@ -1208,8 +1216,8 @@
|
||||
" except Exception as e:\n",
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
" if \"BUCKET_NAME\" in globals():\n",
|
||||
" ! gsutil rm -r $BUCKET_NAME"
|
||||
" if \"BUCKET_URI\" in globals():\n",
|
||||
" ! gsutil rm -r $BUCKET_URI"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
Reference in New Issue
Block a user