mirror of
https://github.com/GoogleCloudPlatform/vertex-ai-samples.git
synced 2026-09-30 10:01:58 +00:00
Compare commits
6
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Ivan CheungandGitHub
|
9094dacd62 | ||
|
|
07b4c37135 | ||
|
|
f48fb1c650 | ||
|
|
3bbd59311c | ||
|
Krishna Chaitanya MovvaandGitHub
|
15bb4cea73 | ||
|
Andrew FerlitschandGitHub
|
1511cc9fd1 |
@@ -149,8 +149,8 @@
|
||||
"source": [
|
||||
"### Set up your local development environment\n",
|
||||
"\n",
|
||||
"**If you are using Colab or Google Cloud Notebooks**, your environment already meets\n",
|
||||
"all the requirements to run this notebook. You can skip this step.\n",
|
||||
"**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets\n",
|
||||
"all the requirements to run this notebook. \n",
|
||||
"\n",
|
||||
"**Otherwise**, make sure your environment meets this notebook's requirements.\n",
|
||||
"You need the following:\n",
|
||||
|
||||
@@ -138,7 +138,7 @@
|
||||
"source": [
|
||||
"### Set up your local development environment\n",
|
||||
"\n",
|
||||
"If you are using Colab or Google Cloud Notebooks, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
|
||||
"If you are using Colab or Vertex AI Workbench Notebooks, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
|
||||
"\n",
|
||||
"Otherwise, make sure your environment meets this notebook's requirements. You need the following:\n",
|
||||
"\n",
|
||||
|
||||
@@ -128,7 +128,7 @@
|
||||
"source": [
|
||||
"### Set up your local development environment\n",
|
||||
"\n",
|
||||
"**If you are using Colab or Google Cloud Notebooks**, your environment already meets\n",
|
||||
"**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets\n",
|
||||
"all the requirements to run this notebook. You can skip this step."
|
||||
]
|
||||
},
|
||||
|
||||
@@ -127,7 +127,7 @@
|
||||
"source": [
|
||||
"### Set up your local development environment\n",
|
||||
"\n",
|
||||
"**If you are using Colab or Google Cloud Notebooks**, your environment already meets\n",
|
||||
"**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets\n",
|
||||
"all the requirements to run this notebook. You can skip this step.\n",
|
||||
"\n",
|
||||
"**Otherwise**, make sure your environment meets this notebook's requirements.\n",
|
||||
|
||||
@@ -62,7 +62,7 @@
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 2 : experimentation: get started with Vertex Vizier."
|
||||
"This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 2 : experimentation: get started with Vertex AI Vizier."
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
+57
-579
File diff suppressed because one or more lines are too long
@@ -151,7 +151,7 @@
|
||||
"source": [
|
||||
"### Set up your local development environment\n",
|
||||
"\n",
|
||||
"If you are using Colab or Google Cloud Notebooks, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
|
||||
"If you are using Colab or Vertex AI Workbench Notebooks, your environment already meets all the requirements to run this notebook. \n",
|
||||
"\n",
|
||||
"Otherwise, make sure your environment meets this notebook's requirements. You need the following:\n",
|
||||
"\n",
|
||||
|
||||
+1
-1
@@ -204,7 +204,7 @@
|
||||
"source": [
|
||||
"### Set up your local development environment\n",
|
||||
"\n",
|
||||
"If you are using Colab or Google Cloud Notebooks, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
|
||||
"If you are using Colab or Vertex AI Workbench Notebooks, your environment already meets all the requirements to run this notebook. \n",
|
||||
"\n",
|
||||
"Otherwise, make sure your environment meets this notebook's requirements. You need the following:\n",
|
||||
"\n",
|
||||
|
||||
@@ -169,15 +169,6 @@
|
||||
"scikit-learn~=0.24"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "faf22f3af1ce"
|
||||
},
|
||||
"source": [
|
||||
"**The model you deploy will have a different set of dependencies pre-installed than your notebook environment has. You should not assume that because things work in the notebook, they will work in the model. Instead, you will be very explicit about the dependencies for the model by listing them in requirements.txt and then use `pip install` to install the exact same dependencies in the notebook. Please note, of course, that there is a chance that a dependency is missed in requirements.txt that already exists in the notebook. If that's the case, things will run in the notebook, but not in the model. To guard against that, you will test the model locally before deploying to the cloud.**"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
|
||||
@@ -143,7 +143,7 @@
|
||||
"source": [
|
||||
"### Set up your local development environment\n",
|
||||
"\n",
|
||||
"**If you are using Colab or Google Cloud Notebooks**, your environment already meets\n",
|
||||
"**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets\n",
|
||||
"all the requirements to run this notebook. You can skip this step."
|
||||
]
|
||||
},
|
||||
@@ -424,7 +424,7 @@
|
||||
"### Authenticate your Google Cloud account\n",
|
||||
"\n",
|
||||
"**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
|
||||
"authenticated. Skip this step."
|
||||
"authenticated. "
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1355,8 +1355,6 @@
|
||||
"source": [
|
||||
"QUERY_EMBEDDING_PATH = f\"{BUCKET_URI}/embeddings/train.jsonl\"\n",
|
||||
"\n",
|
||||
"import tensorflow as tf\n",
|
||||
"\n",
|
||||
"with tf.io.gfile.GFile(QUERY_EMBEDDING_PATH, \"w\") as f:\n",
|
||||
" for i in range(1, 200001):\n",
|
||||
" query = str(i)\n",
|
||||
@@ -1417,7 +1415,7 @@
|
||||
"MAX_NODES = 4\n",
|
||||
"\n",
|
||||
"batch_predict_job = model.batch_predict(\n",
|
||||
" job_display_name=f\"batch_predict_swivel\",\n",
|
||||
" job_display_name=\"batch_predict_swivel\",\n",
|
||||
" gcs_source=[QUERY_EMBEDDING_PATH],\n",
|
||||
" gcs_destination_prefix=f\"{BUCKET_URI}/embeddings/output\",\n",
|
||||
" machine_type=DEPLOY_COMPUTE,\n",
|
||||
|
||||
@@ -142,7 +142,7 @@
|
||||
"source": [
|
||||
"### Set up your local development environment\n",
|
||||
"\n",
|
||||
"**If you are using Colab or Google Cloud Notebooks**, your environment already meets\n",
|
||||
"**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets\n",
|
||||
"all the requirements to run this notebook. You can skip this step."
|
||||
]
|
||||
},
|
||||
@@ -422,7 +422,7 @@
|
||||
"### Authenticate your Google Cloud account\n",
|
||||
"\n",
|
||||
"**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
|
||||
"authenticated. Skip this step."
|
||||
"authenticated. "
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
@@ -134,7 +134,7 @@
|
||||
"source": [
|
||||
"### Set up your local development environment\n",
|
||||
"\n",
|
||||
"**If you are using Colab or Google Cloud Notebooks**, your environment already meets\n",
|
||||
"**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets\n",
|
||||
"all the requirements to run this notebook. You can skip this step."
|
||||
]
|
||||
},
|
||||
@@ -395,7 +395,7 @@
|
||||
"source": [
|
||||
"### Authenticate your Google Cloud account\n",
|
||||
"\n",
|
||||
"**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
|
||||
"**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. \n",
|
||||
"\n",
|
||||
"**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
|
||||
"\n",
|
||||
|
||||
@@ -23,6 +23,15 @@
|
||||
"# limitations under the License."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "2d7a1a97d1ee"
|
||||
},
|
||||
"source": [
|
||||
"# Vertex AI: SDK BigQuery Custom Container Training"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
@@ -58,18 +67,61 @@
|
||||
},
|
||||
"source": [
|
||||
"### Overview \n",
|
||||
"To use this Jupyter notebook, copy the notebook to a Google Cloud Notebooks instance and open it. You can run each step, or cell, and see its results. To run a cell, use Shift+Enter. Jupyter automatically displays the return value of the last line in each cell. For more information about running notebooks in Google Cloud Notebook, see the Google Cloud Notebook guide.. \n",
|
||||
"\n",
|
||||
"### Objective \n",
|
||||
"This Note book creates a custom container using bigquery dataset it will train container and crete ,train and Deploy the model to perform prediction. \n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "989999fbdab3"
|
||||
},
|
||||
"source": [
|
||||
"### Objective\n",
|
||||
"\n",
|
||||
"This notebook demonstrate how to create a Custom Model using Custom Container Training and a Big Query Dataset. It will require you provide a bucket where the dataset will be stored.\n",
|
||||
"In this notebook, you will learn how to use Vertex AI Experiments to \n",
|
||||
"\n",
|
||||
"Costs \n",
|
||||
"This tutorial uses billable components of Google Cloud: \n",
|
||||
"* Log Pipeline Job\n",
|
||||
"* Compare different Pipeline Jobs\n",
|
||||
"\n",
|
||||
"Vertex AI\n",
|
||||
"Cloud Storage\n",
|
||||
"Learn about Vertex AI pricing and Cloud Storage pricing, and use the Pricing Calculator to generate a cost estimate based on your projected usage."
|
||||
"The steps covered include:\n",
|
||||
"\n",
|
||||
"* Formalize a training component\n",
|
||||
"* Build a training a Model\n",
|
||||
"* Run several Pipeline jobs and log their results\n",
|
||||
"* Train the model for prediction\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "3d29af7e49d8"
|
||||
},
|
||||
"source": [
|
||||
"### Dataset\n",
|
||||
"\n",
|
||||
"The dataset used for this tutorial is the [Iris dataset](https://www.tensorflow.org/datasets/catalog/iris) from [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/overview). This dataset does not require any feature engineering. The version of the dataset you use in this tutorial is stored in a public Cloud Storage bucket. The trained model predicts the type of Iris flower species from a class of three species: setosa, virginica, or versicolor."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "e3e924989cce"
|
||||
},
|
||||
"source": [
|
||||
"### Costs \n",
|
||||
"\n",
|
||||
"This tutorial uses billable components of Google Cloud:\n",
|
||||
"\n",
|
||||
"* Vertex AI\n",
|
||||
"* Cloud Storage\n",
|
||||
"\n",
|
||||
"Learn about [Vertex AI\n",
|
||||
"pricing](https://cloud.google.com/vertex-ai/pricing) [Bigquery\n",
|
||||
"pricing](https://cloud.google.com/bigquery/pricing/) and [Cloud Storage\n",
|
||||
"pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
|
||||
"Calculator](https://cloud.google.com/products/calculator/)\n",
|
||||
"to generate a cost estimate based on your projected usage."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -139,10 +191,9 @@
|
||||
"id": "xOMNWzTbftDr"
|
||||
},
|
||||
"source": [
|
||||
"# Install Vertex AI SDK for Python\n",
|
||||
"### Install additional packages\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"After the SDK installation the kernel will be automatically restarted."
|
||||
"Install additional package dependencies not installed in your notebook environment, such as XGBoost, AdaNet, or TensorFlow Hub. Use the latest major GA version of each package."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -161,6 +212,36 @@
|
||||
"app.kernel.do_shutdown(True)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "d4f317591f55"
|
||||
},
|
||||
"source": [
|
||||
"### Restart the kernel\n",
|
||||
"\n",
|
||||
"After you install the additional packages, you need to restart the notebook kernel so it can find the packages."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "f731803a16c0"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Automatically restart kernel after installs\n",
|
||||
"import os\n",
|
||||
"\n",
|
||||
"if not os.getenv(\"IS_TESTING\"):\n",
|
||||
" # Automatically restart kernel after installs\n",
|
||||
" import IPython\n",
|
||||
"\n",
|
||||
" app = IPython.Application.instance()\n",
|
||||
" app.kernel.do_shutdown(True)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
@@ -172,6 +253,17 @@
|
||||
"Enter your Project Id in the cell below. Then run the cell to make sure the Cloud SDK uses the right project for all the commands in this notebook."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "3c8049930470"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"PROJECT_ID = \"[your-project-id]\" # @param {type:\"string\"}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
@@ -183,14 +275,11 @@
|
||||
"import os\n",
|
||||
"\n",
|
||||
"PROJECT_ID = \"\"\n",
|
||||
"\n",
|
||||
"# Get your Google Cloud project ID from gcloud\n",
|
||||
"if not os.getenv(\"IS_TESTING\"):\n",
|
||||
" shell_output = !gcloud config list --format 'value(core.project)' 2>/dev/null\n",
|
||||
" PROJECT_ID = shell_output[0]\n",
|
||||
" print(\"Project ID: \", PROJECT_ID)\n",
|
||||
"\n",
|
||||
"MY_STAGING_BUCKET = \"gs://YOUR BUCKET\" # bucket should be in same region as ucaip"
|
||||
" print(\"Project ID: \", PROJECT_ID)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -218,21 +307,14 @@
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "6x6CSodKjMmg"
|
||||
"id": "ZaQd5jNwjP_0"
|
||||
},
|
||||
"source": [
|
||||
"### Authenticate your Google Cloud account\n",
|
||||
"\n",
|
||||
"**If you are using Google Cloud Notebooks**, your environment is already\n",
|
||||
"authenticated. Skip this step"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "ZaQd5jNwjP_0"
|
||||
},
|
||||
"source": [
|
||||
"**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
|
||||
"authenticated. Skip this step\n",
|
||||
"\n",
|
||||
"**If you are using Colab**, run the cell below and follow the instructions\n",
|
||||
"when prompted to authenticate your account via oAuth.\n",
|
||||
"\n",
|
||||
@@ -296,9 +378,9 @@
|
||||
"id": "r2lr6-MVpXLP"
|
||||
},
|
||||
"source": [
|
||||
"#### Timestamp\n",
|
||||
"#### UUID\n",
|
||||
"\n",
|
||||
"If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append it onto the name of resources you create in this tutorial."
|
||||
"If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -309,9 +391,16 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from datetime import datetime\n",
|
||||
"import random\n",
|
||||
"import string\n",
|
||||
"\n",
|
||||
"TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
|
||||
"\n",
|
||||
"# Generate a uuid of a specifed length(default=8)\n",
|
||||
"def generate_uuid(length: int = 8) -> str:\n",
|
||||
" return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"UUID = generate_uuid()"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -343,15 +432,24 @@
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "GF076Vmoioll"
|
||||
"id": "2f6f0f6ec383"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"BUCKET_URI = \"gs://[your-bucket-name]\" # @param {type:\"string\"}\n",
|
||||
"REGION = \"[your-region]\" # @param {type:\"string\"}\n",
|
||||
"\n",
|
||||
"REGION = \"[your-region]\" # @param {type:\"string\"}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "GF076Vmoioll"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
|
||||
" BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP\n",
|
||||
" BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + UUID\n",
|
||||
"\n",
|
||||
"if REGION == \"[your-region]\":\n",
|
||||
" REGION = \"us-central1\""
|
||||
@@ -403,8 +501,9 @@
|
||||
"id": "5T1d5uBoftDw"
|
||||
},
|
||||
"source": [
|
||||
"# Copy Big Query Iris Dataset\n",
|
||||
"We will make a Big Query dataset and copy Big Query's public iris table to that dataset. For more information about this dataset please visit: https://archive.ics.uci.edu/ml/datasets/iris "
|
||||
"# Copy bigquery iris dataset\n",
|
||||
"\n",
|
||||
"You make a BigQuery dataset and copy BigQuery's public iris table to that dataset. For more information about this dataset please visit: https://archive.ics.uci.edu/ml/datasets/iris "
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -543,6 +642,15 @@
|
||||
"### Write the entrypoint script to invoke trainer"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "c5cdc477cd73"
|
||||
},
|
||||
"source": [
|
||||
"The entrypoint script train adn validates the data and also compiles the model."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
@@ -697,9 +805,9 @@
|
||||
"id": "736ddff8408b"
|
||||
},
|
||||
"source": [
|
||||
"# Create a Managed Tabular Dataset from Big Query Dataset\n",
|
||||
"# Create a managed tabular dataset from bigquery dataset\n",
|
||||
"\n",
|
||||
"This section will create a managed Tabular dataset from the iris Big Query table we copied above."
|
||||
"This section create a managed Tabular dataset from the iris BigQuery table we copied above.The param's used are BigQuery's public iris dataset."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -723,7 +831,7 @@
|
||||
"source": [
|
||||
"# Launch The Training Job to Create a Model\n",
|
||||
"\n",
|
||||
"We will train a model with the container we built above."
|
||||
"We will train a model with the container we built above.To train the model you use the CustomeContanier TrainingJob method with Container Image and Container_uri as parametrs."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -753,9 +861,9 @@
|
||||
"id": "a7fa9b59f919"
|
||||
},
|
||||
"source": [
|
||||
"# Deploy The Model\n",
|
||||
"# Deploy the model\n",
|
||||
"\n",
|
||||
"Deploy your model, then wait until the model FINISHES deployment before proceeding to prediction."
|
||||
"Deploy your model, then wait until the model Finishes deployment before proceeding to prediction.For prediction deploy method takes machine_type as parameter."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -778,6 +886,15 @@
|
||||
"# Make a prediction\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "e4b04d246ba9"
|
||||
},
|
||||
"source": [
|
||||
"Endpoint predict method publish the prediction based on length and width feature parameters."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
@@ -786,9 +903,11 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"endpoint.predict(\n",
|
||||
"prediction = endpoint.predict(\n",
|
||||
" [{\"sepal_length\": 5.1, \"sepal_width\": 2.5, \"petal_length\": 3.0, \"petal_width\": 1.1}]\n",
|
||||
")"
|
||||
")\n",
|
||||
"\n",
|
||||
"print(prediction)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -797,12 +916,16 @@
|
||||
"id": "MaoIczP8qu--"
|
||||
},
|
||||
"source": [
|
||||
"## Cleaning up\n",
|
||||
"# Cleaning up\n",
|
||||
"\n",
|
||||
"To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
|
||||
"project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
|
||||
"\n",
|
||||
"Otherwise, you can delete the individual resources you created in this tutorial:"
|
||||
"Otherwise, you can delete the individual resources you created in this tutorial:\n",
|
||||
"\n",
|
||||
"- Pipeline\n",
|
||||
"- Endpoint\n",
|
||||
"- Cloud Storage Bucket"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -813,16 +936,28 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"delete_pipeline = True\n",
|
||||
"delete_endpoint = True\n",
|
||||
"\n",
|
||||
"# Delete endpoint resource\n",
|
||||
"! gcloud ai endpoints delete $ENDPOINT_NAME --quiet --region $REGION_NAME\n",
|
||||
"\n",
|
||||
"# Delete Cloud Storage objects that were created\n",
|
||||
"! gsutil -m rm -r $JOB_DIR\n",
|
||||
"if delete_pipeline:\n",
|
||||
" job.delete()\n",
|
||||
"\n",
|
||||
"if os.getenv(\"IS_TESTING\"):\n",
|
||||
"! gsutil -m rm -r $BUCKET_URI "
|
||||
" if delete_endpoint and \"DISPLAY_NAME\" in globals():\n",
|
||||
" endpoints = aip.Endpoint.list(\n",
|
||||
" filter=f\"display_name={DISPLAY_NAME}_endpoint\", order_by=\"create_time\"\n",
|
||||
" )\n",
|
||||
" if endpoints:\n",
|
||||
" endpoint = endpoints[0]\n",
|
||||
" endpoint.undeploy_all()\n",
|
||||
" aip.Endpoint.delete(endpoint.resource_name)\n",
|
||||
" print(\"Deleted endpoint:\", endpoint)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Delete bucket\n",
|
||||
"delete_bucket = False\n",
|
||||
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
|
||||
" ! gsutil rm -rf {BUCKET_URI}"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
@@ -25,7 +25,11 @@ args = parser.parse_args()
|
||||
|
||||
if args.errors_codes:
|
||||
args.errors_codes = args.errors_codes.split(',')
|
||||
|
||||
args.errors = True
|
||||
|
||||
if args.errors_csv:
|
||||
args.errors = True
|
||||
|
||||
def parse_dir(directory):
|
||||
entries = os.scandir(directory)
|
||||
for entry in entries:
|
||||
@@ -38,7 +42,7 @@ def parse_dir(directory):
|
||||
parse_dir(entry.path)
|
||||
elif entry.name.endswith('.ipynb'):
|
||||
parse_notebook(entry.path)
|
||||
|
||||
|
||||
def parse_notebook(path):
|
||||
with open(path, 'r') as f:
|
||||
try:
|
||||
@@ -158,6 +162,11 @@ def parse_notebook(path):
|
||||
if cell['cell_type'] != 'code':
|
||||
report_error(path, 22, "Installation code section not found")
|
||||
else:
|
||||
if cell['source'][0].startswith('! mkdir'):
|
||||
cell, nth = get_cell(path, cells, nth)
|
||||
if 'requirements.txt' in cell['source'][0]:
|
||||
cell, nth = get_cell(path, cells, nth)
|
||||
|
||||
text = ''
|
||||
for line in cell['source']:
|
||||
text += line
|
||||
@@ -238,7 +247,7 @@ def parse_notebook(path):
|
||||
if cell['source'][0].startswith("### Region"):
|
||||
report_error(path, 34, "Region section not found")
|
||||
'''
|
||||
|
||||
|
||||
|
||||
def get_cell(path, cells, nth):
|
||||
while empty_cell(path, cells, nth):
|
||||
@@ -249,15 +258,38 @@ def get_cell(path, cells, nth):
|
||||
check_text_cell(path, cell)
|
||||
return cell, nth + 1
|
||||
|
||||
|
||||
|
||||
def empty_cell(path, cells, nth):
|
||||
if len(cells[nth]['source']) == 0:
|
||||
report_error(path, 10, f'empty cell: cell #{nth}')
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
|
||||
def check_text_cell(path, cell):
|
||||
|
||||
branding = {
|
||||
'Vertex SDK': 'Vertex AI SDK',
|
||||
'Vertex Training': 'Vertex AI Training',
|
||||
'Vertex Prediction': 'Vertex AI Prediction',
|
||||
'Vertex Batch Prediction': 'Vertex AI Batch Prediction',
|
||||
'Vertex XAI': 'Vertex Explainable AI',
|
||||
'Vertex Experiments': 'Vertex AI Experiments',
|
||||
'Vertex TensorBoard': 'Vertex AI TensorBoard',
|
||||
'Vertex Pipelines': 'Vertex AI Pipelines',
|
||||
'Vertex Hyperparameter Tuning': 'Vertex AI Hyperparameter Tuning',
|
||||
'Vertex Metadata': 'Vertex ML Metadata',
|
||||
'Vertex AI Metadata': 'Vertex ML Metadata',
|
||||
'Vertex Vizier': 'Vertex AI Vizier',
|
||||
'Vertex Dataset': 'Vertex AI Dataset',
|
||||
'Vertex Model': 'Vertex AI Model',
|
||||
'Vertex Endpoint': 'Vertex AI Endpoint',
|
||||
'Vertex Private Endpoint': 'Vertex AI Private Endpoint',
|
||||
'Tensorflow': 'TensorFlow',
|
||||
'Tensorboard': 'TensorBoard',
|
||||
'Google Cloud Notebooks': 'Vertex AI Workbench Notebooks'
|
||||
}
|
||||
|
||||
for line in cell['source']:
|
||||
if 'TODO' in line:
|
||||
report_error(path, 14, f'TODO in cell: {line}')
|
||||
@@ -266,28 +298,9 @@ def check_text_cell(path, cell):
|
||||
if 'will' in line.lower() or 'would' in line.lower():
|
||||
report_error(path, 16, f'Do not use future tense (e.g., will), replace with present tense: {line}')
|
||||
|
||||
if 'Vertex SDK' in line:
|
||||
report_error(path, 27, f"Branding: Vertex AI SDK: {line}")
|
||||
if 'Vertex Training' in line:
|
||||
report_error(path, 27, f"Branding: Vertex AI Training: {line}")
|
||||
if 'Vertex Prediction' in line:
|
||||
report_error(path, 27, f"Branding: Vertex AI Prediction: {line}")
|
||||
if 'Vertex Batch Prediction' in line:
|
||||
report_error(path, 27, f"Branding: Vertex AI Batch Prediction {line}")
|
||||
if 'Vertex XAI' in line:
|
||||
report_error(path, 27, f"Branding: Vertex Explainable AI: {line}")
|
||||
if 'Vertex Experiments' in line:
|
||||
report_error(path, 27, f"Branding: Vertex AI Experiments: {line}")
|
||||
if 'Vertex TensorBoard' in line:
|
||||
report_error(path, 27, f"Branding: Vertex AI TensorBoard: {line}")
|
||||
if 'Vertex Pipelines' in line:
|
||||
report_error(path, 27, f"Branding: Vertex AI Pipelines: {line}")
|
||||
if 'Vertex Hyperparameter Tuning' in line:
|
||||
report_error(path, 27, f"Branding: Vertex AI Hyperparameter Tuning: {line}")
|
||||
if 'Tensorflow' in line:
|
||||
report_error(path, 27, f"Branding: TensorFlow: {line}")
|
||||
if 'Tensorboard' in line:
|
||||
report_error(path, 27, f"Branding: TensorBoard: {line}")
|
||||
for mistake, brand in branding.items():
|
||||
if mistake in line:
|
||||
report_error(path, 27, f"Branding {brand}: {line}")
|
||||
|
||||
|
||||
def check_sentence_case(path, heading):
|
||||
@@ -298,12 +311,12 @@ def check_sentence_case(path, heading):
|
||||
for word in words[1:]:
|
||||
word = word.replace(':', '').replace('(', '').replace(')', '')
|
||||
if word in ['E2E', 'Vertex', 'AutoML', 'ML', 'AI', 'GCP', 'API', 'R', 'CMEK', 'TFX', 'TFDV', 'SDK',
|
||||
'VM', 'CPR', 'NVIDIA', 'ID']:
|
||||
'VM', 'CPR', 'NVIDIA', 'ID', 'DASK']:
|
||||
continue
|
||||
if word.isupper():
|
||||
report_error(path, 3, f"heading is not sentence case: {word}")
|
||||
|
||||
|
||||
|
||||
|
||||
def report_error(notebook, code, msg):
|
||||
if args.errors:
|
||||
if args.errors_codes:
|
||||
@@ -314,7 +327,7 @@ def report_error(notebook, code, msg):
|
||||
print(notebook, ',', code)
|
||||
else:
|
||||
print(f"{notebook}: ERROR ({code}): {msg}")
|
||||
|
||||
|
||||
def parse_objective(path, cell):
|
||||
desc = ''
|
||||
in_desc = True
|
||||
@@ -392,7 +405,7 @@ def add_index(path, title, desc, uses, steps):
|
||||
|
||||
if args.steps:
|
||||
print(steps)
|
||||
|
||||
|
||||
|
||||
if args.notebook_dir:
|
||||
if not os.path.isdir(args.notebook_dir):
|
||||
@@ -406,4 +419,4 @@ elif args.notebook:
|
||||
parse_notebook(args.notebook)
|
||||
else:
|
||||
print("Error: must specify a directory or notebook")
|
||||
exit(1)
|
||||
exit(1)
|
||||
|
||||
@@ -27,5 +27,5 @@
|
||||
/pipelines/google_cloud_pipelines_dataproc_tabular @inardini
|
||||
/automl/automl_forecasting_bqml_arima_plus_comparison.ipynb @TheMichaelHu
|
||||
/automl/automl_tabular_on_vertex_pipelines.ipynb @helinwang
|
||||
/custom/custom_training_tensorboard_profiler.ipynb @itseric
|
||||
/workbench/spark/spark_sample_notebook.ipynb @bmiro
|
||||
/custom/custom_training_tensorboard_profiler.ipynb @gericdong
|
||||
/workbench/spark/spark_sample_notebook.ipynb @bradmiro
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "91417fdd",
|
||||
"metadata": {
|
||||
"id": "copyright"
|
||||
},
|
||||
@@ -25,11 +26,12 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "f2902dac",
|
||||
"metadata": {
|
||||
"id": "title"
|
||||
},
|
||||
"source": [
|
||||
"# Vertex SDK: AutoML training video classification model for batch prediction\n",
|
||||
"# Vertex AI SDK: AutoML training video classification model for batch prediction\n",
|
||||
"\n",
|
||||
"<table align=\"left\">\n",
|
||||
" <td>\n",
|
||||
@@ -55,6 +57,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "42cfbec0",
|
||||
"metadata": {
|
||||
"id": "overview:automl"
|
||||
},
|
||||
@@ -62,29 +65,25 @@
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"This tutorial demonstrates how to use the Vertex SDK to create video classification models and do batch prediction using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "dataset:hmdb,vcn"
|
||||
},
|
||||
"source": [
|
||||
"### Dataset\n",
|
||||
"\n",
|
||||
"The dataset used for this tutorial is the golf swing recognition portion of the [Human Motion dataset from MIT](http://cbcl.mit.edu/publications/ps/Kuehne_etal_iccv11.pdf). The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket. The trained model will predict the start frame where a golf swing begins.\n"
|
||||
"This tutorial demonstrates how to use the Vertex AI SDK to create video classification models and do batch prediction using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "90b9b726",
|
||||
"metadata": {
|
||||
"id": "objective:automl,training,batch_prediction"
|
||||
},
|
||||
"source": [
|
||||
"### Objective\n",
|
||||
"\n",
|
||||
"In this tutorial, you create an AutoML video classification model from a Python script, and then do a batch prediction using the Vertex SDK. You can alternatively create and deploy models using the `gcloud` command-line tool or online using the Cloud Console.\n",
|
||||
"In this tutorial, you create an AutoML video classification model from a Python script, and then do a batch prediction using the Vertex AI SDK. You can alternatively create and deploy models using the `gcloud` command-line tool or online using the Cloud Console.\n",
|
||||
"\n",
|
||||
"This tutorial uses the following Google Cloud ML services and resources:\n",
|
||||
"\n",
|
||||
"- Vertex AI Training\n",
|
||||
"- Vertex AI Models\n",
|
||||
"- Vertex AI Batch Prediction\n",
|
||||
"\n",
|
||||
"The steps performed include:\n",
|
||||
"\n",
|
||||
@@ -102,6 +101,19 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "44940826",
|
||||
"metadata": {
|
||||
"id": "dataset:hmdb,vcn"
|
||||
},
|
||||
"source": [
|
||||
"### Dataset\n",
|
||||
"\n",
|
||||
"The dataset used for this tutorial is the golf swing recognition portion of the [Human Motion dataset from MIT](http://cbcl.mit.edu/publications/ps/Kuehne_etal_iccv11.pdf). The version of the dataset you use in this tutorial is stored in a public Cloud Storage bucket. The trained model predicts the start frame where a golf swing begins.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "7183fc01",
|
||||
"metadata": {
|
||||
"id": "costs"
|
||||
},
|
||||
@@ -122,89 +134,96 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "b88c255b-df72-4666-9403-0c96d7e657ca",
|
||||
"metadata": {
|
||||
"id": "setup_local"
|
||||
"id": "384b53dfdb54"
|
||||
},
|
||||
"source": [
|
||||
"### Set up your local development environment\n",
|
||||
"\n",
|
||||
"If you are using Colab or Google Cloud Notebooks, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
|
||||
"\n",
|
||||
"Otherwise, make sure your environment meets this notebook's requirements. You need the following:\n",
|
||||
"\n",
|
||||
"- The Cloud Storage SDK\n",
|
||||
"- Git\n",
|
||||
"- Python 3\n",
|
||||
"- virtualenv\n",
|
||||
"- Jupyter notebook running in a virtual environment with Python 3\n",
|
||||
"\n",
|
||||
"The Cloud Storage guide to [Setting up a Python development environment](https://cloud.google.com/python/setup) and the [Jupyter installation guide](https://jupyter.org/install) provide detailed instructions for meeting these requirements. The following steps provide a condensed set of instructions:\n",
|
||||
"\n",
|
||||
"1. [Install and initialize the SDK](https://cloud.google.com/sdk/docs/).\n",
|
||||
"\n",
|
||||
"2. [Install Python 3](https://cloud.google.com/python/setup#installing_python).\n",
|
||||
"\n",
|
||||
"3. [Install virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv) and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
|
||||
"\n",
|
||||
"4. To install Jupyter, run `pip3 install jupyter` on the command-line in a terminal shell.\n",
|
||||
"\n",
|
||||
"5. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
|
||||
"\n",
|
||||
"6. Open this notebook in the Jupyter Notebook Dashboard.\n"
|
||||
"**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets\n",
|
||||
"all the requirements to run this notebook. You can skip this step."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "8c1be8fc",
|
||||
"metadata": {
|
||||
"id": "setup_local"
|
||||
},
|
||||
"source": [
|
||||
"**Otherwise**, make sure your environment meets this notebook's requirements.\n",
|
||||
"You need the following:\n",
|
||||
"\n",
|
||||
"* The Google Cloud SDK\n",
|
||||
"* Git\n",
|
||||
"* Python 3\n",
|
||||
"* virtualenv\n",
|
||||
"* Jupyter notebook running in a virtual environment with Python 3\n",
|
||||
"\n",
|
||||
"The Google Cloud guide to [Setting up a Python development\n",
|
||||
"environment](https://cloud.google.com/python/setup) and the [Jupyter\n",
|
||||
"installation guide](https://jupyter.org/install) provide detailed instructions\n",
|
||||
"for meeting these requirements. The following steps provide a condensed set of\n",
|
||||
"instructions:\n",
|
||||
"\n",
|
||||
"1. [Install and initialize the Cloud SDK.](https://cloud.google.com/sdk/docs/)\n",
|
||||
"\n",
|
||||
"1. [Install Python 3.](https://cloud.google.com/python/setup#installing_python)\n",
|
||||
"\n",
|
||||
"1. [Install\n",
|
||||
" virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv)\n",
|
||||
" and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
|
||||
"\n",
|
||||
"1. To install Jupyter, run `pip3 install jupyter` on the\n",
|
||||
"command-line in a terminal shell.\n",
|
||||
"\n",
|
||||
"1. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
|
||||
"\n",
|
||||
"1. Open this notebook in the Jupyter Notebook Dashboard.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "e131fbee",
|
||||
"metadata": {
|
||||
"id": "install_aip:mbsdk"
|
||||
},
|
||||
"source": [
|
||||
"## Installation\n",
|
||||
"\n",
|
||||
"Install the latest version of Vertex SDK for Python."
|
||||
"Install the latest versions of Vertex AI and Cloud Storage SDK for Python."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "484dcd52-ef9e-4928-b0f2-7940001bbc2e",
|
||||
"metadata": {
|
||||
"id": "install_aip:mbsdk"
|
||||
"id": "2abdd254e90f"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"\n",
|
||||
"# Google Cloud Notebook\n",
|
||||
"if os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
|
||||
" USER_FLAG = \"--user\"\n",
|
||||
"else:\n",
|
||||
" USER_FLAG = \"\"\n",
|
||||
"# The Vertex AI Workbench Notebook product has specific requirements\n",
|
||||
"IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
|
||||
"IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
|
||||
" \"/opt/deeplearning/metadata/env_version\"\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "install_storage"
|
||||
},
|
||||
"source": [
|
||||
"Install the latest GA version of *google-cloud-storage* library as well."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "install_storage"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! pip3 install -U google-cloud-storage $USER_FLAG"
|
||||
"# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
|
||||
"USER_FLAG = \"\"\n",
|
||||
"if IS_WORKBENCH_NOTEBOOK:\n",
|
||||
" USER_FLAG = \"--user\"\n",
|
||||
"\n",
|
||||
"! pip3 install --upgrade google-cloud-aiplatform {USER_FLAG} -q\n",
|
||||
"! pip3 install -U google-cloud-storage {USER_FLAG} -q"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "aa8cefcd",
|
||||
"metadata": {
|
||||
"id": "restart"
|
||||
},
|
||||
@@ -217,6 +236,7 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "4f079854",
|
||||
"metadata": {
|
||||
"id": "restart"
|
||||
},
|
||||
@@ -234,6 +254,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "e96a43b8",
|
||||
"metadata": {
|
||||
"id": "before_you_begin:nogpu"
|
||||
},
|
||||
@@ -254,19 +275,32 @@
|
||||
"\n",
|
||||
"3. [Enable the following APIs: Vertex AI APIs, Compute Engine APIs, and Cloud Storage.](https://console.cloud.google.com/flows/enableapi?apiid=ml.googleapis.com,compute_component,storage-component.googleapis.com)\n",
|
||||
"\n",
|
||||
"4. If you are running this notebook locally, you will need to install the [Cloud SDK]((https://cloud.google.com/sdk)).\n",
|
||||
"4. If you are running this notebook locally, you need to install the [Cloud SDK]((https://cloud.google.com/sdk)).\n",
|
||||
"\n",
|
||||
"5. Enter your project ID in the cell below. Then run the cell to make sure the\n",
|
||||
"Cloud SDK uses the right project for all the commands in this notebook.\n",
|
||||
"\n",
|
||||
"**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$`."
|
||||
"**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these conmmands."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "305e7fa5-dcaf-477a-b20d-d9b69ecba381",
|
||||
"metadata": {
|
||||
"id": "1460fd744366"
|
||||
},
|
||||
"source": [
|
||||
"#### Set your project ID\n",
|
||||
"\n",
|
||||
"**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "ffd7caab-c2f8-41d3-a0e3-d2519f0bcf2c",
|
||||
"metadata": {
|
||||
"id": "set_project_id"
|
||||
"id": "cd85f5c794e5"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -276,8 +310,9 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "ffb8077b",
|
||||
"metadata": {
|
||||
"id": "autoset_project_id"
|
||||
"id": "set_project_id"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -291,6 +326,7 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "3c30f77a",
|
||||
"metadata": {
|
||||
"id": "set_gcloud_project_id"
|
||||
},
|
||||
@@ -301,6 +337,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "61221789",
|
||||
"metadata": {
|
||||
"id": "region"
|
||||
},
|
||||
@@ -322,68 +359,99 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "e05b6148",
|
||||
"metadata": {
|
||||
"id": "region"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"REGION = \"us-central1\" # @param {type: \"string\"}"
|
||||
"REGION = \"[your-region]\" # @param {type: \"string\"}\n",
|
||||
"\n",
|
||||
"if REGION == \"[your-region]\":\n",
|
||||
" REGION = \"us-central1\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "dab6b689",
|
||||
"metadata": {
|
||||
"id": "timestamp"
|
||||
},
|
||||
"source": [
|
||||
"#### Timestamp\n",
|
||||
"#### UUID\n",
|
||||
"\n",
|
||||
"If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
|
||||
"If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "6dac7084",
|
||||
"metadata": {
|
||||
"id": "timestamp"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from datetime import datetime\n",
|
||||
"import random\n",
|
||||
"import string\n",
|
||||
"\n",
|
||||
"TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
|
||||
"\n",
|
||||
"# Generate a uuid of a specifed length(default=8)\n",
|
||||
"def generate_uuid(length: int = 8) -> str:\n",
|
||||
" return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"UUID = generate_uuid()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "1bd3f05b-f17f-4341-be85-0bdcef3e6f13",
|
||||
"metadata": {
|
||||
"id": "gcp_authenticate"
|
||||
"id": "79055ac4078d"
|
||||
},
|
||||
"source": [
|
||||
"### Authenticate your Google Cloud account\n",
|
||||
"\n",
|
||||
"**If you are using Google Cloud Notebooks**, your environment is already authenticated. Skip this step.\n",
|
||||
"\n",
|
||||
"**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
|
||||
"**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
|
||||
"authenticated. Skip this step."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "c38fbff8",
|
||||
"metadata": {
|
||||
"id": "gcp_authenticate"
|
||||
},
|
||||
"source": [
|
||||
"**If you are using Colab**, run the cell below and follow the instructions\n",
|
||||
"when prompted to authenticate your account via oAuth.\n",
|
||||
"\n",
|
||||
"**Otherwise**, follow these steps:\n",
|
||||
"\n",
|
||||
"In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
|
||||
"1. In the Cloud Console, go to the [**Create service account key**\n",
|
||||
" page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
|
||||
"\n",
|
||||
"**Click Create service account**.\n",
|
||||
"2. Click **Create service account**.\n",
|
||||
"\n",
|
||||
"In the **Service account name** field, enter a name, and click **Create**.\n",
|
||||
"3. In the **Service account name** field, enter a name, and\n",
|
||||
" click **Create**.\n",
|
||||
"\n",
|
||||
"In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex\" into the filter box, and select **Vertex Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
|
||||
"4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
|
||||
"into the filter box, and select\n",
|
||||
" **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
|
||||
"\n",
|
||||
"Click Create. A JSON file that contains your key downloads to your local environment.\n",
|
||||
"5. Click *Create*. A JSON file that contains your key downloads to your\n",
|
||||
"local environment.\n",
|
||||
"\n",
|
||||
"Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
|
||||
"6. Enter the path to your service account key as the\n",
|
||||
"`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "8bae9ca0",
|
||||
"metadata": {
|
||||
"id": "gcp_authenticate"
|
||||
},
|
||||
@@ -397,8 +465,11 @@
|
||||
"import os\n",
|
||||
"import sys\n",
|
||||
"\n",
|
||||
"# If on Google Cloud Notebook, then don't execute this code\n",
|
||||
"if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
|
||||
"# If on Vertex AI Workbench, then don't execute this code\n",
|
||||
"IS_COLAB = \"google.colab\" in sys.modules\n",
|
||||
"if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
|
||||
" \"DL_ANACONDA_HOME\"\n",
|
||||
"):\n",
|
||||
" if \"google.colab\" in sys.modules:\n",
|
||||
" from google.colab import auth as google_auth\n",
|
||||
"\n",
|
||||
@@ -413,6 +484,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "bbda1639",
|
||||
"metadata": {
|
||||
"id": "bucket:mbsdk"
|
||||
},
|
||||
@@ -421,36 +493,42 @@
|
||||
"\n",
|
||||
"**The following steps are required, regardless of your notebook environment.**\n",
|
||||
"\n",
|
||||
"When you initialize the Vertex SDK for Python, you specify a Cloud Storage staging bucket. The staging bucket is where all the data associated with your dataset and model resources are retained across sessions.\n",
|
||||
"When you create a model in Vertex AI using the Cloud SDK, you give a Cloud Storage path where the trained model is saved. In this tutorial, you create a batch prediction job using the Vertex AI model. For this purpose, you need to save your test instances to a Cloud Storage bucket and give a destination Cloud Storage path to write the batch predictions.\n",
|
||||
"\n",
|
||||
"Set the name of your Cloud Storage bucket below. Bucket names must be globally unique across all Google Cloud projects, including those outside of your organization."
|
||||
"Set the name of your Cloud Storage bucket below. It must be unique across all\n",
|
||||
"Cloud Storage buckets."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "be69ad8c",
|
||||
"metadata": {
|
||||
"id": "bucket"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"BUCKET_NAME = \"gs://[your-bucket-name]\" # @param {type:\"string\"}"
|
||||
"BUCKET_NAME = \"[your-bucket-name]\" # @param {type:\"string\"}\n",
|
||||
"BUCKET_URI = f\"gs://{BUCKET_NAME}\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "2d0d674c",
|
||||
"metadata": {
|
||||
"id": "autoset_bucket"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
|
||||
" BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
|
||||
"if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
|
||||
" BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
|
||||
" BUCKET_URI = f\"gs://{BUCKET_NAME}\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "9307a615",
|
||||
"metadata": {
|
||||
"id": "create_bucket"
|
||||
},
|
||||
@@ -461,16 +539,18 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "709e7b95",
|
||||
"metadata": {
|
||||
"id": "create_bucket"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil mb -l $REGION $BUCKET_NAME"
|
||||
"! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "b52bb2e6",
|
||||
"metadata": {
|
||||
"id": "validate_bucket"
|
||||
},
|
||||
@@ -481,72 +561,76 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "e86c8b22",
|
||||
"metadata": {
|
||||
"id": "validate_bucket"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil ls -al $BUCKET_NAME"
|
||||
"! gsutil ls -al $BUCKET_URI"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "cf0222d3",
|
||||
"metadata": {
|
||||
"id": "setup_vars"
|
||||
},
|
||||
"source": [
|
||||
"### Set up variables\n",
|
||||
"\n",
|
||||
"Next, set up some variables used throughout the tutorial.\n",
|
||||
"### Import libraries and define constants"
|
||||
"### Import libraries"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 13,
|
||||
"id": "7534d1a5",
|
||||
"metadata": {
|
||||
"id": "import_aip:mbsdk"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import google.cloud.aiplatform as aiplatform"
|
||||
"from google.cloud import aiplatform"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "15e5e61a",
|
||||
"metadata": {
|
||||
"id": "init_aip:mbsdk"
|
||||
},
|
||||
"source": [
|
||||
"## Initialize Vertex SDK for Python\n",
|
||||
"### Initialize Vertex AI SDK for Python\n",
|
||||
"\n",
|
||||
"Initialize the Vertex SDK for Python for your project and corresponding bucket."
|
||||
"Initialize the Vertex AI SDK for Python for your project and corresponding bucket."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "9df9b0b9",
|
||||
"metadata": {
|
||||
"id": "init_aip:mbsdk"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"aiplatform.init(project=PROJECT_ID, staging_bucket=BUCKET_NAME)"
|
||||
"aiplatform.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "866ae45f",
|
||||
"metadata": {
|
||||
"id": "tutorial_start:automl"
|
||||
},
|
||||
"source": [
|
||||
"# Tutorial\n",
|
||||
"## Tutorial\n",
|
||||
"\n",
|
||||
"Now you are ready to start creating your own AutoML video classification model."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "0adbd455",
|
||||
"metadata": {
|
||||
"id": "import_file:u_dataset,csv"
|
||||
},
|
||||
@@ -559,6 +643,7 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "ab42c2d4",
|
||||
"metadata": {
|
||||
"id": "import_file:hmdb,csv,vcn"
|
||||
},
|
||||
@@ -569,6 +654,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "2f7757ea",
|
||||
"metadata": {
|
||||
"id": "quick_peek:csv"
|
||||
},
|
||||
@@ -583,6 +669,7 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "ea7bac53",
|
||||
"metadata": {
|
||||
"id": "quick_peek:csv"
|
||||
},
|
||||
@@ -597,6 +684,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "aeadee6e",
|
||||
"metadata": {
|
||||
"id": "create_dataset:video,vcn"
|
||||
},
|
||||
@@ -614,13 +702,14 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "9c55581d",
|
||||
"metadata": {
|
||||
"id": "create_dataset:video,vcn"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"dataset = aiplatform.VideoDataset.create(\n",
|
||||
" display_name=\"MIT Human Motion\" + \"_\" + TIMESTAMP,\n",
|
||||
" display_name=\"MIT Human Motion\" + \"_\" + UUID,\n",
|
||||
" gcs_source=[IMPORT_FILE],\n",
|
||||
" import_schema_uri=aiplatform.schema.dataset.ioformat.video.classification,\n",
|
||||
")\n",
|
||||
@@ -630,6 +719,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "26f09f81",
|
||||
"metadata": {
|
||||
"id": "create_automl_pipeline:video,vcn"
|
||||
},
|
||||
@@ -652,13 +742,14 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "9f35d88f",
|
||||
"metadata": {
|
||||
"id": "create_automl_pipeline:video,vcn"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"job = aiplatform.AutoMLVideoTrainingJob(\n",
|
||||
" display_name=\"hmdb_\" + TIMESTAMP,\n",
|
||||
" display_name=\"hmdb_\" + UUID,\n",
|
||||
" prediction_type=\"classification\",\n",
|
||||
")\n",
|
||||
"\n",
|
||||
@@ -667,6 +758,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "6bbaaf5f",
|
||||
"metadata": {
|
||||
"id": "run_automl_pipeline:video"
|
||||
},
|
||||
@@ -688,6 +780,7 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "4b3f2c56",
|
||||
"metadata": {
|
||||
"id": "run_automl_pipeline:video"
|
||||
},
|
||||
@@ -695,7 +788,7 @@
|
||||
"source": [
|
||||
"model = job.run(\n",
|
||||
" dataset=dataset,\n",
|
||||
" model_display_name=\"hmdb_\" + TIMESTAMP,\n",
|
||||
" model_display_name=\"hmdb_\" + UUID,\n",
|
||||
" training_fraction_split=0.8,\n",
|
||||
" test_fraction_split=0.2,\n",
|
||||
")"
|
||||
@@ -703,6 +796,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "6d9e9f29",
|
||||
"metadata": {
|
||||
"id": "evaluate_the_model:mbsdk"
|
||||
},
|
||||
@@ -710,35 +804,38 @@
|
||||
"## Review model evaluation scores\n",
|
||||
"After your model has finished training, you can review the evaluation scores for it.\n",
|
||||
"\n",
|
||||
"First, you need to get a reference to the new model. As with datasets, you can either use the reference to the model variable you created when you deployed the model or you can list all of the models in your project."
|
||||
"You can check the model's evaluation results using the `get_model_evaluation` method of the Vertex AI Model resource.\n",
|
||||
"\n",
|
||||
"Just like Vertex AI datasets, you can either use the reference to the model variable you created when you deployed the model or you can filter from the list of all of the models in your project using the model's display name as given below."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "59a76fa5",
|
||||
"metadata": {
|
||||
"id": "evaluate_the_model:mbsdk"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Get model resource ID\n",
|
||||
"models = aiplatform.Model.list(filter=\"display_name=hmdb_\" + TIMESTAMP)\n",
|
||||
"# Get model resource ID using the display_name\n",
|
||||
"models = aiplatform.Model.list(filter=\"display_name=hmdb_\" + UUID)\n",
|
||||
"\n",
|
||||
"# Get a reference to the Model Service client\n",
|
||||
"client_options = {\"api_endpoint\": f\"{REGION}-aiplatform.googleapis.com\"}\n",
|
||||
"model_service_client = aiplatform.gapic.ModelServiceClient(\n",
|
||||
" client_options=client_options\n",
|
||||
")\n",
|
||||
"if len(models) != 0:\n",
|
||||
"\n",
|
||||
"model_evaluations = model_service_client.list_model_evaluations(\n",
|
||||
" parent=models[0].resource_name\n",
|
||||
")\n",
|
||||
"model_evaluation = list(model_evaluations)[0]\n",
|
||||
"print(model_evaluation)"
|
||||
" # Get the model object\n",
|
||||
" model_rsc_name = models[0].resource_name\n",
|
||||
" print(\"Model resource name:\", model_rsc_name)\n",
|
||||
" model = aiplatform.Model(model_rsc_name)\n",
|
||||
"\n",
|
||||
" # Print the model evaluation\n",
|
||||
" model_eval = model.get_model_evaluation()\n",
|
||||
" print(model_eval.to_dict())"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "060d3bae",
|
||||
"metadata": {
|
||||
"id": "make_prediction"
|
||||
},
|
||||
@@ -750,18 +847,20 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "e614b9bf",
|
||||
"metadata": {
|
||||
"id": "get_test_items:batch_prediction"
|
||||
},
|
||||
"source": [
|
||||
"### Get test item(s)\n",
|
||||
"\n",
|
||||
"Now do a batch prediction to your Vertex model. You will use arbitrary examples out of the dataset as a test items. Don't be concerned that the examples were likely used in training the model -- we just want to demonstrate how to make a prediction."
|
||||
"Now do a batch prediction to your Vertex AI model. You use arbitrary examples from the dataset as a test items. Don't be concerned that the examples were likely used in training the model as this tutorial is just about how to make a batch prediction."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "bae97d10",
|
||||
"metadata": {
|
||||
"id": "get_test_items:automl,vcn,csv"
|
||||
},
|
||||
@@ -783,13 +882,14 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "54138ea2",
|
||||
"metadata": {
|
||||
"id": "make_batch_file:automl,video"
|
||||
},
|
||||
"source": [
|
||||
"### Make a batch input file\n",
|
||||
"\n",
|
||||
"Now make a batch input file, which you store in your local Cloud Storage bucket. The batch input file can be either CSV or JSONL. You will use JSONL in this tutorial. For JSONL file, you make one dictionary entry per line for each video. The dictionary contains the key/value pairs:\n",
|
||||
"Now make a batch input file, which you store in your local Cloud Storage bucket. The batch input file can be either CSV or JSONL. You use JSONL in this tutorial. For JSONL file, you make one dictionary entry per line for each video. The dictionary contains the key/value pairs:\n",
|
||||
"\n",
|
||||
"- `content`: The Cloud Storage path to the video.\n",
|
||||
"- `mimeType`: The content type. In our example, it is a `avi` file.\n",
|
||||
@@ -800,6 +900,7 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "ce7da5dd",
|
||||
"metadata": {
|
||||
"id": "make_batch_file:automl,video"
|
||||
},
|
||||
@@ -810,7 +911,7 @@
|
||||
"from google.cloud import storage\n",
|
||||
"\n",
|
||||
"test_filename = \"test.jsonl\"\n",
|
||||
"gcs_input_uri = BUCKET_NAME + \"/\" + test_filename\n",
|
||||
"gcs_input_uri = BUCKET_URI + \"/\" + test_filename\n",
|
||||
"\n",
|
||||
"data_1 = {\n",
|
||||
" \"content\": test_item_1,\n",
|
||||
@@ -826,42 +927,66 @@
|
||||
"}\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"bucket = storage.Client(project=PROJECT_ID).bucket(BUCKET_NAME.replace(\"gs://\", \"\"))\n",
|
||||
"bucket = storage.Client(project=PROJECT_ID).bucket(BUCKET_NAME)\n",
|
||||
"blob = bucket.blob(blob_name=test_filename)\n",
|
||||
"data = json.dumps(data_1) + \"\\n\" + json.dumps(data_2) + \"\\n\"\n",
|
||||
"blob.upload_from_string(data)\n",
|
||||
"print(gcs_input_uri)\n",
|
||||
"print(gcs_input_uri)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "5bbefe4a-e05f-4ed7-acf8-a0588757c376",
|
||||
"metadata": {
|
||||
"id": "d56366168ec5"
|
||||
},
|
||||
"source": [
|
||||
"### Check input content\n",
|
||||
"Check the contents of the `test.jsonl`."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "a98d1c39-29f2-40c7-8267-91afebb8a440",
|
||||
"metadata": {
|
||||
"id": "378131e21a7e"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil cat $gcs_input_uri"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "105f3bc5",
|
||||
"metadata": {
|
||||
"id": "batch_request:mbsdk"
|
||||
},
|
||||
"source": [
|
||||
"### Make the batch prediction request\n",
|
||||
"\n",
|
||||
"Now that your Model resource is trained, you can make a batch prediction by invoking the batch_predict() method, with the following parameters:\n",
|
||||
"Now that your Vertex AI Model resource is trained, you can make a batch prediction by invoking the batch_predict() method, with the following parameters:\n",
|
||||
"\n",
|
||||
"- `job_display_name`: The human readable name for the batch prediction job.\n",
|
||||
"- `gcs_source`: A list of one or more batch request input files.\n",
|
||||
"- `gcs_destination_prefix`: The Cloud Storage location for storing the batch prediction resuls.\n",
|
||||
"- `sync`: If set to True, the call will block while waiting for the asynchronous batch job to complete."
|
||||
"- `sync`: If set to True, the call blocks while waiting for the asynchronous batch job to complete."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "5657e704",
|
||||
"metadata": {
|
||||
"id": "batch_request:mbsdk"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"batch_predict_job = model.batch_predict(\n",
|
||||
" job_display_name=\"hmdb_\" + TIMESTAMP,\n",
|
||||
" job_display_name=\"hmdb_\" + UUID,\n",
|
||||
" gcs_source=gcs_input_uri,\n",
|
||||
" gcs_destination_prefix=BUCKET_NAME,\n",
|
||||
" gcs_destination_prefix=BUCKET_URI,\n",
|
||||
" sync=False,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
@@ -870,6 +995,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "c86ec9ec",
|
||||
"metadata": {
|
||||
"id": "batch_request_wait:mbsdk"
|
||||
},
|
||||
@@ -882,6 +1008,7 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "2f108cc8",
|
||||
"metadata": {
|
||||
"id": "batch_request_wait:mbsdk"
|
||||
},
|
||||
@@ -892,6 +1019,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "63e33110",
|
||||
"metadata": {
|
||||
"id": "get_batch_prediction:mbsdk,vcn"
|
||||
},
|
||||
@@ -914,6 +1042,7 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "a76f3f2c",
|
||||
"metadata": {
|
||||
"id": "get_batch_prediction:mbsdk,vcn"
|
||||
},
|
||||
@@ -928,7 +1057,7 @@
|
||||
"\n",
|
||||
"for prediction_result in prediction_results:\n",
|
||||
" gfile_name = f\"gs://{bp_iter_outputs.bucket.name}/{prediction_result}\".replace(\n",
|
||||
" BUCKET_NAME + \"/\", \"\"\n",
|
||||
" BUCKET_URI + \"/\", \"\"\n",
|
||||
" )\n",
|
||||
" data = bucket.get_blob(gfile_name).download_as_string()\n",
|
||||
" data = json.loads(data)\n",
|
||||
@@ -937,11 +1066,12 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "000413e5",
|
||||
"metadata": {
|
||||
"id": "cleanup:mbsdk"
|
||||
},
|
||||
"source": [
|
||||
"# Cleaning up\n",
|
||||
"## Clean up\n",
|
||||
"\n",
|
||||
"To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
|
||||
"project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
|
||||
@@ -958,6 +1088,7 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "7761ab4d",
|
||||
"metadata": {
|
||||
"id": "cleanup:mbsdk"
|
||||
},
|
||||
@@ -979,8 +1110,8 @@
|
||||
"batch_predict_job.delete()\n",
|
||||
"\n",
|
||||
"# Delete the Cloud storage bucket\n",
|
||||
"if delete_bucket is True:\n",
|
||||
" ! gsutil rm -r $BUCKET_NAME"
|
||||
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
|
||||
" ! gsutil rm -r $BUCKET_URI"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
+2451
File diff suppressed because it is too large
Load Diff
-1388
File diff suppressed because it is too large
Load Diff
-169
@@ -1,169 +0,0 @@
|
||||
# Copyright 2022 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# https://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""
|
||||
data_preprocessing.py is the module for
|
||||
|
||||
- ingest data
|
||||
- do simple preprocessing tasks
|
||||
- upload processed data to gcs
|
||||
"""
|
||||
|
||||
# Libraries --------------------------------------------------------------------------------
|
||||
import logging
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
try:
|
||||
from pyspark import SparkContext, SparkConf
|
||||
from pyspark.sql import SparkSession
|
||||
except ImportError as error:
|
||||
print('WARN: Something wrong with pyspark library. Please check configuration settings!')
|
||||
print(error)
|
||||
|
||||
from pyspark.sql.types import StructType, DoubleType, StringType
|
||||
|
||||
# Variables --------------------------------------------------------------------------------
|
||||
DATA_SCHEMA = (StructType()
|
||||
.add("label", StringType(), True)
|
||||
.add("loan_amount", StringType(), True)
|
||||
.add("loan_term", StringType(), True)
|
||||
.add("property_area", StringType(), True)
|
||||
.add("timestamp", StringType(), True)
|
||||
.add("entity_type_customer_id", StringType(), True)
|
||||
.add("feature_7", DoubleType(), True)
|
||||
.add("feature_3", DoubleType(), True)
|
||||
.add("feature_1", DoubleType(), True)
|
||||
.add("feature_9", DoubleType(), True)
|
||||
.add("feature_5", DoubleType(), True)
|
||||
.add("feature_0", DoubleType(), True)
|
||||
.add("feature_8", DoubleType(), True)
|
||||
.add("feature_4", DoubleType(), True)
|
||||
.add("feature_2", DoubleType(), True)
|
||||
.add("feature_6", DoubleType(), True)
|
||||
)
|
||||
|
||||
ENTITY_CUSTOMER_ID = 'entity_type_customer_id'
|
||||
FEATURE_STORE_IDS = ['timestamp', 'entity_type_customer_id']
|
||||
CATEGORICAL_VARIABLES = ['loan_term', 'property_area']
|
||||
IDX_CATEGORICAL_FEATURES = [f'{col}_idx' for col in CATEGORICAL_VARIABLES]
|
||||
TARGET = 'label'
|
||||
|
||||
|
||||
# Helpers ----------------------------------------------------------------------------------
|
||||
|
||||
def set_logger():
|
||||
"""
|
||||
Set logger for the module
|
||||
Returns:
|
||||
logger: logger object
|
||||
"""
|
||||
fmt_pattern = "%(asctime)s — %(name)s — %(levelname)s —" "%(funcName)s:%(lineno)d — %(message)s"
|
||||
main_logger = logging.getLogger(__name__)
|
||||
main_logger.setLevel(logging.INFO)
|
||||
main_logger.propagate = False
|
||||
stream_handler = logging.StreamHandler(sys.stdout)
|
||||
stream_handler.setLevel(logging.INFO)
|
||||
formatter = logging.Formatter(fmt_pattern)
|
||||
stream_handler.setFormatter(formatter)
|
||||
main_logger.addHandler(stream_handler)
|
||||
return main_logger
|
||||
|
||||
|
||||
def get_args():
|
||||
"""
|
||||
Get arguments from command line
|
||||
Returns:
|
||||
args: arguments from command line
|
||||
"""
|
||||
args_parser = argparse.ArgumentParser()
|
||||
args_parser.add_argument(
|
||||
'--train-data-path',
|
||||
help='The GCS path of training sample',
|
||||
type=str,
|
||||
required=True)
|
||||
args_parser.add_argument(
|
||||
'--out-process-path',
|
||||
help='''
|
||||
The path to load processed data.
|
||||
Format:
|
||||
- locally: /path/to/dir
|
||||
- cloud: gs://bucket/path
|
||||
''',
|
||||
type=str,
|
||||
required=True)
|
||||
return args_parser.parse_args()
|
||||
|
||||
|
||||
# Main -------------------------------------------------------------------------------------
|
||||
|
||||
def main(logger, args):
|
||||
"""
|
||||
Main function
|
||||
Args:
|
||||
logger: logger object
|
||||
args: arguments from command line
|
||||
Returns:
|
||||
None
|
||||
"""
|
||||
# variables
|
||||
train_data_path = args.train_data_path
|
||||
output_data_path = args.out_process_path
|
||||
|
||||
logger.info('initializing data preprocessing.')
|
||||
logger.info('start spark session.')
|
||||
|
||||
spark = (SparkSession.builder
|
||||
.master("local[*]")
|
||||
.appName("spark go live")
|
||||
.config('spark.ui.port', '4050')
|
||||
.getOrCreate())
|
||||
try:
|
||||
logger.info(f'spark version: {spark.sparkContext.version}')
|
||||
logger.info('start ingesting data.')
|
||||
|
||||
training_data_raw_df = (spark.read.option("header", True)
|
||||
.option("delimiter", ',')
|
||||
.schema(DATA_SCHEMA)
|
||||
.csv(train_data_path)
|
||||
.drop(*FEATURE_STORE_IDS))
|
||||
|
||||
training_data_raw_df = training_data_raw_df.withColumn("label",
|
||||
training_data_raw_df.label.cast('double'))
|
||||
training_data_raw_df = training_data_raw_df.withColumn("loan_amount",
|
||||
training_data_raw_df.loan_amount.cast('double'))
|
||||
training_data_raw_df.show(truncate=False)
|
||||
|
||||
logger.info(f'load prepared data to {output_data_path}.')
|
||||
if output_data_path.startswith('gs://'):
|
||||
training_data_raw_df.write.mode('overwrite').csv(str(output_data_path), header=True)
|
||||
else:
|
||||
output_file_path = Path(output_data_path)
|
||||
output_file_path.mkdir(parents=True, exist_ok=True)
|
||||
training_data_raw_df.write.mode('overwrite').csv(str(output_file_path), header=True)
|
||||
except RuntimeError as main_error:
|
||||
logger.error(main_error)
|
||||
else:
|
||||
logger.info('data preprocessing successfully completed!')
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
runtime_args = get_args()
|
||||
runtime_logger = set_logger()
|
||||
main(runtime_logger, runtime_args)
|
||||
-366
@@ -1,366 +0,0 @@
|
||||
# Copyright 2022 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# https://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""
|
||||
hp_model_tuning.py is the module for hypertune the spark pipeline
|
||||
"""
|
||||
|
||||
# Libraries --------------------------------------------------------------------------------
|
||||
import logging
|
||||
import sys
|
||||
import argparse
|
||||
from os import environ
|
||||
from datetime import datetime
|
||||
from pathlib import Path as path
|
||||
import tempfile
|
||||
from urllib.parse import urlparse, urljoin
|
||||
import json
|
||||
|
||||
try:
|
||||
from pyspark import SparkContext, SparkConf
|
||||
from pyspark.sql import SparkSession
|
||||
except ImportError as e:
|
||||
print('WARN: Something wrong with pyspark library. Please check configuration settings!')
|
||||
print(e)
|
||||
|
||||
from pyspark.sql.types import StructType, DoubleType, StringType
|
||||
from pyspark.sql.functions import col, udf
|
||||
from pyspark.sql.functions import round as spark_round
|
||||
from pyspark.ml.feature import StringIndexer, StandardScaler, VectorAssembler
|
||||
from pyspark.ml.classification import RandomForestClassifier
|
||||
from pyspark.ml.evaluation import BinaryClassificationEvaluator, MulticlassClassificationEvaluator
|
||||
from pyspark.ml.tuning import ParamGridBuilder, CrossValidator
|
||||
from pyspark.ml import Pipeline
|
||||
|
||||
from google.cloud import storage
|
||||
|
||||
# Variables --------------------------------------------------------------------------------
|
||||
|
||||
# Data schema
|
||||
DATA_SCHEMA = (StructType()
|
||||
.add("label", DoubleType(), True)
|
||||
.add("loan_amount", DoubleType(), True)
|
||||
.add("loan_term", StringType(), True)
|
||||
.add("property_area", StringType(), True)
|
||||
.add("feature_7", DoubleType(), True)
|
||||
.add("feature_3", DoubleType(), True)
|
||||
.add("feature_1", DoubleType(), True)
|
||||
.add("feature_9", DoubleType(), True)
|
||||
.add("feature_5", DoubleType(), True)
|
||||
.add("feature_0", DoubleType(), True)
|
||||
.add("feature_8", DoubleType(), True)
|
||||
.add("feature_4", DoubleType(), True)
|
||||
.add("feature_2", DoubleType(), True)
|
||||
.add("feature_6", DoubleType(), True)
|
||||
)
|
||||
|
||||
# Training
|
||||
TARGET = 'label'
|
||||
CATEGORICAL_VARIABLES = ['loan_term', 'property_area']
|
||||
IDX_CATEGORICAL_FEATURES = [f'{col}_idx' for col in CATEGORICAL_VARIABLES]
|
||||
REAL_TIME_FEATURES_VECTOR = 'real_time_features_vector'
|
||||
REAL_TIME_FEATURES = 'real_time_features'
|
||||
FEATURES_SELECTED = ['feature_0', 'feature_1', 'feature_2', 'feature_3', 'feature_4', 'feature_5',
|
||||
'feature_6', 'feature_7', 'feature_8', 'feature_9', 'real_time_features']
|
||||
FEATURES = 'features'
|
||||
RANDOM_SEED = 8
|
||||
RANDOM_QUOTAS = [0.8, 0.2]
|
||||
MAX_DEPTH = [5, 10, 15]
|
||||
MAX_BINS = [24, 32, 40]
|
||||
N_TREES = [25, 30, 35]
|
||||
N_FOLDS = 5
|
||||
|
||||
|
||||
# Helpers ----------------------------------------------------------------------------------
|
||||
def set_logger():
|
||||
"""
|
||||
Set logger for the module
|
||||
Returns:
|
||||
logger: logger object
|
||||
"""
|
||||
fmt_pattern = "%(asctime)s — %(name)s — %(levelname)s —" "%(funcName)s:%(lineno)d — %(message)s"
|
||||
main_logger = logging.getLogger(__name__)
|
||||
main_logger.setLevel(logging.INFO)
|
||||
main_logger.propagate = False
|
||||
stream_handler = logging.StreamHandler(sys.stdout)
|
||||
stream_handler.setLevel(logging.INFO)
|
||||
formatter = logging.Formatter(fmt_pattern)
|
||||
stream_handler.setFormatter(formatter)
|
||||
main_logger.addHandler(stream_handler)
|
||||
return main_logger
|
||||
|
||||
|
||||
def get_args():
|
||||
"""
|
||||
Get arguments from command line
|
||||
Returns:
|
||||
args: arguments from command line
|
||||
"""
|
||||
args_parser = argparse.ArgumentParser()
|
||||
args_parser.add_argument(
|
||||
'--train-path',
|
||||
help='''
|
||||
The GCS path of training data'
|
||||
Format:
|
||||
- locally: /path/to/dir
|
||||
- cloud: gs://bucket/path
|
||||
''',
|
||||
type=str,
|
||||
required=False)
|
||||
args_parser.add_argument(
|
||||
'--model-path',
|
||||
help='''
|
||||
The GCS path to store the trained model.
|
||||
Format:
|
||||
- locally: /path/to/dir
|
||||
- cloud: gs://bucket/path
|
||||
''',
|
||||
type=str,
|
||||
required=False)
|
||||
args_parser.add_argument(
|
||||
'--metrics-path',
|
||||
help='''
|
||||
The GCS path to store the metrics of model.
|
||||
Format:
|
||||
- locally: /path/to/dir
|
||||
- cloud: gs://bucket/path
|
||||
''',
|
||||
type=str,
|
||||
required=True)
|
||||
return args_parser.parse_args()
|
||||
|
||||
|
||||
def build_preprocessing_components():
|
||||
"""
|
||||
Build preprocessing components
|
||||
Returns:
|
||||
preprocessing_components: preprocessing components
|
||||
"""
|
||||
loan_term_indexer = StringIndexer(inputCol=CATEGORICAL_VARIABLES[0], outputCol=IDX_CATEGORICAL_FEATURES[0],
|
||||
stringOrderType='frequencyDesc', handleInvalid='keep')
|
||||
property_area_indexer = StringIndexer(inputCol=CATEGORICAL_VARIABLES[1], outputCol=IDX_CATEGORICAL_FEATURES[1],
|
||||
stringOrderType='frequencyDesc', handleInvalid='keep')
|
||||
data_preprocessing_stages = [loan_term_indexer, property_area_indexer]
|
||||
return data_preprocessing_stages
|
||||
|
||||
|
||||
def build_feature_engineering_components():
|
||||
"""
|
||||
Build feature engineering components
|
||||
Returns:
|
||||
feature_engineering_components: feature engineering components
|
||||
"""
|
||||
feature_engineering_stages = []
|
||||
realtime_vector_assembler = VectorAssembler(inputCols=IDX_CATEGORICAL_FEATURES, outputCol=REAL_TIME_FEATURES_VECTOR)
|
||||
realtime_scaler = StandardScaler(inputCol=REAL_TIME_FEATURES_VECTOR, outputCol=REAL_TIME_FEATURES)
|
||||
features_vector_assembler = VectorAssembler(inputCols=FEATURES_SELECTED, outputCol=FEATURES)
|
||||
feature_engineering_stages.extend((realtime_vector_assembler,
|
||||
realtime_scaler,
|
||||
features_vector_assembler))
|
||||
return feature_engineering_stages
|
||||
|
||||
|
||||
def build_training_model_component():
|
||||
"""
|
||||
Build training model component
|
||||
Returns:
|
||||
training_model_component: training model component
|
||||
"""
|
||||
model_training_stage = []
|
||||
rfor = RandomForestClassifier(featuresCol=FEATURES, labelCol=TARGET, seed=RANDOM_SEED)
|
||||
model_training_stage.append(rfor)
|
||||
return model_training_stage
|
||||
|
||||
|
||||
def build_hp_pipeline(data_preprocessing_stages, feature_engineering_stages, model_training_stage):
|
||||
"""
|
||||
Build hyperparameter pipeline
|
||||
Args:
|
||||
data_preprocessing_stages: preprocessing components
|
||||
feature_engineering_stages: feature engineering components
|
||||
model_training_stage: training model component
|
||||
Returns:
|
||||
hp_pipeline: hyperparameter pipeline
|
||||
"""
|
||||
pipeline = Pipeline(stages=data_preprocessing_stages + feature_engineering_stages + model_training_stage)
|
||||
params_grid = (ParamGridBuilder()
|
||||
.addGrid(model_training_stage[0].maxDepth, MAX_DEPTH)
|
||||
.addGrid(model_training_stage[0].maxBins, MAX_BINS)
|
||||
.addGrid(model_training_stage[0].numTrees, N_TREES)
|
||||
.build())
|
||||
evaluator = BinaryClassificationEvaluator(labelCol=TARGET)
|
||||
cross_validator = CrossValidator(estimator=pipeline,
|
||||
estimatorParamMaps=params_grid,
|
||||
evaluator=evaluator,
|
||||
numFolds=N_FOLDS)
|
||||
return cross_validator
|
||||
|
||||
|
||||
def get_true_score_prediction(predictions, target):
|
||||
"""
|
||||
Get true score and prediction
|
||||
Args:
|
||||
predictions: predictions
|
||||
target: target column
|
||||
Returns:
|
||||
roc_dict: a dict of roc values for each class
|
||||
"""
|
||||
|
||||
split1_udf = udf(lambda value: value[1].item(), DoubleType())
|
||||
roc_dataset = predictions.select(col(target).alias('true'),
|
||||
spark_round(split1_udf('probability'), 5).alias('score'),
|
||||
'prediction')
|
||||
roc_df = roc_dataset.toPandas()
|
||||
roc_dict = roc_df.to_dict(orient='list')
|
||||
return roc_dict
|
||||
|
||||
|
||||
def get_metrics(predictions, target, mode):
|
||||
"""
|
||||
Get metrics
|
||||
Args:
|
||||
predictions: predictions
|
||||
target: target column
|
||||
mode: train or test
|
||||
Returns:
|
||||
metrics: metrics
|
||||
"""
|
||||
metric_labels = ['area_roc', 'area_prc', 'accuracy', 'f1', 'precision', 'recall']
|
||||
metric_cols = ['true', 'score', 'prediction']
|
||||
metric_keys = [f'{mode}_{ml}' for ml in metric_labels] + metric_cols
|
||||
|
||||
bc_evaluator = BinaryClassificationEvaluator(labelCol=target)
|
||||
mc_evaluator = MulticlassClassificationEvaluator(labelCol=target)
|
||||
|
||||
# areas, acc, f1, prec, rec
|
||||
metric_values = []
|
||||
area_roc = round(bc_evaluator.evaluate(predictions, {bc_evaluator.metricName: 'areaUnderROC'}), 5)
|
||||
area_prc = round(bc_evaluator.evaluate(predictions, {bc_evaluator.metricName: 'areaUnderPR'}), 5)
|
||||
acc = round(mc_evaluator.evaluate(predictions, {mc_evaluator.metricName: "accuracy"}), 5)
|
||||
f1 = round(mc_evaluator.evaluate(predictions, {mc_evaluator.metricName: "f1"}), 5)
|
||||
prec = round(mc_evaluator.evaluate(predictions, {mc_evaluator.metricName: "weightedPrecision"}), 5)
|
||||
rec = round(mc_evaluator.evaluate(predictions, {mc_evaluator.metricName: "weightedRecall"}), 5)
|
||||
|
||||
# true, score, prediction
|
||||
roc_dict = get_true_score_prediction(predictions, target)
|
||||
true = roc_dict['true']
|
||||
score = roc_dict['score']
|
||||
pred = roc_dict['prediction']
|
||||
|
||||
metric_values.extend((area_roc, area_prc, acc, f1, prec, rec, true, score, pred))
|
||||
metrics = dict(zip(metric_keys, metric_values))
|
||||
|
||||
return metrics
|
||||
|
||||
|
||||
def upload_file(bucket_name, source_file_name, destination_blob_name):
|
||||
storage_client = storage.Client()
|
||||
bucket = storage_client.bucket(bucket_name)
|
||||
blob = bucket.blob(destination_blob_name)
|
||||
blob.upload_from_filename(source_file_name)
|
||||
|
||||
|
||||
def write_metrics(bucket_name, metrics, destination, dir='/tmp'):
|
||||
temp_dir = tempfile.TemporaryDirectory(dir=dir)
|
||||
temp_metrics_file_path = str(path(temp_dir.name) / path(destination).name)
|
||||
with open(temp_metrics_file_path, 'w') as temp_file:
|
||||
json.dump(metrics, temp_file)
|
||||
upload_file(bucket_name, temp_metrics_file_path, destination)
|
||||
temp_dir.cleanup()
|
||||
|
||||
|
||||
# Main -------------------------------------------------------------------------------------
|
||||
|
||||
def main(logger, args):
|
||||
"""
|
||||
Main function
|
||||
Args:
|
||||
logger: logger
|
||||
args: args
|
||||
Returns:
|
||||
None
|
||||
"""
|
||||
train_path = args.train_path
|
||||
model_path = args.model_path
|
||||
metrics_path = args.metrics_path
|
||||
|
||||
try:
|
||||
logger.info('initializing pipeline training.')
|
||||
logger.info('start spark session.')
|
||||
spark = (SparkSession.builder
|
||||
.master("local[*]")
|
||||
.appName("spark go live")
|
||||
.config('spark.ui.port', '4050')
|
||||
.config('spark.jars.packages', 'ml.combust.mleap:mleap-runtime_2.12:0.19.0')
|
||||
.config('spark.jars.packages', 'ml.combust.mleap:mleap-base_2.12:0.19.0')
|
||||
.config('spark.jars.packages', 'ml.combust.mleap:mleap-spark_2.12:0.19.0')
|
||||
.config('spark.jars.packages', 'ml.combust.mleap:mleap-spark-extension_2.12:0.19.0')
|
||||
.getOrCreate())
|
||||
logger.info(f'spark version: {spark.sparkContext.version}')
|
||||
logger.info('start building pipeline.')
|
||||
preprocessing_stages = build_preprocessing_components()
|
||||
feature_engineering_stages = build_feature_engineering_components()
|
||||
model_training_stage = build_training_model_component()
|
||||
pipeline_cross_validator = build_hp_pipeline(preprocessing_stages, feature_engineering_stages,
|
||||
model_training_stage)
|
||||
logger.info(f'load train data from {train_path}.')
|
||||
if train_path.startswith('bq://'):
|
||||
raw_data = spark.read.format('bigquery') \
|
||||
.option('table', train_path.replace('bq://', '')) \
|
||||
.load()
|
||||
else:
|
||||
raw_data = (spark.read.format('csv')
|
||||
.option("header", "true")
|
||||
.schema(DATA_SCHEMA)
|
||||
.load(train_path))
|
||||
logger.info(f'fit model pipeline.')
|
||||
train, test = raw_data.randomSplit(RANDOM_QUOTAS, seed=RANDOM_SEED)
|
||||
pipeline_model = pipeline_cross_validator.fit(train)
|
||||
predictions = pipeline_model.transform(test)
|
||||
metrics = get_metrics(predictions, TARGET, 'test')
|
||||
for m, v in metrics.items():
|
||||
print(f'{m}: {v}')
|
||||
|
||||
logger.info(f'load model pipeline in {model_path}.')
|
||||
if model_path.startswith('gs://'):
|
||||
pipeline_model.write().overwrite().save(model_path)
|
||||
else:
|
||||
path(model_path).mkdir(parents=True, exist_ok=True)
|
||||
pipeline_model.write().overwrite().save(model_path)
|
||||
|
||||
logger.info(f'Upload metrics under {metrics_path}.')
|
||||
if metrics_path.startswith('gs://'):
|
||||
bucket = urlparse(model_path).netloc
|
||||
metrics_file_path = urlparse(metrics_path).path.strip('/')
|
||||
write_metrics(bucket, metrics, metrics_file_path)
|
||||
else:
|
||||
metrics_version_path = path(metrics_path).parents[0]
|
||||
metrics_version_path.mkdir(parents=True, exist_ok=True)
|
||||
with open(metrics_path, 'w') as json_file:
|
||||
json.dump(metrics, json_file)
|
||||
json_file.close()
|
||||
except RuntimeError as main_error:
|
||||
logger.error(main_error)
|
||||
else:
|
||||
logger.info('model pipeline training successfully completed!')
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
runtime_args = get_args()
|
||||
runtime_logger = set_logger()
|
||||
main(runtime_logger, runtime_args)
|
||||
-359
@@ -1,359 +0,0 @@
|
||||
# Copyright 2022 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# https://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""
|
||||
model_training.py is the module for training spark pipeline
|
||||
"""
|
||||
|
||||
# Libraries --------------------------------------------------------------------------------
|
||||
import logging
|
||||
import sys
|
||||
import argparse
|
||||
from pathlib import Path as path
|
||||
import tempfile
|
||||
import json
|
||||
from urllib.parse import urlparse
|
||||
|
||||
try:
|
||||
from pyspark import SparkContext, SparkConf
|
||||
from pyspark.sql import SparkSession
|
||||
except ImportError as e:
|
||||
print('WARN: Something wrong with pyspark library. Please check configuration settings!')
|
||||
print(e)
|
||||
|
||||
from pyspark.sql.types import StructType, DoubleType, StringType
|
||||
from pyspark.sql.functions import col, udf
|
||||
from pyspark.sql.functions import round as spark_round
|
||||
from pyspark.ml.feature import StringIndexer, StandardScaler, VectorAssembler
|
||||
from pyspark.ml.classification import RandomForestClassifier
|
||||
from pyspark.ml.evaluation import BinaryClassificationEvaluator, MulticlassClassificationEvaluator
|
||||
from pyspark.ml import Pipeline
|
||||
|
||||
from google.cloud import storage
|
||||
|
||||
# Variables --------------------------------------------------------------------------------
|
||||
|
||||
# Data schema
|
||||
DATA_SCHEMA = (StructType()
|
||||
.add("label", DoubleType(), True)
|
||||
.add("loan_amount", DoubleType(), True)
|
||||
.add("loan_term", StringType(), True)
|
||||
.add("property_area", StringType(), True)
|
||||
.add("feature_7", DoubleType(), True)
|
||||
.add("feature_3", DoubleType(), True)
|
||||
.add("feature_1", DoubleType(), True)
|
||||
.add("feature_9", DoubleType(), True)
|
||||
.add("feature_5", DoubleType(), True)
|
||||
.add("feature_0", DoubleType(), True)
|
||||
.add("feature_8", DoubleType(), True)
|
||||
.add("feature_4", DoubleType(), True)
|
||||
.add("feature_2", DoubleType(), True)
|
||||
.add("feature_6", DoubleType(), True)
|
||||
)
|
||||
|
||||
# Training
|
||||
TARGET = 'label'
|
||||
CATEGORICAL_VARIABLES = ['loan_term', 'property_area']
|
||||
IDX_CATEGORICAL_FEATURES = [f'{col}_idx' for col in CATEGORICAL_VARIABLES]
|
||||
REAL_TIME_FEATURES_VECTOR = 'real_time_features_vector'
|
||||
REAL_TIME_FEATURES = 'real_time_features'
|
||||
FEATURES_SELECTED = ['feature_0', 'feature_1', 'feature_2', 'feature_3', 'feature_4', 'feature_5',
|
||||
'feature_6', 'feature_7', 'feature_8', 'feature_9', 'real_time_features']
|
||||
FEATURES = 'features'
|
||||
RANDOM_SEED = 8
|
||||
RANDOM_QUOTAS = [0.8, 0.2]
|
||||
|
||||
|
||||
# Helpers ----------------------------------------------------------------------------------
|
||||
def set_logger():
|
||||
"""
|
||||
Set logger
|
||||
Returns:
|
||||
logger: logger
|
||||
"""
|
||||
fmt_pattern = "%(asctime)s — %(name)s — %(levelname)s —" "%(funcName)s:%(lineno)d — %(message)s"
|
||||
main_logger = logging.getLogger(__name__)
|
||||
main_logger.setLevel(logging.INFO)
|
||||
main_logger.propagate = False
|
||||
stream_handler = logging.StreamHandler(sys.stdout)
|
||||
stream_handler.setLevel(logging.INFO)
|
||||
formatter = logging.Formatter(fmt_pattern)
|
||||
stream_handler.setFormatter(formatter)
|
||||
main_logger.addHandler(stream_handler)
|
||||
return main_logger
|
||||
|
||||
|
||||
def get_args():
|
||||
"""
|
||||
Get arguments
|
||||
Returns:
|
||||
args: arguments
|
||||
"""
|
||||
args_parser = argparse.ArgumentParser()
|
||||
args_parser.add_argument(
|
||||
'--train-path',
|
||||
help='''
|
||||
The GCS path of training data'
|
||||
Format:
|
||||
- locally: /path/to/dir
|
||||
- cloud: gs://bucket/path
|
||||
''',
|
||||
type=str,
|
||||
required=True)
|
||||
args_parser.add_argument(
|
||||
'--model-path',
|
||||
help='''
|
||||
The GCS path to store the trained model.
|
||||
Format:
|
||||
- locally: /path/to/dir
|
||||
- cloud: gs://bucket/path
|
||||
''',
|
||||
type=str,
|
||||
required=True)
|
||||
args_parser.add_argument(
|
||||
'--metrics-path',
|
||||
help='''
|
||||
The GCS path to store the metrics of model.
|
||||
Format:
|
||||
- locally: /path/to/dir
|
||||
- cloud: gs://bucket/path
|
||||
''',
|
||||
type=str,
|
||||
required=True)
|
||||
return args_parser.parse_args()
|
||||
|
||||
|
||||
def build_preprocessing_components():
|
||||
"""
|
||||
Build preprocessing components
|
||||
Returns:
|
||||
data_preprocessing_stages: data preprocessing stages
|
||||
"""
|
||||
loan_term_indexer = StringIndexer(inputCol=CATEGORICAL_VARIABLES[0], outputCol=IDX_CATEGORICAL_FEATURES[0],
|
||||
stringOrderType='frequencyDesc', handleInvalid='keep')
|
||||
property_area_indexer = StringIndexer(inputCol=CATEGORICAL_VARIABLES[1], outputCol=IDX_CATEGORICAL_FEATURES[1],
|
||||
stringOrderType='frequencyDesc', handleInvalid='keep')
|
||||
data_preprocessing_stages = [loan_term_indexer, property_area_indexer]
|
||||
return data_preprocessing_stages
|
||||
|
||||
|
||||
def build_feature_engineering_components():
|
||||
"""
|
||||
Build feature engineering components
|
||||
Returns:
|
||||
feature_engineering_stages: feature engineering stages
|
||||
"""
|
||||
feature_engineering_stages = []
|
||||
realtime_vector_assembler = VectorAssembler(inputCols=IDX_CATEGORICAL_FEATURES, outputCol=REAL_TIME_FEATURES_VECTOR)
|
||||
realtime_scaler = StandardScaler(inputCol=REAL_TIME_FEATURES_VECTOR, outputCol=REAL_TIME_FEATURES)
|
||||
features_vector_assembler = VectorAssembler(inputCols=FEATURES_SELECTED, outputCol=FEATURES)
|
||||
feature_engineering_stages.extend((realtime_vector_assembler,
|
||||
realtime_scaler,
|
||||
features_vector_assembler))
|
||||
return feature_engineering_stages
|
||||
|
||||
|
||||
def build_training_model_component():
|
||||
"""
|
||||
Build training model component
|
||||
Returns:
|
||||
model_training_stage: model_training_stage
|
||||
"""
|
||||
model_training_stage = []
|
||||
rfor = RandomForestClassifier(featuresCol=FEATURES, labelCol=TARGET, seed=RANDOM_SEED)
|
||||
model_training_stage.append(rfor)
|
||||
return model_training_stage
|
||||
|
||||
|
||||
def build_pipeline(data_preprocessing_stages, feature_engineering_stages, model_training_stage):
|
||||
"""
|
||||
Build pipeline
|
||||
Args:
|
||||
data_preprocessing_stages: data preprocessing stages
|
||||
feature_engineering_stages: feature engineering stages
|
||||
model_training_stage: model_training_stage
|
||||
Returns:
|
||||
pipeline: pipeline
|
||||
"""
|
||||
pipeline = Pipeline(stages=data_preprocessing_stages + feature_engineering_stages + model_training_stage)
|
||||
return pipeline
|
||||
|
||||
|
||||
def get_true_score_prediction(predictions, target):
|
||||
"""
|
||||
Get true score prediction
|
||||
Args:
|
||||
predictions: predictions
|
||||
target: target
|
||||
Returns:
|
||||
roc_dict: a dict of roc values for each class
|
||||
"""
|
||||
split1_udf = udf(lambda value: value[1].item(), DoubleType())
|
||||
roc_dataset = predictions.select(col(target).alias('true'),
|
||||
spark_round(split1_udf('probability'), 5).alias('score'),
|
||||
'prediction')
|
||||
roc_df = roc_dataset.toPandas()
|
||||
roc_dict = roc_df.to_dict(orient='list')
|
||||
return roc_dict
|
||||
|
||||
|
||||
def get_metrics(predictions, target, mode):
|
||||
"""
|
||||
Get metrics
|
||||
Args:
|
||||
predictions: predictions
|
||||
target: target column name
|
||||
mode: train or test
|
||||
Returns:
|
||||
metrics: metrics
|
||||
"""
|
||||
metric_labels = ['area_roc', 'area_prc', 'accuracy', 'f1', 'precision', 'recall']
|
||||
metric_cols = ['true', 'score', 'prediction']
|
||||
metric_keys = [f'{mode}_{ml}' for ml in metric_labels] + metric_cols
|
||||
bc_evaluator = BinaryClassificationEvaluator(labelCol=target)
|
||||
mc_evaluator = MulticlassClassificationEvaluator(labelCol=target)
|
||||
|
||||
# areas, acc, f1, prec, rec
|
||||
metric_values = []
|
||||
area_roc = round(bc_evaluator.evaluate(predictions, {bc_evaluator.metricName: 'areaUnderROC'}), 5)
|
||||
area_prc = round(bc_evaluator.evaluate(predictions, {bc_evaluator.metricName: 'areaUnderPR'}), 5)
|
||||
acc = round(mc_evaluator.evaluate(predictions, {mc_evaluator.metricName: "accuracy"}), 5)
|
||||
f1 = round(mc_evaluator.evaluate(predictions, {mc_evaluator.metricName: "f1"}), 5)
|
||||
prec = round(mc_evaluator.evaluate(predictions, {mc_evaluator.metricName: "weightedPrecision"}), 5)
|
||||
rec = round(mc_evaluator.evaluate(predictions, {mc_evaluator.metricName: "weightedRecall"}), 5)
|
||||
|
||||
# true, score, prediction
|
||||
roc_dict = get_true_score_prediction(predictions, target)
|
||||
true = roc_dict['true']
|
||||
score = roc_dict['score']
|
||||
pred = roc_dict['prediction']
|
||||
|
||||
metric_values.extend((area_roc, area_prc, acc, f1, prec, rec, true, score, pred))
|
||||
metrics = dict(zip(metric_keys, metric_values))
|
||||
|
||||
return metrics
|
||||
|
||||
|
||||
def upload_file(bucket_name, source_file_name, destination_blob_name):
|
||||
"""
|
||||
Upload file to bucket
|
||||
Args:
|
||||
bucket_name: bucket name
|
||||
source_file_name: source file name
|
||||
destination_blob_name: destination blob name
|
||||
Returns:
|
||||
None
|
||||
"""
|
||||
storage_client = storage.Client()
|
||||
bucket = storage_client.bucket(bucket_name)
|
||||
blob = bucket.blob(destination_blob_name)
|
||||
blob.upload_from_filename(source_file_name)
|
||||
|
||||
|
||||
def write_metrics(bucket_name, metrics, destination, dir='/tmp'):
|
||||
"""
|
||||
Write metrics to file
|
||||
Args:
|
||||
bucket_name: bucket name
|
||||
metrics: metrics
|
||||
destination: destination
|
||||
dir: directory to write file temporarily
|
||||
Returns:
|
||||
None
|
||||
"""
|
||||
temp_dir = tempfile.TemporaryDirectory(dir=dir)
|
||||
temp_metrics_file_path = str(path(temp_dir.name) / path(destination).name)
|
||||
with open(temp_metrics_file_path, 'w') as temp_file:
|
||||
json.dump(metrics, temp_file)
|
||||
upload_file(bucket_name, temp_metrics_file_path, destination)
|
||||
temp_dir.cleanup()
|
||||
|
||||
|
||||
# Main -------------------------------------------------------------------------------------
|
||||
|
||||
def main(logger, args):
|
||||
"""
|
||||
Main function
|
||||
Args:
|
||||
logger: logger
|
||||
args: args
|
||||
Returns:
|
||||
None
|
||||
"""
|
||||
train_path = args.train_path
|
||||
model_path = args.model_path
|
||||
metrics_path = args.metrics_path
|
||||
|
||||
try:
|
||||
logger.info('initializing pipeline training.')
|
||||
logger.info('start spark session.')
|
||||
spark = (SparkSession.builder
|
||||
.master("local[*]")
|
||||
.appName("spark go live")
|
||||
.config('spark.ui.port', '4050')
|
||||
.getOrCreate())
|
||||
logger.info(f'spark version: {spark.sparkContext.version}')
|
||||
logger.info('start bulding pipeline.')
|
||||
preprocessing_stages = build_preprocessing_components()
|
||||
feature_engineering_stages = build_feature_engineering_components()
|
||||
model_training_stage = build_training_model_component()
|
||||
pipeline = build_pipeline(preprocessing_stages, feature_engineering_stages, model_training_stage)
|
||||
|
||||
logger.info(f'load train data from {train_path}.')
|
||||
raw_data = (spark.read.format('csv')
|
||||
.option("header", "true")
|
||||
.schema(DATA_SCHEMA)
|
||||
.load(train_path))
|
||||
|
||||
logger.info(f'fit model pipeline.')
|
||||
train, test = raw_data.randomSplit(RANDOM_QUOTAS, seed=RANDOM_SEED)
|
||||
pipeline_model = pipeline.fit(train)
|
||||
predictions = pipeline_model.transform(test)
|
||||
metrics = get_metrics(predictions, TARGET, 'test')
|
||||
for m, v in metrics.items():
|
||||
print(f'{m}: {v}')
|
||||
|
||||
logger.info(f'load model pipeline in {model_path}.')
|
||||
pipeline.write().overwrite().save(model_path)
|
||||
if model_path.startswith('gs://'):
|
||||
pipeline.write().overwrite().save(model_path)
|
||||
else:
|
||||
path(model_path).mkdir(parents=True, exist_ok=True)
|
||||
pipeline.write().overwrite().save(model_path)
|
||||
|
||||
logger.info(f'Upload metrics under {metrics_path}.')
|
||||
if metrics_path.startswith('gs://'):
|
||||
bucket = urlparse(model_path).netloc
|
||||
metrics_file_path = urlparse(metrics_path).path.strip('/')
|
||||
write_metrics(bucket, metrics, metrics_file_path)
|
||||
else:
|
||||
metrics_version_path = path(metrics_path).parents[0]
|
||||
metrics_version_path.mkdir(parents=True, exist_ok=True)
|
||||
with open(metrics_path, 'w') as json_file:
|
||||
json.dump(metrics, json_file)
|
||||
json_file.close()
|
||||
except RuntimeError as main_error:
|
||||
logger.error(main_error)
|
||||
else:
|
||||
logger.info('model pipeline training successfully completed!')
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
runtime_args = get_args()
|
||||
runtime_logger = set_logger()
|
||||
main(runtime_logger, runtime_args)
|
||||
+1143
-118
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user