diff --git a/community-content/CODEOWNERS b/community-content/CODEOWNERS index 98fe2d2b6..502aaf08f 100644 --- a/community-content/CODEOWNERS +++ b/community-content/CODEOWNERS @@ -1,5 +1,6 @@ * @vertex-ai-samples-contributors @GoogleCloudPlatform/cloudml-samples-owners /tf_agents_bandits_movie_recommendation_with_kfp_and_vertex_sdk @yinghsienwu +/pytorch_pre_built_images_deployment @googleapis/vertex-prediction-team /pytorch_text_classification_using_vertex_sdk_and_gcloud @RajeshThallam /pytorch_text_classification_using_vertex_sdk_and_gcloud @RajeshThallam @ultrons /sklearn_text_classification_from_script_using_vertex_sdk @maxhardt diff --git a/community-content/pytorch_pre_built_images_deployment/README.md b/community-content/pytorch_pre_built_images_deployment/README.md new file mode 100644 index 000000000..fccca8461 --- /dev/null +++ b/community-content/pytorch_pre_built_images_deployment/README.md @@ -0,0 +1,30 @@ +# PyTorch Deployment on Google Cloud: Text Classification + +**This is an Experimental release**, covered by the Pre-GA Offerings Terms of your Google Cloud Platform [Terms of Service](https://cloud.google.com/terms). + +Experiments are focused on validating a prototype and are not guaranteed to be released. They are not intended for production use or covered by any SLA, support obligation, or deprecation policy and might be subject to backward-incompatible changes. + +**Kindly drop us a note before you run any scale tests.** + +**Do not hesitate to contact vertexai-prediction-preview-feedback@google.com if you have any questions or run into any issues.** + +The projects need to be added to the allowlist in order to deploy PyTorch models using Vertex AI Prediction pre-built PyTorch images. If you are interested in the feature, please send an email to vertexai-prediction-preview-feedback@google.com to provide your project numbers and project ids. + +## Overview + +In the PyTorch on Google Cloud series of blog posts, we aim to share how to deploy PyTorch models at scale on [Vertex AI](https://cloud.google.com/vertex-ai). + +This tutorial on text classification shows how to deploy a PyTorch based text classification model on [Vertex AI](https://cloud.google.com/vertex-ai/docs/start/client-libraries#python) using Vertex SDK and [`gcloud ai`](https://cloud.google.com/sdk/gcloud/reference/beta/ai). + +## Notebooks + +|

Notebook

|

Description

| +| :-------- | :------- | +| [pytorch-text-classification-vertex-ai-deploy.ipynb](./pytorch-text-classification-vertex-ai-deploy.ipynb) | Notebook to show deploying a PyTorch model on Vertex AI | + +## Folders + + +|

Folder Name

|

Description

| +| :-------- | :------- | +| [`predictor`](./predictor) | Folder with custom prediction handler to deploy a PyTorch model to Vertex Prediction. In the [notebook](./pytorch-text-classification-vertex-ai-deploy.ipynb), this folder is used for deploying a PyTorch model on Vertex AI using Vertex Prediction pre-built PyTorch images | diff --git a/community-content/pytorch_pre_built_images_deployment/predictor/custom_handler.py b/community-content/pytorch_pre_built_images_deployment/predictor/custom_handler.py new file mode 100644 index 000000000..fba726737 --- /dev/null +++ b/community-content/pytorch_pre_built_images_deployment/predictor/custom_handler.py @@ -0,0 +1,91 @@ + +import os +import json +import logging + +import torch +from transformers import AutoModelForSequenceClassification, AutoTokenizer +from ts.torch_handler.base_handler import BaseHandler + +logger = logging.getLogger(__name__) + + +class TransformersClassifierHandler(BaseHandler): + """ + The handler takes an input string and returns the classification text + based on the serialized transformers checkpoint. + """ + def __init__(self): + super(TransformersClassifierHandler, self).__init__() + self.initialized = False + + def initialize(self, ctx): + """ Loads the model.pt file and initialized the model object. + Instantiates Tokenizer for preprocessor to use + Loads labels to name mapping file for post-processing inference response + """ + self.manifest = ctx.manifest + + properties = ctx.system_properties + model_dir = properties.get("model_dir") + self.device = torch.device("cuda:" + str(properties.get("gpu_id")) if torch.cuda.is_available() else "cpu") + + # Read model serialize/pt file + serialized_file = self.manifest["model"]["serializedFile"] + model_pt_path = os.path.join(model_dir, serialized_file) + if not os.path.isfile(model_pt_path): + raise RuntimeError("Missing the model.pt or pytorch_model.bin file") + + # Load model + self.model = AutoModelForSequenceClassification.from_pretrained(model_dir) + self.model.to(self.device) + self.model.eval() + logger.debug('Transformer model from path {0} loaded successfully'.format(model_dir)) + + # Ensure to use the same tokenizer used during training + self.tokenizer = AutoTokenizer.from_pretrained('bert-base-cased') + + # Read the mapping file, index to object name + mapping_file_path = os.path.join(model_dir, "index_to_name.json") + + if os.path.isfile(mapping_file_path): + with open(mapping_file_path) as f: + self.mapping = json.load(f) + else: + logger.warning('Missing the index_to_name.json file. Inference output will default.') + self.mapping = {"0": "Negative", "1": "Positive"} + + self.initialized = True + + def preprocess(self, data): + """ Preprocessing input request by tokenizing + Extend with your own preprocessing steps as needed + """ + text = data[0].get("data") + if text is None: + text = data[0].get("body") + sentences = text.decode('utf-8') + logger.info("Received text: '%s'", sentences) + + # Tokenize the texts + tokenizer_args = ((sentences,)) + inputs = self.tokenizer(*tokenizer_args, + padding='max_length', + max_length=128, + truncation=True, + return_tensors = "pt") + return inputs + + def inference(self, inputs): + """ Predict the class of a text using a trained transformer model. + """ + prediction = self.model(inputs['input_ids'].to(self.device))[0].argmax().item() + + if self.mapping: + prediction = self.mapping[str(prediction)] + + logger.info("Model predicted: '%s'", prediction) + return [prediction] + + def postprocess(self, inference_output): + return inference_output diff --git a/community-content/pytorch_pre_built_images_deployment/predictor/index_to_name.json b/community-content/pytorch_pre_built_images_deployment/predictor/index_to_name.json new file mode 100644 index 000000000..89b1004b2 --- /dev/null +++ b/community-content/pytorch_pre_built_images_deployment/predictor/index_to_name.json @@ -0,0 +1,5 @@ + +{ + "0": "Negative", + "1": "Positive" +} diff --git a/community-content/pytorch_pre_built_images_deployment/pytorch-text-classification-vertex-ai-deploy.ipynb b/community-content/pytorch_pre_built_images_deployment/pytorch-text-classification-vertex-ai-deploy.ipynb new file mode 100644 index 000000000..614241465 --- /dev/null +++ b/community-content/pytorch_pre_built_images_deployment/pytorch-text-classification-vertex-ai-deploy.ipynb @@ -0,0 +1,1625 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "46d6c164544f" + }, + "outputs": [], + "source": [ + "# Copyright 2022 Google LLC\n", + "#\n", + "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rEJBSTyZIrIb" + }, + "source": [ + "# Deploying a PyTorch Text Classification Model on [Vertex AI](https://cloud.google.com/vertex-ai)\n", + "\n", + "**This is an Experimental release**, covered by the Pre-GA Offerings Terms of your Google Cloud Platform [Terms of Service](https://cloud.google.com/terms).\n", + "\n", + "Experiments are focused on validating a prototype and are not guaranteed to be released. They are not intended for production use or covered by any SLA, support obligation, or deprecation policy and might be subject to backward-incompatible changes.\n", + "\n", + "**Kindly drop us a note before you run any scale tests.**\n", + "\n", + "**Do not hesitate to contact vertexai-prediction-preview-feedback@google.com if you have any questions or run into any issues.**\n", + "\n", + "The usage of the product is free during the Experimental release period: you will still incur charges for other GCP products usage, such as storage.\n", + "\n", + "The projects need to be added to the allowlist in order to deploy PyTorch models using Vertex AI Prediction pre-built PyTorch images. If you are interested in the feature, please send an email to vertexai-prediction-preview-feedback@google.com to provide your project numbers and project ids." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "X4cRE8IbIrIV" + }, + "source": [ + "# Overview\n", + "\n", + "This example focuses on how to deploy a PyTorch text classification model on Vertex AI using Vertex AI Prediction pre-built PyTorch images. This example assumes you have already had a trained model in the GCS uri. If you have not trained a PyTorch text classification model yet, you can follow [this notebook](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/community-content/pytorch_text_classification_using_vertex_sdk_and_gcloud/pytorch-text-classification-vertex-ai-train-tune-deploy.ipynb) to train one.\n", + "\n", + "### Objective\n", + "\n", + "How to **Deploy PyTorch models on [Vertex AI](https://cloud.google.com/vertex-ai)** and emphasize the support for deploying PyTorch models on Vertex AI. In this notebook, you won't focus on support for training PyTorch models on Vertex AI. Check out [the notebooks](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/community-content/pytorch_text_classification_using_vertex_sdk_and_gcloud/pytorch-text-classification-vertex-ai-train-tune-deploy.ipynb) to learn more about support for training on Vertex AI.\n", + "\n", + "### Table of Contents\n", + "\n", + "This notebook covers following sections:\n", + "\n", + "- [Creating Notebooks instance](#Creating-Notebooks-instance-on-Google-Cloud)\n", + "- [Deploying](#Deploying)\n", + " - [Deploying model on Vertex AI Predictions with custom container](#Deploying-the-pre-built-PyTorch-container-to-Vertex-AI-Predictions)\n", + "\n", + "### Costs \n", + "\n", + "This tutorial uses billable components of Google Cloud Platform (GCP):\n", + "\n", + "* [Vertex AI Workbench](https://cloud.google.com/vertex-ai-workbench)\n", + "* [Vertex AI Predictions](https://cloud.google.com/vertex-ai/docs/predictions/getting-predictions)\n", + "* [Cloud Storage](https://cloud.google.com/storage)\n", + "\n", + "Learn about [Vertex AI Pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage Pricing](https://cloud.google.com/storage/pricing), and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "5178273783dd" + }, + "source": [ + "### Set up your local development environment\n", + "\n", + "**If you are using Colab or Google Cloud Notebooks**, your environment already meets\n", + "all the requirements to run this notebook. You can skip this step." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "1a1ead2cb5e3" + }, + "source": [ + "**Otherwise**, make sure your environment meets this notebook's requirements.\n", + "You need the following:\n", + "\n", + "* The Google Cloud SDK\n", + "* Git\n", + "* Python 3\n", + "* virtualenv\n", + "* Jupyter notebook running in a virtual environment with Python 3\n", + "\n", + "The Google Cloud guide to [Setting up a Python development\n", + "environment](https://cloud.google.com/python/setup) and the [Jupyter\n", + "installation guide](https://jupyter.org/install) provide detailed instructions\n", + "for meeting these requirements. The following steps provide a condensed set of\n", + "instructions:\n", + "\n", + "1. [Install and initialize the Cloud SDK.](https://cloud.google.com/sdk/docs/)\n", + "1. [Install Python 3.](https://cloud.google.com/python/setup#installing_python)\n", + "1. [Install virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv) and create a virtual environment that uses Python 3. Activate the virtual environment.\n", + "1. To install Jupyter, run `pip3 install jupyter` on the command-line in a terminal shell.\n", + "1. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n", + "1. Open this notebook in the Jupyter Notebook Dashboard." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "9a864c22307c" + }, + "source": [ + "### Install additional packages\n", + "\n", + "Python dependencies required for this notebook are [Torch](https://pypi.org/project/torch/) and [Torch Model Archiver](https://pypi.org/project/torch-model-archiver/) which will be installed in the Notebooks instance itself." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "1fd00fa70a2a" + }, + "outputs": [], + "source": [ + "import os\n", + "\n", + "# The Google Cloud Notebook product has specific requirements\n", + "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n", + "\n", + "# Google Cloud Notebook requires dependencies to be installed with '--user'\n", + "USER_FLAG = \"\"\n", + "if IS_GOOGLE_CLOUD_NOTEBOOK:\n", + " USER_FLAG = \"--user\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "2974254ea6be" + }, + "outputs": [], + "source": [ + "!pip install {USER_FLAG} --upgrade torch==1.11" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "e0c1dcadc2c8" + }, + "source": [ + "You will be using [Vertex AI SDK for Python](https://cloud.google.com/vertex-ai/docs/start/client-libraries#python) to interact with Vertex AI services. The high-level `aiplatform` library is designed to simplify common data science workflows by using wrapper classes and opinionated defaults. \n", + "\n", + "#### Install Vertex AI SDK for Python" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ab3fb1551abe" + }, + "outputs": [], + "source": [ + "!pip install {USER_FLAG} --upgrade google-cloud-aiplatform[prediction]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "670e3ada279c" + }, + "source": [ + "#### Install Torch Model Archiver\n", + "This is required if you want to deploy the trained model to Vertex AI using Vertex AI Prediction pre-built PyTorch images." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "3eb0c1813278" + }, + "outputs": [], + "source": [ + "!pip install {USER_FLAG} --upgrade torch-model-archiver" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "f361541eff05" + }, + "source": [ + "### Restart the Kernel\n", + "\n", + "After you install the additional packages, you need to restart the notebook kernel so it can find the packages." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "2d77a223d63d" + }, + "outputs": [], + "source": [ + "# Automatically restart kernel after installs\n", + "import os\n", + "\n", + "if not os.getenv(\"IS_TESTING\"):\n", + " # Automatically restart kernel after installs\n", + " import IPython\n", + "\n", + " app = IPython.Application.instance()\n", + " app.kernel.do_shutdown(True)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "9bebb3b46278" + }, + "source": [ + "## Before you begin" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "e1e4d8f0c294" + }, + "source": [ + "### Set up your Google Cloud project\n", + "\n", + "**The following steps are required, regardless of your notebook environment.**\n", + "\n", + "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n", + "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n", + "1. Enable following APIs in your project required for running the tutorial\n", + " - [Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com)\n", + " - [Cloud Storage API](https://console.cloud.google.com/flows/enableapi?apiid=storage.googleapis.com)\n", + "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n", + "1. Enter your project ID in the cell below. Then run the cell to make sure the Cloud SDK uses the right project for all the commands in this notebook.\n", + "\n", + "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "36a4450b7c2e" + }, + "source": [ + "#### Set your project ID\n", + "\n", + "**If you don't know your project ID**, you may be able to get your project ID using `gcloud` or `google.auth`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "019e546007a3" + }, + "outputs": [], + "source": [ + "PROJECT_ID = \"[your-project-id]\" # <---CHANGE THIS TO YOUR PROJECT\n", + "\n", + "import os\n", + "\n", + "# Get your Google Cloud project ID using google.auth\n", + "if not os.getenv(\"IS_TESTING\"):\n", + " import google.auth\n", + "\n", + " _, PROJECT_ID = google.auth.default()\n", + " print(\"Project ID: \", PROJECT_ID)\n", + "\n", + "# validate PROJECT_ID\n", + "if PROJECT_ID == \"\" or not PROJECT_ID or PROJECT_ID == \"[your-project-id]\":\n", + " print(\n", + " f\"Please set your project id before proceeding to next step. Currently it's set as {PROJECT_ID}\"\n", + " )" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "0c9906f72b18" + }, + "source": [ + "#### Timestamp\n", + "\n", + "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append it onto the name of resources you create in this tutorial." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "e90182316f63" + }, + "outputs": [], + "source": [ + "from datetime import datetime\n", + "\n", + "\n", + "def get_timestamp():\n", + " return datetime.now().strftime(\"%Y%m%d%H%M%S\")\n", + "\n", + "\n", + "TIMESTAMP = get_timestamp()\n", + "print(f\"TIMESTAMP = {TIMESTAMP}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "4d96aec55eba" + }, + "source": [ + "### Authenticate your Google Cloud account\n", + "\n", + "---\n", + "\n", + "**If you are using Google Cloud Notebooks**, your environment is already authenticated. Skip this step.\n", + "\n", + "---" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "82f63fb3401f" + }, + "source": [ + "**If you are using Colab**, run the cell below and follow the instructions\n", + "when prompted to authenticate your account via oAuth.\n", + "\n", + "**Otherwise**, follow these steps:\n", + "\n", + "1. In the Cloud Console, go to the [**Create service account key** page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n", + "2. Click **Create service account**.\n", + "3. In the **Service account name** field, enter a name, and click **Create**.\n", + "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\" into the filter box, and select **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n", + "5. Click *Create*. A JSON file that contains your key downloads to your local environment.\n", + "6. Enter the path to your service account key as the `GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "535223fa4b84" + }, + "outputs": [], + "source": [ + "import os\n", + "import sys\n", + "\n", + "# If you are running this notebook in Colab, run this cell and follow the\n", + "# instructions to authenticate your GCP account. This provides access to your\n", + "# Cloud Storage bucket and lets you submit training jobs and prediction\n", + "# requests.\n", + "\n", + "# The Google Cloud Notebook product has specific requirements\n", + "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n", + "\n", + "# If on Google Cloud Notebooks, then don't execute this code\n", + "if not IS_GOOGLE_CLOUD_NOTEBOOK:\n", + " if \"google.colab\" in sys.modules:\n", + " from google.colab import auth as google_auth\n", + "\n", + " google_auth.authenticate_user()\n", + "\n", + " # If you are running this notebook locally, replace the string below with the\n", + " # path to your service account key and run this cell to authenticate your GCP\n", + " # account.\n", + " elif not os.getenv(\"IS_TESTING\"):\n", + " %env GOOGLE_APPLICATION_CREDENTIALS ''" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "35094e21d888" + }, + "source": [ + "### Create a Cloud Storage bucket\n", + "\n", + "**The following steps are required, regardless of your notebook environment.**\n", + "\n", + "Set the name of your Cloud Storage bucket below. It must be unique across all Cloud Storage buckets.\n", + "\n", + "You may also change the `REGION` variable, which is used for operations throughout the rest of this notebook. Make sure to [choose a region where Vertex AI services are available](https://cloud.google.com/vertex-ai/docs/general/locations#available_regions). You may not use a Multi-Regional Storage bucket for prediction with Vertex AI." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "e07102312039" + }, + "outputs": [], + "source": [ + "BUCKET_NAME = \"gs://[your-bucket-name]\" # <---CHANGE THIS TO YOUR BUCKET\n", + "REGION = \"us-central1\" # @param {type:\"string\"}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "13b11a8299d6" + }, + "outputs": [], + "source": [ + "if BUCKET_NAME == \"\" or not BUCKET_NAME or BUCKET_NAME == \"gs://[your-bucket-name]\":\n", + " BUCKET_NAME = f\"gs://{PROJECT_ID}aip-{get_timestamp()}\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "4908f26b84be" + }, + "outputs": [], + "source": [ + "print(f\"PROJECT_ID = {PROJECT_ID}\")\n", + "print(f\"BUCKET_NAME = {BUCKET_NAME}\")\n", + "print(f\"REGION = {REGION}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "6a3aae29644f" + }, + "source": [ + "---\n", + "\n", + "**Only if your bucket doesn't already exist**: Run the following cell to create your Cloud Storage bucket.\n", + "\n", + "---" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "25f9882bab87" + }, + "outputs": [], + "source": [ + "! gsutil mb -l $REGION $BUCKET_NAME" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "2586e4ed72ad" + }, + "source": [ + "Finally, validate access to your Cloud Storage bucket by examining its contents:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "315724257beb" + }, + "outputs": [], + "source": [ + "! gsutil ls -al $BUCKET_NAME" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "a4685723ceaf" + }, + "source": [ + "## Deploying\n", + "\n", + "There is no need to create a custom container to deploy a PyTorch model on [Vertex AI](https://cloud.google.com/vertex-ai/docs/predictions/getting-predictions) that serves online predictions. You deploy a Vertex AI Prediction pre-built PyTorch container in order to serve predictions from a fine-tuned transformer model from Hugging Face Transformers for sentiment analysis task. You can then use Vertex AI to classify sentiment of input texts. \n", + "\n", + "Essentially, to deploy a PyTorch model using Vertex AI Prediction pre-built PyTorch images on Vertex AI following are the steps:\n", + "\n", + "1. Package the trained model artifacts including [default](https://pytorch.org/serve/#default-handlers) or [custom](https://pytorch.org/serve/custom_service.html) handlers by creating an archive file using [Torch model archiver](https://github.com/pytorch/serve/tree/master/model-archiver).\n", + "2. Run the pre-built PyTorch image locally with the model artifacts (optionally).\n", + "3. Upload the model with the pre-built PyTorch image to serve predictions as a Vertex AI Model resource.\n", + "4. Create a Vertex AI Endpoint and [deploy the model](https://cloud.google.com/vertex-ai/docs/predictions/deploy-model-api) resource." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "a36e65dd4489" + }, + "source": [ + "#### **Download model artifacts**\n", + "\n", + "Download model artifacts that were saved as part of the training (or hyperparameter tuning) job from Cloud Storage to local directory. Follow [this notebook](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/community-content/pytorch_text_classification_using_vertex_sdk_and_gcloud/pytorch-text-classification-vertex-ai-train-tune-deploy.ipynb) if you haven't had a trained PyTorch text classification model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "255298dbbf51" + }, + "outputs": [], + "source": [ + "GCS_TRAINED_MODEL_URI = \"gs://[your-gcs-path]\" # <---CHANGE THIS TO YOUR GCS PATH THAT CONTAINS MODEL ARTIFACTS" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "c73d4ab771e3" + }, + "source": [ + "Validate model artifact files in the Cloud Storage bucket." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "2b30b1855c3c" + }, + "outputs": [], + "source": [ + "!gsutil ls -r $GCS_TRAINED_MODEL_URI" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "17495d36914d" + }, + "source": [ + "Copy files from Cloud Storage to local directory." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "c814195fe41f" + }, + "outputs": [], + "source": [ + "!mkdir trained_model\n", + "!gsutil -m cp -r $GCS_TRAINED_MODEL_URI/ ./trained_model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "a3e25889f240" + }, + "outputs": [], + "source": [ + "!ls -ltrR ./trained_model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "aa1484354072" + }, + "outputs": [], + "source": [ + "LOCAL_TRAINED_MODEL_DIRECTORY = \"[your-local-directory]\" # <---CHANGE THIS TO YOUR LOCAL DIRECTORY THAT CONTAINS MODEL ARTIFACTS" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "4c931da77e57" + }, + "source": [ + "#### **Create a custom model handler to handle prediction requests**\n", + "\n", + "When predicting sentiments of the input text with the fine-tuned transformer model, it requires pre-processing of the input text and post-processing by adding name (positive/negative) to the target label (1/0) along with probability (or confidence). You create a custom handler script that is packaged with the model artifacts and the pre-built PyTorch image executes the code when it runs. \n", + "\n", + "Custom handler script does the following:\n", + "\n", + "- Pre-process input text before sending it to the model for inference\n", + "- Customize how the model is invoked for inference\n", + "- Post-process output from the model before sending back a response\n", + "\n", + "Please refer to the [TorchServe documentation](https://pytorch.org/serve/custom_service.html) for defining a custom handler." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "219778afb9ad" + }, + "outputs": [], + "source": [ + "PREDICTOR_DIRECTORY = \"./predictor\"\n", + "\n", + "!mkdir $PREDICTOR_DIRECTORY" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "5ceb881740d6" + }, + "outputs": [], + "source": [ + "%%writefile $PREDICTOR_DIRECTORY/custom_handler.py\n", + "\n", + "import os\n", + "import json\n", + "import logging\n", + "\n", + "import torch\n", + "from transformers import AutoModelForSequenceClassification, AutoTokenizer\n", + "from ts.torch_handler.base_handler import BaseHandler\n", + "\n", + "logger = logging.getLogger(__name__)\n", + "\n", + "\n", + "class TransformersClassifierHandler(BaseHandler):\n", + " \"\"\"\n", + " The handler takes an input string and returns the classification text \n", + " based on the serialized transformers checkpoint.\n", + " \"\"\"\n", + " def __init__(self):\n", + " super(TransformersClassifierHandler, self).__init__()\n", + " self.initialized = False\n", + "\n", + " def initialize(self, ctx):\n", + " \"\"\" Loads the model.pt file and initialized the model object.\n", + " Instantiates Tokenizer for preprocessor to use\n", + " Loads labels to name mapping file for post-processing inference response\n", + " \"\"\"\n", + " self.manifest = ctx.manifest\n", + "\n", + " properties = ctx.system_properties\n", + " model_dir = properties.get(\"model_dir\")\n", + " self.device = torch.device(\"cuda:\" + str(properties.get(\"gpu_id\")) if torch.cuda.is_available() else \"cpu\")\n", + "\n", + " # Read model serialize/pt file\n", + " serialized_file = self.manifest[\"model\"][\"serializedFile\"]\n", + " model_pt_path = os.path.join(model_dir, serialized_file)\n", + " if not os.path.isfile(model_pt_path):\n", + " raise RuntimeError(\"Missing the model.pt or pytorch_model.bin file\")\n", + " \n", + " # Load model\n", + " self.model = AutoModelForSequenceClassification.from_pretrained(model_dir)\n", + " self.model.to(self.device)\n", + " self.model.eval()\n", + " logger.debug('Transformer model from path {0} loaded successfully'.format(model_dir))\n", + " \n", + " # Ensure to use the same tokenizer used during training\n", + " self.tokenizer = AutoTokenizer.from_pretrained('bert-base-cased')\n", + "\n", + " # Read the mapping file, index to object name\n", + " mapping_file_path = os.path.join(model_dir, \"index_to_name.json\")\n", + "\n", + " if os.path.isfile(mapping_file_path):\n", + " with open(mapping_file_path) as f:\n", + " self.mapping = json.load(f)\n", + " else:\n", + " logger.warning('Missing the index_to_name.json file. Inference output will default.')\n", + " self.mapping = {\"0\": \"Negative\", \"1\": \"Positive\"}\n", + "\n", + " self.initialized = True\n", + "\n", + " def preprocess(self, data):\n", + " \"\"\" Preprocessing input request by tokenizing\n", + " Extend with your own preprocessing steps as needed\n", + " \"\"\"\n", + " text = data[0].get(\"data\")\n", + " if text is None:\n", + " text = data[0].get(\"body\")\n", + " sentences = text.decode('utf-8')\n", + " logger.info(\"Received text: '%s'\", sentences)\n", + "\n", + " # Tokenize the texts\n", + " tokenizer_args = ((sentences,))\n", + " inputs = self.tokenizer(*tokenizer_args,\n", + " padding='max_length',\n", + " max_length=128,\n", + " truncation=True,\n", + " return_tensors = \"pt\")\n", + " return inputs\n", + "\n", + " def inference(self, inputs):\n", + " \"\"\" Predict the class of a text using a trained transformer model.\n", + " \"\"\"\n", + " prediction = self.model(inputs['input_ids'].to(self.device))[0].argmax().item()\n", + "\n", + " if self.mapping:\n", + " prediction = self.mapping[str(prediction)]\n", + "\n", + " logger.info(\"Model predicted: '%s'\", prediction)\n", + " return [prediction]\n", + "\n", + " def postprocess(self, inference_output):\n", + " return inference_output\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "f32fe0d49feb" + }, + "source": [ + "##### **Generate target label to name file**\n", + "\n", + "In the custom handler, you refer to a mapping file between target labels and their meaningful names that will be used to format the prediction response. Here you are mapping target label \"0\" as \"Negative\" and \"1\" as \"Positive\". " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "1be04c0f8960" + }, + "outputs": [], + "source": [ + "%%writefile $PREDICTOR_DIRECTORY/index_to_name.json\n", + "\n", + "{\n", + " \"0\": \"Negative\", \n", + " \"1\": \"Positive\"\n", + "}" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "321a84b9bb79" + }, + "source": [ + "#### **Package the trained model artifacts**\n", + "\n", + "The pre-built PyTorch images require a model archive file using [Torch model archiver](https://github.com/pytorch/serve/tree/master/model-archiver)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "36cfb88bfbf7" + }, + "outputs": [], + "source": [ + "ARCHIVED_MODEL_PATH = \"./archived_model\"\n", + "\n", + "!mkdir $ARCHIVED_MODEL_PATH" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "efaaf4ca7a01" + }, + "outputs": [], + "source": [ + "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n", + "\n", + "# Google Cloud Notebook requires to add a path to find the installed torch-model-archiver\n", + "if IS_GOOGLE_CLOUD_NOTEBOOK:\n", + " os.environ[\"PATH\"] = f'{os.environ.get(\"PATH\")}:~/.local/bin'" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "14a0874e3bdd" + }, + "source": [ + "Package the trained model artifacts including [default](https://pytorch.org/serve/#default-handlers) or [custom](https://pytorch.org/serve/custom_service.html) handlers by creating an archive file using [Torch model archiver](https://github.com/pytorch/serve/tree/master/model-archiver). The pre-built PyTorch image requires the model archived file named as `model.mar` so you need to set the model-name as `model`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "e094d78adc31" + }, + "outputs": [], + "source": [ + "!torch-model-archiver -f \\\n", + " --model-name=model \\\n", + " --version=1.0 \\\n", + " --serialized-file=$LOCAL_TRAINED_MODEL_DIRECTORY/pytorch_model.bin \\\n", + " --handler=$PREDICTOR_DIRECTORY/custom_handler.py \\\n", + " --extra-files \"$LOCAL_TRAINED_MODEL_DIRECTORY/config.json,$LOCAL_TRAINED_MODEL_DIRECTORY/tokenizer.json,$LOCAL_TRAINED_MODEL_DIRECTORY/training_args.bin,$LOCAL_TRAINED_MODEL_DIRECTORY/tokenizer_config.json,$LOCAL_TRAINED_MODEL_DIRECTORY/special_tokens_map.json,$LOCAL_TRAINED_MODEL_DIRECTORY/vocab.txt,$PREDICTOR_DIRECTORY/index_to_name.json\" \\\n", + " --export-path=$ARCHIVED_MODEL_PATH" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "73e20aa31e0a" + }, + "source": [ + "#### **Run the pre-built PyTorch images locally (optionally)**\n", + "\n", + "Before you upload a model to Vertex AI, you can run the pre-built PyTorch images locally with the model file.\n", + "\n", + "Vertex AI Predictions have pre-built images in the different [Artifact Registry multi-regions](https://cloud.google.com/artifact-registry/docs/repositories/repo-locations). To run the images locally, you could use the image matching our target regions. In this example, you use PyTorch 1.11 on CPU so you can choose either of the images.\n", + "- us-docker.pkg.dev/vertex-ai/prediction/pytorch-cpu.1-11:latest\n", + "- europe-docker.pkg.dev/vertex-ai/prediction/pytorch-cpu.1-11:latest\n", + "- asia-docker.pkg.dev/vertex-ai/prediction/pytorch-cpu.1-11:latest\n", + "\n", + "Vertex AI provides [Vertex SDK](https://github.com/googleapis/python-aiplatform/tree/main/google/cloud/aiplatform/prediction) to help test images locally. You will use Vertex SDK to test the pre-built PyTorch images with the archived model file." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "c6fa027a2290" + }, + "source": [ + "Set up the logging config in the notebook." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "7cb48d5d0612" + }, + "outputs": [], + "source": [ + "import logging\n", + "\n", + "logging.basicConfig(level=logging.INFO)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "0746e14a4773" + }, + "source": [ + "Select the image uri matching the region that the models will be deployed to. For the details of Artifact Registry multi-regions, check out [the documentation](https://cloud.google.com/artifact-registry/docs/repositories/repo-locations)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "4ea9c6fccde1" + }, + "outputs": [], + "source": [ + "serving_container_image_uri = \"[your-multi-region]-docker.pkg.dev/vertex-ai/prediction/pytorch-cpu.1-11:latest\" # <---CHANGE THIS TO YOUR MULTI REGION, COULD BE `us`, `europe`, or `asia`" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "3719b0f0ad24" + }, + "source": [ + "Since you are using pre-built PyTorch images locally, you need to populate the necessary routes and ports. You do not need to populate routes or ports while you upload models to Vertex AI with pre-built PyTorch images." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "26c6b2076a78" + }, + "outputs": [], + "source": [ + "health_route = \"/ping\"\n", + "predict_route = \"/predictions/model\"\n", + "serving_container_ports = [8080]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "54bfff8c5303" + }, + "source": [ + "Create a local model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "1088e221624b" + }, + "outputs": [], + "source": [ + "from google.cloud.aiplatform.prediction import LocalModel\n", + "\n", + "local_model = LocalModel(\n", + " serving_container_image_uri=serving_container_image_uri,\n", + " serving_container_predict_route=predict_route,\n", + " serving_container_health_route=health_route,\n", + " serving_container_ports=serving_container_ports,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "5dd0bf0b1b3c" + }, + "source": [ + "Store test instances. To learn more about formatting input instances in JSON, [read the documentation](https://cloud.google.com/vertex-ai/docs/predictions/online-predictions-custom-models#request-body-details)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "8850305d7095" + }, + "outputs": [], + "source": [ + "INPUT_FILE = \"./instances.json\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "6ac646297f55" + }, + "outputs": [], + "source": [ + "%%bash\n", + "\n", + "cat > ./instances.json <\"\n", + " }\n", + " }\n", + "]\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "646e9dfe8add" + }, + "source": [ + "Define sample texts to test predictions" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "edc4658b2df1" + }, + "outputs": [], + "source": [ + "test_instances = [\n", + " b\"Jaw dropping visual affects and action! One of the best I have seen to date.\",\n", + " b\"Take away the CGI and the A-list cast and you end up with film with less punch.\",\n", + "]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "52303da7c048" + }, + "source": [ + "##### **Sending an online prediction request**\n", + "\n", + "Format input text string and call prediction endpoint with formatted input request and get the response" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "a5808801dbcb" + }, + "outputs": [], + "source": [ + "import base64\n", + "import json\n", + "\n", + "print(\"=\" * 100)\n", + "for instance in test_instances:\n", + " print(f\"Input text: \\n\\t{instance.decode('utf-8')}\\n\")\n", + " b64_encoded = base64.b64encode(instance)\n", + " test_instance = [{\"data\": {\"b64\": f\"{str(b64_encoded.decode('utf-8'))}\"}}]\n", + " print(f\"Formatted input: \\n{json.dumps(test_instance, indent=4)}\\n\")\n", + " prediction = endpoint.predict(instances=test_instance)\n", + " print(f\"Prediction response: \\n\\t{prediction}\")\n", + " print(\"=\" * 100)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "21664e69720a" + }, + "source": [ + "##### ***[Optional]*** **Make prediction requests using gcloud CLI**\n", + "You can also call the Vertex AI Endpoint to make predictions using [`gcloud beta ai endpoints predict`](https://cloud.google.com/sdk/gcloud/reference/beta/ai/endpoints/predict). \n", + "\n", + "The following cell shows how to make a prediction request to Vertex AI Endpoints using `gcloud` CLI: " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "950c04742e87" + }, + "outputs": [], + "source": [ + "%%bash -s $REGION $endpoint_display_name\n", + "\n", + "REGION=$1\n", + "endpoint_display_name=$2\n", + "\n", + "# get endpoint id\n", + "echo \"REGION = ${REGION}\"\n", + "echo \"ENDPOINT DISPLAY NAME = ${endpoint_display_name}\"\n", + "endpoint_id=$(gcloud beta ai endpoints list --region ${REGION} --filter \"display_name=${endpoint_display_name}\" --format \"value(ENDPOINT_ID)\")\n", + "echo \"ENDPOINT_ID = ${endpoint_id}\"\n", + "\n", + "# call prediction endpoint\n", + "input_text=\"Take away the CGI and the A-list cast and you end up with film with less punch.\"\n", + "echo \"INPUT TEXT = ${input_text}\"\n", + "\n", + "prediction=$(\n", + "echo \"\"\"\n", + "{ \n", + " \"instances\": [\n", + " { \n", + " \"data\": {\n", + " \"b64\": \"$(echo ${input_text} | base64 --wrap=0)\"\n", + " }\n", + " }\n", + " ]\n", + "}\n", + "\"\"\" | gcloud beta ai endpoints predict ${endpoint_id} --region=$REGION --json-request -)\n", + "\n", + "echo \"PREDICTION RESPONSE = ${prediction}\"" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "e636b52e5913" + }, + "source": [ + "## Cleaning up \n", + "\n", + "### Cleaning up training and deployment resources\n", + "\n", + "To clean up all Google Cloud resources used in this notebook, you can [delete the Google Cloud project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n", + "\n", + "Otherwise, you can delete the individual resources you created in this tutorial:\n", + "\n", + "- Model\n", + "- Endpoint\n", + "- Cloud Storage Bucket\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "7b89f5a348d1" + }, + "source": [ + "Set flags for the resource type to be deleted" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "26ee41d02418" + }, + "outputs": [], + "source": [ + "delete_endpoint = True\n", + "delete_model = True\n", + "delete_bucket = False" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "dc21008bb215" + }, + "source": [ + "#### **Undeploy models and Delete endpoints**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "51684df7dc6b" + }, + "outputs": [], + "source": [ + "if delete_endpoint:\n", + " endpoint.delete(force=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "4a6719f8f8a2" + }, + "source": [ + "#### **Deleting models**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "087a5322f387" + }, + "outputs": [], + "source": [ + "if delete_model:\n", + " model.delete()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "938a7cd172ab" + }, + "source": [ + "#### **Delete contents from the staging bucket**\n", + "\n", + "---\n", + "\n", + "***NOTE: Everything in this Cloud Storage bucket will be DELETED. Please run it with caution.***\n", + "\n", + "---" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "a50bd867c02d" + }, + "outputs": [], + "source": [ + "if delete_bucket and \"BUCKET_NAME\" in globals():\n", + " print(f\"Deleting all contents from the bucket {BUCKET_NAME}\")\n", + "\n", + " shell_output = ! gsutil du -as $BUCKET_NAME\n", + " print(\n", + " f\"Size of the bucket {BUCKET_NAME} before deleting = {shell_output[0].split()[0]} bytes\"\n", + " )\n", + "\n", + " # uncomment below line to delete contents of the bucket\n", + " # ! gsutil rm -r $BUCKET_NAME\n", + "\n", + " shell_output = ! gsutil du -as $BUCKET_NAME\n", + " if float(shell_output[0].split()[0]) > 0:\n", + " print(\n", + " \"PLEASE UNCOMMENT LINE TO DELETE BUCKET. CONTENT FROM THE BUCKET NOT DELETED\"\n", + " )" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "3541e1629c02" + }, + "source": [ + "### Cleaning up Notebook Environment\n", + "\n", + "After you are done experimenting, you can either [STOP](https://cloud.google.com/ai-platform/notebooks/docs/shut-down) or DELETE the AI Notebook instance to prevent any charges. If you want to save your work, you can choose to stop the instance instead.\n", + "\n", + "```\n", + "# Stopping Notebook instance\n", + "gcloud notebooks instances stop example-instance --location=us-central1-a\n", + "\n", + "\n", + "# Deleting Notebook instance\n", + "gcloud notebooks instances delete example-instance --location=us-central1-a\n", + "```" + ] + } + ], + "metadata": { + "colab": { + "collapsed_sections": [ + "n9qywopnIrJH", + "545PP3o8IrJV", + "7k8ge1L1IrJk" + ], + "name": "pytorch-text-classification-vertex-ai-deploy.ipynb", + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +}