From 7a15b8807136041f766ce0eebb0675a0f5e9ffc4 Mon Sep 17 00:00:00 2001 From: TJ-Liu Date: Fri, 28 Jun 2024 17:35:54 -0700 Subject: [PATCH] feat: Add notebook/colab example for prediction PSC based private (#3080) endpoint. Co-authored-by: TJ(Tianjiao) Liu --- notebooks/official/CODEOWNERS | 1 + ...et_started_with_psc_private_endpoint.ipynb | 727 ++++++++++++++++++ 2 files changed, 728 insertions(+) create mode 100644 notebooks/official/prediction/get_started_with_psc_private_endpoint.ipynb diff --git a/notebooks/official/CODEOWNERS b/notebooks/official/CODEOWNERS index c9a65af79..9db191f2c 100644 --- a/notebooks/official/CODEOWNERS +++ b/notebooks/official/CODEOWNERS @@ -55,3 +55,4 @@ /ray_on_vertex_ai/get_started_with_pytorch_rov.ipynb @inardini /training/tpuv5e_gemma_peft_finetuning_and_serving.ipynb @brianchunkang /training/tpuv5e_llama2_pytorch_finetuning_and_serving.ipynb @brianchunkang @chiefkarlin +/prediction/get_started_with_psc_private_endpoint.ipynb @tianjiaoliu diff --git a/notebooks/official/prediction/get_started_with_psc_private_endpoint.ipynb b/notebooks/official/prediction/get_started_with_psc_private_endpoint.ipynb new file mode 100644 index 000000000..f690cb633 --- /dev/null +++ b/notebooks/official/prediction/get_started_with_psc_private_endpoint.ipynb @@ -0,0 +1,727 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "UUTFwSh5u9OK" + }, + "outputs": [], + "source": [ + "# Copyright 2024 Google LLC\n", + "#\n", + "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "qsLff0QbdE8D" + }, + "source": [ + "# Online Prediction PSC based private endpint\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + "
\n", + " \n", + " \"Google
Open in Colab\n", + " \n", + " 		\n", + " \n", + " \"Google
Open in Colab Enterprise\n", + " \n", + " 		\n", + " \n", + " \"Vertex
Open in Workbench\n", + " \n", + " 		\n", + " \n", + " \"GitHub
View on GitHub\n", + " \n", + "
" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "0C7kTlrH1bO8" + }, + "source": [ + "## Overview\n", + "\n", + "Compared to the current PSA Private Endpoint, PSC based Private Endpoint has the following benefits:\n", + "1. Simpler setup process: Currently, the only extra step user need to do is to create an Endpoint in their VPC. And this will be done by PSC automatically before our GA launch.\n", + "\n", + "2. No more IP exhuasted issue: GKE cluster will be hosted in tenant project VPC, so we can create much bigger cluster and won't affected by ip exhuasted issue in User's VPC.\n", + "\n", + "3. Unified experience with public endpoint: The API is the same as public endpoint, so user can use our SDK/client library. We also provide quota, IAM and monitoring metrics as public endpoint does.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "D2sWuTr81v9m" + }, + "source": [ + "## Get started" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "RakMIliNYh8O" + }, + "source": [ + "### Install Vertex AI SDK for Python and other required packages" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "nSHmJT9cTggu" + }, + "outputs": [], + "source": [ + "! pip3 install --upgrade --user --quiet google-cloud-aiplatform" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "936Zz5YI2NeA" + }, + "source": [ + "### Restart runtime (Colab only)\n", + "\n", + "To use the newly installed packages, you must restart the runtime on Google Colab." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "66oJ55lG2Tiq" + }, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "if \"google.colab\" in sys.modules:\n", + "\n", + " import IPython\n", + "\n", + " app = IPython.Application.instance()\n", + " app.kernel.do_shutdown(True)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Clr61ben2WwY" + }, + "source": [ + "
\n", + "⚠️ The kernel is going to restart. Wait until it's finished before continuing to the next step. ⚠️\n", + "
\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "v848aGbn2acH" + }, + "source": [ + "### Authenticate your notebook environment (Colab only)\n", + "\n", + "Authenticate your environment on Google Colab.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "IVeoyQPz2cfh" + }, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "if \"google.colab\" in sys.modules:\n", + "\n", + " from google.colab import auth\n", + "\n", + " auth.authenticate_user()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "HId-ySlY2jlI" + }, + "source": [ + "### Set Google Cloud project information and initialize Vertex AI SDK for Python\n", + "\n", + "To get started using Vertex AI, you must have an existing Google Cloud project and [enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com). Learn more about [setting up a project and a development environment](https://cloud.google.com/vertex-ai/docs/start/cloud-environment)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Y4gnZI9OX6VJ" + }, + "outputs": [], + "source": [ + "PROJECT_ID = \"[your-project-id]\" # @param {type:\"string\"}\n", + "LOCATION = \"us-central1\" # @param {type:\"string\"}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "moS794OKaaCt" + }, + "outputs": [], + "source": [ + "# Create GCS Bucket\n", + "BUCKET_URI = \"gs://your-bucket-name-unique\" # @param {type:\"string\"}\n", + "! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "-NrpFROTjoVL" + }, + "outputs": [], + "source": [ + "from google.cloud import aiplatform\n", + "\n", + "aiplatform.init(project=PROJECT_ID, location=LOCATION, staging_bucket=BUCKET_URI)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "swERjzZ-a_Nd" + }, + "source": [ + "## Prepare Test Models\n", + "\n", + "We prepared some test models, feel free to use your own models." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "3PtFCQNHbloQ" + }, + "outputs": [], + "source": [ + "# Copy Models to the Bucket\n", + "! gsutil cp -r \"gs://cloud-samples-data/vertex-ai/prediction/test-models-requests/*\" {BUCKET_URI}" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "W7sbcii_iZ7x" + }, + "source": [ + "### Upload Model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "bhgBogbTgF_5" + }, + "outputs": [], + "source": [ + "# Depending on which model you wanna use, uncomment the corresponding section below and run the block.\n", + "\n", + "# TF Model\n", + "DISPLAY_NAME = \"tensorflow model\" # @param {type:\"string\"}\n", + "ARTIFACT_URI = BUCKET_URI + \"/tensorflow\"\n", + "IMAGE_URI = \"us-docker.pkg.dev/vertex-ai/prediction/tf2-cpu.2-12:latest\"\n", + "REQUEST_FILE = \"tensorflow_request.json\"\n", + "\n", + "\n", + "# Pytorch Model\n", + "# DISPLAY_NAME=\"Pytorch model\"\n", + "# ARTIFACT_URI=BUCKET_URI+\"/pytorch\"\n", + "# IMAGE_URI=\"us-docker.pkg.dev/vertex-ai/prediction/pytorch-cpu.2-0:latest\"\n", + "# REQUEST_FILE=\"pytorch_request.json\"\n", + "\n", + "\n", + "# Sklearn Model\n", + "# DISPLAY_NAME=\"Sklearn model\"\n", + "# ARTIFACT_URI=BUCKET_URI+\"/sklearn\"\n", + "# IMAGE_URI=\"us-docker.pkg.dev/vertex-ai/prediction/sklearn-cpu.1-2:latest\"\n", + "# REQUEST_FILE=\"sklearn_request.json\"\n", + "\n", + "\n", + "# xgboost Model\n", + "# DISPLAY_NAME=\"xgboost model\"\n", + "# ARTIFACT_URI=BUCKET_URI+\"/xgboost\"\n", + "# IMAGE_URI=\"us-docker.pkg.dev/vertex-ai/prediction/xgboost-cpu.1-7:latest\"\n", + "# REQUEST_FILE=\"xgboost_request.json\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "49Dak6icicSu" + }, + "outputs": [], + "source": [ + "model = aiplatform.Model.upload(\n", + " display_name=DISPLAY_NAME,\n", + " artifact_uri=ARTIFACT_URI,\n", + " serving_container_image_uri=IMAGE_URI,\n", + " sync=False,\n", + ")\n", + "\n", + "model.wait()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "JN2NkhEljbse" + }, + "source": [ + "### Create PSC based Prediction Private Endpoint\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "BqMtuRgPjqfD" + }, + "outputs": [], + "source": [ + "psc_endpoint = aiplatform.PrivateEndpoint.create(\n", + " display_name=\"psc-endpoint\",\n", + " project=PROJECT_ID,\n", + " location=LOCATION,\n", + " private_service_connect_config=aiplatform.PrivateEndpoint.PrivateServiceConnectConfig(\n", + " project_allowlist=[PROJECT_ID],\n", + " ),\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Mp-W67qKHIN6" + }, + "source": [ + "Alternatively, send http call to create endpoint. You need to manually replace ALL the variables below" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ellzpZ43jZVm" + }, + "outputs": [], + "source": [ + "# ! curl -X POST -H \"Content-Type: application/json\" -H \"Authorization: Bearer `gcloud auth print-access-token`\" https://${LOCATION}-aiplatform.googleapis.com/v1/projects/${PROJECT_ID}/locations/{LOCATION}/endpoints -d \\\n", + "# '{ \\\n", + "# displayName: \"psc-endpoint\", \\\n", + "# privateServiceConnectConfig: { \\\n", + "# enablePrivateServiceConnect: true, \\\n", + "# projectAllowlist: [\"{PROJECT_ID}\"] \\\n", + "# }, \\\n", + "# }'" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "USWCW-SNo-9M" + }, + "source": [ + "### Deploy Model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "S_eRJglhpVfL" + }, + "outputs": [], + "source": [ + "psc_endpoint.deploy(model=model, traffic_percentage=100, machine_type=\"e2-standard-8\")\n", + "\n", + "psc_endpoint.list_models()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "5uLFsbxpwzvN" + }, + "source": [ + "### Create Forwarding Rule in Consumer Project\n", + "\n", + "First, find the service attachment from the endpoint and deployed model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "EgjKUSAMnqvI" + }, + "outputs": [], + "source": [ + "service_attachment = psc_endpoint.list_models()[0].private_endpoints.service_attachment\n", + "print(service_attachment)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "5R2z2mUlMrl9" + }, + "source": [ + "Then, create an address and a forwarding rule targeting at the service attachment. In this example, default network and subnet are used, replace it with your VPC network and subnet if running in your VPC." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "EF7XCp0t1_AY" + }, + "outputs": [], + "source": [ + "! gcloud compute addresses create psc-prediction \\\n", + " --region={LOCATION} \\\n", + " --subnet=default\n", + "\n", + "! gcloud compute forwarding-rules create op-psc-endpoint \\\n", + " --network=default \\\n", + " --address=psc-prediction \\\n", + " --target-service-attachment={service_attachment} \\\n", + " --region={LOCATION}" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "oL-74S0kVkym" + }, + "source": [ + "Save the IP address above." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "bEtkfw1dTbvh" + }, + "outputs": [], + "source": [ + "IP_ADDRESS = ! gcloud compute forwarding-rules describe op-psc-endpoint --region={LOCATION} --format='value(IPAddress)'\n", + "IP_ADDRESS = IP_ADDRESS[0]\n", + "print(IP_ADDRESS)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "civyNQaPr4QD" + }, + "source": [ + "## Make Predictions\n", + "\n", + "From this point, all the code below must be run from a GCP VM in the same VPC, same region as your PSC Endpoint.\n", + "\n", + "If you're using Vertex AI Workbench or Colab Enterprise, you should be good.\n", + "\n", + "If you're creating a GCE VM, please make sure Cloud Platform access scope is enabled." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "9Y7Zr9hQuZxC" + }, + "outputs": [], + "source": [ + "# Download the requests files:\n", + "! gsutil cp {BUCKET_URI}/requests/* ./" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "I-O9U63juoWE" + }, + "outputs": [], + "source": [ + "import os\n", + "\n", + "if not os.getenv(\"IS_TESTING\"):\n", + " import json\n", + "\n", + " import urllib3\n", + "\n", + " urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)\n", + "\n", + " with open(REQUEST_FILE) as json_file:\n", + " data = json.load(json_file)\n", + " response = psc_endpoint.predict(\n", + " instances=data[\"instances\"], endpoint_override=IP_ADDRESS\n", + " )\n", + " print(response)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "U1sbaYPbueQc" + }, + "source": [ + "### Predict Requests" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ngiWwtuOM1PL" + }, + "source": [ + "Alternatively, you can send HTTP requests directly to the IP address. Make sure to replace all variabled in the requests" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "IjLZV-hZoNy2" + }, + "outputs": [], + "source": [ + "ENDPOINT_RESOURCE_NAME = psc_endpoint.resource_name" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "_V-Zz4N5tMev" + }, + "outputs": [], + "source": [ + "import os\n", + "\n", + "if not os.getenv(\"IS_TESTING\"):\n", + " # Predict\n", + " ! curl --insecure -H \"Content-Type: application/json\" -H \"Authorization: Bearer `gcloud auth print-access-token`\" https://{IP_ADDRESS}/v1/{ENDPOINT_RESOURCE_NAME}:predict -d@{REQUEST_FILE}\n", + "\n", + " # # RawPredict\n", + " # ! curl -v --insecure -H \"Content-Type: application/json\" -H \"Authorization: Bearer `gcloud auth print-access-token`\" https://{IP_ADDRESS}/v1/{ENDPOINT_RESOURCE_NAME}:rawPredict -d@{REQUEST_FILE}" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "_TCktMxbA4mb" + }, + "source": [ + "### Deploy another model and update traffic split\n", + "\n", + "Deploy another model, and update the traffic split to be 50:50, after the deployment is done, you can rerun the prediction again for multiple times, you should be able to see the deployed_model_id are different." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "bVLgpRiRBEL7" + }, + "outputs": [], + "source": [ + "psc_endpoint.deploy(model=model, traffic_percentage=50, machine_type=\"e2-standard-8\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "2dx975IkCv7v" + }, + "outputs": [], + "source": [ + "import os\n", + "\n", + "if not os.getenv(\"IS_TESTING\"):\n", + " import json\n", + "\n", + " import urllib3\n", + "\n", + " urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)\n", + "\n", + " counter = {}\n", + " with open(REQUEST_FILE) as json_file:\n", + " data = json.load(json_file)\n", + " for i in range(1000):\n", + " response = psc_endpoint.predict(\n", + " instances=data[\"instances\"], endpoint_override=IP_ADDRESS\n", + " )\n", + " if response.deployed_model_id in counter.keys():\n", + " counter[response.deployed_model_id] += 1\n", + " else:\n", + " counter[response.deployed_model_id] = 1\n", + " print(counter)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "2Hld7iDmEyiF" + }, + "source": [ + "You can update the traffic split with the following command and run the code above again." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "HXPI-2q9Eh6X" + }, + "outputs": [], + "source": [ + "import os\n", + "\n", + "if not os.getenv(\"IS_TESTING\"):\n", + " deployed_model_id_0 = list(counter)[0]\n", + " deployed_model_id_1 = list(counter)[1]\n", + "\n", + " psc_endpoint.update(\n", + " traffic_split={deployed_model_id_0: 20, deployed_model_id_1: 80}\n", + " )" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "XW_BtPnEFPp4" + }, + "source": [ + "## Cleanup" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "E4Ik3eKqdI_2" + }, + "outputs": [], + "source": [ + "psc_endpoint.undeploy_all()\n", + "psc_endpoint.delete()\n", + "model.delete()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "JRncavA6FSlc" + }, + "outputs": [], + "source": [ + "! gcloud compute forwarding-rules delete op-psc-endpoint --region={LOCATION} --quiet\n", + "\n", + "! gcloud compute addresses delete psc-prediction --region={LOCATION} --quiet" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "BTz-2N1XunXB" + }, + "source": [ + "Delete the bucket if needed." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "NPQT5Wv9lC3O" + }, + "outputs": [], + "source": [ + "! gsutil rm -r {BUCKET_URI}" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "2iR_Q5K_ksWi" + }, + "source": [ + "Optionally, you can use the following command to clean up all private endpoint and models if needed." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "vkyvlwTgky0J" + }, + "outputs": [], + "source": [ + "for pe in aiplatform.PrivateEndpoint.list():\n", + " pe.undeploy_all()\n", + " pe.delete()" + ] + } + ], + "metadata": { + "colab": { + "name": "get_started_with_psc_private_endpoint.ipynb", + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +}